alexvanin/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

march: Fix excessive parallelism when using --no-traverse

Browse source 
When using `--no-traverse` the march routines call NewObject on each
potential object in the destination.

The concurrency limiter was accidentally arranged so that there were
`--checkers` * `--checkers` NewObject calls going on at once.

This became obvious when using the sftp backend which used too many
connections.

Fixes #5824
This commit is contained in:
Nick Craig-Wood 2023-10-10 18:03:56 +01:00
parent c6755aa768
commit 88bd80c1fa
1 changed files with 7 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
fs/march/march.go
View file

@ -35,6 +35,7 @@ type March struct {
srcListDir listDirFn // function to call to list a directory in the src srcListDir listDirFn // function to call to list a directory in the src
dstListDir listDirFn // function to call to list a directory in the dst dstListDir listDirFn // function to call to list a directory in the dst
transforms []matchTransformFn transforms []matchTransformFn
limiter chan struct{} // make sure we don't do too many operations at once
} }
// Marcher is called on each match // Marcher is called on each match
@ -69,6 +70,8 @@ func (m *March) init(ctx context.Context) {
if m.Fdst.Features().CaseInsensitive || ci.IgnoreCaseSync { if m.Fdst.Features().CaseInsensitive || ci.IgnoreCaseSync {
m.transforms = append(m.transforms, strings.ToLower) m.transforms = append(m.transforms, strings.ToLower)
} }
// Limit parallelism for operations
m.limiter = make(chan struct{}, ci.Checkers)
} }
// list a directory into entries, err // list a directory into entries, err
@ -429,13 +432,11 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
// If NoTraverse is set, then try to find a matching object // If NoTraverse is set, then try to find a matching object
// for each item in the srcList to head dst object // for each item in the srcList to head dst object
ci := fs.GetConfig(m.Ctx)
limiter := make(chan struct{}, ci.Checkers)
if m.NoTraverse && !m.NoCheckDest { if m.NoTraverse && !m.NoCheckDest {
for _, src := range srcList { for _, src := range srcList {
wg.Add(1) wg.Add(1)
limiter <- struct{}{} m.limiter <- struct{}{}
go func(limiter chan struct{}, src fs.DirEntry) { go func(src fs.DirEntry) {
defer wg.Done() defer wg.Done()
if srcObj, ok := src.(fs.Object); ok { if srcObj, ok := src.(fs.Object); ok {
leaf := path.Base(srcObj.Remote()) leaf := path.Base(srcObj.Remote())
@ -446,8 +447,8 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
mu.Unlock() mu.Unlock()
} }
} }
<-limiter <-m.limiter
}(limiter, src) }(src)
} }
wg.Wait() wg.Wait()
} }