Use --fast-list flag for sync/copy/move - fixes #1277

Redo test framework to take a -fast-list flag and test remotes with that flag.

fs/operations_test.go

@@ -52,6 +52,7 @@ var (
 	DumpBodies      = flag.Bool("dump-bodies", false, "Set to dump bodies (needs -verbose)")
 	Individual      = flag.Bool("individual", false, "Make individual bucket/container/directory for each test - much slower")
 	LowLevelRetries = flag.Int("low-level-retries", 10, "Number of low level retries")
+	UseListR        = flag.Bool("fast-list", false, "Use recursive list if available. Uses more memory but fewer transactions.")
 )
 
 // Some times used in the tests
@@ -113,6 +114,7 @@ func newRun() *Run {
 	fs.Config.DumpHeaders = *DumpHeaders
 	fs.Config.DumpBodies = *DumpBodies
 	fs.Config.LowLevelRetries = *LowLevelRetries
+	fs.Config.UseListR = *UseListR
 	var err error
 	r.fremote, r.fremoteName, r.cleanRemote, err = fstest.RandomRemote(*RemoteName, *SubDir)
 	if err != nil {

fs/sync.go

@@ -49,6 +49,8 @@ type syncCopyMove struct {
 	renameCheck    []Object            // accumulate files to check for rename here
 	backupDir      Fs                  // place to store overwrites/deletes
 	suffix         string              // suffix to add to files placed in backupDir
+	srcListDir     listDirFn           // function to call to list a directory in the src
+	dstListDir     listDirFn           // function to call to list a directory in the dst
 }
 
 func newSyncCopyMove(fdst, fsrc Fs, deleteMode DeleteMode, DoMove bool) (*syncCopyMove, error) {
@@ -117,9 +119,47 @@ func newSyncCopyMove(fdst, fsrc Fs, deleteMode DeleteMode, DoMove bool) (*syncCo
 		}
 		s.suffix = Config.Suffix
 	}
+	s.srcListDir = s.makeListDir(fsrc, false)
+	s.dstListDir = s.makeListDir(fdst, Config.Filter.DeleteExcluded)
 	return s, nil
 }
 
+// list a directory into entries, err
+type listDirFn func(dir string) (entries DirEntries, err error)
+
+// makeListDir makes a listing function for the given fs and includeAll flags
+func (s *syncCopyMove) makeListDir(f Fs, includeAll bool) listDirFn {
+	if !Config.UseListR || f.Features().ListR == nil {
+		return func(dir string) (entries DirEntries, err error) {
+			return ListDirSorted(f, includeAll, dir)
+		}
+	}
+	var (
+		mu      sync.Mutex
+		started bool
+		dirs    DirTree
+		dirsErr error
+	)
+	return func(dir string) (entries DirEntries, err error) {
+		mu.Lock()
+		defer mu.Unlock()
+		if !started {
+			dirs, dirsErr = NewDirTree(f, s.dir, includeAll, Config.MaxDepth)
+			started = true
+		}
+		if dirsErr != nil {
+			return nil, dirsErr
+		}
+		entries, ok := dirs[dir]
+		if !ok {
+			err = ErrorDirNotFound
+		} else {
+			delete(dirs, dir)
+		}
+		return entries, err
+	}
+}
+
 // Check to see if have set the abort flag
 func (s *syncCopyMove) aborting() bool {
 	select {
@@ -985,14 +1025,14 @@ func (s *syncCopyMove) _runDirAtATime(job listDirJob) (jobs []listDirJob) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			srcList, srcListErr = ListDirSorted(s.fsrc, false, job.remote)
+			srcList, srcListErr = s.srcListDir(job.remote)
 		}()
 	}
 	if !job.noDst {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			dstList, dstListErr = ListDirSorted(s.fdst, Config.Filter.DeleteExcluded, job.remote)
+			dstList, dstListErr = s.dstListDir(job.remote)
 		}()
 	}
 

fs/test_all.go

@@ -20,22 +20,84 @@ import (
 	"github.com/ncw/rclone/fstest"
 )
 
+type remoteConfig struct {
+	Name     string
+	SubDir   bool
+	FastList bool
+}
+
 var (
-	remotes = []string{
+	remotes = []remoteConfig{
-		"TestAmazonCloudDrive:",
+		{
-		"TestB2:",
+			Name:     "TestAmazonCloudDrive:",
-		"TestCryptDrive:",
+			SubDir:   false,
-		"TestCryptSwift:",
+			FastList: false,
-		"TestDrive:",
+		},
-		"TestDropbox:",
+		{
-		"TestGoogleCloudStorage:",
+			Name:     "TestB2:",
-		"TestHubic:",
+			SubDir:   true,
-		"TestOneDrive:",
+			FastList: true,
-		"TestS3:",
+		},
-		"TestSftp:",
+		{
-		"TestSwift:",
+			Name:     "TestCryptDrive:",
-		"TestYandex:",
+			SubDir:   false,
-		"TestFTP:",
+			FastList: false,
+		},
+		{
+			Name:     "TestCryptSwift:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestDrive:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestDropbox:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestGoogleCloudStorage:",
+			SubDir:   true,
+			FastList: true,
+		},
+		{
+			Name:     "TestHubic:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestOneDrive:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestS3:",
+			SubDir:   true,
+			FastList: true,
+		},
+		{
+			Name:     "TestSftp:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestSwift:",
+			SubDir:   true,
+			FastList: true,
+		},
+		{
+			Name:     "TestYandex:",
+			SubDir:   false,
+			FastList: false,
+		},
+		{
+			Name:     "TestFTP:",
+			SubDir:   false,
+			FastList: false,
+		},
 	}
 	binary = "fs.test"
 	// Flags
@@ -60,7 +122,7 @@ type test struct {
 }
 
 // newTest creates a new test
-func newTest(remote string, subdir bool) *test {
+func newTest(remote string, subdir bool, fastlist bool) *test {
 	t := &test{
 		remote:  remote,
 		subdir:  subdir,
@@ -77,6 +139,9 @@ func newTest(remote string, subdir bool) *test {
 	if subdir {
 		t.cmdLine = append(t.cmdLine, "-subdir")
 	}
+	if fastlist {
+		t.cmdLine = append(t.cmdLine, "-fast-list")
+	}
 	t.cmdString = strings.Join(t.cmdLine, " ")
 	return t
 }
@@ -223,9 +288,25 @@ func removeTestBinary() {
 func main() {
 	flag.Parse()
 	if *runTests != "" {
-		remotes = strings.Split(*runTests, ",")
+		newRemotes := []remoteConfig{}
+		for _, name := range strings.Split(*runTests, ",") {
+			for i := range remotes {
+				if remotes[i].Name == name {
+					newRemotes = append(newRemotes, remotes[i])
+					goto found
+				}
+			}
+			log.Printf("Remote %q not found - inserting with default flags", name)
+			newRemotes = append(newRemotes, remoteConfig{Name: name})
+		found:
+		}
+		remotes = newRemotes
 	}
-	log.Printf("Testing remotes: %s", strings.Join(remotes, ", "))
+	var names []string
+	for _, remote := range remotes {
+		names = append(names, remote.Name)
+	}
+	log.Printf("Testing remotes: %s", strings.Join(names, ", "))
 
 	start := time.Now()
 	if *clean {
@@ -239,9 +320,14 @@ func main() {
 	results := make(chan *test, 8)
 	awaiting := 0
 	for _, remote := range remotes {
-		awaiting += 2
+		for _, subdir := range []bool{false, true} {
-		go newTest(remote, false).run(results)
+			for _, fastlist := range []bool{false, true} {
-		go newTest(remote, true).run(results)
+				if (!subdir || subdir && remote.SubDir) && (!fastlist || fastlist && remote.FastList) {
+					go newTest(remote.Name, subdir, fastlist).run(results)
+					awaiting++
+				}
+			}
+		}
 	}
 
 	// Wait for the tests to finish
@@ -261,6 +347,7 @@ func main() {
 		log.Printf("FAIL: %d tests failed in %v", len(failed), duration)
 		for _, t := range failed {
 			log.Printf("  * %s", t.cmdString)
+			log.Printf("    * Failed tests: %v", t.failedTests)
 		}
 		os.Exit(1)
 	}