diff --git a/cmd/copy/copy.go b/cmd/copy/copy.go index 9c919c32c..3e758d1e9 100644 --- a/cmd/copy/copy.go +++ b/cmd/copy/copy.go @@ -51,6 +51,17 @@ written a trailing / - meaning "copy the contents of this directory". This applies to all commands and whether you are talking about the source or destination. +See the [--no-traverse](/docs/#no-traverse) option for controlling +whether rclone lists the destination directory or not. Supplying this +option when copying a small number of files into a large destination +can speed transfers up greatly. + +For example, if you have many files in /path/to/src but only a few of +them change every day, you can to copy all the files which have +changed recently very efficiently like this: + + rclone copy --max-age 24h --no-traverse /path/to/src remote: + **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics `, Run: func(command *cobra.Command, args []string) { diff --git a/cmd/move/move.go b/cmd/move/move.go index 38c654cb8..e70caf6ff 100644 --- a/cmd/move/move.go +++ b/cmd/move/move.go @@ -37,6 +37,11 @@ into ` + "`dest:path`" + ` then delete the original (if no errors on copy) in If you want to delete empty source directories after move, use the --delete-empty-src-dirs flag. +See the [--no-traverse](/docs/#no-traverse) option for controlling +whether rclone lists the destination directory or not. Supplying this +option when moving a small number of files into a large destination +can speed transfers up greatly. + **Important**: Since this can cause data loss, test first with the --dry-run flag. diff --git a/docs/content/docs.md b/docs/content/docs.md index f84e57fbd..495ab259f 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -842,8 +842,8 @@ will fall back to the default behaviour and log an error level message to the console. Note: Encrypted destinations are not supported by `--track-renames`. -Note that `--track-renames` uses extra memory to keep track of all -the rename candidates. +Note that `--track-renames` is incompatible with `--no-traverse` and +that it uses extra memory to keep track of all the rename candidates. Note also that `--track-renames` is incompatible with `--delete-before` and will select `--delete-after` instead of @@ -1132,6 +1132,24 @@ This option defaults to `false`. **This should be used only for testing.** +### --no-traverse ### + +The `--no-traverse` flag controls whether the destination file system +is traversed when using the `copy` or `move` commands. +`--no-traverse` is not compatible with `sync` and will be ignored if +you supply it with `sync`. + +If you are only copying a small number of files (or are filtering most +of the files) and/or have a large number of files on the destination +then `--no-traverse` will stop rclone listing the destination and save +time. + +However, if you are copying a large number of files, especially if you +are doing a copy where lots of the files under consideration haven't +changed and won't need copying then you shouldn't use `--no-traverse`. + +See [rclone copy](/commands/rclone_copy/) for an example of how to use it. + Filtering --------- diff --git a/fs/config.go b/fs/config.go index cf608b25d..aaa392ae4 100644 --- a/fs/config.go +++ b/fs/config.go @@ -62,6 +62,7 @@ type ConfigInfo struct { MaxDepth int IgnoreSize bool IgnoreChecksum bool + NoTraverse bool NoUpdateModTime bool DataRateUnit string BackupDir string diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 110755c8e..f15fbf2bf 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -27,7 +27,6 @@ var ( deleteAfter bool bindAddr string disableFeatures string - noTraverse bool ) // AddFlags adds the non filing system specific flags to the command @@ -65,7 +64,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.IntVarP(flagSet, &fs.Config.MaxDepth, "max-depth", "", fs.Config.MaxDepth, "If set limits the recursion depth to this.") flags.BoolVarP(flagSet, &fs.Config.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.") flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.") - flags.BoolVarP(flagSet, &noTraverse, "no-traverse", "", noTraverse, "Obsolete - does nothing.") + flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.") flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.") flags.StringVarP(flagSet, &fs.Config.BackupDir, "backup-dir", "", fs.Config.BackupDir, "Make backups into hierarchy based in DIR.") flags.StringVarP(flagSet, &fs.Config.Suffix, "suffix", "", fs.Config.Suffix, "Suffix for use with --backup-dir.") @@ -113,10 +112,6 @@ func SetFlags() { } } - if noTraverse { - fs.Logf(nil, "--no-traverse is obsolete and no longer needed - please remove") - } - if dumpHeaders { fs.Config.Dump |= fs.DumpHeaders fs.Logf(nil, "--dump-headers is obsolete - please use --dump headers instead") diff --git a/fs/march/march.go b/fs/march/march.go index 4056297f3..01e1570ef 100644 --- a/fs/march/march.go +++ b/fs/march/march.go @@ -23,6 +23,7 @@ type March struct { Fdst fs.Fs // source Fs Fsrc fs.Fs // dest Fs Dir string // directory + NoTraverse bool // don't traverse the destination SrcIncludeAll bool // don't include all files in the src DstIncludeAll bool // don't include all files in the destination Callback Marcher // object to call with results @@ -45,7 +46,9 @@ type Marcher interface { // init sets up a march over opt.Fsrc, and opt.Fdst calling back callback for each match func (m *March) init() { m.srcListDir = m.makeListDir(m.Fsrc, m.SrcIncludeAll) - m.dstListDir = m.makeListDir(m.Fdst, m.DstIncludeAll) + if !m.NoTraverse { + m.dstListDir = m.makeListDir(m.Fdst, m.DstIncludeAll) + } // Now create the matching transform // ..normalise the UTF8 first m.transforms = append(m.transforms, norm.NFC.String) @@ -344,7 +347,7 @@ func (m *March) processJob(job listDirJob) (jobs []listDirJob) { srcList, srcListErr = m.srcListDir(job.srcRemote) }() } - if !job.noDst { + if !m.NoTraverse && !job.noDst { wg.Add(1) go func() { defer wg.Done() @@ -367,6 +370,20 @@ func (m *March) processJob(job listDirJob) (jobs []listDirJob) { return nil } + // If NoTraverse is set, then try to find a matching object + // for each item in the srcList + if m.NoTraverse { + for _, src := range srcList { + if srcObj, ok := src.(fs.Object); ok { + leaf := path.Base(srcObj.Remote()) + dstObj, err := m.Fdst.NewObject(path.Join(job.dstRemote, leaf)) + if err == nil { + dstList = append(dstList, dstObj) + } + } + } + } + // Work out what to do and do it srcOnly, dstOnly, matches := matchListings(srcList, dstList, m.transforms) for _, src := range srcOnly { diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 856e19cf9..15ee2fd56 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -29,6 +29,7 @@ type syncCopyMove struct { // internal state ctx context.Context // internal context for controlling go-routines cancel func() // cancel the context + noTraverse bool // if set don't traverse the dst deletersWg sync.WaitGroup // for delete before go routine deleteFilesCh chan fs.Object // channel to receive deletes if delete before trackRenames bool // set if we should do server side renames @@ -75,6 +76,7 @@ func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de dstFilesResult: make(chan error, 1), dstEmptyDirs: make(map[string]fs.DirEntry), srcEmptyDirs: make(map[string]fs.DirEntry), + noTraverse: fs.Config.NoTraverse, toBeChecked: newPipe(accounting.Stats.SetCheckQueue, fs.Config.MaxBacklog), toBeUploaded: newPipe(accounting.Stats.SetTransferQueue, fs.Config.MaxBacklog), deleteFilesCh: make(chan fs.Object, fs.Config.Checkers), @@ -84,6 +86,10 @@ func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de trackRenamesCh: make(chan fs.Object, fs.Config.Checkers), } s.ctx, s.cancel = context.WithCancel(context.Background()) + if s.noTraverse && s.deleteMode != fs.DeleteModeOff { + fs.Errorf(nil, "Ignoring --no-traverse with sync") + s.noTraverse = false + } if s.trackRenames { // Don't track renames for remotes without server-side move support. if !operations.CanServerSideMove(fdst) { @@ -104,6 +110,10 @@ func newSyncCopyMove(fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, de if s.deleteMode != fs.DeleteModeOff { s.deleteMode = fs.DeleteModeAfter } + if s.noTraverse { + fs.Errorf(nil, "Ignoring --no-traverse with --track-renames") + s.noTraverse = false + } } // Make Fs for --backup-dir if required if fs.Config.BackupDir != "" { @@ -651,6 +661,7 @@ func (s *syncCopyMove) run() error { Fdst: s.fdst, Fsrc: s.fsrc, Dir: s.dir, + NoTraverse: s.noTraverse, Callback: s, DstIncludeAll: filter.Active.Opt.DeleteExcluded, } diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 019b171ca..f98988387 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -61,6 +61,41 @@ func TestCopy(t *testing.T) { fstest.CheckItems(t, r.Fremote, file1) } +// Now with --no-traverse +func TestCopyNoTraverse(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + fs.Config.NoTraverse = true + defer func() { fs.Config.NoTraverse = false }() + + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + + err := CopyDir(r.Fremote, r.Flocal) + require.NoError(t, err) + + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) +} + +// Now with --no-traverse +func TestSyncNoTraverse(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + fs.Config.NoTraverse = true + defer func() { fs.Config.NoTraverse = false }() + + file1 := r.WriteFile("sub dir/hello world", "hello world", t1) + + accounting.Stats.ResetCounters() + err := Sync(r.Fremote, r.Flocal) + require.NoError(t, err) + + fstest.CheckItems(t, r.Flocal, file1) + fstest.CheckItems(t, r.Fremote, file1) +} + // Test copy with depth func TestCopyWithDepth(t *testing.T) { r := fstest.NewRun(t)