diff --git a/docs/content/docs.md b/docs/content/docs.md index 46a7955e3..b7c194801 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -770,6 +770,23 @@ in effect (the defaults): - 500MB..750MB files will be downloaded with 3 streams - 750MB+ files will be downloaded with 4 streams +### --no-check-dest ### + +The `--no-check-dest` can be used with `move` or `copy` and it causes +rclone not to check the destination at all when copying files. + +This means that: + +- the destination is not listed minimising the API calls +- files are always transferred +- this can cause duplicates on remotes which allow it (eg Google Drive) +- `--retries 1` is recommended otherwise you'll transfer everything again on a retry + +This flag is useful to minimise the transactions if you know that none +of the files are on the destination. + +This is a specialized flag which should be ignored by most users! + ### --no-gzip-encoding ### Don't set `Accept-Encoding: gzip`. This means that rclone won't ask diff --git a/fs/config.go b/fs/config.go index 0e7f30f7b..52df901dd 100644 --- a/fs/config.go +++ b/fs/config.go @@ -67,6 +67,7 @@ type ConfigInfo struct { IgnoreChecksum bool IgnoreCaseSync bool NoTraverse bool + NoCheckDest bool NoUpdateModTime bool DataRateUnit string CompareDest string diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 8aa868dd1..f515e0e3e 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -68,6 +68,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.") flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing") flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.") + flags.BoolVarP(flagSet, &fs.Config.NoCheckDest, "no-check-dest", "", fs.Config.NoCheckDest, "Don't check the destination, copy regardless.") flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.") flags.StringVarP(flagSet, &fs.Config.CompareDest, "compare-dest", "", fs.Config.CompareDest, "Include additional server-side path during comparison.") flags.StringVarP(flagSet, &fs.Config.CopyDest, "copy-dest", "", fs.Config.CopyDest, "Implies --compare-dest but also copies files from path into destination.") diff --git a/fs/march/march.go b/fs/march/march.go index d27337b86..cf50812b0 100644 --- a/fs/march/march.go +++ b/fs/march/march.go @@ -30,6 +30,7 @@ type March struct { SrcIncludeAll bool // don't include all files in the src DstIncludeAll bool // don't include all files in the destination Callback Marcher // object to call with results + NoCheckDest bool // transfer all objects regardless without checking dst // internal state srcListDir listDirFn // function to call to list a directory in the src dstListDir listDirFn // function to call to list a directory in the dst @@ -188,6 +189,7 @@ func (m *March) Run() error { srcDepth: srcDepth - 1, dstRemote: m.Dir, dstDepth: dstDepth - 1, + noDst: m.NoCheckDest, } go func() { // when the context is cancelled discard the remaining jobs @@ -406,7 +408,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) { // If NoTraverse is set, then try to find a matching object // for each item in the srcList - if m.NoTraverse { + if m.NoTraverse && !m.NoCheckDest { for _, src := range srcList { if srcObj, ok := src.(fs.Object); ok { leaf := path.Base(srcObj.Remote()) diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 0df993486..c90c67399 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1705,11 +1705,14 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str } // Find dst object if it exists - dstObj, err := fdst.NewObject(ctx, dstFileName) - if err == fs.ErrorObjectNotFound { - dstObj = nil - } else if err != nil { - return err + var dstObj fs.Object + if !fs.Config.NoCheckDest { + dstObj, err = fdst.NewObject(ctx, dstFileName) + if err == fs.ErrorObjectNotFound { + dstObj = nil + } else if err != nil { + return err + } } // Special case for changing case of a file on a case insensitive remote diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 850d63381..ca89533c1 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -31,6 +31,7 @@ type syncCopyMove struct { ctx context.Context // internal context for controlling go-routines cancel func() // cancel the context noTraverse bool // if set don't traverse the dst + noCheckDest bool // if set transfer all objects regardless without checking dst deletersWg sync.WaitGroup // for delete before go routine deleteFilesCh chan fs.Object // channel to receive deletes if delete before trackRenames bool // set if we should do server side renames @@ -82,6 +83,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete dstEmptyDirs: make(map[string]fs.DirEntry), srcEmptyDirs: make(map[string]fs.DirEntry), noTraverse: fs.Config.NoTraverse, + noCheckDest: fs.Config.NoCheckDest, toBeChecked: newPipe(accounting.Stats(ctx).SetCheckQueue, fs.Config.MaxBacklog), toBeUploaded: newPipe(accounting.Stats(ctx).SetTransferQueue, fs.Config.MaxBacklog), deleteFilesCh: make(chan fs.Object, fs.Config.Checkers), @@ -95,6 +97,17 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete fs.Errorf(nil, "Ignoring --no-traverse with sync") s.noTraverse = false } + if s.noCheckDest { + if s.deleteMode != fs.DeleteModeOff { + return nil, errors.New("can't use --no-check-dest with sync: use copy instead") + } + if fs.Config.Immutable { + return nil, errors.New("can't use --no-check-dest with --immutable") + } + if s.backupDir != nil { + return nil, errors.New("can't use --no-check-dest with --backup-dir") + } + } if s.trackRenames { // Don't track renames for remotes without server-side move support. if !operations.CanServerSideMove(fdst) { @@ -667,6 +680,7 @@ func (s *syncCopyMove) run() error { NoTraverse: s.noTraverse, Callback: s, DstIncludeAll: filter.Active.Opt.DeleteExcluded, + NoCheckDest: s.noCheckDest, } s.processError(m.Run())