diff --git a/docs/content/docs.md b/docs/content/docs.md index d5032e08c..1dc29885c 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -291,6 +291,21 @@ This sets the interval. The default is `1m`. Use 0 to disable. +### --delete-(before,during,after) ### + +This option allows you to specify when files on your destination are +deleted when you sync folders. + +Specifying the value `--delete-before` will delete all files present on the +destination, but not on the source *before* starting the transfer +of any new or updated files. + +Specifying `--delete-during` (default value) will delete files while checking +and uploading files. This is usually the fastest option. + +Specifying `--delete-after` will delay deletion of files until all new/updated +files have been successfully transfered. + ### --timeout=TIME ### This sets the IO idle timeout. If a transfer has started but then diff --git a/fs/config.go b/fs/config.go index 7b6fb4c9e..956365b5a 100644 --- a/fs/config.go +++ b/fs/config.go @@ -69,6 +69,9 @@ var ( dumpHeaders = pflag.BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info") dumpBodies = pflag.BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info") skipVerify = pflag.BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.") + deleteBefore = pflag.BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering") + deleteDuring = pflag.BoolP("delete-during", "", false, "When synchronizing, delete files during transfer (default)") + deleteAfter = pflag.BoolP("delete-after", "", false, "When synchronizing, delete files on destination after transfering") bwLimit SizeSuffix ) @@ -179,6 +182,9 @@ type ConfigInfo struct { DumpBodies bool Filter *Filter InsecureSkipVerify bool // Skip server certificate verification + DeleteBefore bool // Delete before checking + DeleteDuring bool // Delete during checking/transfer + DeleteAfter bool // Delete after successful transfer. } // Transport returns an http.RoundTripper with the correct timeouts @@ -270,6 +276,20 @@ func LoadConfig() { ConfigPath = *configFile + Config.DeleteBefore = *deleteBefore + Config.DeleteDuring = *deleteDuring + Config.DeleteAfter = *deleteAfter + + switch { + case *deleteBefore && (*deleteDuring || *deleteAfter), + *deleteDuring && *deleteAfter: + log.Fatalf(`Only one of --delete-before, --delete-during or --delete-after can be used.`) + + // If none are specified, use "during". + case !*deleteBefore && !*deleteDuring && !*deleteAfter: + Config.DeleteDuring = true + } + // Load configuration file. var err error ConfigFile, err = goconfig.LoadConfigFile(ConfigPath) diff --git a/fs/operations.go b/fs/operations.go index b9bd96935..89a89b27b 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -402,14 +402,16 @@ func DeleteFiles(toBeDeleted ObjectsChan) { wg.Wait() } -// Read a map of Object.Remote to Object for the given Fs -func readFilesMap(fs Fs) map[string]Object { +// Read a map of Object.Remote to Object for the given Fs. +// If includeAll is specified all files will be added, +// otherwise only files passing the filter will be added. +func readFilesMap(fs Fs, includeAll bool) map[string]Object { files := make(map[string]Object) for o := range fs.List() { remote := o.Remote() if _, ok := files[remote]; !ok { // Make sure we don't delete excluded files if not required - if Config.Filter.DeleteExcluded || Config.Filter.IncludeObject(o) { + if includeAll || Config.Filter.IncludeObject(o) { files[remote] = o } else { Debug(o, "Excluded from sync (and deletion)") @@ -446,9 +448,78 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error { Log(fdst, "Building file list") - // Read the destination files first - // FIXME could do this in parallel and make it use less memory - delFiles := readFilesMap(fdst) + // Read the files of both source and destination + var listWg sync.WaitGroup + listWg.Add(2) + + var dstFiles map[string]Object + var srcFiles map[string]Object + var srcObjects = make(ObjectsChan, Config.Transfers) + + go func() { + dstFiles = readFilesMap(fdst, Config.Filter.DeleteExcluded) + listWg.Done() + }() + + go func() { + srcFiles = readFilesMap(fsrc, false) + listWg.Done() + for _, v := range srcFiles { + srcObjects <- v + } + close(srcObjects) + }() + + startDeletion := make(chan struct{}, 0) + + // Delete files if asked + var delWg sync.WaitGroup + delWg.Add(1) + go func() { + if !Delete { + return + } + defer func() { + Debug(fdst, "Deletion finished") + delWg.Done() + }() + + _ = <-startDeletion + Debug(fdst, "Starting deletion") + + if Stats.Errored() { + ErrorLog(fdst, "Not deleting files as there were IO errors") + return + } + + // Delete the spare files + toDelete := make(ObjectsChan, Config.Transfers) + + go func() { + for key, fs := range dstFiles { + _, exists := srcFiles[key] + if !exists { + toDelete <- fs + } + } + close(toDelete) + }() + DeleteFiles(toDelete) + }() + + // Wait for all files to be read + listWg.Wait() + + // Start deleting, unless we must delete after transfer + if Delete && !Config.DeleteAfter { + close(startDeletion) + } + + // If deletes must finish before starting transfers, we must wait now. + if Delete && Config.DeleteBefore { + Log(fdst, "Waiting for deletes to finish (before)") + delWg.Wait() + } // Read source files checking them off against dest files toBeChecked := make(ObjectPairChan, Config.Transfers) @@ -471,13 +542,12 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error { } go func() { - for src := range fsrc.List() { + for src := range srcObjects { if !Config.Filter.IncludeObject(src) { Debug(src, "Excluding from sync") } else { remote := src.Remote() - if dst, dstFound := delFiles[remote]; dstFound { - delete(delFiles, remote) + if dst, dstFound := dstFiles[remote]; dstFound { toBeChecked <- ObjectPair{src, dst} } else { // No need to check since doesn't exist @@ -494,23 +564,16 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error { Log(fdst, "Waiting for transfers to finish") copierWg.Wait() - // Delete files if asked - if Delete { - if Stats.Errored() { - ErrorLog(fdst, "Not deleting files as there were IO errors") - return nil - } - - // Delete the spare files - toDelete := make(ObjectsChan, Config.Transfers) - go func() { - for _, fs := range delFiles { - toDelete <- fs - } - close(toDelete) - }() - DeleteFiles(toDelete) + // If deleting after, start deletion now + if Delete && Config.DeleteAfter { + close(startDeletion) } + // Unless we have already waited, wait for deletion to finish. + if Delete && !Config.DeleteBefore { + Log(fdst, "Waiting for deletes to finish (during+after)") + delWg.Wait() + } + return nil } @@ -570,7 +633,7 @@ func Check(fdst, fsrc Fs) error { defer wg.Done() // Read the destination files Log(fdst, "Building file list") - dstFiles = readFilesMap(fdst) + dstFiles = readFilesMap(fdst, false) Debug(fdst, "Done building file list") }() @@ -578,7 +641,7 @@ func Check(fdst, fsrc Fs) error { defer wg.Done() // Read the source files Log(fsrc, "Building file list") - srcFiles = readFilesMap(fsrc) + srcFiles = readFilesMap(fsrc, false) Debug(fdst, "Done building file list") }()