Implement --delete-before, --delete-during, --delete-after - fixes #252.

docs/content/docs.md

@@ -291,6 +291,21 @@ This sets the interval.
 
 The default is `1m`. Use 0 to disable.
 
+### --delete-(before,during,after) ###
+
+This option allows you to specify when files on your destination are
+deleted when you sync folders.
+
+Specifying the value `--delete-before` will delete all files present on the
+destination, but not on the source *before* starting the transfer
+of any new or updated files.
+
+Specifying `--delete-during` (default value) will delete files while checking
+and uploading files. This is usually the fastest option.
+
+Specifying `--delete-after` will delay deletion of files until all new/updated
+files have been successfully transfered.
+
 ### --timeout=TIME ###
 
 This sets the IO idle timeout.  If a transfer has started but then

fs/config.go

@@ -69,6 +69,9 @@ var (
 	dumpHeaders    = pflag.BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info")
 	dumpBodies     = pflag.BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info")
 	skipVerify     = pflag.BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.")
+	deleteBefore   = pflag.BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering")
+	deleteDuring   = pflag.BoolP("delete-during", "", false, "When synchronizing, delete files during transfer (default)")
+	deleteAfter    = pflag.BoolP("delete-after", "", false, "When synchronizing, delete files on destination after transfering")
 	bwLimit        SizeSuffix
 )
 
@@ -179,6 +182,9 @@ type ConfigInfo struct {
 	DumpBodies         bool
 	Filter             *Filter
 	InsecureSkipVerify bool // Skip server certificate verification
+	DeleteBefore       bool // Delete before checking
+	DeleteDuring       bool // Delete during checking/transfer
+	DeleteAfter        bool // Delete after successful transfer.
 }
 
 // Transport returns an http.RoundTripper with the correct timeouts
@@ -270,6 +276,20 @@ func LoadConfig() {
 
 	ConfigPath = *configFile
 
+	Config.DeleteBefore = *deleteBefore
+	Config.DeleteDuring = *deleteDuring
+	Config.DeleteAfter = *deleteAfter
+
+	switch {
+	case *deleteBefore && (*deleteDuring || *deleteAfter),
+		*deleteDuring && *deleteAfter:
+		log.Fatalf(`Only one of --delete-before, --delete-during or --delete-after can be used.`)
+
+	// If none are specified, use "during".
+	case !*deleteBefore && !*deleteDuring && !*deleteAfter:
+		Config.DeleteDuring = true
+	}
+
 	// Load configuration file.
 	var err error
 	ConfigFile, err = goconfig.LoadConfigFile(ConfigPath)

fs/operations.go

@@ -402,14 +402,16 @@ func DeleteFiles(toBeDeleted ObjectsChan) {
 	wg.Wait()
 }
 
-// Read a map of Object.Remote to Object for the given Fs
+// Read a map of Object.Remote to Object for the given Fs.
-func readFilesMap(fs Fs) map[string]Object {
+// If includeAll is specified all files will be added,
+// otherwise only files passing the filter will be added.
+func readFilesMap(fs Fs, includeAll bool) map[string]Object {
 	files := make(map[string]Object)
 	for o := range fs.List() {
 		remote := o.Remote()
 		if _, ok := files[remote]; !ok {
 			// Make sure we don't delete excluded files if not required
-			if Config.Filter.DeleteExcluded || Config.Filter.IncludeObject(o) {
+			if includeAll || Config.Filter.IncludeObject(o) {
 				files[remote] = o
 			} else {
 				Debug(o, "Excluded from sync (and deletion)")
@@ -446,9 +448,78 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
 
 	Log(fdst, "Building file list")
 
-	// Read the destination files first
+	// Read the files of both source and destination
-	// FIXME could do this in parallel and make it use less memory
+	var listWg sync.WaitGroup
-	delFiles := readFilesMap(fdst)
+	listWg.Add(2)
+
+	var dstFiles map[string]Object
+	var srcFiles map[string]Object
+	var srcObjects = make(ObjectsChan, Config.Transfers)
+
+	go func() {
+		dstFiles = readFilesMap(fdst, Config.Filter.DeleteExcluded)
+		listWg.Done()
+	}()
+
+	go func() {
+		srcFiles = readFilesMap(fsrc, false)
+		listWg.Done()
+		for _, v := range srcFiles {
+			srcObjects <- v
+		}
+		close(srcObjects)
+	}()
+
+	startDeletion := make(chan struct{}, 0)
+
+	// Delete files if asked
+	var delWg sync.WaitGroup
+	delWg.Add(1)
+	go func() {
+		if !Delete {
+			return
+		}
+		defer func() {
+			Debug(fdst, "Deletion finished")
+			delWg.Done()
+		}()
+
+		_ = <-startDeletion
+		Debug(fdst, "Starting deletion")
+
+		if Stats.Errored() {
+			ErrorLog(fdst, "Not deleting files as there were IO errors")
+			return
+		}
+
+		// Delete the spare files
+		toDelete := make(ObjectsChan, Config.Transfers)
+
+		go func() {
+			for key, fs := range dstFiles {
+				_, exists := srcFiles[key]
+				if !exists {
+					toDelete <- fs
+				}
+			}
+			close(toDelete)
+		}()
+		DeleteFiles(toDelete)
+	}()
+
+	// Wait for all files to be read
+	listWg.Wait()
+
+	// Start deleting, unless we must delete after transfer
+	if Delete && !Config.DeleteAfter {
+		close(startDeletion)
+	}
+
+	// If deletes must finish before starting transfers, we must wait now.
+	if Delete && Config.DeleteBefore {
+		Log(fdst, "Waiting for deletes to finish (before)")
+		delWg.Wait()
+	}
 
 	// Read source files checking them off against dest files
 	toBeChecked := make(ObjectPairChan, Config.Transfers)
@@ -471,13 +542,12 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
 	}
 
 	go func() {
-		for src := range fsrc.List() {
+		for src := range srcObjects {
 			if !Config.Filter.IncludeObject(src) {
 				Debug(src, "Excluding from sync")
 			} else {
 				remote := src.Remote()
-				if dst, dstFound := delFiles[remote]; dstFound {
+				if dst, dstFound := dstFiles[remote]; dstFound {
-					delete(delFiles, remote)
 					toBeChecked <- ObjectPair{src, dst}
 				} else {
 					// No need to check since doesn't exist
@@ -494,23 +564,16 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
 	Log(fdst, "Waiting for transfers to finish")
 	copierWg.Wait()
 
-	// Delete files if asked
+	// If deleting after, start deletion now
-	if Delete {
+	if Delete && Config.DeleteAfter {
-		if Stats.Errored() {
+		close(startDeletion)
-			ErrorLog(fdst, "Not deleting files as there were IO errors")
+	}
-			return nil
+	// Unless we have already waited, wait for deletion to finish.
+	if Delete && !Config.DeleteBefore {
+		Log(fdst, "Waiting for deletes to finish (during+after)")
+		delWg.Wait()
 	}
 
-		// Delete the spare files
-		toDelete := make(ObjectsChan, Config.Transfers)
-		go func() {
-			for _, fs := range delFiles {
-				toDelete <- fs
-			}
-			close(toDelete)
-		}()
-		DeleteFiles(toDelete)
-	}
 	return nil
 }
 
@@ -570,7 +633,7 @@ func Check(fdst, fsrc Fs) error {
 		defer wg.Done()
 		// Read the destination files
 		Log(fdst, "Building file list")
-		dstFiles = readFilesMap(fdst)
+		dstFiles = readFilesMap(fdst, false)
 		Debug(fdst, "Done building file list")
 	}()
 
@@ -578,7 +641,7 @@ func Check(fdst, fsrc Fs) error {
 		defer wg.Done()
 		// Read the source files
 		Log(fsrc, "Building file list")
-		srcFiles = readFilesMap(fsrc)
+		srcFiles = readFilesMap(fsrc, false)
 		Debug(fdst, "Done building file list")
 	}()