bisync: merge copies and deletes, support --track-renames and --backup-dir -- fixes #5690 fixes #5685

Before this change, bisync handled copies and deletes in separate operations.
After this change, they are combined in one sync operation, which is faster
and also allows bisync to support --track-renames and --backup-dir.

Bisync uses a --files-from filter containing only the paths bisync has
determined need to be synced. Just like in sync (but in both directions),
if a path is present on the dst but not the src, it's interpreted as a delete
rather than a copy.

cmd/bisync/deltas.go

@@ -327,6 +327,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 			if !in2 {
 				b.indent("Path2", p2, "Queue delete")
 				delete2.Add(file)
+				copy1to2.Add(file)
 			} else if d2.is(deltaOther) {
 				b.indent("Path2", p1, "Queue copy to Path1")
 				copy2to1.Add(file)
@@ -351,6 +352,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 			// Deleted
 			b.indent("Path1", p1, "Queue delete")
 			delete1.Add(file)
+			copy2to1.Add(file)
 		}
 	}
 
@@ -380,25 +382,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
 	}
 
 	if delete1.NotEmpty() {
-		changes1 = true
+		if err = b.saveQueue(delete1, "delete1"); err != nil {
-		b.indent("", "Path1", "Do queued deletes on")
-		err = b.fastDelete(ctx, b.fs1, delete1, "delete1")
-		if err != nil {
 			return
 		}
-
 		//propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs)
 		b.syncEmptyDirs(ctx, b.fs1, delete1, dirs1, "remove")
 	}
 
 	if delete2.NotEmpty() {
-		changes2 = true
+		if err = b.saveQueue(delete2, "delete2"); err != nil {
-		b.indent("", "Path2", "Do queued deletes on")
-		err = b.fastDelete(ctx, b.fs2, delete2, "delete2")
-		if err != nil {
 			return
 		}
-
 		//propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs)
 		b.syncEmptyDirs(ctx, b.fs2, delete2, dirs2, "remove")
 	}

cmd/bisync/queue.go

@@ -24,39 +24,11 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.
 		}
 	}
 
-	return sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
+	var err error
-}
+	if b.opt.Resync {
-
+		err = sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
-func (b *bisyncRun) fastDelete(ctx context.Context, f fs.Fs, files bilib.Names, queueName string) error {
+	} else {
-	if err := b.saveQueue(files, queueName); err != nil {
+		err = sync.Sync(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
-		return err
-	}
-
-	transfers := fs.GetConfig(ctx).Transfers
-
-	ctxRun, filterDelete := filter.AddConfig(b.opt.setDryRun(ctx))
-
-	for _, file := range files.ToList() {
-		if err := filterDelete.AddFile(file); err != nil {
-			return err
-		}
-	}
-
-	objChan := make(fs.ObjectsChan, transfers)
-	errChan := make(chan error, 1)
-	go func() {
-		errChan <- operations.DeleteFiles(ctxRun, objChan)
-	}()
-	err := operations.ListFn(ctxRun, f, func(obj fs.Object) {
-		remote := obj.Remote()
-		if files.Has(remote) {
-			objChan <- obj
-		}
-	})
-	close(objChan)
-	opErr := <-errChan
-	if err == nil {
-		err = opErr
 	}
 	return err
 }
@@ -75,8 +47,9 @@ func (b *bisyncRun) syncEmptyDirs(ctx context.Context, dst fs.Fs, candidates bil
 			var direrr error
 			if dirsList.has(s) { //make sure it's a dir, not a file
 				if operation == "remove" {
-					//note: we need to use Rmdirs instead of Rmdir because directories will fail to delete if they have other empty dirs inside of them.
+					// directories made empty by the sync will have already been deleted during the sync
-					direrr = operations.Rmdirs(ctx, dst, s, false)
+					// this just catches the already-empty ones (excluded from sync by --files-from filter)
+					direrr = operations.TryRmdir(ctx, dst, s)
 				} else if operation == "make" {
 					direrr = operations.Mkdir(ctx, dst, s)
 				} else {

cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que

@@ -1,4 +1,5 @@
 "file11.txt"
 "file2.txt"
+"file4.txt"
 "file5.txt..path1"
 "file7.txt"

cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que

@@ -1,4 +1,5 @@
 "file1.txt"
 "file10.txt"
+"file3.txt"
 "file5.txt..path2"
 "file6.txt"

cmd/bisync/testdata/test_changes/golden/test.log

@@ -88,8 +88,6 @@ INFO  : - Path2    Queue copy to Path1                 - {path1/}file10.txt
 INFO  : - Path1    Queue delete                        - {path1/}file3.txt
 INFO  : - Path2    Do queued copies to                 - Path1
 INFO  : - Path1    Do queued copies to                 - Path2
-INFO  : -          Do queued deletes on                - Path1
-INFO  : -          Do queued deletes on                - Path2
 INFO  : Updating listings
 INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
 INFO  : Bisync successful

cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log

@@ -77,7 +77,7 @@ INFO  : - Path2    File was deleted                    - subdir
 INFO  : Path2:    1 changes:    0 new,    0 newer,    0 older,    1 deleted
 INFO  : Applying changes
 INFO  : - Path2    Queue delete                        - {path2/}RCLONE_TEST
-INFO  : -          Do queued deletes on                - Path2
+INFO  : - Path1    Do queued copies to                 - Path2
 INFO  : Updating listings
 INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
 INFO  : Bisync successful
@@ -121,7 +121,7 @@ INFO  : Path1:    1 changes:    0 new,    0 newer,    0 older,    1 deleted
 INFO  : Path2 checking for diffs
 INFO  : Applying changes
 INFO  : - Path2    Queue delete                        - {path2/}subdir
-INFO  : -          Do queued deletes on                - Path2
+INFO  : - Path1    Do queued copies to                 - Path2
 INFO  : subdir: Removing directory
 INFO  : Updating listings
 INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"

cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que

@@ -1,4 +1,5 @@
 "file11.txt"
 "file2.txt"
+"file4.txt"
 "file5.txt..path1"
 "file7.txt"

cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que

@@ -1,4 +1,5 @@
 "file1.txt"
 "file10.txt"
+"file3.txt"
 "file5.txt..path2"
 "file6.txt"

cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que

@@ -1,4 +1,5 @@
 "file11.txt"
 "file2.txt"
+"file4.txt"
 "file5.txt..path1"
 "file7.txt"

cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que

@@ -1,4 +1,5 @@
 "file1.txt"
 "file10.txt"
+"file3.txt"
 "file5.txt..path2"
 "file6.txt"

cmd/bisync/testdata/test_dry_run/golden/test.log

@@ -106,15 +106,13 @@ INFO  : - Path1    Queue delete                        - {path1/}file3.txt
 INFO  : - Path2    Do queued copies to                 - Path1
 NOTICE: file1.txt: Skipped copy as --dry-run is set (size 19)
 NOTICE: file10.txt: Skipped copy as --dry-run is set (size 19)
+NOTICE: file3.txt: Skipped delete as --dry-run is set (size 0)
 NOTICE: file6.txt: Skipped copy as --dry-run is set (size 19)
 INFO  : - Path1    Do queued copies to                 - Path2
 NOTICE: file11.txt: Skipped copy as --dry-run is set (size 19)
 NOTICE: file2.txt: Skipped copy as --dry-run is set (size 13)
-NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19)
-INFO  : -          Do queued deletes on                - Path1
-NOTICE: file3.txt: Skipped delete as --dry-run is set (size 0)
-INFO  : -          Do queued deletes on                - Path2
 NOTICE: file4.txt: Skipped delete as --dry-run is set (size 0)
+NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19)
 INFO  : Updating listings
 INFO  : Bisync successful
 (32)  : copy-listings dryrun
@@ -159,8 +157,6 @@ INFO  : - Path2    Queue copy to Path1                 - {path1/}file10.txt
 INFO  : - Path1    Queue delete                        - {path1/}file3.txt
 INFO  : - Path2    Do queued copies to                 - Path1
 INFO  : - Path1    Do queued copies to                 - Path2
-INFO  : -          Do queued deletes on                - Path1
-INFO  : -          Do queued deletes on                - Path2
 INFO  : Updating listings
 INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
 INFO  : Bisync successful

cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que

@@ -1,7 +1,9 @@
 "New_top_level_mañana_funcionará.txt"
 "file_enconde_mañana_funcionará.txt"
+"filename_contains_ࢺ_.txt"
 "subdir with␊white space.txt/file2 with␊white space.txt"
 "subdir_rawchars_␙_\x81_\xfe/file3_␙_\x81_\xfe"
 "subdir_with_ࢺ_/filechangedbothpaths_ࢺ_.txt..path2"
+"subdir_with_ࢺ_/filename_contains_ě_.txt"
 "subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt"
 "Русский.txt"

cmd/bisync/testdata/test_extended_filenames/golden/test.log

@@ -84,7 +84,6 @@ INFO  : - Path1    Queue delete                        - {path1/}subdir_with_ࢺ
 INFO  : - Path2    Queue copy to Path1                 - {path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt
 INFO  : - Path2    Do queued copies to                 - Path1
 INFO  : - Path1    Do queued copies to                 - Path2
-INFO  : -          Do queued deletes on                - Path1
 INFO  : Updating listings
 INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
 INFO  : Bisync successful

cmd/bisync/testdata/test_max_delete_path1/golden/_testdir_path1.._testdir_path2.copy1to2.que

@@ -0,0 +1,5 @@
+"file1.txt"
+"file2.txt"
+"file3.txt"
+"file4.txt"
+"file5.txt"

cmd/bisync/testdata/test_max_delete_path1/golden/test.log

@@ -49,7 +49,7 @@ INFO  : - Path2    Queue delete                        - {path2/}file2.txt
 INFO  : - Path2    Queue delete                        - {path2/}file3.txt
 INFO  : - Path2    Queue delete                        - {path2/}file4.txt
 INFO  : - Path2    Queue delete                        - {path2/}file5.txt
-INFO  : -          Do queued deletes on                - Path2
+INFO  : - Path1    Do queued copies to                 - Path2
 INFO  : Updating listings
 INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
 INFO  : Bisync successful

cmd/bisync/testdata/test_max_delete_path2_force/golden/_testdir_path1.._testdir_path2.copy2to1.que

@@ -0,0 +1,5 @@
+"file1.txt"
+"file2.txt"
+"file3.txt"
+"file4.txt"
+"file5.txt"

cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log

@@ -49,7 +49,7 @@ INFO  : - Path1    Queue delete                        - {path1/}file2.txt
 INFO  : - Path1    Queue delete                        - {path1/}file3.txt
 INFO  : - Path1    Queue delete                        - {path1/}file4.txt
 INFO  : - Path1    Queue delete                        - {path1/}file5.txt
-INFO  : -          Do queued deletes on                - Path1
+INFO  : - Path2    Do queued copies to                 - Path1
 INFO  : Updating listings
 INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
 INFO  : Bisync successful

cmd/bisync/testdata/test_rmdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que

@@ -0,0 +1 @@
+"subdir/file20.txt"

cmd/bisync/testdata/test_rmdirs/golden/test.log

@@ -21,7 +21,7 @@ INFO  : Path1:    1 changes:    0 new,    0 newer,    0 older,    1 deleted
 INFO  : Path2 checking for diffs
 INFO  : Applying changes
 INFO  : - Path2    Queue delete                        - {path2/}subdir/file20.txt
-INFO  : -          Do queued deletes on                - Path2
+INFO  : - Path1    Do queued copies to                 - Path2
 INFO  : Updating listings
 INFO  : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
 INFO  : Bisync successful

docs/content/bisync.md

@@ -552,11 +552,17 @@ and use `--resync` when you need to switch.
 
 ### Renamed directories
 
-Renaming a folder on the Path1 side results in deleting all files on
+By default, renaming a folder on the Path1 side results in deleting all files on
 the Path2 side and then copying all files again from Path1 to Path2.
 Bisync sees this as all files in the old directory name as deleted and all
 files in the new directory name as new. 
-Currently, the most effective and efficient method of renaming a directory
+
+A recommended solution is to use [`--track-renames`](/docs/#track-renames),
+which is now supported in bisync as of `rclone v1.65`.
+Note that `--track-renames` is not available during `--resync`,
+as `--resync` does not delete anything (`--track-renames` only supports `sync`, not `copy`.)
+
+Otherwise, the most effective and efficient method of renaming a directory
 is to rename it to the same name on both sides. (As of `rclone v1.64`, 
 a `--resync` is no longer required after doing so, as bisync will automatically
 detect that Path1 and Path2 are in agreement.)
@@ -1263,6 +1269,10 @@ about _Unison_ and synchronization in general.
 
 ## Changelog
 
+### `v1.65`
+* Copies and deletes are now handled in one operation instead of two
+* `--track-renames` and `--backup-dir` are now supported
+
 ### `v1.64`
 * Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry) 
 causing dry runs to inadvertently commit filter changes