TrueCloudLab/rclone
16
0
Fork 2
Code Issues 1 Pull requests 1 Projects Releases Packages Wiki Activity Actions

bisync: merge copies and deletes, support --track-renames and --backup-dir -- fixes #5690 fixes #5685

Browse source 
Before this change, bisync handled copies and deletes in separate operations.
After this change, they are combined in one sync operation, which is faster
and also allows bisync to support --track-renames and --backup-dir.

Bisync uses a --files-from filter containing only the paths bisync has
determined need to be synced. Just like in sync (but in both directions),
if a path is present on the dst but not the src, it's interpreted as a delete
rather than a copy.
This commit is contained in:
nielash 2023-10-01 04:47:46 -04:00
parent 5c7ba0bfd3
commit 0e5f12126f
20 changed files with 50 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
cmd/bisync/deltas.go
View file

@ -327,6 +327,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
if !in2 {
b.indent("Path2", p2, "Queue delete")
delete2.Add(file)
copy1to2.Add(file)
} else if d2.is(deltaOther) {
b.indent("Path2", p1, "Queue copy to Path1")
copy2to1.Add(file)
@ -351,6 +352,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
// Deleted
b.indent("Path1", p1, "Queue delete")
delete1.Add(file)
copy2to1.Add(file)
}
}
@ -380,25 +382,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
}
if delete1.NotEmpty() {
changes1 = true
b.indent("", "Path1", "Do queued deletes on")
err = b.fastDelete(ctx, b.fs1, delete1, "delete1")
if err != nil {
if err = b.saveQueue(delete1, "delete1"); err != nil {
return
}
//propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs)
b.syncEmptyDirs(ctx, b.fs1, delete1, dirs1, "remove")
}
if delete2.NotEmpty() {
changes2 = true
b.indent("", "Path2", "Do queued deletes on")
err = b.fastDelete(ctx, b.fs2, delete2, "delete2")
if err != nil {
if err = b.saveQueue(delete2, "delete2"); err != nil {
return
}
//propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs)
b.syncEmptyDirs(ctx, b.fs2, delete2, dirs2, "remove")
}

43
cmd/bisync/queue.go
View file

@ -24,39 +24,11 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.
}
}
return sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
}
func (b *bisyncRun) fastDelete(ctx context.Context, f fs.Fs, files bilib.Names, queueName string) error {
if err := b.saveQueue(files, queueName); err != nil {
return err
}
transfers := fs.GetConfig(ctx).Transfers
ctxRun, filterDelete := filter.AddConfig(b.opt.setDryRun(ctx))
for _, file := range files.ToList() {
if err := filterDelete.AddFile(file); err != nil {
return err
}
}
objChan := make(fs.ObjectsChan, transfers)
errChan := make(chan error, 1)
go func() {
errChan <- operations.DeleteFiles(ctxRun, objChan)
}()
err := operations.ListFn(ctxRun, f, func(obj fs.Object) {
remote := obj.Remote()
if files.Has(remote) {
objChan <- obj
}
})
close(objChan)
opErr := <-errChan
if err == nil {
err = opErr
var err error
if b.opt.Resync {
err = sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
} else {
err = sync.Sync(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
}
return err
}
@ -75,8 +47,9 @@ func (b *bisyncRun) syncEmptyDirs(ctx context.Context, dst fs.Fs, candidates bil
var direrr error
if dirsList.has(s) { //make sure it's a dir, not a file
if operation == "remove" {
//note: we need to use Rmdirs instead of Rmdir because directories will fail to delete if they have other empty dirs inside of them.
direrr = operations.Rmdirs(ctx, dst, s, false)
// directories made empty by the sync will have already been deleted during the sync
// this just catches the already-empty ones (excluded from sync by --files-from filter)
direrr = operations.TryRmdir(ctx, dst, s)
} else if operation == "make" {
direrr = operations.Mkdir(ctx, dst, s)
} else {

1
cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que vendored
View file

@ -1,4 +1,5 @@
"file11.txt"
"file2.txt"
"file4.txt"
"file5.txt..path1"
"file7.txt"

1
cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que vendored
View file

@ -1,4 +1,5 @@
"file1.txt"
"file10.txt"
"file3.txt"
"file5.txt..path2"
"file6.txt"

2
cmd/bisync/testdata/test_changes/golden/test.log vendored
View file

@ -88,8 +88,6 @@ INFO : - Path2 Queue copy to Path1 - {path1/}file10.txt
INFO : - Path1 Queue delete - {path1/}file3.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : - Do queued deletes on - Path1
INFO : - Do queued deletes on - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

4
cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log vendored
View file

@ -77,7 +77,7 @@ INFO : - Path2 File was deleted - subdir
INFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deleted
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}RCLONE_TEST
INFO : - Do queued deletes on - Path2
INFO : - Path1 Do queued copies to - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@ -121,7 +121,7 @@ INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted
INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}subdir
INFO : - Do queued deletes on - Path2
INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Removing directory
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"

1
cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que vendored
View file

@ -1,4 +1,5 @@
"file11.txt"
"file2.txt"
"file4.txt"
"file5.txt..path1"
"file7.txt"

1
cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que vendored
View file

@ -1,4 +1,5 @@
"file1.txt"
"file10.txt"
"file3.txt"
"file5.txt..path2"
"file6.txt"

1
cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que vendored
View file

@ -1,4 +1,5 @@
"file11.txt"
"file2.txt"
"file4.txt"
"file5.txt..path1"
"file7.txt"

1
cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que vendored
View file

@ -1,4 +1,5 @@
"file1.txt"
"file10.txt"
"file3.txt"
"file5.txt..path2"
"file6.txt"

8
cmd/bisync/testdata/test_dry_run/golden/test.log vendored
View file

@ -106,15 +106,13 @@ INFO : - Path1 Queue delete - {path1/}file3.txt
INFO : - Path2 Do queued copies to - Path1
NOTICE: file1.txt: Skipped copy as --dry-run is set (size 19)
NOTICE: file10.txt: Skipped copy as --dry-run is set (size 19)
NOTICE: file3.txt: Skipped delete as --dry-run is set (size 0)
NOTICE: file6.txt: Skipped copy as --dry-run is set (size 19)
INFO : - Path1 Do queued copies to - Path2
NOTICE: file11.txt: Skipped copy as --dry-run is set (size 19)
NOTICE: file2.txt: Skipped copy as --dry-run is set (size 13)
NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19)
INFO : - Do queued deletes on - Path1
NOTICE: file3.txt: Skipped delete as --dry-run is set (size 0)
INFO : - Do queued deletes on - Path2
NOTICE: file4.txt: Skipped delete as --dry-run is set (size 0)
NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19)
INFO : Updating listings
INFO : Bisync successful
(32) : copy-listings dryrun
@ -159,8 +157,6 @@ INFO : - Path2 Queue copy to Path1 - {path1/}file10.txt
INFO : - Path1 Queue delete - {path1/}file3.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : - Do queued deletes on - Path1
INFO : - Do queued deletes on - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

2
cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que vendored
View file

@ -1,7 +1,9 @@
"New_top_level_mañana_funcionará.txt"
"file_enconde_mañana_funcionará.txt"
"filename_contains_ࢺ_.txt"
"subdir with␊white space.txt/file2 with␊white space.txt"
"subdir_rawchars_␙_\x81_\xfe/file3_␙_\x81_\xfe"
"subdir_with_ࢺ_/filechangedbothpaths_ࢺ_.txt..path2"
"subdir_with_ࢺ_/filename_contains_ě_.txt"
"subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt"
"Русский.txt"

1
cmd/bisync/testdata/test_extended_filenames/golden/test.log vendored
View file

@ -84,7 +84,6 @@ INFO : - Path1 Queue delete - {path1/}subdir_with_ࢺ
INFO : - Path2 Queue copy to Path1 - {path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
INFO : - Do queued deletes on - Path1
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

5
cmd/bisync/testdata/test_max_delete_path1/golden/_testdir_path1.._testdir_path2.copy1to2.que vendored Normal file
View file

@ -0,0 +1,5 @@
"file1.txt"
"file2.txt"
"file3.txt"
"file4.txt"
"file5.txt"

2
cmd/bisync/testdata/test_max_delete_path1/golden/test.log vendored
View file

@ -49,7 +49,7 @@ INFO : - Path2 Queue delete - {path2/}file2.txt
INFO : - Path2 Queue delete - {path2/}file3.txt
INFO : - Path2 Queue delete - {path2/}file4.txt
INFO : - Path2 Queue delete - {path2/}file5.txt
INFO : - Do queued deletes on - Path2
INFO : - Path1 Do queued copies to - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

5
cmd/bisync/testdata/test_max_delete_path2_force/golden/_testdir_path1.._testdir_path2.copy2to1.que vendored Normal file
View file

@ -0,0 +1,5 @@
"file1.txt"
"file2.txt"
"file3.txt"
"file4.txt"
"file5.txt"

2
cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log vendored
View file

@ -49,7 +49,7 @@ INFO : - Path1 Queue delete - {path1/}file2.txt
INFO : - Path1 Queue delete - {path1/}file3.txt
INFO : - Path1 Queue delete - {path1/}file4.txt
INFO : - Path1 Queue delete - {path1/}file5.txt
INFO : - Do queued deletes on - Path1
INFO : - Path2 Do queued copies to - Path1
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

1
cmd/bisync/testdata/test_rmdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que vendored Normal file
View file

@ -0,0 +1 @@
"subdir/file20.txt"

2
cmd/bisync/testdata/test_rmdirs/golden/test.log vendored
View file

@ -21,7 +21,7 @@ INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted
INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}subdir/file20.txt
INFO : - Do queued deletes on - Path2
INFO : - Path1 Do queued copies to - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful

14
docs/content/bisync.md
View file

@ -552,11 +552,17 @@ and use `--resync` when you need to switch.
### Renamed directories
Renaming a folder on the Path1 side results in deleting all files on
By default, renaming a folder on the Path1 side results in deleting all files on
the Path2 side and then copying all files again from Path1 to Path2.
Bisync sees this as all files in the old directory name as deleted and all
files in the new directory name as new.
Currently, the most effective and efficient method of renaming a directory
A recommended solution is to use [`--track-renames`](/docs/#track-renames),
which is now supported in bisync as of `rclone v1.65`.
Note that `--track-renames` is not available during `--resync`,
as `--resync` does not delete anything (`--track-renames` only supports `sync`, not `copy`.)
Otherwise, the most effective and efficient method of renaming a directory
is to rename it to the same name on both sides. (As of `rclone v1.64`,
a `--resync` is no longer required after doing so, as bisync will automatically
detect that Path1 and Path2 are in agreement.)
@ -1263,6 +1269,10 @@ about _Unison_ and synchronization in general.
## Changelog
### `v1.65`
* Copies and deletes are now handled in one operation instead of two
* `--track-renames` and `--backup-dir` are now supported
### `v1.64`
* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)
causing dry runs to inadvertently commit filter changes