diff --git a/cmd/bisync/bilib/files.go b/cmd/bisync/bilib/files.go index e2c7bb1e8..0d0134635 100644 --- a/cmd/bisync/bilib/files.go +++ b/cmd/bisync/bilib/files.go @@ -39,7 +39,7 @@ func FileExists(file string) bool { return !os.IsNotExist(err) } -// CopyFileIfExists is like CopyFile but does to fail if source does not exist +// CopyFileIfExists is like CopyFile but does not fail if source does not exist func CopyFileIfExists(srcFile, dstFile string) error { if !FileExists(srcFile) { return nil diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go index 87e274d0e..3167dfb00 100644 --- a/cmd/bisync/deltas.go +++ b/cmd/bisync/deltas.go @@ -137,8 +137,9 @@ func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter } // findDeltas -func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing, newListing, msg string) (ds *deltaSet, err error) { - var old, now *fileList +func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string, now *fileList, msg string) (ds *deltaSet, err error) { + var old *fileList + newListing := oldListing + "-new" old, err = b.loadListing(oldListing) if err != nil { @@ -150,7 +151,6 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing, newLis return } - now, err = b.makeListing(fctx, f, newListing) if err == nil { err = b.checkListing(now, newListing, "current "+msg) } @@ -235,6 +235,8 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change renamed2 := bilib.Names{} renameSkipped := bilib.Names{} deletedonboth := bilib.Names{} + skippedDirs1 := newFileList() + skippedDirs2 := newFileList() ctxMove := b.opt.setDryRun(ctx) @@ -304,6 +306,8 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change //if files are identical, leave them alone instead of renaming if dirs1.has(file) && dirs2.has(file) { fs.Debugf(nil, "This is a directory, not a file. Skipping equality check and will not rename: %s", file) + ls1.getPut(file, skippedDirs1) + ls2.getPut(file, skippedDirs2) } else { equal := matches.Has(file) if equal { @@ -424,6 +428,8 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change queues.renamed2 = renamed2 queues.renameSkipped = renameSkipped queues.deletedonboth = deletedonboth + queues.skippedDirs1 = skippedDirs1 + queues.skippedDirs2 = skippedDirs2 return } diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.go index 25953a4ec..7875036fa 100644 --- a/cmd/bisync/listing.go +++ b/cmd/bisync/listing.go @@ -12,7 +12,6 @@ import ( "sort" "strconv" "strings" - "sync" "time" "github.com/rclone/rclone/cmd/bisync/bilib" @@ -20,7 +19,6 @@ import ( "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" - "github.com/rclone/rclone/fs/walk" "golang.org/x/exp/slices" ) @@ -70,6 +68,9 @@ func newFileList() *fileList { } func (ls *fileList) empty() bool { + if ls == nil { + return true + } return len(ls.list) == 0 } @@ -99,6 +100,12 @@ func (ls *fileList) getPut(file string, dest *fileList) { dest.put(file, f.size, f.time, f.hash, f.id, f.flags) } +func (ls *fileList) getPutAll(dest *fileList) { + for file, f := range ls.info { + dest.put(file, f.size, f.time, f.hash, f.id, f.flags) + } +} + func (ls *fileList) remove(file string) { if ls.has(file) { ls.list = slices.Delete(ls.list, slices.Index(ls.list, file), slices.Index(ls.list, file)+1) @@ -292,13 +299,16 @@ func (b *bisyncRun) loadListing(listing string) (*fileList, error) { return ls, nil } +// saveOldListings saves the most recent successful listing, in case we need to rollback on error func (b *bisyncRun) saveOldListings() { - if err := bilib.CopyFileIfExists(b.listing1, b.listing1+"-old"); err != nil { - fs.Debugf(b.listing1, "error saving old listing1: %v", err) - } - if err := bilib.CopyFileIfExists(b.listing2, b.listing2+"-old"); err != nil { - fs.Debugf(b.listing1, "error saving old listing2: %v", err) - } + b.handleErr(b.listing1, "error saving old Path1 listing", bilib.CopyFileIfExists(b.listing1, b.listing1+"-old"), true, true) + b.handleErr(b.listing2, "error saving old Path2 listing", bilib.CopyFileIfExists(b.listing2, b.listing2+"-old"), true, true) +} + +// replaceCurrentListings saves both ".lst-new" listings as ".lst" +func (b *bisyncRun) replaceCurrentListings() { + b.handleErr(b.newListing1, "error replacing Path1 listing", bilib.CopyFileIfExists(b.newListing1, b.listing1), true, true) + b.handleErr(b.newListing2, "error replacing Path2 listing", bilib.CopyFileIfExists(b.newListing2, b.listing2), true, true) } func parseHash(str string) (string, string, error) { @@ -314,71 +324,6 @@ func parseHash(str string) (string, string, error) { return "", "", fmt.Errorf("invalid hash %q", str) } -// makeListing will produce listing from directory tree and write it to a file -func (b *bisyncRun) makeListing(ctx context.Context, f fs.Fs, listing string) (ls *fileList, err error) { - ci := fs.GetConfig(ctx) - depth := ci.MaxDepth - hashType := hash.None - if !b.opt.IgnoreListingChecksum { - // Currently bisync just honors --ignore-listing-checksum - // (note that this is different from --ignore-checksum) - // TODO add full support for checksums and related flags - hashType = f.Hashes().GetOne() - } - ls = newFileList() - ls.hash = hashType - var lock sync.Mutex - listType := walk.ListObjects - if b.opt.CreateEmptySrcDirs { - listType = walk.ListAll - } - err = walk.ListR(ctx, f, "", false, depth, listType, func(entries fs.DirEntries) error { - var firstErr error - entries.ForObject(func(o fs.Object) { - //tr := accounting.Stats(ctx).NewCheckingTransfer(o) // TODO - var ( - hashVal string - hashErr error - ) - if hashType != hash.None { - hashVal, hashErr = o.Hash(ctx, hashType) - if firstErr == nil { - firstErr = hashErr - } - } - time := o.ModTime(ctx).In(TZ) - id := "" // TODO - flags := "-" // "-" for a file and "d" for a directory - lock.Lock() - ls.put(o.Remote(), o.Size(), time, hashVal, id, flags) - lock.Unlock() - //tr.Done(ctx, nil) // TODO - }) - if b.opt.CreateEmptySrcDirs { - entries.ForDir(func(o fs.Directory) { - var ( - hashVal string - ) - time := o.ModTime(ctx).In(TZ) - id := "" // TODO - flags := "d" // "-" for a file and "d" for a directory - lock.Lock() - //record size as 0 instead of -1, so bisync doesn't think it's a google doc - ls.put(o.Remote(), 0, time, hashVal, id, flags) - lock.Unlock() - }) - } - return firstErr - }) - if err == nil { - err = ls.save(ctx, listing) - } - if err != nil { - b.abort = true - } - return -} - // checkListing verifies that listing is not empty (unless resynching) func (b *bisyncRun) checkListing(ls *fileList, listing, msg string) error { if b.opt.Resync || !ls.empty() { @@ -542,8 +487,6 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res updateLists("src", srcWinners, srcList) updateLists("dst", dstWinners, dstList) - // TODO: rollback on error - // account for "deltaOthers" we handled separately if queues.deletedonboth.NotEmpty() { for file := range queues.deletedonboth { @@ -587,7 +530,7 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res // recheck the ones we skipped because they were equal // we never got their info because they were never synced. - // TODO: add flag to skip this for people who don't care and would rather avoid? + // TODO: add flag to skip this? (since it re-lists) if queues.renameSkipped.NotEmpty() { skippedList := queues.renameSkipped.ToList() for _, file := range skippedList { @@ -596,6 +539,20 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res } } } + // skipped dirs -- nothing to recheck, just add them + // (they are not necessarily there already, if they are new) + path1List := srcList + path2List := dstList + if !is1to2 { + path1List = dstList + path2List = srcList + } + if !queues.skippedDirs1.empty() { + queues.skippedDirs1.getPutAll(path1List) + } + if !queues.skippedDirs2.empty() { + queues.skippedDirs2.getPutAll(path2List) + } if filterRecheck.HaveFilesFrom() { b.recheck(ctxRecheck, src, dst, srcList, dstList, is1to2) @@ -665,9 +622,9 @@ func (b *bisyncRun) recheck(ctxRecheck context.Context, src, dst fs.Fs, srcList, if len(toRollback) > 0 { srcListing, dstListing := b.getListingNames(is1to2) oldSrc, err := b.loadListing(srcListing + "-old") - handleErr(oldSrc, "error loading old src listing", err) // TODO: make this critical? + b.handleErr(oldSrc, "error loading old src listing", err, true, true) oldDst, err := b.loadListing(dstListing + "-old") - handleErr(oldDst, "error loading old dst listing", err) // TODO: make this critical? + b.handleErr(oldDst, "error loading old dst listing", err, true, true) for _, item := range toRollback { rollback(item, oldSrc, srcList) @@ -683,13 +640,6 @@ func (b *bisyncRun) getListingNames(is1to2 bool) (srcListing string, dstListing return b.listing2, b.listing1 } -func handleErr(o interface{}, msg string, err error) { - // TODO: add option to make critical? - if err != nil { - fs.Debugf(o, "%s: %v", msg, err) - } -} - func rollback(item string, oldList, newList *fileList) { if oldList.has(item) { oldList.getPut(item, newList) diff --git a/cmd/bisync/march.go b/cmd/bisync/march.go new file mode 100644 index 000000000..229f3ed63 --- /dev/null +++ b/cmd/bisync/march.go @@ -0,0 +1,189 @@ +package bisync + +import ( + "context" + "sync" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/march" +) + +var ls1 = newFileList() +var ls2 = newFileList() +var err error +var firstErr error +var marchLsLock sync.Mutex +var marchErrLock sync.Mutex +var marchCtx context.Context + +func (b *bisyncRun) makeMarchListing(ctx context.Context) (*fileList, *fileList, error) { + ci := fs.GetConfig(ctx) + marchCtx = ctx + b.setupListing() + fs.Debugf(b, "starting to march!") + + // set up a march over fdst (Path2) and fsrc (Path1) + m := &march.March{ + Ctx: ctx, + Fdst: b.fs2, + Fsrc: b.fs1, + Dir: "", + NoTraverse: false, + Callback: b, + DstIncludeAll: false, + NoCheckDest: false, + NoUnicodeNormalization: ci.NoUnicodeNormalization, + } + err = m.Run(ctx) + + fs.Debugf(b, "march completed. err: %v", err) + if err == nil { + err = firstErr + } + if err != nil { + b.abort = true + } + + // save files + err = ls1.save(ctx, b.newListing1) + if err != nil { + b.abort = true + } + err = ls2.save(ctx, b.newListing2) + if err != nil { + b.abort = true + } + + return ls1, ls2, err +} + +// SrcOnly have an object which is on path1 only +func (b *bisyncRun) SrcOnly(o fs.DirEntry) (recurse bool) { + fs.Debugf(o, "path1 only") + b.parse(o, true) + return isDir(o) +} + +// DstOnly have an object which is on path2 only +func (b *bisyncRun) DstOnly(o fs.DirEntry) (recurse bool) { + fs.Debugf(o, "path2 only") + b.parse(o, false) + return isDir(o) +} + +// Match is called when object exists on both path1 and path2 (whether equal or not) +func (b *bisyncRun) Match(ctx context.Context, o2, o1 fs.DirEntry) (recurse bool) { + fs.Debugf(o1, "both path1 and path2") + b.parse(o1, true) + b.parse(o2, false) + return isDir(o1) +} + +func isDir(e fs.DirEntry) bool { + switch x := e.(type) { + case fs.Object: + fs.Debugf(x, "is Object") + return false + case fs.Directory: + fs.Debugf(x, "is Dir") + return true + default: + fs.Debugf(e, "is unknown") + } + return false +} + +func (b *bisyncRun) parse(e fs.DirEntry, isPath1 bool) { + switch x := e.(type) { + case fs.Object: + b.ForObject(x, isPath1) + case fs.Directory: + if b.opt.CreateEmptySrcDirs { + b.ForDir(x, isPath1) + } + default: + fs.Debugf(e, "is unknown") + } +} + +func (b *bisyncRun) setupListing() { + ls1 = newFileList() + ls2 = newFileList() + + hashType1 := hash.None + hashType2 := hash.None + if !b.opt.IgnoreListingChecksum { + // Currently bisync just honors --ignore-listing-checksum + // (note that this is different from --ignore-checksum) + // TODO add full support for checksums and related flags + hashType1 = b.fs1.Hashes().GetOne() + hashType2 = b.fs2.Hashes().GetOne() + } + + ls1.hash = hashType1 + ls2.hash = hashType2 +} + +func (b *bisyncRun) ForObject(o fs.Object, isPath1 bool) { + tr := accounting.Stats(marchCtx).NewCheckingTransfer(o, "listing file - "+whichPath(isPath1)) + defer func() { + tr.Done(marchCtx, nil) + }() + var ( + hashVal string + hashErr error + ) + ls := whichLs(isPath1) + hashType := ls.hash + if hashType != hash.None { + hashVal, hashErr = o.Hash(marchCtx, hashType) + marchErrLock.Lock() + if firstErr == nil { + firstErr = hashErr + } + marchErrLock.Unlock() + } + time := o.ModTime(marchCtx).In(TZ) + id := "" // TODO + flags := "-" // "-" for a file and "d" for a directory + marchLsLock.Lock() + ls.put(o.Remote(), o.Size(), time, hashVal, id, flags) + marchLsLock.Unlock() +} + +func (b *bisyncRun) ForDir(o fs.Directory, isPath1 bool) { + tr := accounting.Stats(marchCtx).NewCheckingTransfer(o, "listing dir - "+whichPath(isPath1)) + defer func() { + tr.Done(marchCtx, nil) + }() + ls := whichLs(isPath1) + time := o.ModTime(marchCtx).In(TZ) + id := "" // TODO + flags := "d" // "-" for a file and "d" for a directory + marchLsLock.Lock() + //record size as 0 instead of -1, so bisync doesn't think it's a google doc + ls.put(o.Remote(), 0, time, "", id, flags) + marchLsLock.Unlock() +} + +func whichLs(isPath1 bool) *fileList { + ls := ls1 + if !isPath1 { + ls = ls2 + } + return ls +} + +func whichPath(isPath1 bool) string { + s := "Path1" + if !isPath1 { + s = "Path2" + } + return s +} + +// TODO: +// equality check? +// unicode stuff diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go index e3f590b45..67cb5b440 100644 --- a/cmd/bisync/operations.go +++ b/cmd/bisync/operations.go @@ -45,6 +45,8 @@ type queues struct { renamed1 bilib.Names // renamed on 1 and copied to 2 renamed2 bilib.Names // renamed on 2 and copied to 1 renameSkipped bilib.Names // not renamed because it was equal + skippedDirs1 *fileList + skippedDirs2 *fileList deletedonboth bilib.Names } @@ -217,10 +219,15 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) { return errors.New("cannot find prior Path1 or Path2 listings, likely due to critical error on prior run") } + fs.Infof(nil, "Building Path1 and Path2 listings") + ls1, ls2, err = b.makeMarchListing(fctx) + if err != nil { + return err + } + // Check for Path1 deltas relative to the prior sync fs.Infof(nil, "Path1 checking for diffs") - newListing1 := b.listing1 + "-new" - ds1, err := b.findDeltas(fctx, b.fs1, b.listing1, newListing1, "Path1") + ds1, err := b.findDeltas(fctx, b.fs1, b.listing1, ls1, "Path1") if err != nil { return err } @@ -228,8 +235,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) { // Check for Path2 deltas relative to the prior sync fs.Infof(nil, "Path2 checking for diffs") - newListing2 := b.listing2 + "-new" - ds2, err := b.findDeltas(fctx, b.fs2, b.listing2, newListing2, "Path2") + ds2, err := b.findDeltas(fctx, b.fs2, b.listing2, ls2, "Path2") if err != nil { return err } @@ -298,8 +304,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) { b.saveOldListings() // save new listings if noChanges { - err1 = bilib.CopyFileIfExists(newListing1, b.listing1) - err2 = bilib.CopyFileIfExists(newListing2, b.listing2) + b.replaceCurrentListings() } else { if changes1 { // 2to1 err1 = b.modifyListing(fctx, b.fs2, b.fs1, results2to1, queues, false) @@ -360,7 +365,10 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) { func (b *bisyncRun) resync(octx, fctx context.Context) error { fs.Infof(nil, "Copying unique Path2 files to Path1") - filesNow1, err := b.makeListing(fctx, b.fs1, b.newListing1) + // TODO: remove this listing eventually. + // Listing here is only really necessary for our --ignore-existing logic + // which would be more efficiently implemented by setting ci.IgnoreExisting + filesNow1, filesNow2, err := b.makeMarchListing(fctx) if err == nil { err = b.checkListing(filesNow1, b.newListing1, "current Path1") } @@ -368,10 +376,7 @@ func (b *bisyncRun) resync(octx, fctx context.Context) error { return err } - filesNow2, err := b.makeListing(fctx, b.fs2, b.newListing2) - if err == nil { - err = b.checkListing(filesNow2, b.newListing2, "current Path2") - } + err = b.checkListing(filesNow2, b.newListing2, "current Path2") if err != nil { return err } @@ -468,13 +473,8 @@ func (b *bisyncRun) resync(octx, fctx context.Context) error { } fs.Infof(nil, "Resync updating listings") - b.saveOldListings() // TODO: also make replaceCurrentListings? - if err := bilib.CopyFileIfExists(b.newListing1, b.listing1); err != nil { - return err - } - if err := bilib.CopyFileIfExists(b.newListing2, b.listing2); err != nil { - return err - } + b.saveOldListings() + b.replaceCurrentListings() // resync 2to1 queues.copy2to1 = bilib.ToNames(copy2to1) @@ -574,3 +574,17 @@ func (b *bisyncRun) testFn() { b.opt.TestFn() } } + +func (b *bisyncRun) handleErr(o interface{}, msg string, err error, critical, retryable bool) { + if err != nil { + if retryable { + b.retryable = true + } + if critical { + b.critical = true + fs.Errorf(o, "%s: %v", msg, err) + } else { + fs.Debugf(o, "%s: %v", msg, err) + } + } +} diff --git a/cmd/bisync/testdata/test_all_changed/golden/test.log b/cmd/bisync/testdata/test_all_changed/golden/test.log index c2daac34d..9922112e7 100644 --- a/cmd/bisync/testdata/test_all_changed/golden/test.log +++ b/cmd/bisync/testdata/test_all_changed/golden/test.log @@ -16,6 +16,7 @@ INFO : Bisync successful (07) : test sync should pass (08) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is newer - file1.copy1.txt INFO : - Path1 File is newer - file1.copy2.txt @@ -46,6 +47,7 @@ INFO : Bisync successful (12) : test sync should fail (13) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is newer - RCLONE_TEST INFO : - Path1 File is OLDER - file1.copy1.txt @@ -64,6 +66,7 @@ Bisync error: all files were changed (14) : test sync with force should pass (15) : bisync force INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is newer - RCLONE_TEST INFO : - Path1 File is OLDER - file1.copy1.txt diff --git a/cmd/bisync/testdata/test_basic/golden/test.log b/cmd/bisync/testdata/test_basic/golden/test.log index 57d6541a0..8eb3e47f4 100644 --- a/cmd/bisync/testdata/test_basic/golden/test.log +++ b/cmd/bisync/testdata/test_basic/golden/test.log @@ -17,6 +17,7 @@ INFO : Bisync successful (07) : test bisync run (08) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is newer - subdir/file20.txt INFO : Path1: 1 changes: 0 new, 1 newer, 0 older, 0 deleted diff --git a/cmd/bisync/testdata/test_changes/golden/test.log b/cmd/bisync/testdata/test_changes/golden/test.log index aff7b456d..e595678da 100644 --- a/cmd/bisync/testdata/test_changes/golden/test.log +++ b/cmd/bisync/testdata/test_changes/golden/test.log @@ -49,6 +49,7 @@ INFO : Bisync successful (31) : test bisync run (32) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is newer - file2.txt INFO : - Path1 File was deleted - file4.txt diff --git a/cmd/bisync/testdata/test_check_access/golden/test.log b/cmd/bisync/testdata/test_check_access/golden/test.log index d7382b9af..d99ef694d 100644 --- a/cmd/bisync/testdata/test_check_access/golden/test.log +++ b/cmd/bisync/testdata/test_check_access/golden/test.log @@ -12,6 +12,7 @@ INFO : Bisync successful (04) : test 1. see that check-access passes with the initial setup (05) : bisync check-access INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : Checking access health @@ -25,6 +26,7 @@ INFO : Bisync successful (07) : delete-file {path2/}subdir/RCLONE_TEST (08) : bisync check-access INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : - Path2 File was deleted - subdir/RCLONE_TEST @@ -49,6 +51,7 @@ INFO : Bisync successful (13) : test 4. run sync with check-access. should pass. (14) : bisync check-access INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : Checking access health @@ -62,6 +65,7 @@ INFO : Bisync successful (16) : delete-file {path1/}RCLONE_TEST (17) : bisync check-access INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File was deleted - RCLONE_TEST INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted @@ -95,6 +99,7 @@ INFO : Bisync successful (24) : test 8. run sync with --check-access. should pass. (25) : bisync check-access INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : Checking access health diff --git a/cmd/bisync/testdata/test_check_access_filters/golden/test.log b/cmd/bisync/testdata/test_check_access_filters/golden/test.log index cd7753a9f..c12126374 100644 --- a/cmd/bisync/testdata/test_check_access_filters/golden/test.log +++ b/cmd/bisync/testdata/test_check_access_filters/golden/test.log @@ -18,6 +18,7 @@ INFO : Bisync successful (07) : bisync check-access filters-file={workdir/}exclude-other-filtersfile.txt INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Using filters file {workdir/}exclude-other-filtersfile.txt +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : Checking access health @@ -38,6 +39,7 @@ INFO : Bisync successful (15) : bisync check-access filters-file={workdir/}exclude-other-filtersfile.txt INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Using filters file {workdir/}exclude-other-filtersfile.txt +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : Checking access health @@ -56,6 +58,7 @@ INFO : Bisync successful (21) : bisync check-access filters-file={workdir/}exclude-other-filtersfile.txt INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Using filters file {workdir/}exclude-other-filtersfile.txt +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File was deleted - subdir/RCLONE_TEST INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted @@ -88,6 +91,7 @@ INFO : Bisync successful (30) : bisync check-access filters-file={workdir/}include-other-filtersfile.txt INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Using filters file {workdir/}include-other-filtersfile.txt +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : Checking access health @@ -107,6 +111,7 @@ INFO : Bisync successful (37) : bisync check-access filters-file={workdir/}include-other-filtersfile.txt INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Using filters file {workdir/}include-other-filtersfile.txt +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : Checking access health @@ -126,6 +131,7 @@ INFO : Bisync successful (44) : bisync check-access filters-file={workdir/}include-other-filtersfile.txt INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Using filters file {workdir/}include-other-filtersfile.txt +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File was deleted - subdir/RCLONE_TEST INFO : - Path1 File was deleted - subdirX/subdirX1/RCLONE_TEST @@ -136,8 +142,8 @@ INFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deleted INFO : Checking access health ERROR : Access test failed: Path1 count 3, Path2 count 4 - RCLONE_TEST ERROR : -  Access test failed: Path1 file not found in Path2 - RCLONE_TEST -ERROR : -  Access test failed: Path2 file not found in Path1 - subdirX/subdirX1/RCLONE_TEST ERROR : -  Access test failed: Path2 file not found in Path1 - subdir/RCLONE_TEST +ERROR : -  Access test failed: Path2 file not found in Path1 - subdirX/subdirX1/RCLONE_TEST ERROR : Bisync critical error: check file check failed ERROR : Bisync aborted. Must run --resync to recover. Bisync error: bisync aborted diff --git a/cmd/bisync/testdata/test_check_filename/golden/test.log b/cmd/bisync/testdata/test_check_filename/golden/test.log index f03e1fdc2..c860b6e20 100644 --- a/cmd/bisync/testdata/test_check_filename/golden/test.log +++ b/cmd/bisync/testdata/test_check_filename/golden/test.log @@ -12,6 +12,7 @@ INFO : Bisync successful (04) : test 1. see that check-access passes with the initial setup (05) : bisync check-access check-filename=.chk_file INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : Checking access health @@ -26,6 +27,7 @@ INFO : Bisync successful (08) : delete-file {path2/}subdir/.chk_file (09) : bisync check-access check-filename=.chk_file INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : - Path2 File was deleted - subdir/.chk_file @@ -52,6 +54,7 @@ INFO : Bisync successful (14) : test 4. run sync with check-access. should pass. (15) : bisync check-access check-filename=.chk_file INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : Checking access health diff --git a/cmd/bisync/testdata/test_check_sync/golden/test.log b/cmd/bisync/testdata/test_check_sync/golden/test.log index d70224b29..277b45ff3 100644 --- a/cmd/bisync/testdata/test_check_sync/golden/test.log +++ b/cmd/bisync/testdata/test_check_sync/golden/test.log @@ -51,6 +51,7 @@ INFO : Bisync successful (20) : test 7. run normal sync with check-sync enabled (default) (21) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : No changes found @@ -61,6 +62,7 @@ INFO : Bisync successful (22) : test 8. run normal sync with no-check-sync (23) : bisync no-check-sync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : No changes found diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log index 1b2583d27..dd9be0d67 100644 --- a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log +++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log @@ -24,6 +24,7 @@ INFO : Bisync successful (15) : test 2. Run bisync without --create-empty-src-dirs (16) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : No changes found @@ -39,6 +40,7 @@ subdir/ (20) : test 4.Run bisync WITH --create-empty-src-dirs (21) : bisync create-empty-src-dirs INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is new - subdir INFO : Path1: 1 changes: 1 new, 0 newer, 0 older, 0 deleted @@ -68,6 +70,7 @@ subdir/ (33) : test 7. Run bisync without --create-empty-src-dirs (34) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File was deleted - RCLONE_TEST INFO : - Path1 File was deleted - subdir @@ -115,6 +118,7 @@ subdir/ (51) : bisync create-empty-src-dirs INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File was deleted - subdir INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted @@ -134,6 +138,7 @@ INFO : Bisync successful (55) : test 11. bisync again (because if we leave subdir in listings, test will fail due to mismatched modtime) (56) : bisync create-empty-src-dirs INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : No changes found diff --git a/cmd/bisync/testdata/test_dry_run/golden/test.log b/cmd/bisync/testdata/test_dry_run/golden/test.log index db5a487fa..f29698f0a 100644 --- a/cmd/bisync/testdata/test_dry_run/golden/test.log +++ b/cmd/bisync/testdata/test_dry_run/golden/test.log @@ -66,6 +66,7 @@ INFO : Bisync successful (30) : test sync with dry-run (31) : bisync dry-run INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is newer - file2.txt INFO : - Path1 File was deleted - file4.txt @@ -120,6 +121,7 @@ INFO : Bisync successful (33) : test sync without dry-run (34) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is newer - file2.txt INFO : - Path1 File was deleted - file4.txt diff --git a/cmd/bisync/testdata/test_equal/golden/test.log b/cmd/bisync/testdata/test_equal/golden/test.log index 37bd0d41a..fe32ce710 100644 --- a/cmd/bisync/testdata/test_equal/golden/test.log +++ b/cmd/bisync/testdata/test_equal/golden/test.log @@ -23,6 +23,7 @@ INFO : Bisync successful (13) : test bisync run (14) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is newer - file1.txt INFO : - Path1 File is newer - file2.txt diff --git a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log b/cmd/bisync/testdata/test_extended_char_paths/golden/test.log index 772d88ad6..e0ff286c2 100644 --- a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log +++ b/cmd/bisync/testdata/test_extended_char_paths/golden/test.log @@ -19,6 +19,7 @@ INFO : Bisync successful (09) : test normal sync of subdirs with extended chars (10) : bisync subdir=測試_Русский_{spc}_{spc}_ě_áñ INFO : Synching Path1 "{path1/}測試_Русский_ _ _ě_áñ/" with Path2 "{path2/}測試_Русский_ _ _ě_áñ/" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is new - 測試_file1p1 INFO : Path1: 1 changes: 1 new, 0 newer, 0 older, 0 deleted @@ -45,6 +46,7 @@ INFO : Bisync successful (14) : delete-file {path1/}測試_Русский_{spc}_{spc}_ě_áñ/測試_check{spc}file (15) : bisync check-access check-filename=測試_check{spc}file INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File was deleted - 測試_Русский_ _ _ě_áñ/測試_check file INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted @@ -68,6 +70,7 @@ INFO : Resync updating listings INFO : Bisync successful (19) : bisync check-access check-filename=測試_check{spc}file INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : Checking access health @@ -92,6 +95,7 @@ INFO : Bisync successful (25) : bisync filters-file={workdir/}測試_filtersfile.txt INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Using filters file {workdir/}測試_filtersfile.txt +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : No changes found diff --git a/cmd/bisync/testdata/test_extended_filenames/golden/test.log b/cmd/bisync/testdata/test_extended_filenames/golden/test.log index 039ed9e82..790a722da 100644 --- a/cmd/bisync/testdata/test_extended_filenames/golden/test.log +++ b/cmd/bisync/testdata/test_extended_filenames/golden/test.log @@ -38,6 +38,7 @@ INFO : Bisync successful (26) : test bisync run (27) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File was deleted - Русский.txt INFO : - Path1 File is new - file1_with white space.txt diff --git a/cmd/bisync/testdata/test_filters/golden/test.log b/cmd/bisync/testdata/test_filters/golden/test.log index dfea30ddf..2821e617b 100644 --- a/cmd/bisync/testdata/test_filters/golden/test.log +++ b/cmd/bisync/testdata/test_filters/golden/test.log @@ -24,6 +24,7 @@ INFO : Bisync successful (11) : bisync filters-file={workdir/}filtersfile.flt INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Using filters file {workdir/}filtersfile.flt +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is new - subdir/fileZ.txt INFO : Path1: 1 changes: 1 new, 0 newer, 0 older, 0 deleted diff --git a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log index 937d4fbee..b4494cbdf 100644 --- a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log +++ b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log @@ -41,6 +41,7 @@ INFO : Bisync successful (13) : bisync filters-file={workdir/}filtersfile.txt INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" INFO : Using filters file {workdir/}filtersfile.txt +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : No changes found diff --git a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log index 6e383f784..4baad3309 100644 --- a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log +++ b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log @@ -23,6 +23,7 @@ INFO : Bisync successful (08) : test bisync run (09) : bisync ignore-listing-checksum INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is newer - subdir/file20.txt INFO : Path1: 1 changes: 0 new, 1 newer, 0 older, 0 deleted diff --git a/cmd/bisync/testdata/test_max_delete_path1/golden/test.log b/cmd/bisync/testdata/test_max_delete_path1/golden/test.log index c6d802a2f..83c33ef2d 100644 --- a/cmd/bisync/testdata/test_max_delete_path1/golden/test.log +++ b/cmd/bisync/testdata/test_max_delete_path1/golden/test.log @@ -19,6 +19,7 @@ INFO : Bisync successful (10) : test sync should fail due to too many local deletes (11) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File was deleted - file1.txt INFO : - Path1 File was deleted - file2.txt @@ -35,6 +36,7 @@ Bisync error: too many deletes (13) : test change max-delete limit to 60%. sync should run. (14) : bisync max-delete=60 INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File was deleted - file1.txt INFO : - Path1 File was deleted - file2.txt diff --git a/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log b/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log index ca5a0f636..326987c4b 100644 --- a/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log +++ b/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log @@ -19,6 +19,7 @@ INFO : Bisync successful (10) : test sync should fail due to too many path2 deletes (11) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : - Path2 File was deleted - file1.txt @@ -35,6 +36,7 @@ Bisync error: too many deletes (13) : test apply force option. sync should run. (14) : bisync force INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : - Path2 File was deleted - file1.txt diff --git a/cmd/bisync/testdata/test_rclone_args/golden/test.log b/cmd/bisync/testdata/test_rclone_args/golden/test.log index f0caf729a..f92ecfe1d 100644 --- a/cmd/bisync/testdata/test_rclone_args/golden/test.log +++ b/cmd/bisync/testdata/test_rclone_args/golden/test.log @@ -23,6 +23,7 @@ INFO : Bisync successful (10) : test run bisync with custom options (11) : bisync size-only INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is newer - file1.txt INFO : - Path1 File is newer - subdir/file20.txt diff --git a/cmd/bisync/testdata/test_resync/golden/test.log b/cmd/bisync/testdata/test_resync/golden/test.log index 39f35b09e..41e84ce58 100644 --- a/cmd/bisync/testdata/test_resync/golden/test.log +++ b/cmd/bisync/testdata/test_resync/golden/test.log @@ -72,6 +72,7 @@ INFO : Bisync successful (32) : test run normal bisync (33) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : No changes found @@ -83,6 +84,7 @@ INFO : Bisync successful (35) : purge-children {path2/} (36) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs ERROR : Empty current Path2 listing. Cannot sync to an empty directory: {workdir/}{session}.path2.lst-new diff --git a/cmd/bisync/testdata/test_rmdirs/golden/test.log b/cmd/bisync/testdata/test_rmdirs/golden/test.log index eca4e4fba..5ea1bacdd 100644 --- a/cmd/bisync/testdata/test_rmdirs/golden/test.log +++ b/cmd/bisync/testdata/test_rmdirs/golden/test.log @@ -15,6 +15,7 @@ INFO : Bisync successful (06) : test 2. run bisync without remove-empty-dirs (07) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File was deleted - subdir/file20.txt INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted @@ -35,6 +36,7 @@ subdir/ (11) : test 4. run bisync with remove-empty-dirs (12) : bisync remove-empty-dirs INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : Path2 checking for diffs INFO : No changes found diff --git a/cmd/bisync/testdata/test_volatile/golden/test.log b/cmd/bisync/testdata/test_volatile/golden/test.log index b65c8a26e..97fce76af 100644 --- a/cmd/bisync/testdata/test_volatile/golden/test.log +++ b/cmd/bisync/testdata/test_volatile/golden/test.log @@ -18,6 +18,7 @@ INFO : Bisync successful (10) : test-func (11) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is newer - file5.txt INFO : Path1: 1 changes: 0 new, 1 newer, 0 older, 0 deleted @@ -51,6 +52,7 @@ INFO : Bisync successful (18) : test-func (19) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is new - file100.txt INFO : - Path1 File is new - file5.txt @@ -268,6 +270,7 @@ INFO : Bisync successful (26) : test-func (27) : bisync INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings INFO : Path1 checking for diffs INFO : - Path1 File is new - file5.txt INFO : Path1: 1 changes: 1 new, 0 newer, 0 older, 0 deleted diff --git a/docs/content/bisync.md b/docs/content/bisync.md index c34073890..b01e82cc0 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -1275,6 +1275,9 @@ about _Unison_ and synchronization in general. * Final listings are now generated from sync results, to avoid needing to re-list * Bisync is now much more resilient to changes that happen during a bisync run, and far less prone to critical errors / undetected changes * Bisync is now capable of rolling a file listing back in cases of uncertainty, essentially marking the file as needing to be rechecked next time. +* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`) +* Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`, +for performance improvements and less [risk of error](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=4.%20Listings%20should%20alternate%20between%20paths%20to%20minimize%20errors). ### `v1.64` * Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)