diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.go index d08583c7d..5c4fddf48 100644 --- a/cmd/bisync/cmd.go +++ b/cmd/bisync/cmd.go @@ -27,18 +27,19 @@ import ( // Options keep bisync options type Options struct { - Resync bool - CheckAccess bool - CheckFilename string - CheckSync CheckSyncMode - RemoveEmptyDirs bool - MaxDelete int // percentage from 0 to 100 - Force bool - FiltersFile string - Workdir string - DryRun bool - NoCleanup bool - SaveQueues bool // save extra debugging files (test only flag) + Resync bool + CheckAccess bool + CheckFilename string + CheckSync CheckSyncMode + RemoveEmptyDirs bool + MaxDelete int // percentage from 0 to 100 + Force bool + FiltersFile string + Workdir string + DryRun bool + NoCleanup bool + SaveQueues bool // save extra debugging files (test only flag) + IgnoreListingChecksum bool } // Default values @@ -108,6 +109,7 @@ func init() { flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "") flags.BoolVarP(cmdFlags, &tzLocal, "localtime", "", tzLocal, "Use local time in listings (default: UTC)", "") flags.BoolVarP(cmdFlags, &Opt.NoCleanup, "no-cleanup", "", Opt.NoCleanup, "Retain working files (useful for troubleshooting and testing).", "") + flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "") } // bisync command definition @@ -213,9 +215,9 @@ func (opt *Options) applyFilters(ctx context.Context) (context.Context, error) { if opt.DryRun { fs.Infof(nil, "Skipped storing filters file hash to %s as --dry-run is set", hashFile) } else { - fs.Infof(nil, "Storing filters file hash to %s", hashFile) - if err := os.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil { - return ctx, err + fs.Infof(nil, "Storing filters file hash to %s", hashFile) + if err := os.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil { + return ctx, err } } } diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.go index 066b927e8..557e44d5d 100644 --- a/cmd/bisync/listing.go +++ b/cmd/bisync/listing.go @@ -253,8 +253,9 @@ func (b *bisyncRun) makeListing(ctx context.Context, f fs.Fs, listing string) (l ci := fs.GetConfig(ctx) depth := ci.MaxDepth hashType := hash.None - if !ci.IgnoreChecksum { - // Currently bisync just honors --ignore-checksum + if !b.opt.IgnoreListingChecksum { + // Currently bisync just honors --ignore-listing-checksum + // (note that this is different from --ignore-checksum) // TODO add full support for checksums and related flags hashType = f.Hashes().GetOne() } diff --git a/docs/content/bisync.md b/docs/content/bisync.md index 6c0d7fa86..ec2851762 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -95,6 +95,8 @@ Optional Flags: -1, --resync Performs the resync run. Warning: Path1 files may overwrite Path2 versions. Consider using `--verbose` or `--dry-run` first. + --ignore-listing-checksum Do not use checksums for listings + (add --ignore-checksum to additionally skip post-copy checksum checks) --localtime Use local time in listings (default: UTC) --no-cleanup Retain working files (useful for troubleshooting and testing). --workdir PATH Use custom working directory (useful for testing). @@ -255,6 +257,34 @@ sync run times for very large numbers of files. The check may be run manually with `--check-sync=only`. It runs only the integrity check and terminates without actually synching. + +#### --ignore-listing-checksum + +By default, bisync will retrieve (or generate) checksums (for backends that support them) +when creating the listings for both paths, and store the checksums in the listing files. +`--ignore-listing-checksum` will disable this behavior, which may speed things up considerably, +especially on backends (such as [local](/local/)) where hashes must be computed on the fly instead of retrieved. +Please note the following: + +* While checksums are (by default) generated and stored in the listing files, +they are NOT currently used for determining diffs (deltas). +It is anticipated that full checksum support will be added in a future version. +* `--ignore-listing-checksum` is NOT the same as [`--ignore-checksum`](/docs/#ignore-checksum), +and you may wish to use one or the other, or both. In a nutshell: +`--ignore-listing-checksum` controls whether checksums are considered when scanning for diffs, +while `--ignore-checksum` controls whether checksums are considered during the copy/sync operations that follow, +if there ARE diffs. +* Unless `--ignore-listing-checksum` is passed, bisync currently computes hashes for one path +*even when there's no common hash with the other path* +(for example, a [crypt](/crypt/#modified-time-and-hashes) remote.) +* If both paths support checksums and have a common hash, +AND `--ignore-listing-checksum` was not specified when creating the listings, +`--check-sync=only` can be used to compare Path1 vs. Path2 checksums (as of the time the previous listings were created.) +However, `--check-sync=only` will NOT include checksums if the previous listings +were generated on a run using `--ignore-listing-checksum`. For a more robust integrity check of the current state, +consider using [`check`](commands/rclone_check/) +(or [`cryptcheck`](/commands/rclone_cryptcheck/), if at least one path is a `crypt` remote.) + ## Operation ### Runtime flow details @@ -1134,3 +1164,5 @@ causing dry runs to inadvertently commit filter changes causing bisync to consider more files than necessary due to overbroad filters during delete operations * [Improved detection of false positive change conflicts](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Identical%20files%20should%20be%20left%20alone%2C%20even%20if%20new/newer/changed%20on%20both%20sides) (identical files are now left alone instead of renamed) +* Added [new `--ignore-listing-checksum` flag](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20%2D%2Dignore%2Dchecksum%20should%20be%20split%20into%20two%20flags%20for%20separate%20purposes) +to distinguish from `--ignore-checksum`