From b14269fd23a309e86b5e0d019501cb897a83ba6a Mon Sep 17 00:00:00 2001 From: nielash Date: Tue, 9 Jan 2024 10:07:53 -0500 Subject: [PATCH] bisync: add support for --retries-sleep - fixes #7555 Before this change, bisync supported --retries but not --retries-sleep. This change adds support for --retries-sleep. --- cmd/bisync/cmd.go | 4 +++- cmd/bisync/operations.go | 27 ++++++++++++++------------- cmd/bisync/queue.go | 23 +++++++++++++++++++++++ docs/content/bisync.md | 4 +++- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.go index 805e9897d..e50ae9e16 100644 --- a/cmd/bisync/cmd.go +++ b/cmd/bisync/cmd.go @@ -52,6 +52,7 @@ type Options struct { Recover bool TestFn TestFunc // test-only option, for mocking errors Retries int + RetriesInterval time.Duration Compare CompareOpt CompareFlag string DebugName string @@ -143,7 +144,8 @@ func init() { flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "") flags.BoolVarP(cmdFlags, &Opt.Resilient, "resilient", "", Opt.Resilient, "Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!", "") flags.BoolVarP(cmdFlags, &Opt.Recover, "recover", "", Opt.Recover, "Automatically recover from interruptions without requiring --resync.", "") - flags.IntVarP(cmdFlags, &Opt.Retries, "retries", "", Opt.Retries, "Retry operations this many times if they fail", "") + flags.IntVarP(cmdFlags, &Opt.Retries, "retries", "", Opt.Retries, "Retry operations this many times if they fail (requires --resilient).", "") + flags.DurationVarP(cmdFlags, &Opt.RetriesInterval, "retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)", "") flags.StringVarP(cmdFlags, &Opt.CompareFlag, "compare", "", Opt.CompareFlag, "Comma-separated list of bisync-specific compare options ex. 'size,modtime,checksum' (default: 'size,modtime')", "") flags.BoolVarP(cmdFlags, &Opt.Compare.NoSlowHash, "no-slow-hash", "", Opt.Compare.NoSlowHash, "Ignore listing checksums only on backends where they are slow", "") flags.BoolVarP(cmdFlags, &Opt.Compare.SlowHashSyncOnly, "slow-hash-sync-only", "", Opt.Compare.SlowHashSyncOnly, "Ignore slow checksums for listings and deltas, but still consider them during sync calls.", "") diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go index 432ceaee4..4b5a288b1 100644 --- a/cmd/bisync/operations.go +++ b/cmd/bisync/operations.go @@ -127,19 +127,6 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) { // Handle SIGINT var finaliseOnce gosync.Once - // waitFor runs fn() until it returns true or the timeout expires - waitFor := func(msg string, totalWait time.Duration, fn func() bool) (ok bool) { - const individualWait = 1 * time.Second - for i := 0; i < int(totalWait/individualWait); i++ { - ok = fn() - if ok { - return ok - } - fs.Infof(nil, Color(terminal.YellowFg, "%s: %v"), msg, int(totalWait/individualWait)-i) - time.Sleep(individualWait) - } - return false - } finalise := func() { finaliseOnce.Do(func() { if atexit.Signalled() { @@ -648,6 +635,20 @@ func (b *bisyncRun) debugFn(nametocheck string, fn func()) { } } +// waitFor runs fn() until it returns true or the timeout expires +func waitFor(msg string, totalWait time.Duration, fn func() bool) (ok bool) { + const individualWait = 1 * time.Second + for i := 0; i < int(totalWait/individualWait); i++ { + ok = fn() + if ok { + return ok + } + fs.Infof(nil, Color(terminal.YellowFg, "%s: %vs"), msg, int(totalWait/individualWait)-i) + time.Sleep(individualWait) + } + return false +} + // mainly to make sure tests don't interfere with each other when running more than one func resetGlobals() { downloadHash = false diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.go index 73d9f18e6..bdd85b665 100644 --- a/cmd/bisync/queue.go +++ b/cmd/bisync/queue.go @@ -266,6 +266,16 @@ func (b *bisyncRun) retryFastCopy(ctx context.Context, fsrc, fdst fs.Fs, files b for tries := 1; tries <= b.opt.Retries; tries++ { fs.Logf(queueName, Color(terminal.YellowFg, "Received error: %v - retrying as --resilient is set. Retry %d/%d"), err, tries, b.opt.Retries) accounting.GlobalStats().ResetErrors() + if retryAfter := accounting.GlobalStats().RetryAfter(); !retryAfter.IsZero() { + d := time.Until(retryAfter) + if d > 0 { + fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d) + time.Sleep(d) + } + } + if b.opt.RetriesInterval > 0 { + naptime(b.opt.RetriesInterval) + } results, err = b.fastCopy(ctx, fsrc, fdst, files, queueName) if err == nil || b.InGracefulShutdown { return results, err @@ -362,3 +372,16 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error { queueFile := fmt.Sprintf("%s.%s.que", b.basePath, jobName) return files.Save(queueFile) } + +func naptime(totalWait time.Duration) { + expireTime := time.Now().Add(totalWait) + fs.Logf(nil, "will retry in %v at %v", totalWait, expireTime.Format("2006-01-02 15:04:05 MST")) + for i := 0; time.Until(expireTime) > 0; i++ { + if i > 0 && i%10 == 0 { + fs.Infof(nil, Color(terminal.Dim, "retrying in %v..."), time.Until(expireTime).Round(1*time.Second)) + } else { + fs.Debugf(nil, Color(terminal.Dim, "retrying in %v..."), time.Until(expireTime).Round(1*time.Second)) + } + time.Sleep(1 * time.Second) + } +} diff --git a/docs/content/bisync.md b/docs/content/bisync.md index 93b819297..85250985c 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -114,7 +114,8 @@ Optional Flags: --resilient Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk! -1, --resync Performs the resync run. Equivalent to --resync-mode path1. Consider using --verbose or --dry-run first. --resync-mode string During resync, prefer the version that is: path1, path2, newer, older, larger, smaller (default: path1 if --resync, otherwise none for no resync.) (default "none") - --retries int Retry operations this many times if they fail (default 3) + --retries int Retry operations this many times if they fail (requires --resilient). (default 3) + --retries-sleep Duration Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) (default 0s) --slow-hash-sync-only Ignore slow checksums for listings and deltas, but still consider them during sync calls. --workdir string Use custom working dir - useful for testing. (default: {WORKDIR}) --max-delete PERCENT Safety check on maximum percentage of deleted files allowed. If exceeded, the bisync run will abort. (default: 50%) @@ -1833,6 +1834,7 @@ instead of of `--size-only`, when `check` is not available. * A new `--max-lock` setting allows lock files to automatically renew and expire, for better automatic recovery when a run is interrupted. * Bisync now supports auto-resolving sync conflicts and customizing rename behavior with new [`--conflict-resolve`](#conflict-resolve), [`--conflict-loser`](#conflict-loser), and [`--conflict-suffix`](#conflict-suffix) flags. * A new [`--resync-mode`](#resync-mode) flag allows more control over which version of a file gets kept during a `--resync`. +* Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-time) (when [`--resilient`](#resilient) is set.) ### `v1.64` * Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)