From b14269fd23a309e86b5e0d019501cb897a83ba6a Mon Sep 17 00:00:00 2001
From: nielash <nielronash@gmail.com>
Date: Tue, 9 Jan 2024 10:07:53 -0500
Subject: [PATCH] bisync: add support for --retries-sleep - fixes #7555
Before this change, bisync supported --retries but not --retries-sleep.
This change adds support for --retries-sleep.
---
cmd/bisync/cmd.go | 4 +++-
cmd/bisync/operations.go | 27 ++++++++++++++-------------
cmd/bisync/queue.go | 23 +++++++++++++++++++++++
docs/content/bisync.md | 4 +++-
4 files changed, 43 insertions(+), 15 deletions(-)
diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.go
index 805e9897d..e50ae9e16 100644
--- a/cmd/bisync/cmd.go
+++ b/cmd/bisync/cmd.go
@@ -52,6 +52,7 @@ type Options struct {
Recover bool
TestFn TestFunc // test-only option, for mocking errors
Retries int
+ RetriesInterval time.Duration
Compare CompareOpt
CompareFlag string
DebugName string
@@ -143,7 +144,8 @@ func init() {
flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "")
flags.BoolVarP(cmdFlags, &Opt.Resilient, "resilient", "", Opt.Resilient, "Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!", "")
flags.BoolVarP(cmdFlags, &Opt.Recover, "recover", "", Opt.Recover, "Automatically recover from interruptions without requiring --resync.", "")
- flags.IntVarP(cmdFlags, &Opt.Retries, "retries", "", Opt.Retries, "Retry operations this many times if they fail", "")
+ flags.IntVarP(cmdFlags, &Opt.Retries, "retries", "", Opt.Retries, "Retry operations this many times if they fail (requires --resilient).", "")
+ flags.DurationVarP(cmdFlags, &Opt.RetriesInterval, "retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)", "")
flags.StringVarP(cmdFlags, &Opt.CompareFlag, "compare", "", Opt.CompareFlag, "Comma-separated list of bisync-specific compare options ex. 'size,modtime,checksum' (default: 'size,modtime')", "")
flags.BoolVarP(cmdFlags, &Opt.Compare.NoSlowHash, "no-slow-hash", "", Opt.Compare.NoSlowHash, "Ignore listing checksums only on backends where they are slow", "")
flags.BoolVarP(cmdFlags, &Opt.Compare.SlowHashSyncOnly, "slow-hash-sync-only", "", Opt.Compare.SlowHashSyncOnly, "Ignore slow checksums for listings and deltas, but still consider them during sync calls.", "")
diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go
index 432ceaee4..4b5a288b1 100644
--- a/cmd/bisync/operations.go
+++ b/cmd/bisync/operations.go
@@ -127,19 +127,6 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
// Handle SIGINT
var finaliseOnce gosync.Once
- // waitFor runs fn() until it returns true or the timeout expires
- waitFor := func(msg string, totalWait time.Duration, fn func() bool) (ok bool) {
- const individualWait = 1 * time.Second
- for i := 0; i < int(totalWait/individualWait); i++ {
- ok = fn()
- if ok {
- return ok
- }
- fs.Infof(nil, Color(terminal.YellowFg, "%s: %v"), msg, int(totalWait/individualWait)-i)
- time.Sleep(individualWait)
- }
- return false
- }
finalise := func() {
finaliseOnce.Do(func() {
if atexit.Signalled() {
@@ -648,6 +635,20 @@ func (b *bisyncRun) debugFn(nametocheck string, fn func()) {
}
}
+// waitFor runs fn() until it returns true or the timeout expires
+func waitFor(msg string, totalWait time.Duration, fn func() bool) (ok bool) {
+ const individualWait = 1 * time.Second
+ for i := 0; i < int(totalWait/individualWait); i++ {
+ ok = fn()
+ if ok {
+ return ok
+ }
+ fs.Infof(nil, Color(terminal.YellowFg, "%s: %vs"), msg, int(totalWait/individualWait)-i)
+ time.Sleep(individualWait)
+ }
+ return false
+}
+
// mainly to make sure tests don't interfere with each other when running more than one
func resetGlobals() {
downloadHash = false
diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.go
index 73d9f18e6..bdd85b665 100644
--- a/cmd/bisync/queue.go
+++ b/cmd/bisync/queue.go
@@ -266,6 +266,16 @@ func (b *bisyncRun) retryFastCopy(ctx context.Context, fsrc, fdst fs.Fs, files b
for tries := 1; tries <= b.opt.Retries; tries++ {
fs.Logf(queueName, Color(terminal.YellowFg, "Received error: %v - retrying as --resilient is set. Retry %d/%d"), err, tries, b.opt.Retries)
accounting.GlobalStats().ResetErrors()
+ if retryAfter := accounting.GlobalStats().RetryAfter(); !retryAfter.IsZero() {
+ d := time.Until(retryAfter)
+ if d > 0 {
+ fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d)
+ time.Sleep(d)
+ }
+ }
+ if b.opt.RetriesInterval > 0 {
+ naptime(b.opt.RetriesInterval)
+ }
results, err = b.fastCopy(ctx, fsrc, fdst, files, queueName)
if err == nil || b.InGracefulShutdown {
return results, err
@@ -362,3 +372,16 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {
queueFile := fmt.Sprintf("%s.%s.que", b.basePath, jobName)
return files.Save(queueFile)
}
+
+func naptime(totalWait time.Duration) {
+ expireTime := time.Now().Add(totalWait)
+ fs.Logf(nil, "will retry in %v at %v", totalWait, expireTime.Format("2006-01-02 15:04:05 MST"))
+ for i := 0; time.Until(expireTime) > 0; i++ {
+ if i > 0 && i%10 == 0 {
+ fs.Infof(nil, Color(terminal.Dim, "retrying in %v..."), time.Until(expireTime).Round(1*time.Second))
+ } else {
+ fs.Debugf(nil, Color(terminal.Dim, "retrying in %v..."), time.Until(expireTime).Round(1*time.Second))
+ }
+ time.Sleep(1 * time.Second)
+ }
+}
diff --git a/docs/content/bisync.md b/docs/content/bisync.md
index 93b819297..85250985c 100644
--- a/docs/content/bisync.md
+++ b/docs/content/bisync.md
@@ -114,7 +114,8 @@ Optional Flags:
--resilient Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!
-1, --resync Performs the resync run. Equivalent to --resync-mode path1. Consider using --verbose or --dry-run first.
--resync-mode string During resync, prefer the version that is: path1, path2, newer, older, larger, smaller (default: path1 if --resync, otherwise none for no resync.) (default "none")
- --retries int Retry operations this many times if they fail (default 3)
+ --retries int Retry operations this many times if they fail (requires --resilient). (default 3)
+ --retries-sleep Duration Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) (default 0s)
--slow-hash-sync-only Ignore slow checksums for listings and deltas, but still consider them during sync calls.
--workdir string Use custom working dir - useful for testing. (default: {WORKDIR})
--max-delete PERCENT Safety check on maximum percentage of deleted files allowed. If exceeded, the bisync run will abort. (default: 50%)
@@ -1833,6 +1834,7 @@ instead of of `--size-only`, when `check` is not available.
* A new `--max-lock` setting allows lock files to automatically renew and expire, for better automatic recovery when a run is interrupted.
* Bisync now supports auto-resolving sync conflicts and customizing rename behavior with new [`--conflict-resolve`](#conflict-resolve), [`--conflict-loser`](#conflict-loser), and [`--conflict-suffix`](#conflict-suffix) flags.
* A new [`--resync-mode`](#resync-mode) flag allows more control over which version of a file gets kept during a `--resync`.
+* Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-time) (when [`--resilient`](#resilient) is set.)
### `v1.64`
* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)