bisync: add support for --retries-sleep - fixes #7555

Before this change, bisync supported --retries but not --retries-sleep.
This change adds support for --retries-sleep.

cmd/bisync/cmd.go

@@ -52,6 +52,7 @@ type Options struct {
 	Recover               bool
 	TestFn                TestFunc // test-only option, for mocking errors
 	Retries               int
+	RetriesInterval       time.Duration
 	Compare               CompareOpt
 	CompareFlag           string
 	DebugName             string
@@ -143,7 +144,8 @@ func init() {
 	flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "")
 	flags.BoolVarP(cmdFlags, &Opt.Resilient, "resilient", "", Opt.Resilient, "Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!", "")
 	flags.BoolVarP(cmdFlags, &Opt.Recover, "recover", "", Opt.Recover, "Automatically recover from interruptions without requiring --resync.", "")
-	flags.IntVarP(cmdFlags, &Opt.Retries, "retries", "", Opt.Retries, "Retry operations this many times if they fail", "")
+	flags.IntVarP(cmdFlags, &Opt.Retries, "retries", "", Opt.Retries, "Retry operations this many times if they fail (requires --resilient).", "")
+	flags.DurationVarP(cmdFlags, &Opt.RetriesInterval, "retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)", "")
 	flags.StringVarP(cmdFlags, &Opt.CompareFlag, "compare", "", Opt.CompareFlag, "Comma-separated list of bisync-specific compare options ex. 'size,modtime,checksum' (default: 'size,modtime')", "")
 	flags.BoolVarP(cmdFlags, &Opt.Compare.NoSlowHash, "no-slow-hash", "", Opt.Compare.NoSlowHash, "Ignore listing checksums only on backends where they are slow", "")
 	flags.BoolVarP(cmdFlags, &Opt.Compare.SlowHashSyncOnly, "slow-hash-sync-only", "", Opt.Compare.SlowHashSyncOnly, "Ignore slow checksums for listings and deltas, but still consider them during sync calls.", "")

cmd/bisync/operations.go

@@ -127,19 +127,6 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 	// Handle SIGINT
 	var finaliseOnce gosync.Once
 
-	// waitFor runs fn() until it returns true or the timeout expires
-	waitFor := func(msg string, totalWait time.Duration, fn func() bool) (ok bool) {
-		const individualWait = 1 * time.Second
-		for i := 0; i < int(totalWait/individualWait); i++ {
-			ok = fn()
-			if ok {
-				return ok
-			}
-			fs.Infof(nil, Color(terminal.YellowFg, "%s: %v"), msg, int(totalWait/individualWait)-i)
-			time.Sleep(individualWait)
-		}
-		return false
-	}
 	finalise := func() {
 		finaliseOnce.Do(func() {
 			if atexit.Signalled() {
@@ -648,6 +635,20 @@ func (b *bisyncRun) debugFn(nametocheck string, fn func()) {
 	}
 }
 
+// waitFor runs fn() until it returns true or the timeout expires
+func waitFor(msg string, totalWait time.Duration, fn func() bool) (ok bool) {
+	const individualWait = 1 * time.Second
+	for i := 0; i < int(totalWait/individualWait); i++ {
+		ok = fn()
+		if ok {
+			return ok
+		}
+		fs.Infof(nil, Color(terminal.YellowFg, "%s: %vs"), msg, int(totalWait/individualWait)-i)
+		time.Sleep(individualWait)
+	}
+	return false
+}
+
 // mainly to make sure tests don't interfere with each other when running more than one
 func resetGlobals() {
 	downloadHash = false

cmd/bisync/queue.go

@@ -266,6 +266,16 @@ func (b *bisyncRun) retryFastCopy(ctx context.Context, fsrc, fdst fs.Fs, files b
 		for tries := 1; tries <= b.opt.Retries; tries++ {
 			fs.Logf(queueName, Color(terminal.YellowFg, "Received error: %v - retrying as --resilient is set. Retry %d/%d"), err, tries, b.opt.Retries)
 			accounting.GlobalStats().ResetErrors()
+			if retryAfter := accounting.GlobalStats().RetryAfter(); !retryAfter.IsZero() {
+				d := time.Until(retryAfter)
+				if d > 0 {
+					fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d)
+					time.Sleep(d)
+				}
+			}
+			if b.opt.RetriesInterval > 0 {
+				naptime(b.opt.RetriesInterval)
+			}
 			results, err = b.fastCopy(ctx, fsrc, fdst, files, queueName)
 			if err == nil || b.InGracefulShutdown {
 				return results, err
@@ -362,3 +372,16 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {
 	queueFile := fmt.Sprintf("%s.%s.que", b.basePath, jobName)
 	return files.Save(queueFile)
 }
+
+func naptime(totalWait time.Duration) {
+	expireTime := time.Now().Add(totalWait)
+	fs.Logf(nil, "will retry in %v at %v", totalWait, expireTime.Format("2006-01-02 15:04:05 MST"))
+	for i := 0; time.Until(expireTime) > 0; i++ {
+		if i > 0 && i%10 == 0 {
+			fs.Infof(nil, Color(terminal.Dim, "retrying in %v..."), time.Until(expireTime).Round(1*time.Second))
+		} else {
+			fs.Debugf(nil, Color(terminal.Dim, "retrying in %v..."), time.Until(expireTime).Round(1*time.Second))
+		}
+		time.Sleep(1 * time.Second)
+	}
+}

docs/content/bisync.md

@@ -114,7 +114,8 @@ Optional Flags:
       --resilient                            Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!
   -1, --resync                               Performs the resync run. Equivalent to --resync-mode path1. Consider using --verbose or --dry-run first.
       --resync-mode string                   During resync, prefer the version that is: path1, path2, newer, older, larger, smaller (default: path1 if --resync, otherwise none for no resync.) (default "none")
-      --retries int                          Retry operations this many times if they fail (default 3)
+      --retries int                          Retry operations this many times if they fail (requires --resilient). (default 3)
+      --retries-sleep Duration               Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) (default 0s)
       --slow-hash-sync-only                  Ignore slow checksums for listings and deltas, but still consider them during sync calls.
       --workdir string                       Use custom working dir - useful for testing. (default: {WORKDIR})
       --max-delete PERCENT                   Safety check on maximum percentage of deleted files allowed. If exceeded, the bisync run will abort. (default: 50%)
@@ -1833,6 +1834,7 @@ instead of of `--size-only`, when `check` is not available.
 * A new `--max-lock` setting allows lock files to automatically renew and expire, for better automatic recovery when a run is interrupted.
 * Bisync now supports auto-resolving sync conflicts and customizing rename behavior with new [`--conflict-resolve`](#conflict-resolve), [`--conflict-loser`](#conflict-loser), and [`--conflict-suffix`](#conflict-suffix) flags.
 * A new [`--resync-mode`](#resync-mode) flag allows more control over which version of a file gets kept during a `--resync`.
+* Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-time) (when [`--resilient`](#resilient) is set.)
 
 ### `v1.64`
 * Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)