s3 migrate layout: Retry on errors

src/restic/migrations/s3_layout.go

@@ -2,6 +2,8 @@ package migrations
 
 import (
 	"context"
+	"fmt"
+	"os"
 	"path"
 	"restic"
 	"restic/backend"
@@ -34,13 +36,35 @@ func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, err
 	return true, nil
 }
 
+func retry(max int, fail func(err error), f func() error) error {
+	var err error
+	for i := 0; i < max; i++ {
+		err = f()
+		if err == nil {
+			return err
+		}
+		if fail != nil {
+			fail(err)
+		}
+	}
+	return err
+}
+
+// maxErrors for retrying renames on s3.
+const maxErrors = 20
+
 func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layout, t restic.FileType) error {
+	printErr := func(err error) {
+		fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err)
+	}
+
 	for name := range be.List(ctx, t) {
 		h := restic.Handle{Type: t, Name: name}
 		debug.Log("move %v", h)
-		if err := be.Rename(h, l); err != nil {
-			return err
-		}
+
+		retry(maxErrors, printErr, func() error {
+			return be.Rename(h, l)
+		})
 	}
 
 	return nil