Merge pull request #1579 from ifedorenko/RetryBackend-List

Implement RetryBackend.List()

changelog/0.8.2/pull-1579

@@ -0,0 +1,3 @@
+Enhancement: Retry Backend.List() in case of errors
+
+https://github.com/restic/restic/pull/1579

internal/backend/backend_retry.go

@@ -128,3 +128,17 @@ func (be *RetryBackend) Test(ctx context.Context, h restic.Handle) (exists bool,
 	})
 	return exists, err
 }
+
+// List runs fn for each file in the backend which has the type t.
+func (be *RetryBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
+	listed := make(map[string]struct{})
+	return be.retry(ctx, fmt.Sprintf("List(%v)", t), func() error {
+		return be.Backend.List(ctx, t, func(fi restic.FileInfo) error {
+			if _, ok := listed[fi.Name]; ok {
+				return nil
+			}
+			listed[fi.Name] = struct{}{}
+			return fn(fi)
+		})
+	})
+}

internal/backend/backend_retry_test.go

@@ -88,3 +88,38 @@ func TestBackendSaveRetry(t *testing.T) {
 		t.Errorf("wrong data written to backend")
 	}
 }
+
+func TestBackendListRetry(t *testing.T) {
+	const (
+		ID1 = "id1"
+		ID2 = "id2"
+	)
+
+	retry := 0
+	be := &mock.Backend{
+		ListFn: func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
+			// fail during first retry, succeed during second
+			retry++
+			if retry == 1 {
+				fn(restic.FileInfo{Name: ID1})
+				return errors.New("test list error")
+			}
+			fn(restic.FileInfo{Name: ID1})
+			fn(restic.FileInfo{Name: ID2})
+			return nil
+		},
+	}
+
+	retryBackend := RetryBackend{
+		Backend: be,
+	}
+
+	var listed []string
+	err := retryBackend.List(context.TODO(), restic.DataFile, func(fi restic.FileInfo) error {
+		listed = append(listed, fi.Name)
+		return nil
+	})
+	test.OK(t, err)                            // assert overall success
+	test.Equals(t, 2, retry)                   // assert retried once
+	test.Equals(t, []string{ID1, ID2}, listed) // assert no duplicate files
+}

internal/restic/backend.go

@@ -35,6 +35,9 @@ type Backend interface {
 	// List runs fn for each file in the backend which has the type t. When an
 	// error occurs (or fn returns an error), List stops and returns it.
 	//
+	// The function fn is called exactly once for each file during successful
+	// execution and at most once in case of an error.
+	//
 	// The function fn is called in the same Goroutine that List() is called
 	// from.
 	List(ctx context.Context, t FileType, fn func(FileInfo) error) error