retry: add circuit breaker to load method

If a file exhausts its retry attempts, then it is likely not accessible
the next time. Thus, immediately fail all load calls for that file to
avoid useless retries.

internal/backend/retry/backend_retry.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"sync"
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
@@ -20,6 +21,8 @@ type Backend struct {
 	MaxTries int
 	Report   func(string, error, time.Duration)
 	Success  func(string, int)
+
+	failedLoads sync.Map
 }
 
 // statically ensure that RetryBackend implements backend.Backend.
@@ -132,15 +135,39 @@ func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewind
 	})
 }
 
+// Failed loads expire after an hour
+var failedLoadExpiry = time.Hour
+
 // Load returns a reader that yields the contents of the file at h at the
 // given offset. If length is larger than zero, only a portion of the file
 // is returned. rd must be closed after use. If an error is returned, the
 // ReadCloser must be nil.
 func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
-	return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset),
+	key := h
+	key.IsMetadata = false
+
+	// Implement the circuit breaker pattern for files that exhausted all retries due to a non-permanent error
+	if v, ok := be.failedLoads.Load(key); ok {
+		if time.Since(v.(time.Time)) > failedLoadExpiry {
+			be.failedLoads.Delete(key)
+		} else {
+			// fail immediately if the file was already problematic during the last hour
+			return fmt.Errorf("circuit breaker open for file %v", h)
+		}
+	}
+
+	err = be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset),
 		func() error {
 			return be.Backend.Load(ctx, h, length, offset, consumer)
 		})
+
+	if feature.Flag.Enabled(feature.BackendErrorRedesign) && err != nil && !be.IsPermanentError(err) {
+		// We've exhausted the retries, the file is likely inaccessible. By excluding permanent
+		// errors, not found or truncated files are not recorded.
+		be.failedLoads.LoadOrStore(key, time.Now())
+	}
+
+	return err
 }
 
 // Stat returns information about the File identified by h.