From 4ccd5e806b17736afd0f0f117b241656e9bea785 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 15 Oct 2022 16:23:39 +0200 Subject: [PATCH 1/4] backend: split layout code into own subpackage --- internal/backend/azure/azure.go | 5 +++-- internal/backend/b2/b2.go | 7 ++++--- internal/backend/gs/gs.go | 5 +++-- internal/backend/{ => layout}/layout.go | 2 +- internal/backend/{ => layout}/layout_default.go | 2 +- internal/backend/{ => layout}/layout_rest.go | 2 +- internal/backend/{ => layout}/layout_s3legacy.go | 2 +- internal/backend/{ => layout}/layout_test.go | 16 ++++++++-------- internal/backend/local/local.go | 5 +++-- internal/backend/rest/rest.go | 6 +++--- internal/backend/s3/s3.go | 7 ++++--- internal/backend/sftp/sftp.go | 7 ++++--- internal/backend/swift/swift.go | 5 +++-- internal/migrations/s3_layout.go | 8 ++++---- 14 files changed, 43 insertions(+), 36 deletions(-) rename internal/backend/{ => layout}/layout.go (99%) rename internal/backend/{ => layout}/layout_default.go (99%) rename internal/backend/{ => layout}/layout_rest.go (98%) rename internal/backend/{ => layout}/layout_s3legacy.go (99%) rename internal/backend/{ => layout}/layout_test.go (95%) diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 7f3539f07..e6dc226cb 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -13,6 +13,7 @@ import ( "strings" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" @@ -30,7 +31,7 @@ type Backend struct { sem sema.Semaphore prefix string listMaxItems int - backend.Layout + layout.Layout } const defaultListMaxItems = 5000 @@ -85,7 +86,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { connections: cfg.Connections, sem: sem, prefix: cfg.Prefix, - Layout: &backend.DefaultLayout{ + Layout: &layout.DefaultLayout{ Path: cfg.Prefix, Join: path.Join, }, diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 4a6c79abb..b95506856 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -10,6 +10,7 @@ import ( "time" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" @@ -25,7 +26,7 @@ type b2Backend struct { bucket *b2.Bucket cfg Config listMaxItems int - backend.Layout + layout.Layout sem sema.Semaphore } @@ -97,7 +98,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend client: client, bucket: bucket, cfg: cfg, - Layout: &backend.DefaultLayout{ + Layout: &layout.DefaultLayout{ Join: path.Join, Path: cfg.Prefix, }, @@ -138,7 +139,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe client: client, bucket: bucket, cfg: cfg, - Layout: &backend.DefaultLayout{ + Layout: &layout.DefaultLayout{ Join: path.Join, Path: cfg.Prefix, }, diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 8ffc96af8..f4e4a350b 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -14,6 +14,7 @@ import ( "cloud.google.com/go/storage" "github.com/pkg/errors" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -41,7 +42,7 @@ type Backend struct { bucket *storage.BucketHandle prefix string listMaxItems int - backend.Layout + layout.Layout } // Ensure that *Backend implements restic.Backend. @@ -111,7 +112,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { bucketName: cfg.Bucket, bucket: gcsClient.Bucket(cfg.Bucket), prefix: cfg.Prefix, - Layout: &backend.DefaultLayout{ + Layout: &layout.DefaultLayout{ Path: cfg.Prefix, Join: path.Join, }, diff --git a/internal/backend/layout.go b/internal/backend/layout/layout.go similarity index 99% rename from internal/backend/layout.go rename to internal/backend/layout/layout.go index 421e85e8e..14fb8dcdc 100644 --- a/internal/backend/layout.go +++ b/internal/backend/layout/layout.go @@ -1,4 +1,4 @@ -package backend +package layout import ( "context" diff --git a/internal/backend/layout_default.go b/internal/backend/layout/layout_default.go similarity index 99% rename from internal/backend/layout_default.go rename to internal/backend/layout/layout_default.go index 3bc3087ed..17c250e8f 100644 --- a/internal/backend/layout_default.go +++ b/internal/backend/layout/layout_default.go @@ -1,4 +1,4 @@ -package backend +package layout import ( "encoding/hex" diff --git a/internal/backend/layout_rest.go b/internal/backend/layout/layout_rest.go similarity index 98% rename from internal/backend/layout_rest.go rename to internal/backend/layout/layout_rest.go index 1d65828a8..2aa869995 100644 --- a/internal/backend/layout_rest.go +++ b/internal/backend/layout/layout_rest.go @@ -1,4 +1,4 @@ -package backend +package layout import "github.com/restic/restic/internal/restic" diff --git a/internal/backend/layout_s3legacy.go b/internal/backend/layout/layout_s3legacy.go similarity index 99% rename from internal/backend/layout_s3legacy.go rename to internal/backend/layout/layout_s3legacy.go index f83355860..ac88e77ad 100644 --- a/internal/backend/layout_s3legacy.go +++ b/internal/backend/layout/layout_s3legacy.go @@ -1,4 +1,4 @@ -package backend +package layout import "github.com/restic/restic/internal/restic" diff --git a/internal/backend/layout_test.go b/internal/backend/layout/layout_test.go similarity index 95% rename from internal/backend/layout_test.go rename to internal/backend/layout/layout_test.go index d319a0b2d..554f8a418 100644 --- a/internal/backend/layout_test.go +++ b/internal/backend/layout/layout_test.go @@ -1,4 +1,4 @@ -package backend +package layout import ( "context" @@ -362,15 +362,15 @@ func TestDetectLayout(t *testing.T) { filename string want string }{ - {"repo-layout-default.tar.gz", "*backend.DefaultLayout"}, - {"repo-layout-s3legacy.tar.gz", "*backend.S3LegacyLayout"}, + {"repo-layout-default.tar.gz", "*layout.DefaultLayout"}, + {"repo-layout-s3legacy.tar.gz", "*layout.S3LegacyLayout"}, } var fs = &LocalFilesystem{} for _, test := range tests { for _, fs := range []Filesystem{fs, nil} { t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) { - rtest.SetupTarTestFixture(t, path, filepath.Join("testdata", test.filename)) + rtest.SetupTarTestFixture(t, path, filepath.Join("../testdata", test.filename)) layout, err := DetectLayout(context.TODO(), fs, filepath.Join(path, "repo")) if err != nil { @@ -401,12 +401,12 @@ func TestParseLayout(t *testing.T) { defaultLayoutName string want string }{ - {"default", "", "*backend.DefaultLayout"}, - {"s3legacy", "", "*backend.S3LegacyLayout"}, - {"", "", "*backend.DefaultLayout"}, + {"default", "", "*layout.DefaultLayout"}, + {"s3legacy", "", "*layout.S3LegacyLayout"}, + {"", "", "*layout.DefaultLayout"}, } - rtest.SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-default.tar.gz")) + rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", "repo-layout-default.tar.gz")) for _, test := range tests { t.Run(test.layoutName, func(t *testing.T) { diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index bb644c949..bce83d768 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -10,6 +10,7 @@ import ( "syscall" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" @@ -23,7 +24,7 @@ import ( type Local struct { Config sem sema.Semaphore - backend.Layout + layout.Layout backend.Modes } @@ -33,7 +34,7 @@ var _ restic.Backend = &Local{} const defaultLayout = "default" func open(ctx context.Context, cfg Config) (*Local, error) { - l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) + l, err := layout.ParseLayout(ctx, &layout.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err } diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 6444e8655..3431fd681 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -14,7 +14,7 @@ import ( "strconv" "strings" - "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" @@ -32,7 +32,7 @@ type Backend struct { connections uint sem sema.Semaphore client http.Client - backend.Layout + layout.Layout } // the REST API protocol version is decided by HTTP request headers, these are the constants. @@ -57,7 +57,7 @@ func Open(cfg Config, rt http.RoundTripper) (*Backend, error) { be := &Backend{ url: cfg.URL, client: http.Client{Transport: rt}, - Layout: &backend.RESTLayout{URL: url, Join: path.Join}, + Layout: &layout.RESTLayout{URL: url, Join: path.Join}, connections: cfg.Connections, sem: sem, } diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 0b3816c06..32a175676 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -13,6 +13,7 @@ import ( "time" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" @@ -28,7 +29,7 @@ type Backend struct { client *minio.Client sem sema.Semaphore cfg Config - backend.Layout + layout.Layout } // make sure that *Backend implements backend.Backend @@ -113,7 +114,7 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro cfg: cfg, } - l, err := backend.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix) + l, err := layout.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix) if err != nil { return nil, err } @@ -514,7 +515,7 @@ func (be *Backend) Delete(ctx context.Context) error { func (be *Backend) Close() error { return nil } // Rename moves a file based on the new layout l. -func (be *Backend) Rename(ctx context.Context, h restic.Handle, l backend.Layout) error { +func (be *Backend) Rename(ctx context.Context, h restic.Handle, l layout.Layout) error { debug.Log("Rename %v to %v", h, l) oldname := be.Filename(h) newname := l.Filename(h) diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index f6e82a402..deebb6e7c 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -14,6 +14,7 @@ import ( "time" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" @@ -35,7 +36,7 @@ type SFTP struct { posixRename bool sem sema.Semaphore - backend.Layout + layout.Layout Config backend.Modes } @@ -144,7 +145,7 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) { return nil, err } - sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) + sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err } @@ -243,7 +244,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) { return nil, err } - sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) + sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err } diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index a739b2c6b..f97724ee4 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -14,6 +14,7 @@ import ( "time" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" @@ -30,7 +31,7 @@ type beSwift struct { sem sema.Semaphore container string // Container name prefix string // Prefix of object names in the container - backend.Layout + layout.Layout } // ensure statically that *beSwift implements restic.Backend. @@ -74,7 +75,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend sem: sem, container: cfg.Container, prefix: cfg.Prefix, - Layout: &backend.DefaultLayout{ + Layout: &layout.DefaultLayout{ Path: cfg.Prefix, Join: path.Join, }, diff --git a/internal/migrations/s3_layout.go b/internal/migrations/s3_layout.go index be11733db..d42b94bf8 100644 --- a/internal/migrations/s3_layout.go +++ b/internal/migrations/s3_layout.go @@ -6,7 +6,7 @@ import ( "os" "path" - "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" @@ -74,7 +74,7 @@ func retry(max int, fail func(err error), f func() error) error { // maxErrors for retrying renames on s3. const maxErrors = 20 -func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layout, t restic.FileType) error { +func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layout, t restic.FileType) error { printErr := func(err error) { fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err) } @@ -97,12 +97,12 @@ func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error { return errors.New("backend is not s3") } - oldLayout := &backend.S3LegacyLayout{ + oldLayout := &layout.S3LegacyLayout{ Path: be.Path(), Join: path.Join, } - newLayout := &backend.DefaultLayout{ + newLayout := &layout.DefaultLayout{ Path: be.Path(), Join: path.Join, } From 8c18c65b3b0d4d318c21c2aac7964444d4f60599 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 15 Oct 2022 16:27:04 +0200 Subject: [PATCH 2/4] backend: remove unused Paths variable --- internal/backend/paths.go | 19 ------------------- internal/backend/sftp/sftp.go | 4 ++-- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/internal/backend/paths.go b/internal/backend/paths.go index eaa1a433a..7e511be9c 100644 --- a/internal/backend/paths.go +++ b/internal/backend/paths.go @@ -2,25 +2,6 @@ package backend import "os" -// Paths contains the default paths for file-based backends (e.g. local). -var Paths = struct { - Data string - Snapshots string - Index string - Locks string - Keys string - Temp string - Config string -}{ - "data", - "snapshots", - "index", - "locks", - "keys", - "tmp", - "config", -} - type Modes struct { Dir os.FileMode File os.FileMode diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index deebb6e7c..417b6681a 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -152,7 +152,7 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) { debug.Log("layout: %v\n", sftp.Layout) - fi, err := sftp.c.Stat(Join(cfg.Path, backend.Paths.Config)) + fi, err := sftp.c.Stat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile})) m := backend.DeriveModesFromFileInfo(fi, err) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) @@ -252,7 +252,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) { sftp.Modes = backend.DefaultModes // test if config file already exists - _, err = sftp.c.Lstat(Join(cfg.Path, backend.Paths.Config)) + _, err = sftp.c.Lstat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile})) if err == nil { return nil, errors.New("config file already exists") } From 32603d49c41cc7b5e71767d939356df2dc6fae9f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 15 Oct 2022 16:27:43 +0200 Subject: [PATCH 3/4] backend: remove unused ErrorBackend --- internal/backend/backend_error.go | 84 ------------------------------- 1 file changed, 84 deletions(-) delete mode 100644 internal/backend/backend_error.go diff --git a/internal/backend/backend_error.go b/internal/backend/backend_error.go deleted file mode 100644 index 77a931858..000000000 --- a/internal/backend/backend_error.go +++ /dev/null @@ -1,84 +0,0 @@ -package backend - -import ( - "context" - "io" - "io/ioutil" - "math/rand" - "sync" - - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" -) - -// ErrorBackend is used to induce errors into various function calls and test -// the retry functions. -type ErrorBackend struct { - FailSave float32 - FailSaveRead float32 - FailLoad float32 - FailStat float32 - restic.Backend - - r *rand.Rand - m sync.Mutex -} - -// statically ensure that ErrorBackend implements restic.Backend. -var _ restic.Backend = &ErrorBackend{} - -// NewErrorBackend wraps be with a backend that returns errors according to -// given probabilities. -func NewErrorBackend(be restic.Backend, seed int64) *ErrorBackend { - return &ErrorBackend{ - Backend: be, - r: rand.New(rand.NewSource(seed)), - } -} - -func (be *ErrorBackend) fail(p float32) bool { - be.m.Lock() - v := be.r.Float32() - be.m.Unlock() - - return v < p -} - -// Save stores the data in the backend under the given handle. -func (be *ErrorBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if be.fail(be.FailSave) { - return errors.Errorf("Save(%v) random error induced", h) - } - - if be.fail(be.FailSaveRead) { - _, err := io.CopyN(ioutil.Discard, rd, be.r.Int63n(1000)) - if err != nil { - return err - } - - return errors.Errorf("Save(%v) random error with partial read induced", h) - } - - return be.Backend.Save(ctx, h, rd) -} - -// Load returns a reader that yields the contents of the file at h at the -// given offset. If length is larger than zero, only a portion of the file -// is returned. rd must be closed after use. If an error is returned, the -// ReadCloser must be nil. -func (be *ErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { - if be.fail(be.FailLoad) { - return errors.Errorf("Load(%v, %v, %v) random error induced", h, length, offset) - } - - return be.Backend.Load(ctx, h, length, offset, consumer) -} - -// Stat returns information about the File identified by h. -func (be *ErrorBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - if be.fail(be.FailLoad) { - return restic.FileInfo{}, errors.Errorf("Stat(%v) random error induced", h) - } - - return be.Stat(ctx, h) -} From 5c7a9a739ad3c5b2a1a2cb0f181a836e274398a4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 15 Oct 2022 16:33:15 +0200 Subject: [PATCH 4/4] backend: Split RetryBackend into own package The RetryBackend tests depend on the mock backend. When the Backend interface is eventually split from the restic package, this will lead to a dependency cycle between backend and backend/mock. Thus split the RetryBackend into a separate package to avoid this problem. --- cmd/restic/global.go | 3 +- cmd/restic/integration_helpers_test.go | 4 +-- internal/backend/{ => retry}/backend_retry.go | 28 +++++++++---------- .../backend/{ => retry}/backend_retry_test.go | 16 +++++------ internal/backend/{ => retry}/testing.go | 2 +- 5 files changed, 27 insertions(+), 26 deletions(-) rename internal/backend/{ => retry}/backend_retry.go (82%) rename internal/backend/{ => retry}/backend_retry_test.go (95%) rename internal/backend/{ => retry}/testing.go (90%) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index fb14934f8..25dae55f2 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -22,6 +22,7 @@ import ( "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/rclone" "github.com/restic/restic/internal/backend/rest" + "github.com/restic/restic/internal/backend/retry" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/sftp" "github.com/restic/restic/internal/backend/swift" @@ -445,7 +446,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi success := func(msg string, retries int) { Warnf("%v operation successful after %d retries\n", msg, retries) } - be = backend.NewRetryBackend(be, 10, report, success) + be = retry.New(be, 10, report, success) // wrap backend if a test specified a hook if opts.backendTestHook != nil { diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index 6a4d064a8..1acde5bc2 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -9,7 +9,7 @@ import ( "runtime" "testing" - "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/retry" "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -172,7 +172,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) { repository.TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) - backend.TestFastRetries(t) + retry.TestFastRetries(t) tempdir, err := ioutil.TempDir(rtest.TestTempDir, "restic-test-") rtest.OK(t, err) diff --git a/internal/backend/backend_retry.go b/internal/backend/retry/backend_retry.go similarity index 82% rename from internal/backend/backend_retry.go rename to internal/backend/retry/backend_retry.go index e9a22d75f..334fe4216 100644 --- a/internal/backend/backend_retry.go +++ b/internal/backend/retry/backend_retry.go @@ -1,4 +1,4 @@ -package backend +package retry import ( "context" @@ -11,9 +11,9 @@ import ( "github.com/restic/restic/internal/restic" ) -// RetryBackend retries operations on the backend in case of an error with a +// Backend retries operations on the backend in case of an error with a // backoff. -type RetryBackend struct { +type Backend struct { restic.Backend MaxTries int Report func(string, error, time.Duration) @@ -21,14 +21,14 @@ type RetryBackend struct { } // statically ensure that RetryBackend implements restic.Backend. -var _ restic.Backend = &RetryBackend{} +var _ restic.Backend = &Backend{} -// NewRetryBackend wraps be with a backend that retries operations after a +// New wraps be with a backend that retries operations after a // backoff. report is called with a description and the error, if one occurred. // success is called with the number of retries before a successful operation // (it is not called if it succeeded on the first try) -func NewRetryBackend(be restic.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *RetryBackend { - return &RetryBackend{ +func New(be restic.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend { + return &Backend{ Backend: be, MaxTries: maxTries, Report: report, @@ -57,7 +57,7 @@ func retryNotifyErrorWithSuccess(operation backoff.Operation, b backoff.BackOff, var fastRetries = false -func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) error { +func (be *Backend) retry(ctx context.Context, msg string, f func() error) error { // Don't do anything when called with an already cancelled context. There would be // no retries in that case either, so be consistent and abort always. // This enforces a strict contract for backend methods: Using a cancelled context @@ -92,7 +92,7 @@ func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) e } // Save stores the data in the backend under the given handle. -func (be *RetryBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error { err := rd.Rewind() if err != nil { @@ -125,7 +125,7 @@ func (be *RetryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rew // given offset. If length is larger than zero, only a portion of the file // is returned. rd must be closed after use. If an error is returned, the // ReadCloser must be nil. -func (be *RetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) { +func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) { return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset), func() error { return be.Backend.Load(ctx, h, length, offset, consumer) @@ -133,7 +133,7 @@ func (be *RetryBackend) Load(ctx context.Context, h restic.Handle, length int, o } // Stat returns information about the File identified by h. -func (be *RetryBackend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) { +func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) { err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h), func() error { var innerError error @@ -145,14 +145,14 @@ func (be *RetryBackend) Stat(ctx context.Context, h restic.Handle) (fi restic.Fi } // Remove removes a File with type t and name. -func (be *RetryBackend) Remove(ctx context.Context, h restic.Handle) (err error) { +func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) { return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error { return be.Backend.Remove(ctx, h) }) } // Test a boolean value whether a File with the name and type exists. -func (be *RetryBackend) Test(ctx context.Context, h restic.Handle) (exists bool, err error) { +func (be *Backend) Test(ctx context.Context, h restic.Handle) (exists bool, err error) { err = be.retry(ctx, fmt.Sprintf("Test(%v)", h), func() error { var innerError error exists, innerError = be.Backend.Test(ctx, h) @@ -166,7 +166,7 @@ func (be *RetryBackend) Test(ctx context.Context, h restic.Handle) (exists bool, // error is returned by the underlying backend, the request is retried. When fn // returns an error, the operation is aborted and the error is returned to the // caller. -func (be *RetryBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { // create a new context that we can cancel when fn returns an error, so // that listing is aborted listCtx, cancel := context.WithCancel(ctx) diff --git a/internal/backend/backend_retry_test.go b/internal/backend/retry/backend_retry_test.go similarity index 95% rename from internal/backend/backend_retry_test.go rename to internal/backend/retry/backend_retry_test.go index aca30f61d..b8f2dafca 100644 --- a/internal/backend/backend_retry_test.go +++ b/internal/backend/retry/backend_retry_test.go @@ -1,4 +1,4 @@ -package backend +package retry import ( "bytes" @@ -36,7 +36,7 @@ func TestBackendSaveRetry(t *testing.T) { } TestFastRetries(t) - retryBackend := NewRetryBackend(be, 10, nil, nil) + retryBackend := New(be, 10, nil, nil) data := test.Random(23, 5*1024*1024+11241) err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher())) @@ -72,7 +72,7 @@ func TestBackendSaveRetryAtomic(t *testing.T) { } TestFastRetries(t) - retryBackend := NewRetryBackend(be, 10, nil, nil) + retryBackend := New(be, 10, nil, nil) data := test.Random(23, 5*1024*1024+11241) err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher())) @@ -106,7 +106,7 @@ func TestBackendListRetry(t *testing.T) { } TestFastRetries(t) - retryBackend := NewRetryBackend(be, 10, nil, nil) + retryBackend := New(be, 10, nil, nil) var listed []string err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { @@ -136,7 +136,7 @@ func TestBackendListRetryErrorFn(t *testing.T) { } TestFastRetries(t) - retryBackend := NewRetryBackend(be, 10, nil, nil) + retryBackend := New(be, 10, nil, nil) var ErrTest = errors.New("test error") @@ -193,7 +193,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) { TestFastRetries(t) const maxRetries = 2 - retryBackend := NewRetryBackend(be, maxRetries, nil, nil) + retryBackend := New(be, maxRetries, nil, nil) var listed []string err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { @@ -263,7 +263,7 @@ func TestBackendLoadRetry(t *testing.T) { } TestFastRetries(t) - retryBackend := NewRetryBackend(be, 10, nil, nil) + retryBackend := New(be, 10, nil, nil) var buf []byte err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) { @@ -283,7 +283,7 @@ func TestBackendCanceledContext(t *testing.T) { // unimplemented mock backend functions return an error by default // check that we received the expected context canceled error instead TestFastRetries(t) - retryBackend := NewRetryBackend(mock.NewBackend(), 2, nil, nil) + retryBackend := New(mock.NewBackend(), 2, nil, nil) h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()} // create an already canceled context diff --git a/internal/backend/testing.go b/internal/backend/retry/testing.go similarity index 90% rename from internal/backend/testing.go rename to internal/backend/retry/testing.go index 0d7fa1d76..797573b03 100644 --- a/internal/backend/testing.go +++ b/internal/backend/retry/testing.go @@ -1,4 +1,4 @@ -package backend +package retry import "testing"