From 16fcd071109132f2e360299dce7d522190be2dd5 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 3 Jun 2017 17:39:57 +0200 Subject: [PATCH] Add a Context to the backend --- src/restic/backend.go | 21 +++--- src/restic/backend/b2/b2.go | 49 ++++++-------- src/restic/backend/b2/b2_test.go | 3 +- src/restic/backend/local/layout_test.go | 3 +- src/restic/backend/local/local.go | 26 +++---- src/restic/backend/mem/mem_backend.go | 17 ++--- src/restic/backend/mem/mem_backend_test.go | 3 +- src/restic/backend/rest/rest.go | 59 +++++++++------- src/restic/backend/s3/s3.go | 34 +++++----- src/restic/backend/s3/s3_test.go | 6 +- src/restic/backend/sftp/layout_test.go | 3 +- src/restic/backend/sftp/sftp.go | 17 ++--- src/restic/backend/swift/swift.go | 32 +++++---- src/restic/backend/swift/swift_test.go | 5 +- src/restic/backend/test/benchmarks.go | 15 ++-- src/restic/backend/test/tests.go | 79 +++++++++++----------- src/restic/backend/test/tests_test.go | 3 +- src/restic/backend/utils.go | 5 +- src/restic/backend/utils_test.go | 13 ++-- src/restic/backend_find.go | 15 ++-- src/restic/backend_find_test.go | 11 +-- src/restic/lock.go | 9 +-- src/restic/readerat.go | 3 +- src/restic/repository.go | 9 ++- 24 files changed, 231 insertions(+), 209 deletions(-) diff --git a/src/restic/backend.go b/src/restic/backend.go index 4f776b167..0020a76a9 100644 --- a/src/restic/backend.go +++ b/src/restic/backend.go @@ -1,6 +1,9 @@ package restic -import "io" +import ( + "context" + "io" +) // Backend is used to store and access data. type Backend interface { @@ -9,30 +12,30 @@ type Backend interface { Location() string // Test a boolean value whether a File with the name and type exists. - Test(h Handle) (bool, error) + Test(ctx context.Context, h Handle) (bool, error) // Remove removes a File with type t and name. - Remove(h Handle) error + Remove(ctx context.Context, h Handle) error // Close the backend Close() error // Save stores the data in the backend under the given handle. - Save(h Handle, rd io.Reader) error + Save(ctx context.Context, h Handle, rd io.Reader) error // Load returns a reader that yields the contents of the file at h at the // given offset. If length is larger than zero, only a portion of the file // is returned. rd must be closed after use. If an error is returned, the // ReadCloser must be nil. - Load(h Handle, length int, offset int64) (io.ReadCloser, error) + Load(ctx context.Context, h Handle, length int, offset int64) (io.ReadCloser, error) // Stat returns information about the File identified by h. - Stat(h Handle) (FileInfo, error) + Stat(ctx context.Context, h Handle) (FileInfo, error) // List returns a channel that yields all names of files of type t in an - // arbitrary order. A goroutine is started for this. If the channel done is - // closed, sending stops. - List(t FileType, done <-chan struct{}) <-chan string + // arbitrary order. A goroutine is started for this, which is stopped when + // ctx is cancelled. + List(ctx context.Context, t FileType) <-chan string } // FileInfo is returned by Stat() and contains information about a file in the diff --git a/src/restic/backend/b2/b2.go b/src/restic/backend/b2/b2.go index c209c13ab..9b80f2133 100644 --- a/src/restic/backend/b2/b2.go +++ b/src/restic/backend/b2/b2.go @@ -23,6 +23,9 @@ type b2Backend struct { sem *backend.Semaphore } +// ensure statically that *b2Backend implements restic.Backend. +var _ restic.Backend = &b2Backend{} + func newClient(ctx context.Context, cfg Config) (*b2.Client, error) { opts := []b2.ClientOption{b2.Transport(backend.Transport())} @@ -96,7 +99,7 @@ func Create(cfg Config) (restic.Backend, error) { sem: backend.NewSemaphore(cfg.Connections), } - present, err := be.Test(restic.Handle{Type: restic.ConfigFile}) + present, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile}) if err != nil { return nil, err } @@ -140,7 +143,7 @@ func (wr *wrapReader) Close() error { // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. -func (be *b2Backend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) if err := h.Valid(); err != nil { return nil, err @@ -154,7 +157,7 @@ func (be *b2Backend) Load(h restic.Handle, length int, offset int64) (io.ReadClo return nil, errors.Errorf("invalid length %d", length) } - ctx, cancel := context.WithCancel(context.TODO()) + ctx, cancel := context.WithCancel(ctx) be.sem.GetToken() @@ -191,8 +194,8 @@ func (be *b2Backend) Load(h restic.Handle, length int, offset int64) (io.ReadClo } // Save stores data in the backend at the handle. -func (be *b2Backend) Save(h restic.Handle, rd io.Reader) (err error) { - ctx, cancel := context.WithCancel(context.TODO()) +func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) { + ctx, cancel := context.WithCancel(ctx) defer cancel() if err := h.Valid(); err != nil { @@ -225,12 +228,9 @@ func (be *b2Backend) Save(h restic.Handle, rd io.Reader) (err error) { } // Stat returns information about a blob. -func (be *b2Backend) Stat(h restic.Handle) (bi restic.FileInfo, err error) { +func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { debug.Log("Stat %v", h) - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - be.sem.GetToken() defer be.sem.ReleaseToken() @@ -245,12 +245,9 @@ func (be *b2Backend) Stat(h restic.Handle) (bi restic.FileInfo, err error) { } // Test returns true if a blob of the given type and name exists in the backend. -func (be *b2Backend) Test(h restic.Handle) (bool, error) { +func (be *b2Backend) Test(ctx context.Context, h restic.Handle) (bool, error) { debug.Log("Test %v", h) - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - be.sem.GetToken() defer be.sem.ReleaseToken() @@ -265,12 +262,9 @@ func (be *b2Backend) Test(h restic.Handle) (bool, error) { } // Remove removes the blob with the given name and type. -func (be *b2Backend) Remove(h restic.Handle) error { +func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { debug.Log("Remove %v", h) - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - be.sem.GetToken() defer be.sem.ReleaseToken() @@ -281,11 +275,11 @@ func (be *b2Backend) Remove(h restic.Handle) error { // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. -func (be *b2Backend) List(t restic.FileType, done <-chan struct{}) <-chan string { +func (be *b2Backend) List(ctx context.Context, t restic.FileType) <-chan string { debug.Log("List %v", t) ch := make(chan string) - ctx, cancel := context.WithCancel(context.TODO()) + ctx, cancel := context.WithCancel(ctx) be.sem.GetToken() @@ -315,7 +309,7 @@ func (be *b2Backend) List(t restic.FileType, done <-chan struct{}) <-chan string select { case ch <- m: - case <-done: + case <-ctx.Done(): return } } @@ -330,13 +324,10 @@ func (be *b2Backend) List(t restic.FileType, done <-chan struct{}) <-chan string } // Remove keys for a specified backend type. -func (be *b2Backend) removeKeys(t restic.FileType) error { +func (be *b2Backend) removeKeys(ctx context.Context, t restic.FileType) error { debug.Log("removeKeys %v", t) - - done := make(chan struct{}) - defer close(done) - for key := range be.List(t, done) { - err := be.Remove(restic.Handle{Type: t, Name: key}) + for key := range be.List(ctx, t) { + err := be.Remove(ctx, restic.Handle{Type: t, Name: key}) if err != nil { return err } @@ -345,7 +336,7 @@ func (be *b2Backend) removeKeys(t restic.FileType) error { } // Delete removes all restic keys in the bucket. It will not remove the bucket itself. -func (be *b2Backend) Delete() error { +func (be *b2Backend) Delete(ctx context.Context) error { alltypes := []restic.FileType{ restic.DataFile, restic.KeyFile, @@ -354,12 +345,12 @@ func (be *b2Backend) Delete() error { restic.IndexFile} for _, t := range alltypes { - err := be.removeKeys(t) + err := be.removeKeys(ctx, t) if err != nil { return nil } } - err := be.Remove(restic.Handle{Type: restic.ConfigFile}) + err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) if err != nil && b2.IsNotExist(errors.Cause(err)) { err = nil } diff --git a/src/restic/backend/b2/b2_test.go b/src/restic/backend/b2/b2_test.go index 64c00c9ff..6cf5c1bc6 100644 --- a/src/restic/backend/b2/b2_test.go +++ b/src/restic/backend/b2/b2_test.go @@ -1,6 +1,7 @@ package b2_test import ( + "context" "fmt" "os" "testing" @@ -52,7 +53,7 @@ func newB2TestSuite(t testing.TB) *test.Suite { return err } - if err := be.(restic.Deleter).Delete(); err != nil { + if err := be.(restic.Deleter).Delete(context.TODO()); err != nil { return err } diff --git a/src/restic/backend/local/layout_test.go b/src/restic/backend/local/layout_test.go index 16b6b16e3..3f009b49d 100644 --- a/src/restic/backend/local/layout_test.go +++ b/src/restic/backend/local/layout_test.go @@ -1,6 +1,7 @@ package local import ( + "context" "path/filepath" "restic" . "restic/test" @@ -47,7 +48,7 @@ func TestLayout(t *testing.T) { } datafiles := make(map[string]bool) - for id := range be.List(restic.DataFile, nil) { + for id := range be.List(context.TODO(), restic.DataFile) { datafiles[id] = false } diff --git a/src/restic/backend/local/local.go b/src/restic/backend/local/local.go index 3b97f761e..1a3c0158c 100644 --- a/src/restic/backend/local/local.go +++ b/src/restic/backend/local/local.go @@ -1,6 +1,7 @@ package local import ( + "context" "io" "os" "path/filepath" @@ -75,7 +76,7 @@ func (b *Local) Location() string { } // Save stores data in the backend at the handle. -func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) { +func (b *Local) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) { debug.Log("Save %v", h) if err := h.Valid(); err != nil { return err @@ -100,7 +101,7 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) { return errors.Wrap(err, "MkdirAll") } - return b.Save(h, rd) + return b.Save(ctx, h, rd) } if err != nil { @@ -110,12 +111,12 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) { // save data, then sync _, err = io.Copy(f, rd) if err != nil { - f.Close() + _ = f.Close() return errors.Wrap(err, "Write") } if err = f.Sync(); err != nil { - f.Close() + _ = f.Close() return errors.Wrap(err, "Sync") } @@ -136,7 +137,7 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) { // Load returns a reader that yields the contents of the file at h at the // given offset. If length is nonzero, only a portion of the file is // returned. rd must be closed after use. -func (b *Local) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v", h, length, offset) if err := h.Valid(); err != nil { return nil, err @@ -154,7 +155,7 @@ func (b *Local) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, if offset > 0 { _, err = f.Seek(offset, 0) if err != nil { - f.Close() + _ = f.Close() return nil, err } } @@ -167,7 +168,7 @@ func (b *Local) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, } // Stat returns information about a blob. -func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) { +func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { debug.Log("Stat %v", h) if err := h.Valid(); err != nil { return restic.FileInfo{}, err @@ -182,7 +183,7 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) { } // Test returns true if a blob of the given type and name exists in the backend. -func (b *Local) Test(h restic.Handle) (bool, error) { +func (b *Local) Test(ctx context.Context, h restic.Handle) (bool, error) { debug.Log("Test %v", h) _, err := fs.Stat(b.Filename(h)) if err != nil { @@ -196,7 +197,7 @@ func (b *Local) Test(h restic.Handle) (bool, error) { } // Remove removes the blob with the given name and type. -func (b *Local) Remove(h restic.Handle) error { +func (b *Local) Remove(ctx context.Context, h restic.Handle) error { debug.Log("Remove %v", h) fn := b.Filename(h) @@ -214,9 +215,8 @@ func isFile(fi os.FileInfo) bool { } // List returns a channel that yields all names of blobs of type t. A -// goroutine is started for this. If the channel done is closed, sending -// stops. -func (b *Local) List(t restic.FileType, done <-chan struct{}) <-chan string { +// goroutine is started for this. +func (b *Local) List(ctx context.Context, t restic.FileType) <-chan string { debug.Log("List %v", t) ch := make(chan string) @@ -235,7 +235,7 @@ func (b *Local) List(t restic.FileType, done <-chan struct{}) <-chan string { select { case ch <- filepath.Base(path): - case <-done: + case <-ctx.Done(): return err } diff --git a/src/restic/backend/mem/mem_backend.go b/src/restic/backend/mem/mem_backend.go index 3e96f6a36..bbb4dbd1a 100644 --- a/src/restic/backend/mem/mem_backend.go +++ b/src/restic/backend/mem/mem_backend.go @@ -2,6 +2,7 @@ package mem import ( "bytes" + "context" "io" "io/ioutil" "restic" @@ -37,7 +38,7 @@ func New() *MemoryBackend { } // Test returns whether a file exists. -func (be *MemoryBackend) Test(h restic.Handle) (bool, error) { +func (be *MemoryBackend) Test(ctx context.Context, h restic.Handle) (bool, error) { be.m.Lock() defer be.m.Unlock() @@ -51,7 +52,7 @@ func (be *MemoryBackend) Test(h restic.Handle) (bool, error) { } // Save adds new Data to the backend. -func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error { +func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) error { if err := h.Valid(); err != nil { return err } @@ -81,7 +82,7 @@ func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error { // Load returns a reader that yields the contents of the file at h at the // given offset. If length is nonzero, only a portion of the file is // returned. rd must be closed after use. -func (be *MemoryBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { if err := h.Valid(); err != nil { return nil, err } @@ -117,7 +118,7 @@ func (be *MemoryBackend) Load(h restic.Handle, length int, offset int64) (io.Rea } // Stat returns information about a file in the backend. -func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) { +func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { be.m.Lock() defer be.m.Unlock() @@ -140,7 +141,7 @@ func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) { } // Remove deletes a file from the backend. -func (be *MemoryBackend) Remove(h restic.Handle) error { +func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { be.m.Lock() defer be.m.Unlock() @@ -156,7 +157,7 @@ func (be *MemoryBackend) Remove(h restic.Handle) error { } // List returns a channel which yields entries from the backend. -func (be *MemoryBackend) List(t restic.FileType, done <-chan struct{}) <-chan string { +func (be *MemoryBackend) List(ctx context.Context, t restic.FileType) <-chan string { be.m.Lock() defer be.m.Unlock() @@ -177,7 +178,7 @@ func (be *MemoryBackend) List(t restic.FileType, done <-chan struct{}) <-chan st for _, id := range ids { select { case ch <- id: - case <-done: + case <-ctx.Done(): return } } @@ -192,7 +193,7 @@ func (be *MemoryBackend) Location() string { } // Delete removes all data in the backend. -func (be *MemoryBackend) Delete() error { +func (be *MemoryBackend) Delete(ctx context.Context) error { be.m.Lock() defer be.m.Unlock() diff --git a/src/restic/backend/mem/mem_backend_test.go b/src/restic/backend/mem/mem_backend_test.go index 06da32661..422920da1 100644 --- a/src/restic/backend/mem/mem_backend_test.go +++ b/src/restic/backend/mem/mem_backend_test.go @@ -1,6 +1,7 @@ package mem_test import ( + "context" "restic" "testing" @@ -25,7 +26,7 @@ func newTestSuite() *test.Suite { Create: func(cfg interface{}) (restic.Backend, error) { c := cfg.(*memConfig) if c.be != nil { - ok, err := c.be.Test(restic.Handle{Type: restic.ConfigFile}) + ok, err := c.be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile}) if err != nil { return nil, err } diff --git a/src/restic/backend/rest/rest.go b/src/restic/backend/rest/rest.go index cb3fdadc8..f9ce1f681 100644 --- a/src/restic/backend/rest/rest.go +++ b/src/restic/backend/rest/rest.go @@ -1,6 +1,7 @@ package rest import ( + "context" "encoding/json" "fmt" "io" @@ -11,6 +12,8 @@ import ( "restic" "strings" + "golang.org/x/net/context/ctxhttp" + "restic/debug" "restic/errors" @@ -25,7 +28,7 @@ var _ restic.Backend = &restBackend{} type restBackend struct { url *url.URL connChan chan struct{} - client http.Client + client *http.Client backend.Layout } @@ -36,7 +39,7 @@ func Open(cfg Config) (restic.Backend, error) { connChan <- struct{}{} } - client := http.Client{Transport: backend.Transport()} + client := &http.Client{Transport: backend.Transport()} // use url without trailing slash for layout url := cfg.URL.String() @@ -61,7 +64,7 @@ func Create(cfg Config) (restic.Backend, error) { return nil, err } - _, err = be.Stat(restic.Handle{Type: restic.ConfigFile}) + _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) if err == nil { return nil, errors.Fatal("config file already exists") } @@ -99,22 +102,25 @@ func (b *restBackend) Location() string { } // Save stores data in the backend at the handle. -func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) { +func (b *restBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) { if err := h.Valid(); err != nil { return err } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + // make sure that client.Post() cannot close the reader by wrapping it in // backend.Closer, which has a noop method. rd = backend.Closer{Reader: rd} <-b.connChan - resp, err := b.client.Post(b.Filename(h), "binary/octet-stream", rd) + resp, err := ctxhttp.Post(ctx, b.client, b.Filename(h), "binary/octet-stream", rd) b.connChan <- struct{}{} if resp != nil { defer func() { - io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(ioutil.Discard, resp.Body) e := resp.Body.Close() if err == nil { @@ -137,7 +143,7 @@ func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) { // Load returns a reader that yields the contents of the file at h at the // given offset. If length is nonzero, only a portion of the file is // returned. rd must be closed after use. -func (b *restBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (b *restBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v", h, length, offset) if err := h.Valid(); err != nil { return nil, err @@ -164,20 +170,19 @@ func (b *restBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCl debug.Log("Load(%v) send range %v", h, byteRange) <-b.connChan - resp, err := b.client.Do(req) + resp, err := ctxhttp.Do(ctx, b.client, req) b.connChan <- struct{}{} if err != nil { if resp != nil { - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() + _, _ = io.Copy(ioutil.Discard, resp.Body) + _ = resp.Body.Close() } return nil, errors.Wrap(err, "client.Do") } if resp.StatusCode != 200 && resp.StatusCode != 206 { - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() + _ = resp.Body.Close() return nil, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status) } @@ -185,19 +190,19 @@ func (b *restBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCl } // Stat returns information about a blob. -func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) { +func (b *restBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { if err := h.Valid(); err != nil { return restic.FileInfo{}, err } <-b.connChan - resp, err := b.client.Head(b.Filename(h)) + resp, err := ctxhttp.Head(ctx, b.client, b.Filename(h)) b.connChan <- struct{}{} if err != nil { return restic.FileInfo{}, errors.Wrap(err, "client.Head") } - io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(ioutil.Discard, resp.Body) if err = resp.Body.Close(); err != nil { return restic.FileInfo{}, errors.Wrap(err, "Close") } @@ -218,8 +223,8 @@ func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) { } // Test returns true if a blob of the given type and name exists in the backend. -func (b *restBackend) Test(h restic.Handle) (bool, error) { - _, err := b.Stat(h) +func (b *restBackend) Test(ctx context.Context, h restic.Handle) (bool, error) { + _, err := b.Stat(ctx, h) if err != nil { return false, nil } @@ -228,7 +233,7 @@ func (b *restBackend) Test(h restic.Handle) (bool, error) { } // Remove removes the blob with the given name and type. -func (b *restBackend) Remove(h restic.Handle) error { +func (b *restBackend) Remove(ctx context.Context, h restic.Handle) error { if err := h.Valid(); err != nil { return err } @@ -238,7 +243,7 @@ func (b *restBackend) Remove(h restic.Handle) error { return errors.Wrap(err, "http.NewRequest") } <-b.connChan - resp, err := b.client.Do(req) + resp, err := ctxhttp.Do(ctx, b.client, req) b.connChan <- struct{}{} if err != nil { @@ -249,14 +254,18 @@ func (b *restBackend) Remove(h restic.Handle) error { return errors.Errorf("blob not removed, server response: %v (%v)", resp.Status, resp.StatusCode) } - io.Copy(ioutil.Discard, resp.Body) - return resp.Body.Close() + _, err = io.Copy(ioutil.Discard, resp.Body) + if err != nil { + return errors.Wrap(err, "Copy") + } + + return errors.Wrap(resp.Body.Close(), "Close") } // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. -func (b *restBackend) List(t restic.FileType, done <-chan struct{}) <-chan string { +func (b *restBackend) List(ctx context.Context, t restic.FileType) <-chan string { ch := make(chan string) url := b.Dirname(restic.Handle{Type: t}) @@ -265,12 +274,12 @@ func (b *restBackend) List(t restic.FileType, done <-chan struct{}) <-chan strin } <-b.connChan - resp, err := b.client.Get(url) + resp, err := ctxhttp.Get(ctx, b.client, url) b.connChan <- struct{}{} if resp != nil { defer func() { - io.Copy(ioutil.Discard, resp.Body) + _, _ = io.Copy(ioutil.Discard, resp.Body) e := resp.Body.Close() if err == nil { @@ -296,7 +305,7 @@ func (b *restBackend) List(t restic.FileType, done <-chan struct{}) <-chan strin for _, m := range list { select { case ch <- m: - case <-done: + case <-ctx.Done(): return } } diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 960258392..b7c15f2fe 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -1,6 +1,7 @@ package s3 import ( + "context" "fmt" "io" "os" @@ -31,6 +32,9 @@ type s3 struct { backend.Layout } +// make sure that *s3 implements backend.Backend +var _ restic.Backend = &s3{} + const defaultLayout = "s3legacy" // Open opens the S3 backend at bucket and region. The bucket is created if it @@ -202,7 +206,7 @@ func (wr preventCloser) Close() error { } // Save stores data in the backend at the handle. -func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) { +func (be *s3) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) { if err := h.Valid(); err != nil { return err } @@ -259,7 +263,7 @@ func (wr wrapReader) Close() error { // Load returns a reader that yields the contents of the file at h at the // given offset. If length is nonzero, only a portion of the file is // returned. rd must be closed after use. -func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *s3) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) if err := h.Valid(); err != nil { return nil, err @@ -307,7 +311,7 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er } // Stat returns information about a blob. -func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) { +func (be *s3) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { debug.Log("%v", h) objName := be.Filename(h) @@ -337,7 +341,7 @@ func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) { } // Test returns true if a blob of the given type and name exists in the backend. -func (be *s3) Test(h restic.Handle) (bool, error) { +func (be *s3) Test(ctx context.Context, h restic.Handle) (bool, error) { found := false objName := be.Filename(h) _, err := be.client.StatObject(be.bucketname, objName) @@ -350,7 +354,7 @@ func (be *s3) Test(h restic.Handle) (bool, error) { } // Remove removes the blob with the given name and type. -func (be *s3) Remove(h restic.Handle) error { +func (be *s3) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) err := be.client.RemoveObject(be.bucketname, objName) debug.Log("Remove(%v) at %v -> err %v", h, objName, err) @@ -360,7 +364,7 @@ func (be *s3) Remove(h restic.Handle) error { // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. -func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string { +func (be *s3) List(ctx context.Context, t restic.FileType) <-chan string { debug.Log("listing %v", t) ch := make(chan string) @@ -371,7 +375,7 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string { prefix += "/" } - listresp := be.client.ListObjects(be.bucketname, prefix, true, done) + listresp := be.client.ListObjects(be.bucketname, prefix, true, ctx.Done()) go func() { defer close(ch) @@ -383,7 +387,7 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string { select { case ch <- path.Base(m): - case <-done: + case <-ctx.Done(): return } } @@ -393,11 +397,9 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string { } // Remove keys for a specified backend type. -func (be *s3) removeKeys(t restic.FileType) error { - done := make(chan struct{}) - defer close(done) - for key := range be.List(restic.DataFile, done) { - err := be.Remove(restic.Handle{Type: restic.DataFile, Name: key}) +func (be *s3) removeKeys(ctx context.Context, t restic.FileType) error { + for key := range be.List(ctx, restic.DataFile) { + err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key}) if err != nil { return err } @@ -407,7 +409,7 @@ func (be *s3) removeKeys(t restic.FileType) error { } // Delete removes all restic keys in the bucket. It will not remove the bucket itself. -func (be *s3) Delete() error { +func (be *s3) Delete(ctx context.Context) error { alltypes := []restic.FileType{ restic.DataFile, restic.KeyFile, @@ -416,13 +418,13 @@ func (be *s3) Delete() error { restic.IndexFile} for _, t := range alltypes { - err := be.removeKeys(t) + err := be.removeKeys(ctx, t) if err != nil { return nil } } - return be.Remove(restic.Handle{Type: restic.ConfigFile}) + return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) } // Close does nothing diff --git a/src/restic/backend/s3/s3_test.go b/src/restic/backend/s3/s3_test.go index 787166994..d3f870c0a 100644 --- a/src/restic/backend/s3/s3_test.go +++ b/src/restic/backend/s3/s3_test.go @@ -134,7 +134,7 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite { return nil, err } - exists, err := be.Test(restic.Handle{Type: restic.ConfigFile}) + exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile}) if err != nil { return nil, err } @@ -228,7 +228,7 @@ func newS3TestSuite(t testing.TB) *test.Suite { return nil, err } - exists, err := be.Test(restic.Handle{Type: restic.ConfigFile}) + exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile}) if err != nil { return nil, err } @@ -255,7 +255,7 @@ func newS3TestSuite(t testing.TB) *test.Suite { return err } - if err := be.(restic.Deleter).Delete(); err != nil { + if err := be.(restic.Deleter).Delete(context.TODO()); err != nil { return err } diff --git a/src/restic/backend/sftp/layout_test.go b/src/restic/backend/sftp/layout_test.go index 166fa97e3..aa030ee05 100644 --- a/src/restic/backend/sftp/layout_test.go +++ b/src/restic/backend/sftp/layout_test.go @@ -1,6 +1,7 @@ package sftp_test import ( + "context" "fmt" "path/filepath" "restic" @@ -54,7 +55,7 @@ func TestLayout(t *testing.T) { } datafiles := make(map[string]bool) - for id := range be.List(restic.DataFile, nil) { + for id := range be.List(context.TODO(), restic.DataFile) { datafiles[id] = false } diff --git a/src/restic/backend/sftp/sftp.go b/src/restic/backend/sftp/sftp.go index 8070d01fe..f871da324 100644 --- a/src/restic/backend/sftp/sftp.go +++ b/src/restic/backend/sftp/sftp.go @@ -2,6 +2,7 @@ package sftp import ( "bufio" + "context" "fmt" "io" "os" @@ -262,7 +263,7 @@ func Join(parts ...string) string { } // Save stores data in the backend at the handle. -func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) { +func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) { debug.Log("Save %v", h) if err := r.clientError(); err != nil { return err @@ -283,7 +284,7 @@ func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) { return errors.Wrap(err, "MkdirAll") } - return r.Save(h, rd) + return r.Save(ctx, h, rd) } if err != nil { @@ -315,7 +316,7 @@ func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) { // Load returns a reader that yields the contents of the file at h at the // given offset. If length is nonzero, only a portion of the file is // returned. rd must be closed after use. -func (r *SFTP) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v", h, length, offset) if err := h.Valid(); err != nil { return nil, err @@ -346,7 +347,7 @@ func (r *SFTP) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, e } // Stat returns information about a blob. -func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) { +func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { debug.Log("Stat(%v)", h) if err := r.clientError(); err != nil { return restic.FileInfo{}, err @@ -365,7 +366,7 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) { } // Test returns true if a blob of the given type and name exists in the backend. -func (r *SFTP) Test(h restic.Handle) (bool, error) { +func (r *SFTP) Test(ctx context.Context, h restic.Handle) (bool, error) { debug.Log("Test(%v)", h) if err := r.clientError(); err != nil { return false, err @@ -384,7 +385,7 @@ func (r *SFTP) Test(h restic.Handle) (bool, error) { } // Remove removes the content stored at name. -func (r *SFTP) Remove(h restic.Handle) error { +func (r *SFTP) Remove(ctx context.Context, h restic.Handle) error { debug.Log("Remove(%v)", h) if err := r.clientError(); err != nil { return err @@ -396,7 +397,7 @@ func (r *SFTP) Remove(h restic.Handle) error { // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. -func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string { +func (r *SFTP) List(ctx context.Context, t restic.FileType) <-chan string { debug.Log("List %v", t) ch := make(chan string) @@ -416,7 +417,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string { select { case ch <- path.Base(walker.Path()): - case <-done: + case <-ctx.Done(): return } } diff --git a/src/restic/backend/swift/swift.go b/src/restic/backend/swift/swift.go index 733dc3221..b18b61947 100644 --- a/src/restic/backend/swift/swift.go +++ b/src/restic/backend/swift/swift.go @@ -1,6 +1,7 @@ package swift import ( + "context" "fmt" "io" "net/http" @@ -27,6 +28,9 @@ type beSwift struct { backend.Layout } +// ensure statically that *beSwift implements restic.Backend. +var _ restic.Backend = &beSwift{} + // Open opens the swift backend at a container in region. The container is // created if it does not exist yet. func Open(cfg Config) (restic.Backend, error) { @@ -120,7 +124,7 @@ func (be *beSwift) Location() string { // Load returns a reader that yields the contents of the file at h at the // given offset. If length is nonzero, only a portion of the file is // returned. rd must be closed after use. -func (be *beSwift) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v", h, length, offset) if err := h.Valid(); err != nil { return nil, err @@ -164,7 +168,7 @@ func (be *beSwift) Load(h restic.Handle, length int, offset int64) (io.ReadClose } // Save stores data in the backend at the handle. -func (be *beSwift) Save(h restic.Handle, rd io.Reader) (err error) { +func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) { if err = h.Valid(); err != nil { return err } @@ -201,7 +205,7 @@ func (be *beSwift) Save(h restic.Handle, rd io.Reader) (err error) { } // Stat returns information about a blob. -func (be *beSwift) Stat(h restic.Handle) (bi restic.FileInfo, err error) { +func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { debug.Log("%v", h) objName := be.Filename(h) @@ -216,7 +220,7 @@ func (be *beSwift) Stat(h restic.Handle) (bi restic.FileInfo, err error) { } // Test returns true if a blob of the given type and name exists in the backend. -func (be *beSwift) Test(h restic.Handle) (bool, error) { +func (be *beSwift) Test(ctx context.Context, h restic.Handle) (bool, error) { objName := be.Filename(h) switch _, _, err := be.conn.Object(be.container, objName); err { case nil: @@ -231,7 +235,7 @@ func (be *beSwift) Test(h restic.Handle) (bool, error) { } // Remove removes the blob with the given name and type. -func (be *beSwift) Remove(h restic.Handle) error { +func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) err := be.conn.ObjectDelete(be.container, objName) debug.Log("Remove(%v) -> err %v", h, err) @@ -241,7 +245,7 @@ func (be *beSwift) Remove(h restic.Handle) error { // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. -func (be *beSwift) List(t restic.FileType, done <-chan struct{}) <-chan string { +func (be *beSwift) List(ctx context.Context, t restic.FileType) <-chan string { debug.Log("listing %v", t) ch := make(chan string) @@ -264,7 +268,7 @@ func (be *beSwift) List(t restic.FileType, done <-chan struct{}) <-chan string { select { case ch <- m: - case <-done: + case <-ctx.Done(): return nil, io.EOF } } @@ -280,11 +284,9 @@ func (be *beSwift) List(t restic.FileType, done <-chan struct{}) <-chan string { } // Remove keys for a specified backend type. -func (be *beSwift) removeKeys(t restic.FileType) error { - done := make(chan struct{}) - defer close(done) - for key := range be.List(t, done) { - err := be.Remove(restic.Handle{Type: t, Name: key}) +func (be *beSwift) removeKeys(ctx context.Context, t restic.FileType) error { + for key := range be.List(ctx, t) { + err := be.Remove(ctx, restic.Handle{Type: t, Name: key}) if err != nil { return err } @@ -304,7 +306,7 @@ func (be *beSwift) IsNotExist(err error) bool { // Delete removes all restic objects in the container. // It will not remove the container itself. -func (be *beSwift) Delete() error { +func (be *beSwift) Delete(ctx context.Context) error { alltypes := []restic.FileType{ restic.DataFile, restic.KeyFile, @@ -313,13 +315,13 @@ func (be *beSwift) Delete() error { restic.IndexFile} for _, t := range alltypes { - err := be.removeKeys(t) + err := be.removeKeys(ctx, t) if err != nil { return nil } } - err := be.Remove(restic.Handle{Type: restic.ConfigFile}) + err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) if err != nil && !be.IsNotExist(err) { return err } diff --git a/src/restic/backend/swift/swift_test.go b/src/restic/backend/swift/swift_test.go index b53b7bb64..843efdcd4 100644 --- a/src/restic/backend/swift/swift_test.go +++ b/src/restic/backend/swift/swift_test.go @@ -1,6 +1,7 @@ package swift_test import ( + "context" "fmt" "os" "restic" @@ -44,7 +45,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite { return nil, err } - exists, err := be.Test(restic.Handle{Type: restic.ConfigFile}) + exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile}) if err != nil { return nil, err } @@ -71,7 +72,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite { return err } - if err := be.(restic.Deleter).Delete(); err != nil { + if err := be.(restic.Deleter).Delete(context.TODO()); err != nil { return err } diff --git a/src/restic/backend/test/benchmarks.go b/src/restic/backend/test/benchmarks.go index 2b2b0666d..fb7106561 100644 --- a/src/restic/backend/test/benchmarks.go +++ b/src/restic/backend/test/benchmarks.go @@ -2,6 +2,7 @@ package test import ( "bytes" + "context" "io" "restic" "restic/test" @@ -12,14 +13,14 @@ func saveRandomFile(t testing.TB, be restic.Backend, length int) ([]byte, restic data := test.Random(23, length) id := restic.Hash(data) handle := restic.Handle{Type: restic.DataFile, Name: id.String()} - if err := be.Save(handle, bytes.NewReader(data)); err != nil { + if err := be.Save(context.TODO(), handle, bytes.NewReader(data)); err != nil { t.Fatalf("Save() error: %+v", err) } return data, handle } func remove(t testing.TB, be restic.Backend, h restic.Handle) { - if err := be.Remove(h); err != nil { + if err := be.Remove(context.TODO(), h); err != nil { t.Fatalf("Remove() returned error: %v", err) } } @@ -40,7 +41,7 @@ func (s *Suite) BenchmarkLoadFile(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - rd, err := be.Load(handle, 0, 0) + rd, err := be.Load(context.TODO(), handle, 0, 0) if err != nil { t.Fatal(err) } @@ -82,7 +83,7 @@ func (s *Suite) BenchmarkLoadPartialFile(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - rd, err := be.Load(handle, testLength, 0) + rd, err := be.Load(context.TODO(), handle, testLength, 0) if err != nil { t.Fatal(err) } @@ -126,7 +127,7 @@ func (s *Suite) BenchmarkLoadPartialFileOffset(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - rd, err := be.Load(handle, testLength, int64(testOffset)) + rd, err := be.Load(context.TODO(), handle, testLength, int64(testOffset)) if err != nil { t.Fatal(err) } @@ -171,11 +172,11 @@ func (s *Suite) BenchmarkSave(t *testing.B) { t.Fatal(err) } - if err := be.Save(handle, rd); err != nil { + if err := be.Save(context.TODO(), handle, rd); err != nil { t.Fatal(err) } - if err := be.Remove(handle); err != nil { + if err := be.Remove(context.TODO(), handle); err != nil { t.Fatal(err) } } diff --git a/src/restic/backend/test/tests.go b/src/restic/backend/test/tests.go index 61136cea7..b6da7182d 100644 --- a/src/restic/backend/test/tests.go +++ b/src/restic/backend/test/tests.go @@ -2,6 +2,7 @@ package test import ( "bytes" + "context" "fmt" "io" "io/ioutil" @@ -34,7 +35,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) { // remove a config if present cfgHandle := restic.Handle{Type: restic.ConfigFile} - cfgPresent, err := b.Test(cfgHandle) + cfgPresent, err := b.Test(context.TODO(), cfgHandle) if err != nil { t.Fatalf("unable to test for config: %+v", err) } @@ -53,7 +54,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) { } // remove config - err = b.Remove(restic.Handle{Type: restic.ConfigFile, Name: ""}) + err = b.Remove(context.TODO(), restic.Handle{Type: restic.ConfigFile, Name: ""}) if err != nil { t.Fatalf("unexpected error removing config: %+v", err) } @@ -78,12 +79,12 @@ func (s *Suite) TestConfig(t *testing.T) { var testString = "Config" // create config and read it back - _, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile}) + _, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.ConfigFile}) if err == nil { t.Fatalf("did not get expected error for non-existing config") } - err = b.Save(restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString)) + err = b.Save(context.TODO(), restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString)) if err != nil { t.Fatalf("Save() error: %+v", err) } @@ -92,7 +93,7 @@ func (s *Suite) TestConfig(t *testing.T) { // same config for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { h := restic.Handle{Type: restic.ConfigFile, Name: name} - buf, err := backend.LoadAll(b, h) + buf, err := backend.LoadAll(context.TODO(), b, h) if err != nil { t.Fatalf("unable to read config with name %q: %+v", name, err) } @@ -113,12 +114,12 @@ func (s *Suite) TestLoad(t *testing.T) { b := s.open(t) defer s.close(t, b) - rd, err := b.Load(restic.Handle{}, 0, 0) + rd, err := b.Load(context.TODO(), restic.Handle{}, 0, 0) if err == nil { t.Fatalf("Load() did not return an error for invalid handle") } if rd != nil { - rd.Close() + _ = rd.Close() } err = testLoad(b, restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0) @@ -132,14 +133,14 @@ func (s *Suite) TestLoad(t *testing.T) { id := restic.Hash(data) handle := restic.Handle{Type: restic.DataFile, Name: id.String()} - err = b.Save(handle, bytes.NewReader(data)) + err = b.Save(context.TODO(), handle, bytes.NewReader(data)) if err != nil { t.Fatalf("Save() error: %+v", err) } t.Logf("saved %d bytes as %v", length, handle) - rd, err = b.Load(handle, 100, -1) + rd, err = b.Load(context.TODO(), handle, 100, -1) if err == nil { t.Fatalf("Load() returned no error for negative offset!") } @@ -174,7 +175,7 @@ func (s *Suite) TestLoad(t *testing.T) { d = d[:l] } - rd, err := b.Load(handle, getlen, int64(o)) + rd, err := b.Load(context.TODO(), handle, getlen, int64(o)) if err != nil { t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen) t.Errorf("Load(%d, %d) returned unexpected error: %+v", l, o, err) @@ -235,7 +236,7 @@ func (s *Suite) TestLoad(t *testing.T) { } } - test.OK(t, b.Remove(handle)) + test.OK(t, b.Remove(context.TODO(), handle)) } type errorCloser struct { @@ -276,10 +277,10 @@ func (s *Suite) TestSave(t *testing.T) { Type: restic.DataFile, Name: fmt.Sprintf("%s-%d", id, i), } - err := b.Save(h, bytes.NewReader(data)) + err := b.Save(context.TODO(), h, bytes.NewReader(data)) test.OK(t, err) - buf, err := backend.LoadAll(b, h) + buf, err := backend.LoadAll(context.TODO(), b, h) test.OK(t, err) if len(buf) != len(data) { t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) @@ -289,14 +290,14 @@ func (s *Suite) TestSave(t *testing.T) { t.Fatalf("data not equal") } - fi, err := b.Stat(h) + fi, err := b.Stat(context.TODO(), h) test.OK(t, err) if fi.Size != int64(len(data)) { t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size) } - err = b.Remove(h) + err = b.Remove(context.TODO(), h) if err != nil { t.Fatalf("error removing item: %+v", err) } @@ -324,12 +325,12 @@ func (s *Suite) TestSave(t *testing.T) { // wrap the tempfile in an errorCloser, so we can detect if the backend // closes the reader - err = b.Save(h, errorCloser{t: t, size: int64(length), Reader: tmpfile}) + err = b.Save(context.TODO(), h, errorCloser{t: t, size: int64(length), Reader: tmpfile}) if err != nil { t.Fatal(err) } - err = b.Remove(h) + err = b.Remove(context.TODO(), h) if err != nil { t.Fatalf("error removing item: %+v", err) } @@ -339,7 +340,7 @@ func (s *Suite) TestSave(t *testing.T) { t.Fatal(err) } - err = b.Save(h, tmpfile) + err = b.Save(context.TODO(), h, tmpfile) if err != nil { t.Fatal(err) } @@ -348,7 +349,7 @@ func (s *Suite) TestSave(t *testing.T) { t.Fatal(err) } - err = b.Remove(h) + err = b.Remove(context.TODO(), h) if err != nil { t.Fatalf("error removing item: %+v", err) } @@ -377,13 +378,13 @@ func (s *Suite) TestSaveFilenames(t *testing.T) { for i, test := range filenameTests { h := restic.Handle{Name: test.name, Type: restic.DataFile} - err := b.Save(h, strings.NewReader(test.data)) + err := b.Save(context.TODO(), h, strings.NewReader(test.data)) if err != nil { t.Errorf("test %d failed: Save() returned %+v", i, err) continue } - buf, err := backend.LoadAll(b, h) + buf, err := backend.LoadAll(context.TODO(), b, h) if err != nil { t.Errorf("test %d failed: Load() returned %+v", i, err) continue @@ -393,7 +394,7 @@ func (s *Suite) TestSaveFilenames(t *testing.T) { t.Errorf("test %d: returned wrong bytes", i) } - err = b.Remove(h) + err = b.Remove(context.TODO(), h) if err != nil { t.Errorf("test %d failed: Remove() returned %+v", i, err) continue @@ -414,14 +415,14 @@ var testStrings = []struct { func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle { id := restic.Hash(data) h := restic.Handle{Name: id.String(), Type: tpe} - err := b.Save(h, bytes.NewReader(data)) + err := b.Save(context.TODO(), h, bytes.NewReader(data)) test.OK(t, err) return h } // testLoad loads a blob (but discards its contents). func testLoad(b restic.Backend, h restic.Handle, length int, offset int64) error { - rd, err := b.Load(h, 0, 0) + rd, err := b.Load(context.TODO(), h, 0, 0) if err != nil { return err } @@ -437,14 +438,14 @@ func testLoad(b restic.Backend, h restic.Handle, length int, offset int64) error func delayedRemove(b restic.Backend, h restic.Handle) error { // Some backend (swift, I'm looking at you) may implement delayed // removal of data. Let's wait a bit if this happens. - err := b.Remove(h) + err := b.Remove(context.TODO(), h) if err != nil { return err } - found, err := b.Test(h) + found, err := b.Test(context.TODO(), h) for i := 0; found && i < 20; i++ { - found, err = b.Test(h) + found, err = b.Test(context.TODO(), h) if found { time.Sleep(100 * time.Millisecond) } @@ -468,12 +469,12 @@ func (s *Suite) TestBackend(t *testing.T) { // test if blob is already in repository h := restic.Handle{Type: tpe, Name: id.String()} - ret, err := b.Test(h) + ret, err := b.Test(context.TODO(), h) test.OK(t, err) test.Assert(t, !ret, "blob was found to exist before creating") // try to stat a not existing blob - _, err = b.Stat(h) + _, err = b.Stat(context.TODO(), h) test.Assert(t, err != nil, "blob data could be extracted before creation") // try to read not existing blob @@ -481,7 +482,7 @@ func (s *Suite) TestBackend(t *testing.T) { test.Assert(t, err != nil, "blob could be read before creation") // try to get string out, should fail - ret, err = b.Test(h) + ret, err = b.Test(context.TODO(), h) test.OK(t, err) test.Assert(t, !ret, "id %q was found (but should not have)", ts.id) } @@ -492,7 +493,7 @@ func (s *Suite) TestBackend(t *testing.T) { // test Load() h := restic.Handle{Type: tpe, Name: ts.id} - buf, err := backend.LoadAll(b, h) + buf, err := backend.LoadAll(context.TODO(), b, h) test.OK(t, err) test.Equals(t, ts.data, string(buf)) @@ -502,7 +503,7 @@ func (s *Suite) TestBackend(t *testing.T) { length := end - start buf2 := make([]byte, length) - rd, err := b.Load(h, len(buf2), int64(start)) + rd, err := b.Load(context.TODO(), h, len(buf2), int64(start)) test.OK(t, err) n, err := io.ReadFull(rd, buf2) test.OK(t, err) @@ -522,7 +523,7 @@ func (s *Suite) TestBackend(t *testing.T) { // create blob h := restic.Handle{Type: tpe, Name: ts.id} - err := b.Save(h, strings.NewReader(ts.data)) + err := b.Save(context.TODO(), h, strings.NewReader(ts.data)) test.Assert(t, err != nil, "expected error for %v, got %v", h, err) // remove and recreate @@ -530,12 +531,12 @@ func (s *Suite) TestBackend(t *testing.T) { test.OK(t, err) // test that the blob is gone - ok, err := b.Test(h) + ok, err := b.Test(context.TODO(), h) test.OK(t, err) test.Assert(t, !ok, "removed blob still present") // create blob - err = b.Save(h, strings.NewReader(ts.data)) + err = b.Save(context.TODO(), h, strings.NewReader(ts.data)) test.OK(t, err) // list items @@ -549,7 +550,7 @@ func (s *Suite) TestBackend(t *testing.T) { list := restic.IDs{} - for s := range b.List(tpe, nil) { + for s := range b.List(context.TODO(), tpe) { list = append(list, restic.TestParseID(s)) } @@ -572,13 +573,13 @@ func (s *Suite) TestBackend(t *testing.T) { h := restic.Handle{Type: tpe, Name: id.String()} - found, err := b.Test(h) + found, err := b.Test(context.TODO(), h) test.OK(t, err) test.Assert(t, found, fmt.Sprintf("id %q not found", id)) test.OK(t, delayedRemove(b, h)) - found, err = b.Test(h) + found, err = b.Test(context.TODO(), h) test.OK(t, err) test.Assert(t, !found, fmt.Sprintf("id %q not found after removal", id)) } @@ -600,7 +601,7 @@ func (s *Suite) TestDelete(t *testing.T) { return } - err := be.Delete() + err := be.Delete(context.TODO()) if err != nil { t.Fatalf("error deleting backend: %+v", err) } diff --git a/src/restic/backend/test/tests_test.go b/src/restic/backend/test/tests_test.go index 010fcfe6e..d662e5a32 100644 --- a/src/restic/backend/test/tests_test.go +++ b/src/restic/backend/test/tests_test.go @@ -1,6 +1,7 @@ package test_test import ( + "context" "restic" "restic/errors" "testing" @@ -26,7 +27,7 @@ func newTestSuite(t testing.TB) *test.Suite { Create: func(cfg interface{}) (restic.Backend, error) { c := cfg.(*memConfig) if c.be != nil { - ok, err := c.be.Test(restic.Handle{Type: restic.ConfigFile}) + ok, err := c.be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile}) if err != nil { return nil, err } diff --git a/src/restic/backend/utils.go b/src/restic/backend/utils.go index 3f3a85749..a07c7e86e 100644 --- a/src/restic/backend/utils.go +++ b/src/restic/backend/utils.go @@ -1,14 +1,15 @@ package backend import ( + "context" "io" "io/ioutil" "restic" ) // LoadAll reads all data stored in the backend for the handle. -func LoadAll(be restic.Backend, h restic.Handle) (buf []byte, err error) { - rd, err := be.Load(h, 0, 0) +func LoadAll(ctx context.Context, be restic.Backend, h restic.Handle) (buf []byte, err error) { + rd, err := be.Load(ctx, h, 0, 0) if err != nil { return nil, err } diff --git a/src/restic/backend/utils_test.go b/src/restic/backend/utils_test.go index 51481ed0b..15829f46e 100644 --- a/src/restic/backend/utils_test.go +++ b/src/restic/backend/utils_test.go @@ -2,6 +2,7 @@ package backend_test import ( "bytes" + "context" "math/rand" "restic" "testing" @@ -21,10 +22,10 @@ func TestLoadAll(t *testing.T) { data := Random(23+i, rand.Intn(MiB)+500*KiB) id := restic.Hash(data) - err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data)) + err := b.Save(context.TODO(), restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data)) OK(t, err) - buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}) + buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()}) OK(t, err) if len(buf) != len(data) { @@ -46,10 +47,10 @@ func TestLoadSmallBuffer(t *testing.T) { data := Random(23+i, rand.Intn(MiB)+500*KiB) id := restic.Hash(data) - err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data)) + err := b.Save(context.TODO(), restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data)) OK(t, err) - buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}) + buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()}) OK(t, err) if len(buf) != len(data) { @@ -71,10 +72,10 @@ func TestLoadLargeBuffer(t *testing.T) { data := Random(23+i, rand.Intn(MiB)+500*KiB) id := restic.Hash(data) - err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data)) + err := b.Save(context.TODO(), restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data)) OK(t, err) - buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}) + buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()}) OK(t, err) if len(buf) != len(data) { diff --git a/src/restic/backend_find.go b/src/restic/backend_find.go index 193fd165b..e445972c9 100644 --- a/src/restic/backend_find.go +++ b/src/restic/backend_find.go @@ -1,6 +1,9 @@ package restic -import "restic/errors" +import ( + "context" + "restic/errors" +) // ErrNoIDPrefixFound is returned by Find() when no ID for the given prefix // could be found. @@ -14,13 +17,10 @@ var ErrMultipleIDMatches = errors.New("multiple IDs with prefix found") // start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. // If more than one is found, nil and ErrMultipleIDMatches is returned. func Find(be Lister, t FileType, prefix string) (string, error) { - done := make(chan struct{}) - defer close(done) - match := "" // TODO: optimize by sorting list etc. - for name := range be.List(t, done) { + for name := range be.List(context.TODO(), t) { if prefix == name[:len(prefix)] { if match == "" { match = name @@ -42,12 +42,9 @@ const minPrefixLength = 8 // PrefixLength returns the number of bytes required so that all prefixes of // all names of type t are unique. func PrefixLength(be Lister, t FileType) (int, error) { - done := make(chan struct{}) - defer close(done) - // load all IDs of the given type list := make([]string, 0, 100) - for name := range be.List(t, done) { + for name := range be.List(context.TODO(), t) { list = append(list, name) } diff --git a/src/restic/backend_find_test.go b/src/restic/backend_find_test.go index cc86cd810..032c8a9d9 100644 --- a/src/restic/backend_find_test.go +++ b/src/restic/backend_find_test.go @@ -1,15 +1,16 @@ package restic import ( + "context" "testing" ) type mockBackend struct { - list func(FileType, <-chan struct{}) <-chan string + list func(context.Context, FileType) <-chan string } -func (m mockBackend) List(t FileType, done <-chan struct{}) <-chan string { - return m.list(t, done) +func (m mockBackend) List(ctx context.Context, t FileType) <-chan string { + return m.list(ctx, t) } var samples = IDs{ @@ -27,14 +28,14 @@ func TestPrefixLength(t *testing.T) { list := samples m := mockBackend{} - m.list = func(t FileType, done <-chan struct{}) <-chan string { + m.list = func(ctx context.Context, t FileType) <-chan string { ch := make(chan string) go func() { defer close(ch) for _, id := range list { select { case ch <- id.String(): - case <-done: + case <-ctx.Done(): return } } diff --git a/src/restic/lock.go b/src/restic/lock.go index 97f2d652e..112a7c448 100644 --- a/src/restic/lock.go +++ b/src/restic/lock.go @@ -1,6 +1,7 @@ package restic import ( + "context" "fmt" "os" "os/signal" @@ -186,7 +187,7 @@ func (l *Lock) Unlock() error { return nil } - return l.repo.Backend().Remove(Handle{Type: LockFile, Name: l.lockID.String()}) + return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()}) } var staleTimeout = 30 * time.Minute @@ -234,7 +235,7 @@ func (l *Lock) Refresh() error { return err } - err = l.repo.Backend().Remove(Handle{Type: LockFile, Name: l.lockID.String()}) + err = l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()}) if err != nil { return err } @@ -289,7 +290,7 @@ func RemoveStaleLocks(repo Repository) error { } if lock.Stale() { - return repo.Backend().Remove(Handle{Type: LockFile, Name: id.String()}) + return repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()}) } return nil @@ -299,6 +300,6 @@ func RemoveStaleLocks(repo Repository) error { // RemoveAllLocks removes all locks forcefully. func RemoveAllLocks(repo Repository) error { return eachLock(repo, func(id ID, lock *Lock, err error) error { - return repo.Backend().Remove(Handle{Type: LockFile, Name: id.String()}) + return repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()}) }) } diff --git a/src/restic/readerat.go b/src/restic/readerat.go index a57974473..04c2ba6ae 100644 --- a/src/restic/readerat.go +++ b/src/restic/readerat.go @@ -1,6 +1,7 @@ package restic import ( + "context" "io" "restic/debug" ) @@ -22,7 +23,7 @@ func ReaderAt(be Backend, h Handle) io.ReaderAt { // ReadAt reads from the backend handle h at the given position. func ReadAt(be Backend, h Handle, offset int64, p []byte) (n int, err error) { debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p)) - rd, err := be.Load(h, len(p), offset) + rd, err := be.Load(context.TODO(), h, len(p), offset) if err != nil { return 0, err } diff --git a/src/restic/repository.go b/src/restic/repository.go index 959c0bd3c..36b618cd5 100644 --- a/src/restic/repository.go +++ b/src/restic/repository.go @@ -1,6 +1,9 @@ package restic -import "restic/crypto" +import ( + "context" + "restic/crypto" +) // Repository stores data in a backend. It provides high-level functions and // transparently encrypts/decrypts data. @@ -42,12 +45,12 @@ type Repository interface { // Deleter removes all data stored in a backend/repo. type Deleter interface { - Delete() error + Delete(context.Context) error } // Lister allows listing files in a backend. type Lister interface { - List(FileType, <-chan struct{}) <-chan string + List(context.Context, FileType) <-chan string } // Index keeps track of the blobs are stored within files.