KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Wire context into backend layout detection

Browse source
This commit is contained in:
Michael Eischer 2020-09-19 22:01:32 +02:00
parent 307a6ba3a3
commit 45e9a55c62
11 changed files with 53 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmd/restic/global.go
View file

@ -687,15 +687,15 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
switch loc.Scheme {
case "local":
be, err = local.Open(cfg.(local.Config))
be, err = local.Open(globalOptions.ctx, cfg.(local.Config))
// wrap the backend in a LimitBackend so that the throughput is limited
be = limiter.LimitBackend(be, lim)
case "sftp":
be, err = sftp.Open(cfg.(sftp.Config))
be, err = sftp.Open(globalOptions.ctx, cfg.(sftp.Config))
// wrap the backend in a LimitBackend so that the throughput is limited
be = limiter.LimitBackend(be, lim)
case "s3":
be, err = s3.Open(cfg.(s3.Config), rt)
be, err = s3.Open(globalOptions.ctx, cfg.(s3.Config), rt)
case "gs":
be, err = gs.Open(cfg.(gs.Config), rt)
case "azure":
@ -754,9 +754,9 @@ func create(s string, opts options.Options) (restic.Backend, error) {
switch loc.Scheme {
case "local":
return local.Create(cfg.(local.Config))
return local.Create(globalOptions.ctx, cfg.(local.Config))
case "sftp":
return sftp.Create(cfg.(sftp.Config))
return sftp.Create(globalOptions.ctx, cfg.(sftp.Config))
case "s3":
return s3.Create(globalOptions.ctx, cfg.(s3.Config), rt)
case "gs":

21
internal/backend/layout.go
View file

@ -1,6 +1,7 @@
package backend
import (
"context"
"fmt"
"os"
"path/filepath"
@ -24,7 +25,7 @@ type Layout interface {
// Filesystem is the abstraction of a file system used for a backend.
type Filesystem interface {
Join(...string) string
ReadDir(string) ([]os.FileInfo, error)
ReadDir(context.Context, string) ([]os.FileInfo, error)
IsNotExist(error) bool
}
@ -36,7 +37,7 @@ type LocalFilesystem struct {
}
// ReadDir returns all entries of a directory.
func (l *LocalFilesystem) ReadDir(dir string) ([]os.FileInfo, error) {
func (l *LocalFilesystem) ReadDir(ctx context.Context, dir string) ([]os.FileInfo, error) {
f, err := fs.Open(dir)
if err != nil {
return nil, err
@ -68,8 +69,8 @@ func (l *LocalFilesystem) IsNotExist(err error) bool {
var backendFilenameLength = len(restic.ID{}) * 2
var backendFilename = regexp.MustCompile(fmt.Sprintf("^[a-fA-F0-9]{%d}$", backendFilenameLength))
func hasBackendFile(fs Filesystem, dir string) (bool, error) {
entries, err := fs.ReadDir(dir)
func hasBackendFile(ctx context.Context, fs Filesystem, dir string) (bool, error) {
entries, err := fs.ReadDir(ctx, dir)
if err != nil && fs.IsNotExist(errors.Cause(err)) {
return false, nil
}
@ -94,20 +95,20 @@ var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout
// DetectLayout tries to find out which layout is used in a local (or sftp)
// filesystem at the given path. If repo is nil, an instance of LocalFilesystem
// is used.
func DetectLayout(repo Filesystem, dir string) (Layout, error) {
func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, error) {
debug.Log("detect layout at %v", dir)
if repo == nil {
repo = &LocalFilesystem{}
}
// key file in the "keys" dir (DefaultLayout)
foundKeysFile, err := hasBackendFile(repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
if err != nil {
return nil, err
}
// key file in the "key" dir (S3LegacyLayout)
foundKeyFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
if err != nil {
return nil, err
}
@ -134,7 +135,7 @@ func DetectLayout(repo Filesystem, dir string) (Layout, error) {
// ParseLayout parses the config string and returns a Layout. When layout is
// the empty string, DetectLayout is used. If that fails, defaultLayout is used.
func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) {
func ParseLayout(ctx context.Context, repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) {
debug.Log("parse layout string %q for backend at %v", layout, path)
switch layout {
case "default":
@ -148,12 +149,12 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout,
Join: repo.Join,
}
case "":
l, err = DetectLayout(repo, path)
l, err = DetectLayout(ctx, repo, path)
// use the default layout if auto detection failed
if errors.Cause(err) == ErrLayoutDetectionFailed && defaultLayout != "" {
debug.Log("error: %v, use default layout %v", err, defaultLayout)
return ParseLayout(repo, defaultLayout, "", path)
return ParseLayout(ctx, repo, defaultLayout, "", path)
}
if err != nil {

7
internal/backend/layout_test.go
View file

@ -1,6 +1,7 @@
package backend
import (
"context"
"fmt"
"path"
"path/filepath"
@ -371,7 +372,7 @@ func TestDetectLayout(t *testing.T) {
t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) {
rtest.SetupTarTestFixture(t, path, filepath.Join("testdata", test.filename))
layout, err := DetectLayout(fs, filepath.Join(path, "repo"))
layout, err := DetectLayout(context.TODO(), fs, filepath.Join(path, "repo"))
if err != nil {
t.Fatal(err)
}
@ -409,7 +410,7 @@ func TestParseLayout(t *testing.T) {
for _, test := range tests {
t.Run(test.layoutName, func(t *testing.T) {
layout, err := ParseLayout(&LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo"))
layout, err := ParseLayout(context.TODO(), &LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo"))
if err != nil {
t.Fatal(err)
}
@ -441,7 +442,7 @@ func TestParseLayoutInvalid(t *testing.T) {
for _, name := range invalidNames {
t.Run(name, func(t *testing.T) {
layout, err := ParseLayout(nil, name, "", path)
layout, err := ParseLayout(context.TODO(), nil, name, "", path)
if err == nil {
t.Fatalf("expected error not found for layout name %v, layout is %v", name, layout)
}

2
internal/backend/local/layout_test.go
View file

@ -36,7 +36,7 @@ func TestLayout(t *testing.T) {
rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename))
repo := filepath.Join(path, "repo")
be, err := Open(Config{
be, err := Open(context.TODO(), Config{
Path: repo,
Layout: test.layout,
})

8
internal/backend/local/local.go
View file

@ -27,9 +27,9 @@ var _ restic.Backend = &Local{}
const defaultLayout = "default"
// Open opens the local backend as specified by config.
func Open(cfg Config) (*Local, error) {
func Open(ctx context.Context, cfg Config) (*Local, error) {
debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout)
l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
@ -39,10 +39,10 @@ func Open(cfg Config) (*Local, error) {
// Create creates all the necessary files and directories for a new local
// backend at dir. Afterwards a new config blob should be created.
func Create(cfg Config) (*Local, error) {
func Create(ctx context.Context, cfg Config) (*Local, error) {
debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout)
l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}

7
internal/backend/local/local_test.go
View file

@ -1,6 +1,7 @@
package local_test
import (
"context"
"io/ioutil"
"os"
"path/filepath"
@ -32,13 +33,13 @@ func newTestSuite(t testing.TB) *test.Suite {
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(local.Config)
return local.Create(cfg)
return local.Create(context.TODO(), cfg)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(local.Config)
return local.Open(cfg)
return local.Open(context.TODO(), cfg)
},
// CleanupFn removes data created during the tests.
@ -91,7 +92,7 @@ func empty(t testing.TB, dir string) {
func openclose(t testing.TB, dir string) {
cfg := local.Config{Path: dir}
be, err := local.Open(cfg)
be, err := local.Open(context.TODO(), cfg)
if err != nil {
t.Logf("Open returned error %v", err)
}

14
internal/backend/s3/s3.go
View file

@ -33,7 +33,7 @@ var _ restic.Backend = &Backend{}
const defaultLayout = "default"
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg)
if cfg.MaxRetries > 0 {
@ -87,7 +87,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
cfg: cfg,
}
l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
l, err := backend.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix)
if err != nil {
return nil, err
}
@ -99,14 +99,14 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
return open(cfg, rt)
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
return open(ctx, cfg, rt)
}
// Create opens the S3 backend at bucket and region and creates the bucket if
// it does not exist yet.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
be, err := open(cfg, rt)
be, err := open(ctx, cfg, rt)
if err != nil {
return nil, errors.Wrap(err, "open")
}
@ -179,7 +179,7 @@ func (fi fileInfo) IsDir() bool { return fi.isDir } // abbreviation for
func (fi fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil)
// ReadDir returns the entries for a directory.
func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo, err error) {
debug.Log("ReadDir(%v)", dir)
// make sure dir ends with a slash
@ -187,7 +187,7 @@ func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
dir += "/"
}
ctx, cancel := context.WithCancel(context.TODO())
ctx, cancel := context.WithCancel(ctx)
defer cancel()
for obj := range be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{

2
internal/backend/sftp/layout_test.go
View file

@ -42,7 +42,7 @@ func TestLayout(t *testing.T) {
rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename))
repo := filepath.Join(path, "repo")
be, err := sftp.Open(sftp.Config{
be, err := sftp.Open(context.TODO(), sftp.Config{
Command: fmt.Sprintf("%q -e", sftpServer),
Path: repo,
Layout: test.layout,

22
internal/backend/sftp/sftp.go
View file

@ -109,7 +109,7 @@ func (r *SFTP) clientError() error {
// Open opens an sftp backend as described by the config by running
// "ssh" with the appropriate arguments (or cfg.Command, if set). The function
// preExec is run just before, postExec just after starting a program.
func Open(cfg Config) (*SFTP, error) {
func Open(ctx context.Context, cfg Config) (*SFTP, error) {
debug.Log("open backend with config %#v", cfg)
cmd, args, err := buildSSHCommand(cfg)
@ -123,7 +123,7 @@ func Open(cfg Config) (*SFTP, error) {
return nil, err
}
sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path)
sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
@ -152,7 +152,7 @@ func (r *SFTP) Join(p ...string) string {
}
// ReadDir returns the entries for a directory.
func (r *SFTP) ReadDir(dir string) ([]os.FileInfo, error) {
func (r *SFTP) ReadDir(ctx context.Context, dir string) ([]os.FileInfo, error) {
fi, err := r.c.ReadDir(dir)
// sftp client does not specify dir name on error, so add it here
@ -207,7 +207,7 @@ func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
// Create creates an sftp backend as described by the config by running "ssh"
// with the appropriate arguments (or cfg.Command, if set). The function
// preExec is run just before, postExec just after starting a program.
func Create(cfg Config) (*SFTP, error) {
func Create(ctx context.Context, cfg Config) (*SFTP, error) {
cmd, args, err := buildSSHCommand(cfg)
if err != nil {
return nil, err
@ -219,7 +219,7 @@ func Create(cfg Config) (*SFTP, error) {
return nil, err
}
sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path)
sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
@ -241,7 +241,7 @@ func Create(cfg Config) (*SFTP, error) {
}
// open backend
return Open(cfg)
return Open(ctx, cfg)
}
// Location returns this backend's location (the directory name).
@ -467,8 +467,8 @@ func (r *SFTP) Close() error {
return nil
}
func (r *SFTP) deleteRecursive(name string) error {
entries, err := r.ReadDir(name)
func (r *SFTP) deleteRecursive(ctx context.Context, name string) error {
entries, err := r.ReadDir(ctx, name)
if err != nil {
return errors.Wrap(err, "ReadDir")
}
@ -476,7 +476,7 @@ func (r *SFTP) deleteRecursive(name string) error {
for _, fi := range entries {
itemName := r.Join(name, fi.Name())
if fi.IsDir() {
err := r.deleteRecursive(itemName)
err := r.deleteRecursive(ctx, itemName)
if err != nil {
return errors.Wrap(err, "ReadDir")
}
@ -499,6 +499,6 @@ func (r *SFTP) deleteRecursive(name string) error {
}
// Delete removes all data in the backend.
func (r *SFTP) Delete(context.Context) error {
return r.deleteRecursive(r.p)
func (r *SFTP) Delete(ctx context.Context) error {
return r.deleteRecursive(ctx, r.p)
}

5
internal/backend/sftp/sftp_test.go
View file

@ -1,6 +1,7 @@
package sftp_test
import (
"context"
"fmt"
"io/ioutil"
"os"
@ -50,13 +51,13 @@ func newTestSuite(t testing.TB) *test.Suite {
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(sftp.Config)
return sftp.Create(cfg)
return sftp.Create(context.TODO(), cfg)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(sftp.Config)
return sftp.Open(cfg)
return sftp.Open(context.TODO(), cfg)
},
// CleanupFn removes data created during the tests.

4
internal/repository/testing.go
View file

@ -76,7 +76,7 @@ func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) {
if dir != "" {
_, err := os.Stat(dir)
if err != nil {
be, err := local.Create(local.Config{Path: dir})
be, err := local.Create(context.TODO(), local.Config{Path: dir})
if err != nil {
t.Fatalf("error creating local backend at %v: %v", dir, err)
}
@ -93,7 +93,7 @@ func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) {
// TestOpenLocal opens a local repository.
func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) {
be, err := local.Open(local.Config{Path: dir})
be, err := local.Open(context.TODO(), local.Config{Path: dir})
if err != nil {
t.Fatal(err)
}