From 7d12c292863efbd51c28781ce8be478bb404b994 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Jun 2023 13:04:34 +0200 Subject: [PATCH] backend: Unify backend construction using factory and registry This unified construction removes most backend-specific code from global.go. The backend registry will also enable integration tests to use custom backends if necessary. --- cmd/restic/cmd_init.go | 10 +-- cmd/restic/global.go | 76 ++++++----------- cmd/restic/integration_helpers_test.go | 2 + internal/backend/azure/azure.go | 5 ++ internal/backend/b2/b2.go | 5 ++ internal/backend/gs/gs.go | 5 ++ internal/backend/local/local.go | 10 +++ internal/backend/location/location.go | 67 ++++----------- internal/backend/location/location_test.go | 67 +++++++-------- internal/backend/location/registry.go | 94 ++++++++++++++++++++++ internal/backend/rclone/backend.go | 9 ++- internal/backend/rclone/backend_test.go | 2 +- internal/backend/rest/rest.go | 5 ++ internal/backend/s3/s3.go | 5 ++ internal/backend/sftp/sftp.go | 10 +++ internal/backend/swift/swift.go | 5 ++ 16 files changed, 235 insertions(+), 142 deletions(-) create mode 100644 internal/backend/location/registry.go diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index 43de7ff89..b9dabdc2d 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -87,9 +87,9 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] return err } - be, err := create(ctx, repo, gopts.extended) + be, err := create(ctx, repo, gopts, gopts.extended) if err != nil { - return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err) + return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err) } s, err := repository.New(be, repository.Options{ @@ -102,11 +102,11 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] err = s.Init(ctx, version, gopts.password, chunkerPolynomial) if err != nil { - return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err) + return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err) } if !gopts.JSON { - Verbosef("created restic repository %v at %s", s.Config().ID[:10], location.StripPassword(gopts.Repo)) + Verbosef("created restic repository %v at %s", s.Config().ID[:10], location.StripPassword(gopts.backends, gopts.Repo)) if opts.CopyChunkerParameters && chunkerPolynomial != nil { Verbosef(" with chunker parameters copied from secondary repository\n") } else { @@ -121,7 +121,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] status := initSuccess{ MessageType: "initialized", ID: s.Config().ID, - Repository: location.StripPassword(gopts.Repo), + Repository: location.StripPassword(gopts.backends, gopts.Repo), } return json.NewEncoder(globalOptions.stdout).Encode(status) } diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 3136e8990..f4886ec90 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -75,6 +75,7 @@ type GlobalOptions struct { stdout io.Writer stderr io.Writer + backends *location.Registry backendTestHook, backendInnerTestHook backendWrapper // verbosity is set as follows: @@ -98,6 +99,18 @@ var isReadingPassword bool var internalGlobalCtx context.Context func init() { + backends := location.NewRegistry() + backends.Register("b2", b2.NewFactory()) + backends.Register("local", local.NewFactory()) + backends.Register("sftp", sftp.NewFactory()) + backends.Register("s3", s3.NewFactory()) + backends.Register("gs", gs.NewFactory()) + backends.Register("azure", azure.NewFactory()) + backends.Register("swift", swift.NewFactory()) + backends.Register("rest", rest.NewFactory()) + backends.Register("rclone", rclone.NewFactory()) + globalOptions.backends = backends + var cancel context.CancelFunc internalGlobalCtx, cancel = context.WithCancel(context.Background()) AddCleanupHandler(func(code int) (int, error) { @@ -554,8 +567,8 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro // Open the backend specified by a location config. func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { - debug.Log("parsing location %v", location.StripPassword(s)) - loc, err := location.Parse(s) + debug.Log("parsing location %v", location.StripPassword(gopts.backends, s)) + loc, err := location.Parse(gopts.backends, s) if err != nil { return nil, errors.Fatalf("parsing repository location failed: %v", err) } @@ -576,32 +589,14 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio lim := limiter.NewStaticLimiter(gopts.Limits) rt = lim.Transport(rt) - switch loc.Scheme { - case "local": - be, err = local.Open(ctx, *cfg.(*local.Config)) - case "sftp": - be, err = sftp.Open(ctx, *cfg.(*sftp.Config)) - case "s3": - be, err = s3.Open(ctx, *cfg.(*s3.Config), rt) - case "gs": - be, err = gs.Open(ctx, *cfg.(*gs.Config), rt) - case "azure": - be, err = azure.Open(ctx, *cfg.(*azure.Config), rt) - case "swift": - be, err = swift.Open(ctx, *cfg.(*swift.Config), rt) - case "b2": - be, err = b2.Open(ctx, *cfg.(*b2.Config), rt) - case "rest": - be, err = rest.Open(ctx, *cfg.(*rest.Config), rt) - case "rclone": - be, err = rclone.Open(ctx, *cfg.(*rclone.Config), lim) - - default: + factory := gopts.backends.Lookup(loc.Scheme) + if factory == nil { return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) } + be, err = factory.Open(ctx, cfg, rt, lim) if err != nil { - return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(s), err) + return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(gopts.backends, s), err) } // wrap with debug logging and connection limiting @@ -623,7 +618,7 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio // check if config is there fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) if err != nil { - return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(s)) + return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(gopts.backends, s)) } if fi.Size == 0 { @@ -634,9 +629,9 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio } // Create the backend specified by URI. -func create(ctx context.Context, s string, opts options.Options) (restic.Backend, error) { +func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { debug.Log("parsing location %v", s) - loc, err := location.Parse(s) + loc, err := location.Parse(gopts.backends, s) if err != nil { return nil, err } @@ -651,31 +646,12 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend return nil, err } - var be restic.Backend - switch loc.Scheme { - case "local": - be, err = local.Create(ctx, *cfg.(*local.Config)) - case "sftp": - be, err = sftp.Create(ctx, *cfg.(*sftp.Config)) - case "s3": - be, err = s3.Create(ctx, *cfg.(*s3.Config), rt) - case "gs": - be, err = gs.Create(ctx, *cfg.(*gs.Config), rt) - case "azure": - be, err = azure.Create(ctx, *cfg.(*azure.Config), rt) - case "swift": - be, err = swift.Open(ctx, *cfg.(*swift.Config), rt) - case "b2": - be, err = b2.Create(ctx, *cfg.(*b2.Config), rt) - case "rest": - be, err = rest.Create(ctx, *cfg.(*rest.Config), rt) - case "rclone": - be, err = rclone.Create(ctx, *cfg.(*rclone.Config)) - default: - debug.Log("invalid repository scheme: %v", s) - return nil, errors.Fatalf("invalid scheme %q", loc.Scheme) + factory := gopts.backends.Lookup(loc.Scheme) + if factory == nil { + return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) } + be, err := factory.Create(ctx, cfg, rt, nil) if err != nil { return nil, err } diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index a0e4d49d6..b7cb5b333 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -206,6 +206,8 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) { // replace this hook with "nil" if listing a filetype more than once is necessary backendTestHook: func(r restic.Backend) (restic.Backend, error) { return newOrderedListOnceBackend(r), nil }, + // start with default set of backends + backends: globalOptions.backends, } // always overwrite global options diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 4041d3adc..b33b8dca6 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -14,6 +14,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -43,6 +44,10 @@ const defaultListMaxItems = 5000 // make sure that *Backend implements backend.Backend var _ restic.Backend = &Backend{} +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) +} + func open(cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) var client *azContainer.Client diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 7f4dba831..eb2cfe3c2 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -11,6 +11,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -36,6 +37,10 @@ const defaultListMaxItems = 10 * 1000 // ensure statically that *b2Backend implements restic.Backend. var _ restic.Backend = &b2Backend{} +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) +} + type sniffingRoundTripper struct { sync.Mutex lastErr error diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 022f2534a..445ccc77d 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -15,6 +15,7 @@ import ( "github.com/pkg/errors" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -47,6 +48,10 @@ type Backend struct { // Ensure that *Backend implements restic.Backend. var _ restic.Backend = &Backend{} +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) +} + func getStorageClient(rt http.RoundTripper) (*storage.Client, error) { // create a new HTTP client httpClient := &http.Client{ diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index d6bdef1e4..02ac81b8d 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -10,6 +10,8 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/limiter" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" @@ -28,6 +30,14 @@ type Local struct { // ensure statically that *Local implements restic.Backend. var _ restic.Backend = &Local{} +func NewFactory() location.Factory { + return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*Local, error) { + return Create(ctx, cfg) + }, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*Local, error) { + return Open(ctx, cfg) + }) +} + const defaultLayout = "default" func open(ctx context.Context, cfg Config) (*Local, error) { diff --git a/internal/backend/location/location.go b/internal/backend/location/location.go index 612ae1b4c..947ca17c3 100644 --- a/internal/backend/location/location.go +++ b/internal/backend/location/location.go @@ -4,15 +4,6 @@ package location import ( "strings" - "github.com/restic/restic/internal/backend/azure" - "github.com/restic/restic/internal/backend/b2" - "github.com/restic/restic/internal/backend/gs" - "github.com/restic/restic/internal/backend/local" - "github.com/restic/restic/internal/backend/rclone" - "github.com/restic/restic/internal/backend/rest" - "github.com/restic/restic/internal/backend/s3" - "github.com/restic/restic/internal/backend/sftp" - "github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/errors" ) @@ -23,34 +14,8 @@ type Location struct { Config interface{} } -type parser struct { - scheme string - parse func(string) (interface{}, error) - stripPassword func(string) string -} - -func configToAny[C any](parser func(string) (*C, error)) func(string) (interface{}, error) { - return func(s string) (interface{}, error) { - return parser(s) - } -} - -// parsers is a list of valid config parsers for the backends. The first parser -// is the fallback and should always be set to the local backend. -var parsers = []parser{ - {"b2", configToAny(b2.ParseConfig), noPassword}, - {"local", configToAny(local.ParseConfig), noPassword}, - {"sftp", configToAny(sftp.ParseConfig), noPassword}, - {"s3", configToAny(s3.ParseConfig), noPassword}, - {"gs", configToAny(gs.ParseConfig), noPassword}, - {"azure", configToAny(azure.ParseConfig), noPassword}, - {"swift", configToAny(swift.ParseConfig), noPassword}, - {"rest", configToAny(rest.ParseConfig), rest.StripPassword}, - {"rclone", configToAny(rclone.ParseConfig), noPassword}, -} - -// noPassword returns the repository location unchanged (there's no sensitive information there) -func noPassword(s string) string { +// NoPassword returns the repository location unchanged (there's no sensitive information there) +func NoPassword(s string) string { return s } @@ -88,16 +53,13 @@ func isPath(s string) bool { // starts with a backend name followed by a colon, that backend's Parse() // function is called. Otherwise, the local backend is used which interprets s // as the name of a directory. -func Parse(s string) (u Location, err error) { +func Parse(registry *Registry, s string) (u Location, err error) { scheme := extractScheme(s) u.Scheme = scheme - for _, parser := range parsers { - if parser.scheme != scheme { - continue - } - - u.Config, err = parser.parse(s) + factory := registry.Lookup(scheme) + if factory != nil { + u.Config, err = factory.ParseConfig(s) if err != nil { return Location{}, err } @@ -111,7 +73,12 @@ func Parse(s string) (u Location, err error) { } u.Scheme = "local" - u.Config, err = local.ParseConfig("local:" + s) + factory = registry.Lookup(u.Scheme) + if factory == nil { + return Location{}, errors.New("local backend not available") + } + + u.Config, err = factory.ParseConfig("local:" + s) if err != nil { return Location{}, err } @@ -120,14 +87,12 @@ func Parse(s string) (u Location, err error) { } // StripPassword returns a displayable version of a repository location (with any sensitive information removed) -func StripPassword(s string) string { +func StripPassword(registry *Registry, s string) string { scheme := extractScheme(s) - for _, parser := range parsers { - if parser.scheme != scheme { - continue - } - return parser.stripPassword(s) + factory := registry.Lookup(scheme) + if factory != nil { + return factory.StripPassword(s) } return s } diff --git a/internal/backend/location/location_test.go b/internal/backend/location/location_test.go index 9f5db70c9..6e9042200 100644 --- a/internal/backend/location/location_test.go +++ b/internal/backend/location/location_test.go @@ -1,4 +1,4 @@ -package location +package location_test import ( "net/url" @@ -7,6 +7,7 @@ import ( "github.com/restic/restic/internal/backend/b2" "github.com/restic/restic/internal/backend/local" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/sftp" @@ -24,11 +25,11 @@ func parseURL(s string) *url.URL { var parseTests = []struct { s string - u Location + u location.Location }{ { "local:/srv/repo", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "/srv/repo", Connections: 2, @@ -37,7 +38,7 @@ var parseTests = []struct { }, { "local:dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "dir1/dir2", Connections: 2, @@ -46,7 +47,7 @@ var parseTests = []struct { }, { "local:dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "dir1/dir2", Connections: 2, @@ -55,7 +56,7 @@ var parseTests = []struct { }, { "dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "dir1/dir2", Connections: 2, @@ -64,7 +65,7 @@ var parseTests = []struct { }, { "/dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "/dir1/dir2", Connections: 2, @@ -73,7 +74,7 @@ var parseTests = []struct { }, { "local:../dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "../dir1/dir2", Connections: 2, @@ -82,7 +83,7 @@ var parseTests = []struct { }, { "/dir1/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "/dir1/dir2", Connections: 2, @@ -91,7 +92,7 @@ var parseTests = []struct { }, { "/dir1:foobar/dir2", - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: "/dir1:foobar/dir2", Connections: 2, @@ -100,7 +101,7 @@ var parseTests = []struct { }, { `\dir1\foobar\dir2`, - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: `\dir1\foobar\dir2`, Connections: 2, @@ -109,7 +110,7 @@ var parseTests = []struct { }, { `c:\dir1\foobar\dir2`, - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: `c:\dir1\foobar\dir2`, Connections: 2, @@ -118,7 +119,7 @@ var parseTests = []struct { }, { `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, Connections: 2, @@ -127,7 +128,7 @@ var parseTests = []struct { }, { `c:/dir1/foobar/dir2`, - Location{Scheme: "local", + location.Location{Scheme: "local", Config: &local.Config{ Path: `c:/dir1/foobar/dir2`, Connections: 2, @@ -136,7 +137,7 @@ var parseTests = []struct { }, { "sftp:user@host:/srv/repo", - Location{Scheme: "sftp", + location.Location{Scheme: "sftp", Config: &sftp.Config{ User: "user", Host: "host", @@ -147,7 +148,7 @@ var parseTests = []struct { }, { "sftp:host:/srv/repo", - Location{Scheme: "sftp", + location.Location{Scheme: "sftp", Config: &sftp.Config{ User: "", Host: "host", @@ -158,7 +159,7 @@ var parseTests = []struct { }, { "sftp://user@host/srv/repo", - Location{Scheme: "sftp", + location.Location{Scheme: "sftp", Config: &sftp.Config{ User: "user", Host: "host", @@ -169,7 +170,7 @@ var parseTests = []struct { }, { "sftp://user@host//srv/repo", - Location{Scheme: "sftp", + location.Location{Scheme: "sftp", Config: &sftp.Config{ User: "user", Host: "host", @@ -181,7 +182,7 @@ var parseTests = []struct { { "s3://eu-central-1/bucketname", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "eu-central-1", Bucket: "bucketname", @@ -192,7 +193,7 @@ var parseTests = []struct { }, { "s3://hostname.foo/bucketname", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "bucketname", @@ -203,7 +204,7 @@ var parseTests = []struct { }, { "s3://hostname.foo/bucketname/prefix/directory", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "bucketname", @@ -214,7 +215,7 @@ var parseTests = []struct { }, { "s3:eu-central-1/repo", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "eu-central-1", Bucket: "repo", @@ -225,7 +226,7 @@ var parseTests = []struct { }, { "s3:eu-central-1/repo/prefix/directory", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "eu-central-1", Bucket: "repo", @@ -236,7 +237,7 @@ var parseTests = []struct { }, { "s3:https://hostname.foo/repo", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", @@ -247,7 +248,7 @@ var parseTests = []struct { }, { "s3:https://hostname.foo/repo/prefix/directory", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", @@ -258,7 +259,7 @@ var parseTests = []struct { }, { "s3:http://hostname.foo/repo", - Location{Scheme: "s3", + location.Location{Scheme: "s3", Config: &s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", @@ -270,7 +271,7 @@ var parseTests = []struct { }, { "swift:container17:/", - Location{Scheme: "swift", + location.Location{Scheme: "swift", Config: &swift.Config{ Container: "container17", Prefix: "", @@ -280,7 +281,7 @@ var parseTests = []struct { }, { "swift:container17:/prefix97", - Location{Scheme: "swift", + location.Location{Scheme: "swift", Config: &swift.Config{ Container: "container17", Prefix: "prefix97", @@ -290,7 +291,7 @@ var parseTests = []struct { }, { "rest:http://hostname.foo:1234/", - Location{Scheme: "rest", + location.Location{Scheme: "rest", Config: &rest.Config{ URL: parseURL("http://hostname.foo:1234/"), Connections: 5, @@ -298,7 +299,7 @@ var parseTests = []struct { }, }, { - "b2:bucketname:/prefix", Location{Scheme: "b2", + "b2:bucketname:/prefix", location.Location{Scheme: "b2", Config: &b2.Config{ Bucket: "bucketname", Prefix: "prefix", @@ -307,7 +308,7 @@ var parseTests = []struct { }, }, { - "b2:bucketname", Location{Scheme: "b2", + "b2:bucketname", location.Location{Scheme: "b2", Config: &b2.Config{ Bucket: "bucketname", Prefix: "", @@ -320,7 +321,7 @@ var parseTests = []struct { func TestParse(t *testing.T) { for i, test := range parseTests { t.Run(test.s, func(t *testing.T) { - u, err := Parse(test.s) + u, err := location.Parse(test.s) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -346,7 +347,7 @@ func TestInvalidScheme(t *testing.T) { for _, s := range invalidSchemes { t.Run(s, func(t *testing.T) { - _, err := Parse(s) + _, err := location.Parse(s) if err == nil { t.Fatalf("error for invalid location %q not found", s) } diff --git a/internal/backend/location/registry.go b/internal/backend/location/registry.go new file mode 100644 index 000000000..5d644dfb9 --- /dev/null +++ b/internal/backend/location/registry.go @@ -0,0 +1,94 @@ +package location + +import ( + "context" + "net/http" + + "github.com/restic/restic/internal/backend/limiter" + "github.com/restic/restic/internal/restic" +) + +type Registry struct { + factories map[string]Factory +} + +func NewRegistry() *Registry { + return &Registry{ + factories: make(map[string]Factory), + } +} + +func (r *Registry) Register(scheme string, factory Factory) { + if r.factories[scheme] != nil { + panic("duplicate backend") + } + r.factories[scheme] = factory +} + +func (r *Registry) Lookup(scheme string) Factory { + return r.factories[scheme] +} + +type Factory interface { + ParseConfig(s string) (interface{}, error) + StripPassword(s string) string + Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) + Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) +} + +type GenericBackendFactory[C any, T restic.Backend] struct { + parseConfigFn func(s string) (*C, error) + stripPasswordFn func(s string) string + createFn func(ctx context.Context, cfg C, rt http.RoundTripper, lim limiter.Limiter) (T, error) + openFn func(ctx context.Context, cfg C, rt http.RoundTripper, lim limiter.Limiter) (T, error) +} + +func (f *GenericBackendFactory[C, T]) ParseConfig(s string) (interface{}, error) { + return f.parseConfigFn(s) +} +func (f *GenericBackendFactory[C, T]) StripPassword(s string) string { + if f.stripPasswordFn != nil { + return f.stripPasswordFn(s) + } + return s +} +func (f *GenericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { + return f.createFn(ctx, *cfg.(*C), rt, lim) +} +func (f *GenericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { + return f.openFn(ctx, *cfg.(*C), rt, lim) +} + +func NewHTTPBackendFactory[C any, T restic.Backend](parseConfigFn func(s string) (*C, error), + stripPasswordFn func(s string) string, + createFn func(ctx context.Context, cfg C, rt http.RoundTripper) (T, error), + openFn func(ctx context.Context, cfg C, rt http.RoundTripper) (T, error)) *GenericBackendFactory[C, T] { + + return &GenericBackendFactory[C, T]{ + parseConfigFn: parseConfigFn, + stripPasswordFn: stripPasswordFn, + createFn: func(ctx context.Context, cfg C, rt http.RoundTripper, _ limiter.Limiter) (T, error) { + return createFn(ctx, cfg, rt) + }, + openFn: func(ctx context.Context, cfg C, rt http.RoundTripper, _ limiter.Limiter) (T, error) { + return openFn(ctx, cfg, rt) + }, + } +} + +func NewLimitedBackendFactory[C any, T restic.Backend](parseConfigFn func(s string) (*C, error), + stripPasswordFn func(s string) string, + createFn func(ctx context.Context, cfg C, lim limiter.Limiter) (T, error), + openFn func(ctx context.Context, cfg C, lim limiter.Limiter) (T, error)) *GenericBackendFactory[C, T] { + + return &GenericBackendFactory[C, T]{ + parseConfigFn: parseConfigFn, + stripPasswordFn: stripPasswordFn, + createFn: func(ctx context.Context, cfg C, _ http.RoundTripper, lim limiter.Limiter) (T, error) { + return createFn(ctx, cfg, lim) + }, + openFn: func(ctx context.Context, cfg C, _ http.RoundTripper, lim limiter.Limiter) (T, error) { + return openFn(ctx, cfg, lim) + }, + } +} diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index 881990bc3..f3a97ef75 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -19,6 +19,7 @@ import ( "github.com/cenkalti/backoff/v4" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/limiter" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" @@ -36,6 +37,10 @@ type Backend struct { conn *StdioConn } +func NewFactory() location.Factory { + return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, Create, Open) +} + // run starts command with args and initializes the StdioConn. func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan struct{}, func() error, error) { cmd := exec.Command(command, args...) @@ -283,8 +288,8 @@ func Open(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, error } // Create initializes a new restic repo with rclone. -func Create(ctx context.Context, cfg Config) (*Backend, error) { - be, err := newBackend(ctx, cfg, nil) +func Create(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, error) { + be, err := newBackend(ctx, cfg, lim) if err != nil { return nil, err } diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go index a562a99e6..738462577 100644 --- a/internal/backend/rclone/backend_test.go +++ b/internal/backend/rclone/backend_test.go @@ -27,7 +27,7 @@ func newTestSuite(t testing.TB) *test.Suite[rclone.Config] { // CreateFn is a function that creates a temporary repository for the tests. Create: func(cfg rclone.Config) (restic.Backend, error) { t.Logf("Create()") - be, err := rclone.Create(context.TODO(), cfg) + be, err := rclone.Create(context.TODO(), cfg, nil) var e *exec.Error if errors.As(err, &e) && e.Err == exec.ErrNotFound { t.Skipf("program %q not found", e.Name) diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 0906a7f10..4fb2d54de 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -13,6 +13,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -29,6 +30,10 @@ type Backend struct { layout.Layout } +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, StripPassword, Create, Open) +} + // the REST API protocol version is decided by HTTP request headers, these are the constants. const ( ContentTypeV1 = "application/vnd.x.restic.rest.v1" diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 7b7a761ce..dd5cc36e6 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -13,6 +13,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -31,6 +32,10 @@ type Backend struct { // make sure that *Backend implements backend.Backend var _ restic.Backend = &Backend{} +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Create, Open) +} + const defaultLayout = "default" func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 12c355003..f0a7ef9bc 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -15,6 +15,8 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/limiter" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -41,6 +43,14 @@ type SFTP struct { var _ restic.Backend = &SFTP{} +func NewFactory() location.Factory { + return location.NewLimitedBackendFactory(ParseConfig, location.NoPassword, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*SFTP, error) { + return Create(ctx, cfg) + }, func(ctx context.Context, cfg Config, _ limiter.Limiter) (*SFTP, error) { + return Open(ctx, cfg) + }) +} + const defaultLayout = "default" func startClient(cfg Config) (*SFTP, error) { diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index cfa9ed665..019456be7 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" + "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -34,6 +35,10 @@ type beSwift struct { // ensure statically that *beSwift implements restic.Backend. var _ restic.Backend = &beSwift{} +func NewFactory() location.Factory { + return location.NewHTTPBackendFactory(ParseConfig, location.NoPassword, Open, Open) +} + // Open opens the swift backend at a container in region. The container is // created if it does not exist yet. func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {