KurlesHS/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #4362 from restic/unified-backend-factory

Browse source 
Unified backend initialization
This commit is contained in:
Michael Eischer 2023-06-17 15:52:37 +02:00 committed by GitHub
commit 74ca82a6f8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 591 additions and 1039 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
cmd/restic/cmd_init.go
View file

@ -87,9 +87,9 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
return err
}
be, err := create(ctx, repo, gopts.extended)
be, err := create(ctx, repo, gopts, gopts.extended)
if err != nil {
return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err)
}
s, err := repository.New(be, repository.Options{
@ -102,11 +102,11 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
err = s.Init(ctx, version, gopts.password, chunkerPolynomial)
if err != nil {
return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err)
}
if !gopts.JSON {
Verbosef("created restic repository %v at %s", s.Config().ID[:10], location.StripPassword(gopts.Repo))
Verbosef("created restic repository %v at %s", s.Config().ID[:10], location.StripPassword(gopts.backends, gopts.Repo))
if opts.CopyChunkerParameters && chunkerPolynomial != nil {
Verbosef(" with chunker parameters copied from secondary repository\n")
} else {
@ -121,7 +121,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
status := initSuccess{
MessageType: "initialized",
ID: s.Config().ID,
Repository: location.StripPassword(gopts.Repo),
Repository: location.StripPassword(gopts.backends, gopts.Repo),
}
return json.NewEncoder(globalOptions.stdout).Encode(status)
}

87
cmd/restic/global.go
View file

@ -75,6 +75,7 @@ type GlobalOptions struct {
stdout io.Writer
stderr io.Writer
backends *location.Registry
backendTestHook, backendInnerTestHook backendWrapper
// verbosity is set as follows:
@ -98,6 +99,18 @@ var isReadingPassword bool
var internalGlobalCtx context.Context
func init() {
backends := location.NewRegistry()
backends.Register(azure.NewFactory())
backends.Register(b2.NewFactory())
backends.Register(gs.NewFactory())
backends.Register(local.NewFactory())
backends.Register(rclone.NewFactory())
backends.Register(rest.NewFactory())
backends.Register(s3.NewFactory())
backends.Register(sftp.NewFactory())
backends.Register(swift.NewFactory())
globalOptions.backends = backends
var cancel context.CancelFunc
internalGlobalCtx, cancel = context.WithCancel(context.Background())
AddCleanupHandler(func(code int) (int, error) {
@ -537,9 +550,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
func parseConfig(loc location.Location, opts options.Options) (interface{}, error) {
cfg := loc.Config
if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok {
if err := cfg.ApplyEnvironment(""); err != nil {
return nil, err
}
cfg.ApplyEnvironment("")
}
// only apply options for a particular backend here
@ -554,8 +565,8 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
// Open the backend specified by a location config.
func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
debug.Log("parsing location %v", location.StripPassword(s))
loc, err := location.Parse(s)
debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
loc, err := location.Parse(gopts.backends, s)
if err != nil {
return nil, errors.Fatalf("parsing repository location failed: %v", err)
}
@ -576,32 +587,14 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
lim := limiter.NewStaticLimiter(gopts.Limits)
rt = lim.Transport(rt)
switch loc.Scheme {
case "local":
be, err = local.Open(ctx, *cfg.(*local.Config))
case "sftp":
be, err = sftp.Open(ctx, *cfg.(*sftp.Config))
case "s3":
be, err = s3.Open(ctx, *cfg.(*s3.Config), rt)
case "gs":
be, err = gs.Open(*cfg.(*gs.Config), rt)
case "azure":
be, err = azure.Open(ctx, *cfg.(*azure.Config), rt)
case "swift":
be, err = swift.Open(ctx, *cfg.(*swift.Config), rt)
case "b2":
be, err = b2.Open(ctx, *cfg.(*b2.Config), rt)
case "rest":
be, err = rest.Open(*cfg.(*rest.Config), rt)
case "rclone":
be, err = rclone.Open(*cfg.(*rclone.Config), lim)
default:
factory := gopts.backends.Lookup(loc.Scheme)
if factory == nil {
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
}
be, err = factory.Open(ctx, cfg, rt, lim)
if err != nil {
return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(s), err)
return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(gopts.backends, s), err)
}
// wrap with debug logging and connection limiting
@ -615,15 +608,10 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
}
}
if loc.Scheme == "local" || loc.Scheme == "sftp" {
// wrap the backend in a LimitBackend so that the throughput is limited
be = limiter.LimitBackend(be, lim)
}
// check if config is there
fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil {
return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(s))
return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(gopts.backends, s))
}
if fi.Size == 0 {
@ -634,9 +622,9 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
}
// Create the backend specified by URI.
func create(ctx context.Context, s string, opts options.Options) (restic.Backend, error) {
debug.Log("parsing location %v", s)
loc, err := location.Parse(s)
func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
loc, err := location.Parse(gopts.backends, s)
if err != nil {
return nil, err
}
@ -651,31 +639,12 @@ func create(ctx context.Context, s string, opts options.Options) (restic.Backend
return nil, err
}
var be restic.Backend
switch loc.Scheme {
case "local":
be, err = local.Create(ctx, *cfg.(*local.Config))
case "sftp":
be, err = sftp.Create(ctx, *cfg.(*sftp.Config))
case "s3":
be, err = s3.Create(ctx, *cfg.(*s3.Config), rt)
case "gs":
be, err = gs.Create(ctx, *cfg.(*gs.Config), rt)
case "azure":
be, err = azure.Create(ctx, *cfg.(*azure.Config), rt)
case "swift":
be, err = swift.Open(ctx, *cfg.(*swift.Config), rt)
case "b2":
be, err = b2.Create(ctx, *cfg.(*b2.Config), rt)
case "rest":
be, err = rest.Create(ctx, *cfg.(*rest.Config), rt)
case "rclone":
be, err = rclone.Create(ctx, *cfg.(*rclone.Config))
default:
debug.Log("invalid repository scheme: %v", s)
return nil, errors.Fatalf("invalid scheme %q", loc.Scheme)
factory := gopts.backends.Lookup(loc.Scheme)
if factory == nil {
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
}
be, err := factory.Create(ctx, cfg, rt, nil)
if err != nil {
return nil, err
}

2
cmd/restic/integration_helpers_test.go
View file

@ -206,6 +206,8 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
// replace this hook with "nil" if listing a filetype more than once is necessary
backendTestHook: func(r restic.Backend) (restic.Backend, error) { return newOrderedListOnceBackend(r), nil },
// start with default set of backends
backends: globalOptions.backends,
}
// always overwrite global options

5
internal/backend/azure/azure.go
View file

@ -14,6 +14,7 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -43,6 +44,10 @@ const defaultListMaxItems = 5000
// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("azure", ParseConfig, location.NoPassword, Create, Open)
}
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg)
var client *azContainer.Client

55
internal/backend/azure/azure_test.go
View file

@ -12,18 +12,12 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/azure"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func newAzureTestSuite(t testing.TB) *test.Suite[azure.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
func newAzureTestSuite() *test.Suite[azure.Config] {
return &test.Suite[azure.Config]{
// do not use excessive data
MinimalData: true,
@ -35,51 +29,12 @@ func newAzureTestSuite(t testing.TB) *test.Suite[azure.Config] {
return nil, err
}
err = cfg.ApplyEnvironment("RESTIC_TEST_")
if err != nil {
return nil, err
}
cfg.ApplyEnvironment("RESTIC_TEST_")
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg azure.Config) (restic.Backend, error) {
ctx := context.TODO()
be, err := azure.Create(ctx, cfg, tr)
if err != nil {
return nil, err
}
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if err == nil {
return nil, errors.New("config already exists")
}
return be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg azure.Config) (restic.Backend, error) {
ctx := context.TODO()
return azure.Open(ctx, cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg azure.Config) error {
ctx := context.TODO()
be, err := azure.Open(ctx, cfg, tr)
if err != nil {
return err
}
return be.Delete(context.TODO())
},
Factory: azure.NewFactory(),
}
}
@ -104,7 +59,7 @@ func TestBackendAzure(t *testing.T) {
}
t.Logf("run tests")
newAzureTestSuite(t).RunTests(t)
newAzureTestSuite().RunTests(t)
}
func BenchmarkBackendAzure(t *testing.B) {
@ -122,7 +77,7 @@ func BenchmarkBackendAzure(t *testing.B) {
}
t.Logf("run tests")
newAzureTestSuite(t).RunBenchmarks(t)
newAzureTestSuite().RunBenchmarks(t)
}
func TestUploadLargeFile(t *testing.T) {

3
internal/backend/azure/config.go
View file

@ -59,7 +59,7 @@ func ParseConfig(s string) (*Config, error) {
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
func (cfg *Config) ApplyEnvironment(prefix string) {
if cfg.AccountName == "" {
cfg.AccountName = os.Getenv(prefix + "AZURE_ACCOUNT_NAME")
}
@ -71,5 +71,4 @@ func (cfg *Config) ApplyEnvironment(prefix string) error {
if cfg.AccountSAS.String() == "" {
cfg.AccountSAS = options.NewSecretString(os.Getenv(prefix + "AZURE_ACCOUNT_SAS"))
}
return nil
}

22
internal/backend/b2/b2.go
View file

@ -11,6 +11,7 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -36,6 +37,10 @@ const defaultListMaxItems = 10 * 1000
// ensure statically that *b2Backend implements restic.Backend.
var _ restic.Backend = &b2Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("b2", ParseConfig, location.NoPassword, Create, Open)
}
type sniffingRoundTripper struct {
sync.Mutex
lastErr error
@ -53,6 +58,13 @@ func (s *sniffingRoundTripper) RoundTrip(req *http.Request) (*http.Response, err
}
func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) {
if cfg.AccountID == "" {
return nil, errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty")
}
if cfg.Key.String() == "" {
return nil, errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty")
}
sniffer := &sniffingRoundTripper{RoundTripper: rt}
opts := []b2.ClientOption{b2.Transport(sniffer)}
@ -135,16 +147,6 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe
},
listMaxItems: defaultListMaxItems,
}
_, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if err == nil {
return nil, errors.New("config already exists")
}
return be, nil
}

40
internal/backend/b2/b2_test.go
View file

@ -1,26 +1,18 @@
package b2_test
import (
"context"
"fmt"
"os"
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/b2"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func newB2TestSuite(t testing.TB) *test.Suite[b2.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
func newB2TestSuite() *test.Suite[b2.Config] {
return &test.Suite[b2.Config]{
// do not use excessive data
MinimalData: true,
@ -35,34 +27,12 @@ func newB2TestSuite(t testing.TB) *test.Suite[b2.Config] {
return nil, err
}
err = cfg.ApplyEnvironment("RESTIC_TEST_")
if err != nil {
return nil, err
}
cfg.ApplyEnvironment("RESTIC_TEST_")
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg b2.Config) (restic.Backend, error) {
return b2.Create(context.Background(), cfg, tr)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg b2.Config) (restic.Backend, error) {
return b2.Open(context.Background(), cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg b2.Config) error {
be, err := b2.Open(context.Background(), cfg, tr)
if err != nil {
return err
}
return be.Delete(context.TODO())
},
Factory: b2.NewFactory(),
}
}
@ -89,10 +59,10 @@ func TestBackendB2(t *testing.T) {
}()
testVars(t)
newB2TestSuite(t).RunTests(t)
newB2TestSuite().RunTests(t)
}
func BenchmarkBackendb2(t *testing.B) {
testVars(t)
newB2TestSuite(t).RunBenchmarks(t)
newB2TestSuite().RunBenchmarks(t)
}

12
internal/backend/b2/config.go
View file

@ -85,21 +85,11 @@ func ParseConfig(s string) (*Config, error) {
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
func (cfg *Config) ApplyEnvironment(prefix string) {
if cfg.AccountID == "" {
cfg.AccountID = os.Getenv(prefix + "B2_ACCOUNT_ID")
}
if cfg.AccountID == "" {
return errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty")
}
if cfg.Key.String() == "" {
cfg.Key = options.NewSecretString(os.Getenv(prefix + "B2_ACCOUNT_KEY"))
}
if cfg.Key.String() == "" {
return errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty")
}
return nil
}

3
internal/backend/gs/config.go
View file

@ -62,9 +62,8 @@ func ParseConfig(s string) (*Config, error) {
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
func (cfg *Config) ApplyEnvironment(prefix string) {
if cfg.ProjectID == "" {
cfg.ProjectID = os.Getenv(prefix + "GOOGLE_PROJECT_ID")
}
return nil
}

7
internal/backend/gs/gs.go
View file

@ -15,6 +15,7 @@ import (
"github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
@ -47,6 +48,10 @@ type Backend struct {
// Ensure that *Backend implements restic.Backend.
var _ restic.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open)
}
func getStorageClient(rt http.RoundTripper) (*storage.Client, error) {
// create a new HTTP client
httpClient := &http.Client{
@ -117,7 +122,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
}
// Open opens the gs backend at the specified bucket.
func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
return open(cfg, rt)
}

49
internal/backend/gs/gs_test.go
View file

@ -1,26 +1,17 @@
package gs_test
import (
"context"
"fmt"
"os"
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/gs"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func newGSTestSuite(t testing.TB) *test.Suite[gs.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
func newGSTestSuite() *test.Suite[gs.Config] {
return &test.Suite[gs.Config]{
// do not use excessive data
MinimalData: true,
@ -37,39 +28,7 @@ func newGSTestSuite(t testing.TB) *test.Suite[gs.Config] {
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg gs.Config) (restic.Backend, error) {
be, err := gs.Create(context.Background(), cfg, tr)
if err != nil {
return nil, err
}
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if err == nil {
return nil, errors.New("config already exists")
}
return be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg gs.Config) (restic.Backend, error) {
return gs.Open(cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg gs.Config) error {
be, err := gs.Open(cfg, tr)
if err != nil {
return err
}
return be.Delete(context.TODO())
},
Factory: gs.NewFactory(),
}
}
@ -97,7 +56,7 @@ func TestBackendGS(t *testing.T) {
}
t.Logf("run tests")
newGSTestSuite(t).RunTests(t)
newGSTestSuite().RunTests(t)
}
func BenchmarkBackendGS(t *testing.B) {
@ -118,5 +77,5 @@ func BenchmarkBackendGS(t *testing.B) {
}
t.Logf("run tests")
newGSTestSuite(t).RunBenchmarks(t)
newGSTestSuite().RunBenchmarks(t)
}

15
internal/backend/limiter/limiter_backend.go
View file

@ -7,6 +7,21 @@ import (
"github.com/restic/restic/internal/restic"
)
func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) {
return func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) {
var be restic.Backend
be, err := constructor(ctx, cfg)
if err != nil {
return nil, err
}
if lim != nil {
be = LimitBackend(be, lim)
}
return be, nil
}
}
// LimitBackend wraps a Backend and applies rate limiting to Load() and Save()
// calls on the backend.
func LimitBackend(be restic.Backend, l Limiter) restic.Backend {

28
internal/backend/local/config_test.go
View file

@ -11,6 +11,34 @@ var configTests = []test.ConfigTestData[Config]{
Path: "/some/path",
Connections: 2,
}},
{S: "local:dir1/dir2", Cfg: Config{
Path: "dir1/dir2",
Connections: 2,
}},
{S: "local:../dir1/dir2", Cfg: Config{
Path: "../dir1/dir2",
Connections: 2,
}},
{S: "local:/dir1:foobar/dir2", Cfg: Config{
Path: "/dir1:foobar/dir2",
Connections: 2,
}},
{S: `local:\dir1\foobar\dir2`, Cfg: Config{
Path: `\dir1\foobar\dir2`,
Connections: 2,
}},
{S: `local:c:\dir1\foobar\dir2`, Cfg: Config{
Path: `c:\dir1\foobar\dir2`,
Connections: 2,
}},
{S: `local:C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`, Cfg: Config{
Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
Connections: 2,
}},
{S: `local:c:/dir1/foobar/dir2`, Cfg: Config{
Path: `c:/dir1/foobar/dir2`,
Connections: 2,
}},
}
func TestParseConfig(t *testing.T) {

6
internal/backend/local/local.go
View file

@ -10,6 +10,8 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
@ -28,6 +30,10 @@ type Local struct {
// ensure statically that *Local implements restic.Backend.
var _ restic.Backend = &Local{}
func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
}
const defaultLayout = "default"
func open(ctx context.Context, cfg Config) (*Local, error) {

27
internal/backend/local/local_test.go
View file

@ -8,7 +8,6 @@ import (
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -16,11 +15,7 @@ func newTestSuite(t testing.TB) *test.Suite[local.Config] {
return &test.Suite[local.Config]{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (*local.Config, error) {
dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-local-")
if err != nil {
t.Fatal(err)
}
dir := rtest.TempDir(t)
t.Logf("create new backend at %v", dir)
cfg := &local.Config{
@ -30,25 +25,7 @@ func newTestSuite(t testing.TB) *test.Suite[local.Config] {
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg local.Config) (restic.Backend, error) {
return local.Create(context.TODO(), cfg)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg local.Config) (restic.Backend, error) {
return local.Open(context.TODO(), cfg)
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg local.Config) error {
if !rtest.TestCleanupTempDirs {
t.Logf("leaving test backend dir at %v", cfg.Path)
}
rtest.RemoveAll(t, cfg.Path)
return nil
},
Factory: local.NewFactory(),
}
}

113
internal/backend/location/display_location_test.go
View file

@ -1,96 +1,29 @@
package location
package location_test
import "testing"
import (
"testing"
var passwordTests = []struct {
input string
expected string
}{
{
"local:/srv/repo",
"local:/srv/repo",
},
{
"/dir1/dir2",
"/dir1/dir2",
},
{
`c:\dir1\foobar\dir2`,
`c:\dir1\foobar\dir2`,
},
{
"sftp:user@host:/srv/repo",
"sftp:user@host:/srv/repo",
},
{
"s3://eu-central-1/bucketname",
"s3://eu-central-1/bucketname",
},
{
"swift:container17:/prefix97",
"swift:container17:/prefix97",
},
{
"b2:bucketname:/prefix",
"b2:bucketname:/prefix",
},
{
"rest:",
"rest:/",
},
{
"rest:localhost/",
"rest:localhost/",
},
{
"rest::123/",
"rest::123/",
},
{
"rest:http://",
"rest:http://",
},
{
"rest:http://hostname.foo:1234/",
"rest:http://hostname.foo:1234/",
},
{
"rest:http://user@hostname.foo:1234/",
"rest:http://user@hostname.foo:1234/",
},
{
"rest:http://user:@hostname.foo:1234/",
"rest:http://user:***@hostname.foo:1234/",
},
{
"rest:http://user:p@hostname.foo:1234/",
"rest:http://user:***@hostname.foo:1234/",
},
{
"rest:http://user:pppppaaafhhfuuwiiehhthhghhdkjaoowpprooghjjjdhhwuuhgjsjhhfdjhruuhsjsdhhfhshhsppwufhhsjjsjs@hostname.foo:1234/",
"rest:http://user:***@hostname.foo:1234/",
},
{
"rest:http://user:password@hostname",
"rest:http://user:***@hostname/",
},
{
"rest:http://user:password@:123",
"rest:http://user:***@:123/",
},
{
"rest:http://user:password@",
"rest:http://user:***@/",
},
}
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func TestStripPassword(t *testing.T) {
for i, test := range passwordTests {
t.Run(test.input, func(t *testing.T) {
result := StripPassword(test.input)
if result != test.expected {
t.Errorf("test %d: expected '%s' but got '%s'", i, test.expected, result)
}
registry := location.NewRegistry()
registry.Register(
location.NewHTTPBackendFactory[any, restic.Backend]("test", nil,
func(s string) string {
return "cleaned"
}, nil, nil,
),
)
t.Run("valid", func(t *testing.T) {
clean := location.StripPassword(registry, "test:secret")
test.Equals(t, "cleaned", clean)
})
t.Run("unknown", func(t *testing.T) {
clean := location.StripPassword(registry, "invalid:secret")
test.Equals(t, "invalid:secret", clean)
})
}
}

67
internal/backend/location/location.go
View file

@ -4,15 +4,6 @@ package location
import (
"strings"
"github.com/restic/restic/internal/backend/azure"
"github.com/restic/restic/internal/backend/b2"
"github.com/restic/restic/internal/backend/gs"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/rclone"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/sftp"
"github.com/restic/restic/internal/backend/swift"
"github.com/restic/restic/internal/errors"
)
@ -23,34 +14,8 @@ type Location struct {
Config interface{}
}
type parser struct {
scheme string
parse func(string) (interface{}, error)
stripPassword func(string) string
}
func configToAny[C any](parser func(string) (*C, error)) func(string) (interface{}, error) {
return func(s string) (interface{}, error) {
return parser(s)
}
}
// parsers is a list of valid config parsers for the backends. The first parser
// is the fallback and should always be set to the local backend.
var parsers = []parser{
{"b2", configToAny(b2.ParseConfig), noPassword},
{"local", configToAny(local.ParseConfig), noPassword},
{"sftp", configToAny(sftp.ParseConfig), noPassword},
{"s3", configToAny(s3.ParseConfig), noPassword},
{"gs", configToAny(gs.ParseConfig), noPassword},
{"azure", configToAny(azure.ParseConfig), noPassword},
{"swift", configToAny(swift.ParseConfig), noPassword},
{"rest", configToAny(rest.ParseConfig), rest.StripPassword},
{"rclone", configToAny(rclone.ParseConfig), noPassword},
}
// noPassword returns the repository location unchanged (there's no sensitive information there)
func noPassword(s string) string {
// NoPassword returns the repository location unchanged (there's no sensitive information there)
func NoPassword(s string) string {
return s
}
@ -88,16 +53,13 @@ func isPath(s string) bool {
// starts with a backend name followed by a colon, that backend's Parse()
// function is called. Otherwise, the local backend is used which interprets s
// as the name of a directory.
func Parse(s string) (u Location, err error) {
func Parse(registry *Registry, s string) (u Location, err error) {
scheme := extractScheme(s)
u.Scheme = scheme
for _, parser := range parsers {
if parser.scheme != scheme {
continue
}
u.Config, err = parser.parse(s)
factory := registry.Lookup(scheme)
if factory != nil {
u.Config, err = factory.ParseConfig(s)
if err != nil {
return Location{}, err
}
@ -111,7 +73,12 @@ func Parse(s string) (u Location, err error) {
}
u.Scheme = "local"
u.Config, err = local.ParseConfig("local:" + s)
factory = registry.Lookup(u.Scheme)
if factory == nil {
return Location{}, errors.New("local backend not available")
}
u.Config, err = factory.ParseConfig("local:" + s)
if err != nil {
return Location{}, err
}
@ -120,14 +87,12 @@ func Parse(s string) (u Location, err error) {
}
// StripPassword returns a displayable version of a repository location (with any sensitive information removed)
func StripPassword(s string) string {
func StripPassword(registry *Registry, s string) string {
scheme := extractScheme(s)
for _, parser := range parsers {
if parser.scheme != scheme {
continue
}
return parser.stripPassword(s)
factory := registry.Lookup(scheme)
if factory != nil {
return factory.StripPassword(s)
}
return s
}

367
internal/backend/location/location_test.go
View file

@ -1,344 +1,65 @@
package location
package location_test
import (
"net/url"
"reflect"
"testing"
"github.com/restic/restic/internal/backend/b2"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/sftp"
"github.com/restic/restic/internal/backend/swift"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func parseURL(s string) *url.URL {
u, err := url.Parse(s)
if err != nil {
panic(err)
type testConfig struct {
loc string
}
return u
}
var parseTests = []struct {
s string
u Location
}{
{
"local:/srv/repo",
Location{Scheme: "local",
Config: &local.Config{
Path: "/srv/repo",
Connections: 2,
},
},
},
{
"local:dir1/dir2",
Location{Scheme: "local",
Config: &local.Config{
Path: "dir1/dir2",
Connections: 2,
},
},
},
{
"local:dir1/dir2",
Location{Scheme: "local",
Config: &local.Config{
Path: "dir1/dir2",
Connections: 2,
},
},
},
{
"dir1/dir2",
Location{Scheme: "local",
Config: &local.Config{
Path: "dir1/dir2",
Connections: 2,
},
},
},
{
"/dir1/dir2",
Location{Scheme: "local",
Config: &local.Config{
Path: "/dir1/dir2",
Connections: 2,
},
},
},
{
"local:../dir1/dir2",
Location{Scheme: "local",
Config: &local.Config{
Path: "../dir1/dir2",
Connections: 2,
},
},
},
{
"/dir1/dir2",
Location{Scheme: "local",
Config: &local.Config{
Path: "/dir1/dir2",
Connections: 2,
},
},
},
{
"/dir1:foobar/dir2",
Location{Scheme: "local",
Config: &local.Config{
Path: "/dir1:foobar/dir2",
Connections: 2,
},
},
},
{
`\dir1\foobar\dir2`,
Location{Scheme: "local",
Config: &local.Config{
Path: `\dir1\foobar\dir2`,
Connections: 2,
},
},
},
{
`c:\dir1\foobar\dir2`,
Location{Scheme: "local",
Config: &local.Config{
Path: `c:\dir1\foobar\dir2`,
Connections: 2,
},
},
},
{
`C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
Location{Scheme: "local",
Config: &local.Config{
Path: `C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
Connections: 2,
},
},
},
{
`c:/dir1/foobar/dir2`,
Location{Scheme: "local",
Config: &local.Config{
Path: `c:/dir1/foobar/dir2`,
Connections: 2,
},
},
},
{
"sftp:user@host:/srv/repo",
Location{Scheme: "sftp",
Config: &sftp.Config{
User: "user",
Host: "host",
Path: "/srv/repo",
Connections: 5,
},
},
},
{
"sftp:host:/srv/repo",
Location{Scheme: "sftp",
Config: &sftp.Config{
User: "",
Host: "host",
Path: "/srv/repo",
Connections: 5,
},
},
},
{
"sftp://user@host/srv/repo",
Location{Scheme: "sftp",
Config: &sftp.Config{
User: "user",
Host: "host",
Path: "srv/repo",
Connections: 5,
},
},
},
{
"sftp://user@host//srv/repo",
Location{Scheme: "sftp",
Config: &sftp.Config{
User: "user",
Host: "host",
Path: "/srv/repo",
Connections: 5,
},
},
},
{
"s3://eu-central-1/bucketname",
Location{Scheme: "s3",
Config: &s3.Config{
Endpoint: "eu-central-1",
Bucket: "bucketname",
Prefix: "",
Connections: 5,
},
},
},
{
"s3://hostname.foo/bucketname",
Location{Scheme: "s3",
Config: &s3.Config{
Endpoint: "hostname.foo",
Bucket: "bucketname",
Prefix: "",
Connections: 5,
},
},
},
{
"s3://hostname.foo/bucketname/prefix/directory",
Location{Scheme: "s3",
Config: &s3.Config{
Endpoint: "hostname.foo",
Bucket: "bucketname",
Prefix: "prefix/directory",
Connections: 5,
},
},
},
{
"s3:eu-central-1/repo",
Location{Scheme: "s3",
Config: &s3.Config{
Endpoint: "eu-central-1",
Bucket: "repo",
Prefix: "",
Connections: 5,
},
},
},
{
"s3:eu-central-1/repo/prefix/directory",
Location{Scheme: "s3",
Config: &s3.Config{
Endpoint: "eu-central-1",
Bucket: "repo",
Prefix: "prefix/directory",
Connections: 5,
},
},
},
{
"s3:https://hostname.foo/repo",
Location{Scheme: "s3",
Config: &s3.Config{
Endpoint: "hostname.foo",
Bucket: "repo",
Prefix: "",
Connections: 5,
},
},
},
{
"s3:https://hostname.foo/repo/prefix/directory",
Location{Scheme: "s3",
Config: &s3.Config{
Endpoint: "hostname.foo",
Bucket: "repo",
Prefix: "prefix/directory",
Connections: 5,
},
},
},
{
"s3:http://hostname.foo/repo",
Location{Scheme: "s3",
Config: &s3.Config{
Endpoint: "hostname.foo",
Bucket: "repo",
Prefix: "",
UseHTTP: true,
Connections: 5,
},
},
},
{
"swift:container17:/",
Location{Scheme: "swift",
Config: &swift.Config{
Container: "container17",
Prefix: "",
Connections: 5,
},
},
},
{
"swift:container17:/prefix97",
Location{Scheme: "swift",
Config: &swift.Config{
Container: "container17",
Prefix: "prefix97",
Connections: 5,
},
},
},
{
"rest:http://hostname.foo:1234/",
Location{Scheme: "rest",
Config: &rest.Config{
URL: parseURL("http://hostname.foo:1234/"),
Connections: 5,
},
},
},
{
"b2:bucketname:/prefix", Location{Scheme: "b2",
Config: &b2.Config{
Bucket: "bucketname",
Prefix: "prefix",
Connections: 5,
},
},
},
{
"b2:bucketname", Location{Scheme: "b2",
Config: &b2.Config{
Bucket: "bucketname",
Prefix: "",
Connections: 5,
},
},
},
func testFactory() location.Factory {
return location.NewHTTPBackendFactory[testConfig, restic.Backend](
"local",
func(s string) (*testConfig, error) {
return &testConfig{loc: s}, nil
}, nil, nil, nil,
)
}
func TestParse(t *testing.T) {
for i, test := range parseTests {
t.Run(test.s, func(t *testing.T) {
u, err := Parse(test.s)
registry := location.NewRegistry()
registry.Register(testFactory())
path := "local:example"
u, err := location.Parse(registry, path)
test.OK(t, err)
test.Equals(t, "local", u.Scheme)
test.Equals(t, &testConfig{loc: path}, u.Config)
}
func TestParseFallback(t *testing.T) {
fallbackTests := []string{
"dir1/dir2",
"/dir1/dir2",
"/dir1:foobar/dir2",
`\dir1\foobar\dir2`,
`c:\dir1\foobar\dir2`,
`C:\Users\appveyor\AppData\Local\Temp\1\restic-test-879453535\repo`,
`c:/dir1/foobar/dir2`,
}
registry := location.NewRegistry()
registry.Register(testFactory())
for _, path := range fallbackTests {
t.Run(path, func(t *testing.T) {
u, err := location.Parse(registry, path)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if test.u.Scheme != u.Scheme {
t.Errorf("test %d: scheme does not match, want %q, got %q",
i, test.u.Scheme, u.Scheme)
}
if !reflect.DeepEqual(test.u.Config, u.Config) {
t.Errorf("test %d: cfg map does not match, want:\n %#v\ngot: \n %#v",
i, test.u.Config, u.Config)
}
test.Equals(t, "local", u.Scheme)
test.Equals(t, "local:"+path, u.Config.(*testConfig).loc)
})
}
}
func TestInvalidScheme(t *testing.T) {
registry := location.NewRegistry()
var invalidSchemes = []string{
"foobar:xxx",
"foobar:/dir/dir2",
@ -346,7 +67,7 @@ func TestInvalidScheme(t *testing.T) {
for _, s := range invalidSchemes {
t.Run(s, func(t *testing.T) {
_, err := Parse(s)
_, err := location.Parse(registry, s)
if err == nil {
t.Fatalf("error for invalid location %q not found", s)
}

106
internal/backend/location/registry.go Normal file
View file

@ -0,0 +1,106 @@
package location
import (
"context"
"net/http"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/restic"
)
type Registry struct {
factories map[string]Factory
}
func NewRegistry() *Registry {
return &Registry{
factories: make(map[string]Factory),
}
}
func (r *Registry) Register(factory Factory) {
if r.factories[factory.Scheme()] != nil {
panic("duplicate backend")
}
r.factories[factory.Scheme()] = factory
}
func (r *Registry) Lookup(scheme string) Factory {
return r.factories[scheme]
}
type Factory interface {
Scheme() string
ParseConfig(s string) (interface{}, error)
StripPassword(s string) string
Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error)
Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error)
}
type genericBackendFactory[C any, T restic.Backend] struct {
scheme string
parseConfigFn func(s string) (*C, error)
stripPasswordFn func(s string) string
createFn func(ctx context.Context, cfg C, rt http.RoundTripper, lim limiter.Limiter) (T, error)
openFn func(ctx context.Context, cfg C, rt http.RoundTripper, lim limiter.Limiter) (T, error)
}
func (f *genericBackendFactory[C, T]) Scheme() string {
return f.scheme
}
func (f *genericBackendFactory[C, T]) ParseConfig(s string) (interface{}, error) {
return f.parseConfigFn(s)
}
func (f *genericBackendFactory[C, T]) StripPassword(s string) string {
if f.stripPasswordFn != nil {
return f.stripPasswordFn(s)
}
return s
}
func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) {
return f.createFn(ctx, *cfg.(*C), rt, lim)
}
func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) {
return f.openFn(ctx, *cfg.(*C), rt, lim)
}
func NewHTTPBackendFactory[C any, T restic.Backend](
scheme string,
parseConfigFn func(s string) (*C, error),
stripPasswordFn func(s string) string,
createFn func(ctx context.Context, cfg C, rt http.RoundTripper) (T, error),
openFn func(ctx context.Context, cfg C, rt http.RoundTripper) (T, error)) Factory {
return &genericBackendFactory[C, T]{
scheme: scheme,
parseConfigFn: parseConfigFn,
stripPasswordFn: stripPasswordFn,
createFn: func(ctx context.Context, cfg C, rt http.RoundTripper, _ limiter.Limiter) (T, error) {
return createFn(ctx, cfg, rt)
},
openFn: func(ctx context.Context, cfg C, rt http.RoundTripper, _ limiter.Limiter) (T, error) {
return openFn(ctx, cfg, rt)
},
}
}
func NewLimitedBackendFactory[C any, T restic.Backend](
scheme string,
parseConfigFn func(s string) (*C, error),
stripPasswordFn func(s string) string,
createFn func(ctx context.Context, cfg C, lim limiter.Limiter) (T, error),
openFn func(ctx context.Context, cfg C, lim limiter.Limiter) (T, error)) Factory {
return &genericBackendFactory[C, T]{
scheme: scheme,
parseConfigFn: parseConfigFn,
stripPasswordFn: stripPasswordFn,
createFn: func(ctx context.Context, cfg C, _ http.RoundTripper, lim limiter.Limiter) (T, error) {
return createFn(ctx, cfg, lim)
},
openFn: func(ctx context.Context, cfg C, _ http.RoundTripper, lim limiter.Limiter) (T, error) {
return openFn(ctx, cfg, lim)
},
}
}

21
internal/backend/mem/mem_backend.go
View file

@ -6,10 +6,12 @@ import (
"encoding/base64"
"hash"
"io"
"net/http"
"sync"
"github.com/cespare/xxhash/v2"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -20,6 +22,25 @@ type memMap map[restic.Handle][]byte
// make sure that MemoryBackend implements backend.Backend
var _ restic.Backend = &MemoryBackend{}
// NewFactory creates a persistent mem backend
func NewFactory() location.Factory {
be := New()
return location.NewHTTPBackendFactory[struct{}, *MemoryBackend](
"mem",
func(s string) (*struct{}, error) {
return &struct{}{}, nil
},
location.NoPassword,
func(_ context.Context, _ struct{}, _ http.RoundTripper) (*MemoryBackend, error) {
return be, nil
},
func(_ context.Context, _ struct{}, _ http.RoundTripper) (*MemoryBackend, error) {
return be, nil
},
)
}
var errNotFound = errors.New("not found")
const connectionCount = 2

48
internal/backend/mem/mem_backend_test.go
View file

@ -1,58 +1,20 @@
package mem_test
import (
"context"
"testing"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/backend/test"
)
type memConfig struct {
be restic.Backend
}
func newTestSuite() *test.Suite[*memConfig] {
return &test.Suite[*memConfig]{
func newTestSuite() *test.Suite[struct{}] {
return &test.Suite[struct{}]{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (**memConfig, error) {
cfg := &memConfig{}
return &cfg, nil
NewConfig: func() (*struct{}, error) {
return &struct{}{}, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg *memConfig) (restic.Backend, error) {
if cfg.be != nil {
_, err := cfg.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !cfg.be.IsNotExist(err) {
return nil, err
}
if err == nil {
return nil, errors.New("config already exists")
}
}
cfg.be = mem.New()
return cfg.be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg *memConfig) (restic.Backend, error) {
if cfg.be == nil {
cfg.be = mem.New()
}
return cfg.be, nil
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg *memConfig) error {
// no cleanup needed
return nil
},
Factory: mem.NewFactory(),
}
}

19
internal/backend/rclone/backend.go
View file

@ -19,6 +19,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
@ -36,6 +37,10 @@ type Backend struct {
conn *StdioConn
}
func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("rclone", ParseConfig, location.NoPassword, Create, Open)
}
// run starts command with args and initializes the StdioConn.
func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan struct{}, func() error, error) {
cmd := exec.Command(command, args...)
@ -134,7 +139,7 @@ func wrapConn(c *StdioConn, lim limiter.Limiter) *wrappedConn {
}
// New initializes a Backend and starts the process.
func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) {
func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, error) {
var (
args []string
err error
@ -197,7 +202,7 @@ func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) {
wg: wg,
}
ctx, cancel := context.WithCancel(context.Background())
ctx, cancel := context.WithCancel(ctx)
defer cancel()
wg.Add(1)
@ -256,8 +261,8 @@ func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) {
}
// Open starts an rclone process with the given config.
func Open(cfg Config, lim limiter.Limiter) (*Backend, error) {
be, err := newBackend(cfg, lim)
func Open(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, error) {
be, err := newBackend(ctx, cfg, lim)
if err != nil {
return nil, err
}
@ -272,7 +277,7 @@ func Open(cfg Config, lim limiter.Limiter) (*Backend, error) {
URL: url,
}
restBackend, err := rest.Open(restConfig, debug.RoundTripper(be.tr))
restBackend, err := rest.Open(ctx, restConfig, debug.RoundTripper(be.tr))
if err != nil {
_ = be.Close()
return nil, err
@ -283,8 +288,8 @@ func Open(cfg Config, lim limiter.Limiter) (*Backend, error) {
}
// Create initializes a new restic repo with rclone.
func Create(ctx context.Context, cfg Config) (*Backend, error) {
be, err := newBackend(cfg, nil)
func Create(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, error) {
be, err := newBackend(ctx, cfg, lim)
if err != nil {
return nil, err
}

27
internal/backend/rclone/backend_test.go
View file

@ -1,14 +1,11 @@
package rclone_test
import (
"context"
"os/exec"
"testing"
"github.com/restic/restic/internal/backend/rclone"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -24,23 +21,15 @@ func newTestSuite(t testing.TB) *test.Suite[rclone.Config] {
return &cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg rclone.Config) (restic.Backend, error) {
t.Logf("Create()")
be, err := rclone.Create(context.TODO(), cfg)
var e *exec.Error
if errors.As(err, &e) && e.Err == exec.ErrNotFound {
t.Skipf("program %q not found", e.Name)
return nil, nil
Factory: rclone.NewFactory(),
}
}
return be, err
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg rclone.Config) (restic.Backend, error) {
t.Logf("Open()")
return rclone.Open(cfg, nil)
},
func findRclone(t testing.TB) {
// try to find a rclone binary
_, err := exec.LookPath("rclone")
if err != nil {
t.Skip(err)
}
}
@ -51,9 +40,11 @@ func TestBackendRclone(t *testing.T) {
}
}()
findRclone(t)
newTestSuite(t).RunTests(t)
}
func BenchmarkBackendREST(t *testing.B) {
findRclone(t)
newTestSuite(t).RunBenchmarks(t)
}

4
internal/backend/rclone/internal_test.go
View file

@ -15,7 +15,7 @@ func TestRcloneExit(t *testing.T) {
dir := rtest.TempDir(t)
cfg := NewConfig()
cfg.Remote = dir
be, err := Open(cfg, nil)
be, err := Open(context.TODO(), cfg, nil)
var e *exec.Error
if errors.As(err, &e) && e.Err == exec.ErrNotFound {
t.Skipf("program %q not found", e.Name)
@ -45,7 +45,7 @@ func TestRcloneFailedStart(t *testing.T) {
cfg := NewConfig()
// exits with exit code 1
cfg.Program = "false"
_, err := Open(cfg, nil)
_, err := Open(context.TODO(), cfg, nil)
var e *exec.ExitError
if !errors.As(err, &e) {
// unexpected error

68
internal/backend/rest/config_test.go
View file

@ -36,3 +36,71 @@ var configTests = []test.ConfigTestData[Config]{
func TestParseConfig(t *testing.T) {
test.ParseConfigTester(t, ParseConfig, configTests)
}
var passwordTests = []struct {
input string
expected string
}{
{
"rest:",
"rest:/",
},
{
"rest:localhost/",
"rest:localhost/",
},
{
"rest::123/",
"rest::123/",
},
{
"rest:http://",
"rest:http://",
},
{
"rest:http://hostname.foo:1234/",
"rest:http://hostname.foo:1234/",
},
{
"rest:http://user@hostname.foo:1234/",
"rest:http://user@hostname.foo:1234/",
},
{
"rest:http://user:@hostname.foo:1234/",
"rest:http://user:***@hostname.foo:1234/",
},
{
"rest:http://user:p@hostname.foo:1234/",
"rest:http://user:***@hostname.foo:1234/",
},
{
"rest:http://user:pppppaaafhhfuuwiiehhthhghhdkjaoowpprooghjjjdhhwuuhgjsjhhfdjhruuhsjsdhhfhshhsppwufhhsjjsjs@hostname.foo:1234/",
"rest:http://user:***@hostname.foo:1234/",
},
{
"rest:http://user:password@hostname",
"rest:http://user:***@hostname/",
},
{
"rest:http://user:password@:123",
"rest:http://user:***@:123/",
},
{
"rest:http://user:password@",
"rest:http://user:***@/",
},
}
func TestStripPassword(t *testing.T) {
// Make sure that the factory uses the correct method
StripPassword := NewFactory().StripPassword
for i, test := range passwordTests {
t.Run(test.input, func(t *testing.T) {
result := StripPassword(test.input)
if result != test.expected {
t.Errorf("test %d: expected '%s' but got '%s'", i, test.expected, result)
}
})
}
}

9
internal/backend/rest/rest.go
View file

@ -13,6 +13,7 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -29,6 +30,10 @@ type Backend struct {
layout.Layout
}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("rest", ParseConfig, StripPassword, Create, Open)
}
// the REST API protocol version is decided by HTTP request headers, these are the constants.
const (
ContentTypeV1 = "application/vnd.x.restic.rest.v1"
@ -36,7 +41,7 @@ const (
)
// Open opens the REST backend with the given config.
func Open(cfg Config, rt http.RoundTripper) (*Backend, error) {
func Open(_ context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
// use url without trailing slash for layout
url := cfg.URL.String()
if url[len(url)-1] == '/' {
@ -55,7 +60,7 @@ func Open(cfg Config, rt http.RoundTripper) (*Backend, error) {
// Create creates a new REST on server configured in config.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
be, err := Open(cfg, rt)
be, err := Open(ctx, cfg, rt)
if err != nil {
return nil, err
}

2
internal/backend/rest/rest_int_test.go
View file

@ -117,7 +117,7 @@ func TestListAPI(t *testing.T) {
URL: srvURL,
}
be, err := rest.Open(cfg, http.DefaultTransport)
be, err := rest.Open(context.TODO(), cfg, http.DefaultTransport)
if err != nil {
t.Fatal(err)
}

33
internal/backend/rest/rest_test.go
View file

@ -9,10 +9,8 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -67,12 +65,7 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, fun
return url, cleanup
}
func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData bool) *test.Suite[rest.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
func newTestSuite(url *url.URL, minimalData bool) *test.Suite[rest.Config] {
return &test.Suite[rest.Config]{
MinimalData: minimalData,
@ -83,20 +76,7 @@ func newTestSuite(_ context.Context, t testing.TB, url *url.URL, minimalData boo
return &cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg rest.Config) (restic.Backend, error) {
return rest.Create(context.TODO(), cfg, tr)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg rest.Config) (restic.Backend, error) {
return rest.Open(cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg rest.Config) error {
return nil
},
Factory: rest.NewFactory(),
}
}
@ -114,7 +94,7 @@ func TestBackendREST(t *testing.T) {
serverURL, cleanup := runRESTServer(ctx, t, dir)
defer cleanup()
newTestSuite(ctx, t, serverURL, false).RunTests(t)
newTestSuite(serverURL, false).RunTests(t)
}
func TestBackendRESTExternalServer(t *testing.T) {
@ -128,10 +108,7 @@ func TestBackendRESTExternalServer(t *testing.T) {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
newTestSuite(ctx, t, cfg.URL, true).RunTests(t)
newTestSuite(cfg.URL, true).RunTests(t)
}
func BenchmarkBackendREST(t *testing.B) {
@ -142,5 +119,5 @@ func BenchmarkBackendREST(t *testing.B) {
serverURL, cleanup := runRESTServer(ctx, t, dir)
defer cleanup()
newTestSuite(ctx, t, serverURL, false).RunBenchmarks(t)
newTestSuite(serverURL, false).RunBenchmarks(t)
}

12
internal/backend/s3/config.go
View file

@ -97,24 +97,14 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
func (cfg *Config) ApplyEnvironment(prefix string) {
if cfg.KeyID == "" {
cfg.KeyID = os.Getenv(prefix + "AWS_ACCESS_KEY_ID")
}
if cfg.Secret.String() == "" {
cfg.Secret = options.NewSecretString(os.Getenv(prefix + "AWS_SECRET_ACCESS_KEY"))
}
if cfg.KeyID == "" && cfg.Secret.String() != "" {
return errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
} else if cfg.KeyID != "" && cfg.Secret.String() == "" {
return errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
}
if cfg.Region == "" {
cfg.Region = os.Getenv(prefix + "AWS_DEFAULT_REGION")
}
return nil
}

18
internal/backend/s3/config_test.go
View file

@ -56,6 +56,24 @@ var configTests = []test.ConfigTestData[Config]{
Prefix: "prefix/directory",
Connections: 5,
}},
{S: "s3:hostname.foo/foobar", Cfg: Config{
Endpoint: "hostname.foo",
Bucket: "foobar",
Prefix: "",
Connections: 5,
}},
{S: "s3:hostname.foo/foobar/prefix/directory", Cfg: Config{
Endpoint: "hostname.foo",
Bucket: "foobar",
Prefix: "prefix/directory",
Connections: 5,
}},
{S: "s3:https://hostname/foobar", Cfg: Config{
Endpoint: "hostname",
Bucket: "foobar",
Prefix: "",
Connections: 5,
}},
{S: "s3:https://hostname:9999/foobar", Cfg: Config{
Endpoint: "hostname:9999",
Bucket: "foobar",

11
internal/backend/s3/s3.go
View file

@ -13,6 +13,7 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -31,11 +32,21 @@ type Backend struct {
// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
}
const defaultLayout = "default"
func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg)
if cfg.KeyID == "" && cfg.Secret.String() != "" {
return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
} else if cfg.KeyID != "" && cfg.Secret.String() == "" {
return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
}
if cfg.MaxRetries > 0 {
minio.MaxRetry = int(cfg.MaxRetries)
}

151
internal/backend/s3/s3_test.go
View file

@ -4,7 +4,6 @@ import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"net"
@ -15,7 +14,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/options"
@ -98,84 +97,41 @@ func newRandomCredentials(t testing.TB) (key, secret string) {
return key, secret
}
type MinioTestConfig struct {
s3.Config
func newMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) {
ctx, cancel := context.WithCancel(context.Background())
tempdir string
stopServer func()
}
tempdir := rtest.TempDir(t)
key, secret := newRandomCredentials(t)
cleanup := runMinio(ctx, t, tempdir, key, secret)
func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be restic.Backend, err error) {
return &test.Suite[s3.Config]{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (*s3.Config, error) {
cfg := s3.NewConfig()
cfg.Endpoint = "localhost:9000"
cfg.Bucket = "restictestbucket"
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
cfg.UseHTTP = true
cfg.KeyID = key
cfg.Secret = options.NewSecretString(secret)
return &cfg, nil
},
Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be restic.Backend, err error) {
for i := 0; i < 10; i++ {
be, err = s3.Create(context.TODO(), cfg.Config, tr)
be, err = s3.Create(ctx, cfg, rt)
if err != nil {
t.Logf("s3 open: try %d: error %v", i, err)
time.Sleep(500 * time.Millisecond)
continue
}
break
}
return be, err
}
func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite[MinioTestConfig] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite[MinioTestConfig]{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (*MinioTestConfig, error) {
cfg := MinioTestConfig{}
cfg.tempdir = rtest.TempDir(t)
key, secret := newRandomCredentials(t)
cfg.stopServer = runMinio(ctx, t, cfg.tempdir, key, secret)
cfg.Config = s3.NewConfig()
cfg.Config.Endpoint = "localhost:9000"
cfg.Config.Bucket = "restictestbucket"
cfg.Config.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
cfg.Config.UseHTTP = true
cfg.Config.KeyID = key
cfg.Config.Secret = options.NewSecretString(secret)
return &cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg MinioTestConfig) (restic.Backend, error) {
be, err := createS3(t, cfg, tr)
if err != nil {
return nil, err
}
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if err == nil {
return nil, errors.New("config already exists")
}
return be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg MinioTestConfig) (restic.Backend, error) {
return s3.Open(ctx, cfg.Config, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg MinioTestConfig) error {
if cfg.stopServer != nil {
cfg.stopServer()
}
return nil
},
}, s3.Open),
}, func() {
defer cancel()
defer cleanup()
}
}
@ -193,10 +149,10 @@ func TestBackendMinio(t *testing.T) {
return
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
suite, cleanup := newMinioTestSuite(t)
defer cleanup()
newMinioTestSuite(ctx, t).RunTests(t)
suite.RunTests(t)
}
func BenchmarkBackendMinio(t *testing.B) {
@ -207,18 +163,13 @@ func BenchmarkBackendMinio(t *testing.B) {
return
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
suite, cleanup := newMinioTestSuite(t)
defer cleanup()
newMinioTestSuite(ctx, t).RunBenchmarks(t)
}
func newS3TestSuite(t testing.TB) *test.Suite[s3.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
suite.RunBenchmarks(t)
}
func newS3TestSuite() *test.Suite[s3.Config] {
return &test.Suite[s3.Config]{
// do not use excessive data
MinimalData: true,
@ -236,39 +187,7 @@ func newS3TestSuite(t testing.TB) *test.Suite[s3.Config] {
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg s3.Config) (restic.Backend, error) {
be, err := s3.Create(context.TODO(), cfg, tr)
if err != nil {
return nil, err
}
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if err == nil {
return nil, errors.New("config already exists")
}
return be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg s3.Config) (restic.Backend, error) {
return s3.Open(context.TODO(), cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg s3.Config) error {
be, err := s3.Open(context.TODO(), cfg, tr)
if err != nil {
return err
}
return be.Delete(context.TODO())
},
Factory: s3.NewFactory(),
}
}
@ -293,7 +212,7 @@ func TestBackendS3(t *testing.T) {
}
t.Logf("run tests")
newS3TestSuite(t).RunTests(t)
newS3TestSuite().RunTests(t)
}
func BenchmarkBackendS3(t *testing.B) {
@ -311,5 +230,5 @@ func BenchmarkBackendS3(t *testing.B) {
}
t.Logf("run tests")
newS3TestSuite(t).RunBenchmarks(t)
newS3TestSuite().RunBenchmarks(t)
}

6
internal/backend/sftp/sftp.go
View file

@ -15,6 +15,8 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -41,6 +43,10 @@ type SFTP struct {
var _ restic.Backend = &SFTP{}
func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("sftp", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
}
const defaultLayout = "default"
func startClient(cfg Config) (*SFTP, error) {

28
internal/backend/sftp/sftp_test.go
View file

@ -1,7 +1,6 @@
package sftp_test
import (
"context"
"fmt"
"os"
"path/filepath"
@ -11,7 +10,6 @@ import (
"github.com/restic/restic/internal/backend/sftp"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -33,11 +31,7 @@ func newTestSuite(t testing.TB) *test.Suite[sftp.Config] {
return &test.Suite[sftp.Config]{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (*sftp.Config, error) {
dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-sftp-")
if err != nil {
t.Fatal(err)
}
dir := rtest.TempDir(t)
t.Logf("create new backend at %v", dir)
cfg := &sftp.Config{
@ -48,25 +42,7 @@ func newTestSuite(t testing.TB) *test.Suite[sftp.Config] {
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg sftp.Config) (restic.Backend, error) {
return sftp.Create(context.TODO(), cfg)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg sftp.Config) (restic.Backend, error) {
return sftp.Open(context.TODO(), cfg)
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg sftp.Config) error {
if !rtest.TestCleanupTempDirs {
t.Logf("leaving test backend dir at %v", cfg.Path)
}
rtest.RemoveAll(t, cfg.Path)
return nil
},
Factory: sftp.NewFactory(),
}
}

3
internal/backend/swift/config.go
View file

@ -77,7 +77,7 @@ func ParseConfig(s string) (*Config, error) {
var _ restic.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) error {
func (cfg *Config) ApplyEnvironment(prefix string) {
for _, val := range []struct {
s *string
env string
@ -130,5 +130,4 @@ func (cfg *Config) ApplyEnvironment(prefix string) error {
*val.s = options.NewSecretString(os.Getenv(val.env))
}
}
return nil
}

5
internal/backend/swift/swift.go
View file

@ -15,6 +15,7 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@ -34,6 +35,10 @@ type beSwift struct {
// ensure statically that *beSwift implements restic.Backend.
var _ restic.Backend = &beSwift{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("swift", ParseConfig, location.NoPassword, Open, Open)
}
// Open opens the swift backend at a container in region. The container is
// created if it does not exist yet.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {

46
internal/backend/swift/swift_test.go
View file

@ -1,26 +1,18 @@
package swift_test
import (
"context"
"fmt"
"os"
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/swift"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite[swift.Config]{
// do not use excessive data
MinimalData: true,
@ -48,47 +40,13 @@ func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] {
return nil, err
}
if err = cfg.ApplyEnvironment("RESTIC_TEST_"); err != nil {
return nil, err
}
cfg.ApplyEnvironment("RESTIC_TEST_")
cfg.Prefix += fmt.Sprintf("/test-%d", time.Now().UnixNano())
t.Logf("using prefix %v", cfg.Prefix)
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg swift.Config) (restic.Backend, error) {
be, err := swift.Open(context.TODO(), cfg, tr)
if err != nil {
return nil, err
}
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if err == nil {
return nil, errors.New("config already exists")
}
return be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg swift.Config) (restic.Backend, error) {
return swift.Open(context.TODO(), cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg swift.Config) error {
be, err := swift.Open(context.TODO(), cfg, tr)
if err != nil {
return err
}
return be.Delete(context.TODO())
},
Factory: swift.NewFactory(),
}
}

63
internal/backend/test/suite.go
View file

@ -1,11 +1,16 @@
package test
import (
"context"
"fmt"
"reflect"
"strings"
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@ -18,14 +23,8 @@ type Suite[C any] struct {
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig func() (*C, error)
// CreateFn is a function that creates a temporary repository for the tests.
Create func(cfg C) (restic.Backend, error)
// OpenFn is a function that opens a previously created temporary repository.
Open func(cfg C) (restic.Backend, error)
// CleanupFn removes data created during the tests.
Cleanup func(cfg C) error
// Factory contains a factory that can be used to create or open a repository for the tests.
Factory location.Factory
// MinimalData instructs the tests to not use excessive data.
MinimalData bool
@ -60,11 +59,7 @@ func (s *Suite[C]) RunTests(t *testing.T) {
return
}
if s.Cleanup != nil {
if err = s.Cleanup(*s.Config); err != nil {
t.Fatal(err)
}
}
s.cleanup(t)
}
type testFunction struct {
@ -158,13 +153,34 @@ func (s *Suite[C]) RunBenchmarks(b *testing.B) {
return
}
if err = s.Cleanup(*s.Config); err != nil {
b.Fatal(err)
s.cleanup(b)
}
func (s *Suite[C]) createOrError() (restic.Backend, error) {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
return nil, fmt.Errorf("cannot create transport for tests: %v", err)
}
be, err := s.Factory.Create(context.TODO(), s.Config, tr, nil)
if err != nil {
return nil, err
}
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
if err == nil {
return nil, errors.New("config already exists")
}
return be, nil
}
func (s *Suite[C]) create(t testing.TB) restic.Backend {
be, err := s.Create(*s.Config)
be, err := s.createOrError()
if err != nil {
t.Fatal(err)
}
@ -172,13 +188,26 @@ func (s *Suite[C]) create(t testing.TB) restic.Backend {
}
func (s *Suite[C]) open(t testing.TB) restic.Backend {
be, err := s.Open(*s.Config)
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
be, err := s.Factory.Open(context.TODO(), s.Config, tr, nil)
if err != nil {
t.Fatal(err)
}
return be
}
func (s *Suite[C]) cleanup(t testing.TB) {
be := s.open(t)
if err := be.Delete(context.TODO()); err != nil {
t.Fatal(err)
}
s.close(t, be)
}
func (s *Suite[C]) close(t testing.TB, be restic.Backend) {
err := be.Close()
if err != nil {

8
internal/backend/test/tests.go
View file

@ -36,6 +36,12 @@ func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, erro
return err == nil, err
}
// TestStripPasswordCall tests that the StripPassword method of a factory can be called without crashing.
// It does not verify whether passwords are removed correctly
func (s *Suite[C]) TestStripPasswordCall(_ *testing.T) {
s.Factory.StripPassword("some random string")
}
// TestCreateWithConfig tests that creating a backend in a location which already
// has a config file fails.
func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
@ -57,7 +63,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
store(t, b, restic.ConfigFile, []byte("test config"))
// now create the backend again, this must fail
_, err = s.Create(*s.Config)
_, err = s.createOrError()
if err == nil {
t.Fatalf("expected error not found for creating a backend with an existing config file")
}

2
internal/restic/backend.go
View file

@ -83,5 +83,5 @@ type FileInfo struct {
// ApplyEnvironmenter fills in a backend configuration from the environment
type ApplyEnvironmenter interface {
ApplyEnvironment(prefix string) error
ApplyEnvironment(prefix string)
}