diff --git a/doc/Design.md b/doc/Design.md index 52a228a93..ff4d5176e 100644 --- a/doc/Design.md +++ b/doc/Design.md @@ -34,20 +34,14 @@ in a repository are only written once and never modified afterwards. This allows accessing and even writing to the repository with multiple clients in parallel. Only the delete operation removes data from the repository. -At the time of writing, the only implemented repository type is based on -directories and files. Such repositories can be accessed locally on the same -system or via the integrated SFTP client (or any other storage back end). -The directory layout is the same for both access methods. -This repository type is described in the following section. - -Repositories consist of several directories and a file called `config`. For -all other files stored in the repository, the name for the file is the lower -case hexadecimal representation of the storage ID, which is the SHA-256 hash of -the file's contents. This allows for easy verification of files for accidental -modifications, like disk read errors, by simply running the program `sha256sum` -and comparing its output to the file name. If the prefix of a filename is -unique amongst all the other files in the same directory, the prefix may be -used instead of the complete filename. +Repositories consist of several directories and a top-level file called +`config`. For all other files stored in the repository, the name for the file +is the lower case hexadecimal representation of the storage ID, which is the +SHA-256 hash of the file's contents. This allows for easy verification of files +for accidental modifications, like disk read errors, by simply running the +program `sha256sum` on the file and comparing its output to the file name. If +the prefix of a filename is unique amongst all the other files in the same +directory, the prefix may be used instead of the complete filename. Apart from the files stored within the `keys` directory, all files are encrypted with AES-256 in counter mode (CTR). The integrity of the encrypted data is @@ -78,7 +72,15 @@ regardless if it is accessed via SFTP or locally. The field `chunker_polynomial` contains a parameter that is used for splitting large files into smaller chunks (see below). -The basic layout of a sample restic repository is shown here: +Filesystem-Based Repositories +----------------------------- + +The `local` and `sftp` backends are implemented using files and directories +stored in a file system. The directory layout is the same for both backend +types. + +The basic layout of a repository stored in a `local` or `sftp` backend is shown +here: /tmp/restic-repo ├── config @@ -102,12 +104,66 @@ The basic layout of a sample restic repository is shown here: │ └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec └── tmp -A repository can be initialized with the `restic init` command, e.g.: +A local repository can be initialized with the `restic init` command, e.g.: ```console $ restic -r /tmp/restic-repo init ``` +The local and sftp backends will also accept the repository layout described in +the following section, so that remote repositories mounted locally e.g. via +fuse can be accessed. The layout auto-detection can be overridden by specifying +the option `-o local.layout=default`, valid values are `default`, `cloud` and +`s3`. The option for the sftp backend is named `sftp.layout`. + +Object-Storage-Based Repositories +--------------------------------- + +Repositories in a backend based on an object store (e.g. Amazon s3) have the +same basic layout, with the exception that all data pack files are directly +saved in the `data` path, without the sub-directories listed for the +filesystem-based backends as listed in the previous section. The layout looks +like this: + + /config + /data + ├── 2159dd48f8a24f33c307b750592773f8b71ff8d11452132a7b2e2a6a01611be1 + ├── 32ea976bc30771cebad8285cd99120ac8786f9ffd42141d452458089985043a5 + ├── 59fe4bcde59bd6222eba87795e35a90d82cd2f138a27b6835032b7b58173a426 + ├── 73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c + [...] + /index + ├── c38f5fb68307c6a3e3aa945d556e325dc38f5fb68307c6a3e3aa945d556e325d + └── ca171b1b7394d90d330b265d90f506f9984043b342525f019788f97e745c71fd + /keys + └── b02de829beeb3c01a63e6b25cbd421a98fef144f03b9a02e46eff9e2ca3f0bd7 + /locks + /snapshots + └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec + +Unfortunately during development the s3 backend uses slightly different paths +(directory names use singular instead of plural for `key`, `lock`, and +`snapshot` files), for s3 the repository layout looks like this: + + /config + /data + ├── 2159dd48f8a24f33c307b750592773f8b71ff8d11452132a7b2e2a6a01611be1 + ├── 32ea976bc30771cebad8285cd99120ac8786f9ffd42141d452458089985043a5 + ├── 59fe4bcde59bd6222eba87795e35a90d82cd2f138a27b6835032b7b58173a426 + ├── 73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c + [...] + /index + ├── c38f5fb68307c6a3e3aa945d556e325dc38f5fb68307c6a3e3aa945d556e325d + └── ca171b1b7394d90d330b265d90f506f9984043b342525f019788f97e745c71fd + /key + └── b02de829beeb3c01a63e6b25cbd421a98fef144f03b9a02e46eff9e2ca3f0bd7 + /lock + /snapshot + └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec + +The s3 backend understands and accepts both forms, new backends are always +created with the former layout for compatibility reasons. + Pack Format ----------- diff --git a/doc/Manual.md b/doc/Manual.md index bbf2e7714..2d841cf94 100644 --- a/doc/Manual.md +++ b/doc/Manual.md @@ -551,6 +551,10 @@ Then use it in the backend specification: $ restic -r sftp:restic-backup-host:/tmp/backup init ``` +Last, if you'd like to use an entirely different program to create the SFTP +connection, you can specify the command to be run with the option +`-o sftp.command="foobar"`. + # Create a REST server repository In order to backup data to the remote server via HTTP or HTTPS protocol, diff --git a/doc/code.css b/doc/code.css index 2a73b3a93..9f19a2c1d 100644 --- a/doc/code.css +++ b/doc/code.css @@ -1,4 +1,4 @@ -code { +code, pre { font-size: 90%; } diff --git a/src/cmds/restic/cmd_options.go b/src/cmds/restic/cmd_options.go new file mode 100644 index 000000000..ba78941b0 --- /dev/null +++ b/src/cmds/restic/cmd_options.go @@ -0,0 +1,27 @@ +package main + +import ( + "fmt" + "restic/options" + + "github.com/spf13/cobra" +) + +var optionsCmd = &cobra.Command{ + Use: "options", + Short: "print list of extended options", + Long: ` +The "options" command prints a list of extended options. +`, + Hidden: true, + Run: func(cmd *cobra.Command, args []string) { + fmt.Printf("All Extended Options:\n") + for _, opt := range options.List() { + fmt.Printf(" %-15s %s\n", opt.Namespace+"."+opt.Name, opt.Text) + } + }, +} + +func init() { + cmdRoot.AddCommand(optionsCmd) +} diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index 45b41cf33..b88a25ec8 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -388,7 +388,7 @@ func open(s string, opts options.Options) (restic.Backend, error) { case "local": be, err = local.Open(cfg.(local.Config)) case "sftp": - be, err = sftp.OpenWithConfig(cfg.(sftp.Config)) + be, err = sftp.Open(cfg.(sftp.Config)) case "s3": be, err = s3.Open(cfg.(s3.Config)) case "rest": @@ -422,7 +422,7 @@ func create(s string, opts options.Options) (restic.Backend, error) { case "local": return local.Create(cfg.(local.Config)) case "sftp": - return sftp.CreateWithConfig(cfg.(sftp.Config)) + return sftp.Create(cfg.(sftp.Config)) case "s3": return s3.Open(cfg.(s3.Config)) case "rest": diff --git a/src/cmds/restic/global_debug.go b/src/cmds/restic/global_debug.go index 6e443f2a0..998f349b2 100644 --- a/src/cmds/restic/global_debug.go +++ b/src/cmds/restic/global_debug.go @@ -8,6 +8,7 @@ import ( _ "net/http/pprof" "os" "restic/errors" + "restic/repository" "github.com/pkg/profile" ) @@ -16,6 +17,7 @@ var ( listenMemoryProfile string memProfilePath string cpuProfilePath string + insecure bool prof interface { Stop() @@ -27,6 +29,13 @@ func init() { f.StringVar(&listenMemoryProfile, "listen-profile", "", "listen on this `address:port` for memory profiling") f.StringVar(&memProfilePath, "mem-profile", "", "write memory profile to `dir`") f.StringVar(&cpuProfilePath, "cpu-profile", "", "write cpu profile to `dir`") + f.BoolVar(&insecure, "insecure-kdf", false, "use insecure KDF settings") +} + +type fakeTestingTB struct{} + +func (fakeTestingTB) Logf(msg string, args ...interface{}) { + fmt.Fprintf(os.Stderr, msg, args...) } func runDebug() error { @@ -50,6 +59,10 @@ func runDebug() error { prof = profile.Start(profile.Quiet, profile.CPUProfile, profile.ProfilePath(memProfilePath)) } + if insecure { + repository.TestUseLowSecurityKDFParameters(fakeTestingTB{}) + } + return nil } diff --git a/src/cmds/restic/integration_helpers_test.go b/src/cmds/restic/integration_helpers_test.go index ad6acc8a1..8c58028f6 100644 --- a/src/cmds/restic/integration_helpers_test.go +++ b/src/cmds/restic/integration_helpers_test.go @@ -9,6 +9,7 @@ import ( "runtime" "testing" + "restic/options" "restic/repository" . "restic/test" ) @@ -199,6 +200,7 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions)) password: TestPassword, stdout: os.Stdout, stderr: os.Stderr, + extended: make(options.Options), } // always overwrite global options diff --git a/src/cmds/restic/local_layout_test.go b/src/cmds/restic/local_layout_test.go new file mode 100644 index 000000000..eb6268e72 --- /dev/null +++ b/src/cmds/restic/local_layout_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "path/filepath" + . "restic/test" + "testing" +) + +func TestRestoreLocalLayout(t *testing.T) { + withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) { + var tests = []struct { + filename string + layout string + }{ + {"repo-layout-cloud.tar.gz", ""}, + {"repo-layout-local.tar.gz", ""}, + {"repo-layout-s3-old.tar.gz", ""}, + {"repo-layout-cloud.tar.gz", "cloud"}, + {"repo-layout-local.tar.gz", "default"}, + {"repo-layout-s3-old.tar.gz", "s3"}, + } + + for _, test := range tests { + datafile := filepath.Join("..", "..", "restic", "backend", "testdata", test.filename) + + SetupTarTestFixture(t, env.base, datafile) + + gopts.extended["local.layout"] = test.layout + + // check the repo + testRunCheck(t, gopts) + + // restore latest snapshot + target := filepath.Join(env.base, "restore") + testRunRestoreLatest(t, gopts, target, nil, "") + + RemoveAll(t, filepath.Join(env.base, "repo")) + RemoveAll(t, target) + } + }) +} diff --git a/src/restic/backend/layout.go b/src/restic/backend/layout.go new file mode 100644 index 000000000..de72f07d4 --- /dev/null +++ b/src/restic/backend/layout.go @@ -0,0 +1,221 @@ +package backend + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "restic" + "restic/debug" + "restic/errors" + "restic/fs" +) + +// Layout computes paths for file name storage. +type Layout interface { + Filename(restic.Handle) string + Dirname(restic.Handle) string + Basedir(restic.FileType) string + Paths() []string +} + +// Filesystem is the abstraction of a file system used for a backend. +type Filesystem interface { + Join(...string) string + ReadDir(string) ([]os.FileInfo, error) + IsNotExist(error) bool +} + +// ensure statically that *LocalFilesystem implements Filesystem. +var _ Filesystem = &LocalFilesystem{} + +// LocalFilesystem implements Filesystem in a local path. +type LocalFilesystem struct { +} + +// ReadDir returns all entries of a directory. +func (l *LocalFilesystem) ReadDir(dir string) ([]os.FileInfo, error) { + f, err := fs.Open(dir) + if err != nil { + return nil, err + } + + entries, err := f.Readdir(-1) + if err != nil { + return nil, errors.Wrap(err, "Readdir") + } + + err = f.Close() + if err != nil { + return nil, errors.Wrap(err, "Close") + } + + return entries, nil +} + +// Join combines several path components to one. +func (l *LocalFilesystem) Join(paths ...string) string { + return filepath.Join(paths...) +} + +// IsNotExist returns true for errors that are caused by not existing files. +func (l *LocalFilesystem) IsNotExist(err error) bool { + return os.IsNotExist(err) +} + +var backendFilenameLength = len(restic.ID{}) * 2 +var backendFilename = regexp.MustCompile(fmt.Sprintf("^[a-fA-F0-9]{%d}$", backendFilenameLength)) + +func hasBackendFile(fs Filesystem, dir string) (bool, error) { + entries, err := fs.ReadDir(dir) + if err != nil && fs.IsNotExist(errors.Cause(err)) { + return false, nil + } + + if err != nil { + return false, errors.Wrap(err, "ReadDir") + } + + for _, e := range entries { + if backendFilename.MatchString(e.Name()) { + return true, nil + } + } + + return false, nil +} + +var dataSubdirName = regexp.MustCompile("^[a-fA-F0-9]{2}$") + +func hasSubdirBackendFile(fs Filesystem, dir string) (bool, error) { + entries, err := fs.ReadDir(dir) + if err != nil && fs.IsNotExist(errors.Cause(err)) { + return false, nil + } + + if err != nil { + return false, errors.Wrap(err, "ReadDir") + } + + for _, subdir := range entries { + if !dataSubdirName.MatchString(subdir.Name()) { + continue + } + + present, err := hasBackendFile(fs, fs.Join(dir, subdir.Name())) + if err != nil { + return false, err + } + + if present { + return true, nil + } + } + + return false, nil +} + +// ErrLayoutDetectionFailed is returned by DetectLayout() when the layout +// cannot be detected automatically. +var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed") + +// DetectLayout tries to find out which layout is used in a local (or sftp) +// filesystem at the given path. If repo is nil, an instance of LocalFilesystem +// is used. +func DetectLayout(repo Filesystem, dir string) (Layout, error) { + debug.Log("detect layout at %v", dir) + if repo == nil { + repo = &LocalFilesystem{} + } + + // key file in the "keys" dir (DefaultLayout or CloudLayout) + foundKeysFile, err := hasBackendFile(repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile])) + if err != nil { + return nil, err + } + + // key file in the "key" dir (S3Layout) + foundKeyFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile])) + if err != nil { + return nil, err + } + + // data file in "data" directory (S3Layout or CloudLayout) + foundDataFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile])) + if err != nil { + return nil, err + } + + // data file in subdir of "data" directory (DefaultLayout) + foundDataSubdirFile, err := hasSubdirBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile])) + if err != nil { + return nil, err + } + + if foundKeysFile && foundDataFile && !foundKeyFile && !foundDataSubdirFile { + debug.Log("found cloud layout at %v", dir) + return &CloudLayout{ + Path: dir, + Join: repo.Join, + }, nil + } + + if foundKeysFile && foundDataSubdirFile && !foundKeyFile && !foundDataFile { + debug.Log("found default layout at %v", dir) + return &DefaultLayout{ + Path: dir, + Join: repo.Join, + }, nil + } + + if foundKeyFile && foundDataFile && !foundKeysFile && !foundDataSubdirFile { + debug.Log("found s3 layout at %v", dir) + return &S3Layout{ + Path: dir, + Join: repo.Join, + }, nil + } + + debug.Log("layout detection failed") + return nil, ErrLayoutDetectionFailed +} + +// ParseLayout parses the config string and returns a Layout. When layout is +// the empty string, DetectLayout is used. If that fails, defaultLayout is used. +func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) { + debug.Log("parse layout string %q for backend at %v", layout, path) + switch layout { + case "default": + l = &DefaultLayout{ + Path: path, + Join: repo.Join, + } + case "cloud": + l = &CloudLayout{ + Path: path, + Join: repo.Join, + } + case "s3": + l = &S3Layout{ + Path: path, + Join: repo.Join, + } + case "": + l, err = DetectLayout(repo, path) + + // use the default layout if auto detection failed + if errors.Cause(err) == ErrLayoutDetectionFailed && defaultLayout != "" { + debug.Log("error: %v, use default layout %v", defaultLayout) + return ParseLayout(repo, defaultLayout, "", path) + } + + if err != nil { + return nil, err + } + debug.Log("layout detected: %v", l) + default: + return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, cloud, s3", layout) + } + + return l, nil +} diff --git a/src/restic/backend/layout_cloud.go b/src/restic/backend/layout_cloud.go new file mode 100644 index 000000000..6331d4709 --- /dev/null +++ b/src/restic/backend/layout_cloud.go @@ -0,0 +1,46 @@ +package backend + +import "restic" + +// CloudLayout implements the default layout for cloud storage backends, as +// described in the Design document. +type CloudLayout struct { + URL string + Path string + Join func(...string) string +} + +var cloudLayoutPaths = defaultLayoutPaths + +// Dirname returns the directory path for a given file type and name. +func (l *CloudLayout) Dirname(h restic.Handle) string { + if h.Type == restic.ConfigFile { + return l.URL + l.Join(l.Path, "/") + } + + return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type]) + "/" +} + +// Filename returns a path to a file, including its name. +func (l *CloudLayout) Filename(h restic.Handle) string { + name := h.Name + + if h.Type == restic.ConfigFile { + name = "config" + } + + return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type], name) +} + +// Paths returns all directory names +func (l *CloudLayout) Paths() (dirs []string) { + for _, p := range cloudLayoutPaths { + dirs = append(dirs, l.URL+l.Join(l.Path, p)) + } + return dirs +} + +// Basedir returns the base dir name for files of type t. +func (l *CloudLayout) Basedir(t restic.FileType) string { + return l.URL + l.Join(l.Path, cloudLayoutPaths[t]) +} diff --git a/src/restic/backend/layout_default.go b/src/restic/backend/layout_default.go new file mode 100644 index 000000000..77cb27508 --- /dev/null +++ b/src/restic/backend/layout_default.go @@ -0,0 +1,54 @@ +package backend + +import "restic" + +// DefaultLayout implements the default layout for local and sftp backends, as +// described in the Design document. The `data` directory has one level of +// subdirs, two characters each (taken from the first two characters of the +// file name). +type DefaultLayout struct { + Path string + Join func(...string) string +} + +var defaultLayoutPaths = map[restic.FileType]string{ + restic.DataFile: "data", + restic.SnapshotFile: "snapshots", + restic.IndexFile: "index", + restic.LockFile: "locks", + restic.KeyFile: "keys", +} + +// Dirname returns the directory path for a given file type and name. +func (l *DefaultLayout) Dirname(h restic.Handle) string { + p := defaultLayoutPaths[h.Type] + + if h.Type == restic.DataFile && len(h.Name) > 2 { + p = l.Join(p, h.Name[:2]) + } + + return l.Join(l.Path, p) +} + +// Filename returns a path to a file, including its name. +func (l *DefaultLayout) Filename(h restic.Handle) string { + name := h.Name + if h.Type == restic.ConfigFile { + name = "config" + } + + return l.Join(l.Dirname(h), name) +} + +// Paths returns all directory names +func (l *DefaultLayout) Paths() (dirs []string) { + for _, p := range defaultLayoutPaths { + dirs = append(dirs, l.Join(l.Path, p)) + } + return dirs +} + +// Basedir returns the base dir name for type t. +func (l *DefaultLayout) Basedir(t restic.FileType) string { + return l.Join(l.Path, defaultLayoutPaths[t]) +} diff --git a/src/restic/backend/layout_s3.go b/src/restic/backend/layout_s3.go new file mode 100644 index 000000000..9db8ae7f3 --- /dev/null +++ b/src/restic/backend/layout_s3.go @@ -0,0 +1,52 @@ +package backend + +import "restic" + +// S3Layout implements the old layout used for s3 cloud storage backends, as +// described in the Design document. +type S3Layout struct { + URL string + Path string + Join func(...string) string +} + +var s3LayoutPaths = map[restic.FileType]string{ + restic.DataFile: "data", + restic.SnapshotFile: "snapshot", + restic.IndexFile: "index", + restic.LockFile: "lock", + restic.KeyFile: "key", +} + +// Dirname returns the directory path for a given file type and name. +func (l *S3Layout) Dirname(h restic.Handle) string { + if h.Type == restic.ConfigFile { + return l.URL + l.Join(l.Path, "/") + } + + return l.URL + l.Join(l.Path, "/", s3LayoutPaths[h.Type]) + "/" +} + +// Filename returns a path to a file, including its name. +func (l *S3Layout) Filename(h restic.Handle) string { + name := h.Name + + if h.Type == restic.ConfigFile { + name = "config" + } + + return l.URL + l.Join(l.Path, "/", s3LayoutPaths[h.Type], name) +} + +// Paths returns all directory names +func (l *S3Layout) Paths() (dirs []string) { + for _, p := range s3LayoutPaths { + dirs = append(dirs, l.Join(l.Path, p)) + } + return dirs +} + +// Basedir returns the base dir name for type t. +func (l *S3Layout) Basedir(t restic.FileType) string { + return l.Join(l.Path, s3LayoutPaths[t]) +} diff --git a/src/restic/backend/layout_test.go b/src/restic/backend/layout_test.go new file mode 100644 index 000000000..4fee713cb --- /dev/null +++ b/src/restic/backend/layout_test.go @@ -0,0 +1,374 @@ +package backend + +import ( + "fmt" + "path" + "path/filepath" + "reflect" + "restic" + . "restic/test" + "sort" + "testing" +) + +func TestDefaultLayout(t *testing.T) { + path, cleanup := TempDir(t) + defer cleanup() + + var tests = []struct { + restic.Handle + filename string + }{ + { + restic.Handle{Type: restic.DataFile, Name: "0123456"}, + filepath.Join(path, "data", "01", "0123456"), + }, + { + restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + filepath.Join(path, "config"), + }, + { + restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + filepath.Join(path, "snapshots", "123456"), + }, + { + restic.Handle{Type: restic.IndexFile, Name: "123456"}, + filepath.Join(path, "index", "123456"), + }, + { + restic.Handle{Type: restic.LockFile, Name: "123456"}, + filepath.Join(path, "locks", "123456"), + }, + { + restic.Handle{Type: restic.KeyFile, Name: "123456"}, + filepath.Join(path, "keys", "123456"), + }, + } + + l := &DefaultLayout{ + Path: path, + Join: filepath.Join, + } + + t.Run("Paths", func(t *testing.T) { + dirs := l.Paths() + + want := []string{ + filepath.Join(path, "data"), + filepath.Join(path, "snapshots"), + filepath.Join(path, "index"), + filepath.Join(path, "locks"), + filepath.Join(path, "keys"), + } + + sort.Sort(sort.StringSlice(want)) + sort.Sort(sort.StringSlice(dirs)) + + if !reflect.DeepEqual(dirs, want) { + t.Fatalf("wrong paths returned, want:\n %v\ngot:\n %v", want, dirs) + } + }) + + for _, test := range tests { + t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) { + filename := l.Filename(test.Handle) + if filename != test.filename { + t.Fatalf("wrong filename, want %v, got %v", test.filename, filename) + } + }) + } +} + +func TestCloudLayout(t *testing.T) { + path, cleanup := TempDir(t) + defer cleanup() + + var tests = []struct { + restic.Handle + filename string + }{ + { + restic.Handle{Type: restic.DataFile, Name: "0123456"}, + filepath.Join(path, "data", "0123456"), + }, + { + restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + filepath.Join(path, "config"), + }, + { + restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + filepath.Join(path, "snapshots", "123456"), + }, + { + restic.Handle{Type: restic.IndexFile, Name: "123456"}, + filepath.Join(path, "index", "123456"), + }, + { + restic.Handle{Type: restic.LockFile, Name: "123456"}, + filepath.Join(path, "locks", "123456"), + }, + { + restic.Handle{Type: restic.KeyFile, Name: "123456"}, + filepath.Join(path, "keys", "123456"), + }, + } + + l := &CloudLayout{ + Path: path, + Join: filepath.Join, + } + + t.Run("Paths", func(t *testing.T) { + dirs := l.Paths() + + want := []string{ + filepath.Join(path, "data"), + filepath.Join(path, "snapshots"), + filepath.Join(path, "index"), + filepath.Join(path, "locks"), + filepath.Join(path, "keys"), + } + + sort.Sort(sort.StringSlice(want)) + sort.Sort(sort.StringSlice(dirs)) + + if !reflect.DeepEqual(dirs, want) { + t.Fatalf("wrong paths returned, want:\n %v\ngot:\n %v", want, dirs) + } + }) + + for _, test := range tests { + t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) { + filename := l.Filename(test.Handle) + if filename != test.filename { + t.Fatalf("wrong filename, want %v, got %v", test.filename, filename) + } + }) + } +} + +func TestCloudLayoutURLs(t *testing.T) { + var tests = []struct { + l Layout + h restic.Handle + fn string + dir string + }{ + { + &CloudLayout{URL: "https://hostname.foo", Path: "", Join: path.Join}, + restic.Handle{Type: restic.DataFile, Name: "foobar"}, + "https://hostname.foo/data/foobar", + "https://hostname.foo/data/", + }, + { + &CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, + restic.Handle{Type: restic.LockFile, Name: "foobar"}, + "https://hostname.foo:1234/prefix/repo/locks/foobar", + "https://hostname.foo:1234/prefix/repo/locks/", + }, + { + &CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, + restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + "https://hostname.foo:1234/prefix/repo/config", + "https://hostname.foo:1234/prefix/repo/", + }, + { + &S3Layout{URL: "https://hostname.foo", Path: "/", Join: path.Join}, + restic.Handle{Type: restic.DataFile, Name: "foobar"}, + "https://hostname.foo/data/foobar", + "https://hostname.foo/data/", + }, + { + &S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join}, + restic.Handle{Type: restic.LockFile, Name: "foobar"}, + "https://hostname.foo:1234/prefix/repo/lock/foobar", + "https://hostname.foo:1234/prefix/repo/lock/", + }, + { + &S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, + restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + "https://hostname.foo:1234/prefix/repo/config", + "https://hostname.foo:1234/prefix/repo/", + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%T", test.l), func(t *testing.T) { + fn := test.l.Filename(test.h) + if fn != test.fn { + t.Fatalf("wrong filename, want %v, got %v", test.fn, fn) + } + + dir := test.l.Dirname(test.h) + if dir != test.dir { + t.Fatalf("wrong dirname, want %v, got %v", test.dir, dir) + } + }) + } +} + +func TestS3Layout(t *testing.T) { + path, cleanup := TempDir(t) + defer cleanup() + + var tests = []struct { + restic.Handle + filename string + }{ + { + restic.Handle{Type: restic.DataFile, Name: "0123456"}, + filepath.Join(path, "data", "0123456"), + }, + { + restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + filepath.Join(path, "config"), + }, + { + restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + filepath.Join(path, "snapshot", "123456"), + }, + { + restic.Handle{Type: restic.IndexFile, Name: "123456"}, + filepath.Join(path, "index", "123456"), + }, + { + restic.Handle{Type: restic.LockFile, Name: "123456"}, + filepath.Join(path, "lock", "123456"), + }, + { + restic.Handle{Type: restic.KeyFile, Name: "123456"}, + filepath.Join(path, "key", "123456"), + }, + } + + l := &S3Layout{ + Path: path, + Join: filepath.Join, + } + + t.Run("Paths", func(t *testing.T) { + dirs := l.Paths() + + want := []string{ + filepath.Join(path, "data"), + filepath.Join(path, "snapshot"), + filepath.Join(path, "index"), + filepath.Join(path, "lock"), + filepath.Join(path, "key"), + } + + sort.Sort(sort.StringSlice(want)) + sort.Sort(sort.StringSlice(dirs)) + + if !reflect.DeepEqual(dirs, want) { + t.Fatalf("wrong paths returned, want:\n %v\ngot:\n %v", want, dirs) + } + }) + + for _, test := range tests { + t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) { + filename := l.Filename(test.Handle) + if filename != test.filename { + t.Fatalf("wrong filename, want %v, got %v", test.filename, filename) + } + }) + } +} + +func TestDetectLayout(t *testing.T) { + path, cleanup := TempDir(t) + defer cleanup() + + var tests = []struct { + filename string + want string + }{ + {"repo-layout-local.tar.gz", "*backend.DefaultLayout"}, + {"repo-layout-cloud.tar.gz", "*backend.CloudLayout"}, + {"repo-layout-s3-old.tar.gz", "*backend.S3Layout"}, + } + + var fs = &LocalFilesystem{} + for _, test := range tests { + for _, fs := range []Filesystem{fs, nil} { + t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) { + SetupTarTestFixture(t, path, filepath.Join("testdata", test.filename)) + + layout, err := DetectLayout(fs, filepath.Join(path, "repo")) + if err != nil { + t.Fatal(err) + } + + if layout == nil { + t.Fatal("wanted some layout, but detect returned nil") + } + + layoutName := fmt.Sprintf("%T", layout) + if layoutName != test.want { + t.Fatalf("want layout %v, got %v", test.want, layoutName) + } + + RemoveAll(t, filepath.Join(path, "repo")) + }) + } + } +} + +func TestParseLayout(t *testing.T) { + path, cleanup := TempDir(t) + defer cleanup() + + var tests = []struct { + layoutName string + defaultLayoutName string + want string + }{ + {"default", "", "*backend.DefaultLayout"}, + {"cloud", "", "*backend.CloudLayout"}, + {"s3", "", "*backend.S3Layout"}, + {"", "", "*backend.CloudLayout"}, + } + + SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-cloud.tar.gz")) + + for _, test := range tests { + t.Run(test.layoutName, func(t *testing.T) { + layout, err := ParseLayout(&LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo")) + if err != nil { + t.Fatal(err) + } + + if layout == nil { + t.Fatal("wanted some layout, but detect returned nil") + } + + // test that the functions work (and don't panic) + _ = layout.Dirname(restic.Handle{Type: restic.DataFile}) + _ = layout.Filename(restic.Handle{Type: restic.DataFile, Name: "1234"}) + _ = layout.Paths() + + layoutName := fmt.Sprintf("%T", layout) + if layoutName != test.want { + t.Fatalf("want layout %v, got %v", test.want, layoutName) + } + }) + } +} + +func TestParseLayoutInvalid(t *testing.T) { + path, cleanup := TempDir(t) + defer cleanup() + + var invalidNames = []string{ + "foo", "bar", "local", + } + + for _, name := range invalidNames { + t.Run(name, func(t *testing.T) { + layout, err := ParseLayout(nil, name, "", path) + if err == nil { + t.Fatalf("expected error not found for layout name %v, layout is %v", name, layout) + } + }) + } +} diff --git a/src/restic/backend/local/config.go b/src/restic/backend/local/config.go index 746accd27..e693d8c9e 100644 --- a/src/restic/backend/local/config.go +++ b/src/restic/backend/local/config.go @@ -4,11 +4,17 @@ import ( "strings" "restic/errors" + "restic/options" ) // Config holds all information needed to open a local repository. type Config struct { - Path string + Path string + Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` +} + +func init() { + options.Register("local", Config{}) } // ParseConfig parses a local backend config. diff --git a/src/restic/backend/local/layout_test.go b/src/restic/backend/local/layout_test.go new file mode 100644 index 000000000..da0b0bfc8 --- /dev/null +++ b/src/restic/backend/local/layout_test.go @@ -0,0 +1,84 @@ +package local + +import ( + "path/filepath" + "restic" + . "restic/test" + "testing" +) + +func TestLayout(t *testing.T) { + path, cleanup := TempDir(t) + defer cleanup() + + var tests = []struct { + filename string + layout string + failureExpected bool + datafiles map[string]bool + }{ + {"repo-layout-local.tar.gz", "", false, map[string]bool{ + "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, + "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, + "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, + }}, + {"repo-layout-cloud.tar.gz", "", false, map[string]bool{ + "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, + "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, + "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, + }}, + {"repo-layout-s3-old.tar.gz", "", false, map[string]bool{ + "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, + "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, + "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, + }}, + } + + for _, test := range tests { + t.Run(test.filename, func(t *testing.T) { + SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename)) + + repo := filepath.Join(path, "repo") + be, err := Open(Config{ + Path: repo, + Layout: test.layout, + }) + if err != nil { + t.Fatal(err) + } + + if be == nil { + t.Fatalf("Open() returned nil but no error") + } + + datafiles := make(map[string]bool) + for id := range be.List(restic.DataFile, nil) { + datafiles[id] = false + } + + if len(datafiles) == 0 { + t.Errorf("List() returned zero data files") + } + + for id := range test.datafiles { + if _, ok := datafiles[id]; !ok { + t.Errorf("datafile with id %v not found", id) + } + + datafiles[id] = true + } + + for id, v := range datafiles { + if !v { + t.Errorf("unexpected id %v found", id) + } + } + + if err = be.Close(); err != nil { + t.Errorf("Close() returned error %v", err) + } + + RemoveAll(t, filepath.Join(path, "repo")) + }) + } +} diff --git a/src/restic/backend/local/local.go b/src/restic/backend/local/local.go index 510569654..b140fd8f8 100644 --- a/src/restic/backend/local/local.go +++ b/src/restic/backend/local/local.go @@ -2,7 +2,6 @@ package local import ( "io" - "io/ioutil" "os" "path/filepath" "restic" @@ -17,53 +16,64 @@ import ( // Local is a backend in a local directory. type Local struct { Config + backend.Layout } +// ensure statically that *Local implements restic.Backend. var _ restic.Backend = &Local{} -func paths(dir string) []string { - return []string{ - dir, - filepath.Join(dir, backend.Paths.Data), - filepath.Join(dir, backend.Paths.Snapshots), - filepath.Join(dir, backend.Paths.Index), - filepath.Join(dir, backend.Paths.Locks), - filepath.Join(dir, backend.Paths.Keys), - filepath.Join(dir, backend.Paths.Temp), - } -} +const defaultLayout = "default" // Open opens the local backend as specified by config. func Open(cfg Config) (*Local, error) { + debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout) + l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) + if err != nil { + return nil, err + } + + be := &Local{Config: cfg, Layout: l} + // test if all necessary dirs are there - for _, d := range paths(cfg.Path) { + for _, d := range be.Paths() { if _, err := fs.Stat(d); err != nil { return nil, errors.Wrap(err, "Open") } } - return &Local{Config: cfg}, nil + return be, nil } // Create creates all the necessary files and directories for a new local // backend at dir. Afterwards a new config blob should be created. func Create(cfg Config) (*Local, error) { + debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout) + + l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) + if err != nil { + return nil, err + } + + be := &Local{ + Config: cfg, + Layout: l, + } + // test if config file already exists - _, err := fs.Lstat(filepath.Join(cfg.Path, backend.Paths.Config)) + _, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile})) if err == nil { return nil, errors.New("config file already exists") } // create paths for data, refs and temp - for _, d := range paths(cfg.Path) { + for _, d := range be.Paths() { err := fs.MkdirAll(d, backend.Modes.Dir) if err != nil { return nil, errors.Wrap(err, "MkdirAll") } } - // open backend - return Open(cfg) + return be, nil } // Location returns this backend's location (the directory name). @@ -71,60 +81,6 @@ func (b *Local) Location() string { return b.Path } -// Construct path for given Type and name. -func filename(base string, t restic.FileType, name string) string { - if t == restic.ConfigFile { - return filepath.Join(base, "config") - } - - return filepath.Join(dirname(base, t, name), name) -} - -// Construct directory for given Type. -func dirname(base string, t restic.FileType, name string) string { - var n string - switch t { - case restic.DataFile: - n = backend.Paths.Data - if len(name) > 2 { - n = filepath.Join(n, name[:2]) - } - case restic.SnapshotFile: - n = backend.Paths.Snapshots - case restic.IndexFile: - n = backend.Paths.Index - case restic.LockFile: - n = backend.Paths.Locks - case restic.KeyFile: - n = backend.Paths.Keys - } - return filepath.Join(base, n) -} - -// copyToTempfile saves p into a tempfile in tempdir. -func copyToTempfile(tempdir string, rd io.Reader) (filename string, err error) { - tmpfile, err := ioutil.TempFile(tempdir, "temp-") - if err != nil { - return "", errors.Wrap(err, "TempFile") - } - - _, err = io.Copy(tmpfile, rd) - if err != nil { - return "", errors.Wrap(err, "Write") - } - - if err = tmpfile.Sync(); err != nil { - return "", errors.Wrap(err, "Syncn") - } - - err = tmpfile.Close() - if err != nil { - return "", errors.Wrap(err, "Close") - } - - return tmpfile.Name(), nil -} - // Save stores data in the backend at the handle. func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) { debug.Log("Save %v", h) @@ -132,18 +88,7 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) { return err } - tmpfile, err := copyToTempfile(filepath.Join(b.Path, backend.Paths.Temp), rd) - debug.Log("saved %v to %v", h, tmpfile) - if err != nil { - return err - } - - filename := filename(b.Path, h.Type, h.Name) - - // test if new path already exists - if _, err := fs.Stat(filename); err == nil { - return errors.Errorf("Rename(): file %v already exists", filename) - } + filename := b.Filename(h) // create directories if necessary, ignore errors if h.Type == restic.DataFile { @@ -153,12 +98,27 @@ func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) { } } - err = fs.Rename(tmpfile, filename) - debug.Log("save %v: rename %v -> %v: %v", - h, filepath.Base(tmpfile), filepath.Base(filename), err) - + // create new file + f, err := fs.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, backend.Modes.File) if err != nil { - return errors.Wrap(err, "Rename") + return errors.Wrap(err, "OpenFile") + } + + // save data, then sync + _, err = io.Copy(f, rd) + if err != nil { + f.Close() + return errors.Wrap(err, "Write") + } + + if err = f.Sync(); err != nil { + f.Close() + return errors.Wrap(err, "Sync") + } + + err = f.Close() + if err != nil { + return errors.Wrap(err, "Close") } // set mode to read-only @@ -183,7 +143,7 @@ func (b *Local) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, return nil, errors.New("offset is negative") } - f, err := os.Open(filename(b.Path, h.Type, h.Name)) + f, err := fs.Open(b.Filename(h)) if err != nil { return nil, err } @@ -210,7 +170,7 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) { return restic.FileInfo{}, err } - fi, err := fs.Stat(filename(b.Path, h.Type, h.Name)) + fi, err := fs.Stat(b.Filename(h)) if err != nil { return restic.FileInfo{}, errors.Wrap(err, "Stat") } @@ -221,7 +181,7 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) { // Test returns true if a blob of the given type and name exists in the backend. func (b *Local) Test(h restic.Handle) (bool, error) { debug.Log("Test %v", h) - _, err := fs.Stat(filename(b.Path, h.Type, h.Name)) + _, err := fs.Stat(b.Filename(h)) if err != nil { if os.IsNotExist(errors.Cause(err)) { return false, nil @@ -235,7 +195,7 @@ func (b *Local) Test(h restic.Handle) (bool, error) { // Remove removes the blob with the given name and type. func (b *Local) Remove(h restic.Handle) error { debug.Log("Remove %v", h) - fn := filename(b.Path, h.Type, h.Name) + fn := b.Filename(h) // reset read-only flag err := fs.Chmod(fn, 0666) @@ -250,91 +210,30 @@ func isFile(fi os.FileInfo) bool { return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0 } -func readdir(d string) (fileInfos []os.FileInfo, err error) { - f, e := fs.Open(d) - if e != nil { - return nil, errors.Wrap(e, "Open") - } - - defer func() { - e := f.Close() - if err == nil { - err = errors.Wrap(e, "Close") - } - }() - - return f.Readdir(-1) -} - -// listDir returns a list of all files in d. -func listDir(d string) (filenames []string, err error) { - fileInfos, err := readdir(d) - if err != nil { - return nil, err - } - - for _, fi := range fileInfos { - if isFile(fi) { - filenames = append(filenames, fi.Name()) - } - } - - return filenames, nil -} - -// listDirs returns a list of all files in directories within d. -func listDirs(dir string) (filenames []string, err error) { - fileInfos, err := readdir(dir) - if err != nil { - return nil, err - } - - for _, fi := range fileInfos { - if !fi.IsDir() { - continue - } - - files, err := listDir(filepath.Join(dir, fi.Name())) - if err != nil { - continue - } - - filenames = append(filenames, files...) - } - - return filenames, nil -} - // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. func (b *Local) List(t restic.FileType, done <-chan struct{}) <-chan string { debug.Log("List %v", t) - lister := listDir - if t == restic.DataFile { - lister = listDirs - } ch := make(chan string) - items, err := lister(filepath.Join(dirname(b.Path, t, ""))) - if err != nil { - close(ch) - return ch - } go func() { defer close(ch) - for _, m := range items { - if m == "" { - continue + + fs.Walk(b.Basedir(t), func(path string, fi os.FileInfo, err error) error { + if !isFile(fi) { + return err } select { - case ch <- m: + case ch <- filepath.Base(path): case <-done: - return + return err } - } + + return err + }) }() return ch diff --git a/src/restic/backend/location/location_test.go b/src/restic/backend/location/location_test.go index fe07ee506..47260669a 100644 --- a/src/restic/backend/location/location_test.go +++ b/src/restic/backend/location/location_test.go @@ -79,7 +79,7 @@ var parseTests = []struct { Config: sftp.Config{ User: "user", Host: "host", - Dir: "/srv/repo", + Path: "/srv/repo", }, }, }, @@ -89,7 +89,7 @@ var parseTests = []struct { Config: sftp.Config{ User: "", Host: "host", - Dir: "/srv/repo", + Path: "/srv/repo", }, }, }, @@ -99,7 +99,7 @@ var parseTests = []struct { Config: sftp.Config{ User: "user", Host: "host", - Dir: "srv/repo", + Path: "srv/repo", }, }, }, @@ -109,7 +109,7 @@ var parseTests = []struct { Config: sftp.Config{ User: "user", Host: "host", - Dir: "/srv/repo", + Path: "/srv/repo", }, }, }, diff --git a/src/restic/backend/rest/rest.go b/src/restic/backend/rest/rest.go index 63b6a8094..773fad6bd 100644 --- a/src/restic/backend/rest/rest.go +++ b/src/restic/backend/rest/rest.go @@ -22,39 +22,11 @@ const connLimit = 40 // make sure the rest backend implements restic.Backend var _ restic.Backend = &restBackend{} -// restPath returns the path to the given resource. -func restPath(url *url.URL, h restic.Handle) string { - u := *url - - var dir string - - switch h.Type { - case restic.ConfigFile: - dir = "" - h.Name = "config" - case restic.DataFile: - dir = backend.Paths.Data - case restic.SnapshotFile: - dir = backend.Paths.Snapshots - case restic.IndexFile: - dir = backend.Paths.Index - case restic.LockFile: - dir = backend.Paths.Locks - case restic.KeyFile: - dir = backend.Paths.Keys - default: - dir = string(h.Type) - } - - u.Path = path.Join(url.Path, dir, h.Name) - - return u.String() -} - type restBackend struct { url *url.URL connChan chan struct{} client http.Client + backend.Layout } // Open opens the REST backend with the given config. @@ -66,7 +38,20 @@ func Open(cfg Config) (restic.Backend, error) { tr := &http.Transport{MaxIdleConnsPerHost: connLimit} client := http.Client{Transport: tr} - return &restBackend{url: cfg.URL, connChan: connChan, client: client}, nil + // use url without trailing slash for layout + url := cfg.URL.String() + if url[len(url)-1] == '/' { + url = url[:len(url)-1] + } + + be := &restBackend{ + url: cfg.URL, + connChan: connChan, + client: client, + Layout: &backend.CloudLayout{URL: url, Join: path.Join}, + } + + return be, nil } // Create creates a new REST on server configured in config. @@ -124,7 +109,7 @@ func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) { rd = backend.Closer{Reader: rd} <-b.connChan - resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", rd) + resp, err := b.client.Post(b.Filename(h), "binary/octet-stream", rd) b.connChan <- struct{}{} if resp != nil { @@ -167,7 +152,7 @@ func (b *restBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCl return nil, errors.Errorf("invalid length %d", length) } - req, err := http.NewRequest("GET", restPath(b.url, h), nil) + req, err := http.NewRequest("GET", b.Filename(h), nil) if err != nil { return nil, errors.Wrap(err, "http.NewRequest") } @@ -207,7 +192,7 @@ func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) { } <-b.connChan - resp, err := b.client.Head(restPath(b.url, h)) + resp, err := b.client.Head(b.Filename(h)) b.connChan <- struct{}{} if err != nil { return restic.FileInfo{}, errors.Wrap(err, "client.Head") @@ -249,7 +234,7 @@ func (b *restBackend) Remove(h restic.Handle) error { return err } - req, err := http.NewRequest("DELETE", restPath(b.url, h), nil) + req, err := http.NewRequest("DELETE", b.Filename(h), nil) if err != nil { return errors.Wrap(err, "http.NewRequest") } @@ -275,7 +260,7 @@ func (b *restBackend) Remove(h restic.Handle) error { func (b *restBackend) List(t restic.FileType, done <-chan struct{}) <-chan string { ch := make(chan string) - url := restPath(b.url, restic.Handle{Type: t}) + url := b.Dirname(restic.Handle{Type: t}) if !strings.HasSuffix(url, "/") { url += "/" } diff --git a/src/restic/backend/rest/rest_path_test.go b/src/restic/backend/rest/rest_path_test.go deleted file mode 100644 index 8356abfba..000000000 --- a/src/restic/backend/rest/rest_path_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package rest - -import ( - "net/url" - "restic" - "testing" -) - -var restPathTests = []struct { - Handle restic.Handle - URL *url.URL - Result string -}{ - { - URL: parseURL("https://hostname.foo"), - Handle: restic.Handle{ - Type: restic.DataFile, - Name: "foobar", - }, - Result: "https://hostname.foo/data/foobar", - }, - { - URL: parseURL("https://hostname.foo:1234/prefix/repo"), - Handle: restic.Handle{ - Type: restic.LockFile, - Name: "foobar", - }, - Result: "https://hostname.foo:1234/prefix/repo/locks/foobar", - }, - { - URL: parseURL("https://hostname.foo:1234/prefix/repo"), - Handle: restic.Handle{ - Type: restic.ConfigFile, - Name: "foobar", - }, - Result: "https://hostname.foo:1234/prefix/repo/config", - }, -} - -func TestRESTPaths(t *testing.T) { - for i, test := range restPathTests { - result := restPath(test.URL, test.Handle) - if result != test.Result { - t.Errorf("test %d: resulting URL does not match, want:\n %#v\ngot: \n %#v", - i, test.Result, result) - } - } -} diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 684588164..f59a92a15 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -27,6 +27,7 @@ type s3 struct { prefix string cacheMutex sync.RWMutex cacheObjSize map[string]int64 + backend.Layout } // Open opens the S3 backend at bucket and region. The bucket is created if it @@ -44,6 +45,7 @@ func Open(cfg Config) (restic.Backend, error) { bucketname: cfg.Bucket, prefix: cfg.Prefix, cacheObjSize: make(map[string]int64), + Layout: &backend.S3Layout{URL: cfg.Endpoint, Join: path.Join}, } tr := &http.Transport{MaxIdleConnsPerHost: connLimit} @@ -68,13 +70,6 @@ func Open(cfg Config) (restic.Backend, error) { return be, nil } -func (be *s3) s3path(h restic.Handle) string { - if h.Type == restic.ConfigFile { - return path.Join(be.prefix, string(h.Type)) - } - return path.Join(be.prefix, string(h.Type), h.Name) -} - func (be *s3) createConnections() { be.connChan = make(chan struct{}, connLimit) for i := 0; i < connLimit; i++ { @@ -95,7 +90,7 @@ func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) { debug.Log("Save %v", h) - objName := be.s3path(h) + objName := be.Filename(h) // Check key does not already exist _, err = be.client.StatObject(be.bucketname, objName) @@ -149,7 +144,7 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er var obj *minio.Object var size int64 - objName := be.s3path(h) + objName := be.Filename(h) // get token for connection <-be.connChan @@ -242,7 +237,7 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) { debug.Log("%v", h) - objName := be.s3path(h) + objName := be.Filename(h) var obj *minio.Object obj, err = be.client.GetObject(be.bucketname, objName) @@ -271,7 +266,7 @@ func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) { // Test returns true if a blob of the given type and name exists in the backend. func (be *s3) Test(h restic.Handle) (bool, error) { found := false - objName := be.s3path(h) + objName := be.Filename(h) _, err := be.client.StatObject(be.bucketname, objName) if err == nil { found = true @@ -283,7 +278,7 @@ func (be *s3) Test(h restic.Handle) (bool, error) { // Remove removes the blob with the given name and type. func (be *s3) Remove(h restic.Handle) error { - objName := be.s3path(h) + objName := be.Filename(h) err := be.client.RemoveObject(be.bucketname, objName) debug.Log("Remove(%v) -> err %v", h, err) return errors.Wrap(err, "client.RemoveObject") @@ -296,7 +291,7 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string { debug.Log("listing %v", t) ch := make(chan string) - prefix := be.s3path(restic.Handle{Type: t}) + "/" + prefix := be.Dirname(restic.Handle{Type: t}) listresp := be.client.ListObjects(be.bucketname, prefix, true, done) diff --git a/src/restic/backend/sftp/config.go b/src/restic/backend/sftp/config.go index c5b65e639..1ba3dcbbd 100644 --- a/src/restic/backend/sftp/config.go +++ b/src/restic/backend/sftp/config.go @@ -6,11 +6,18 @@ import ( "strings" "restic/errors" + "restic/options" ) // Config collects all information required to connect to an sftp server. type Config struct { - User, Host, Dir string + User, Host, Path string + Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` + Command string `option:"command" help:"specify command to create sftp connection"` +} + +func init() { + options.Register("sftp", Config{}) } // ParseConfig parses the string s and extracts the sftp config. The @@ -60,6 +67,6 @@ func ParseConfig(s string) (interface{}, error) { return Config{ User: user, Host: host, - Dir: path.Clean(dir), + Path: path.Clean(dir), }, nil } diff --git a/src/restic/backend/sftp/config_test.go b/src/restic/backend/sftp/config_test.go index d0350745c..44439005e 100644 --- a/src/restic/backend/sftp/config_test.go +++ b/src/restic/backend/sftp/config_test.go @@ -9,53 +9,53 @@ var configTests = []struct { // first form, user specified sftp://user@host/dir { "sftp://user@host/dir/subdir", - Config{User: "user", Host: "host", Dir: "dir/subdir"}, + Config{User: "user", Host: "host", Path: "dir/subdir"}, }, { "sftp://host/dir/subdir", - Config{Host: "host", Dir: "dir/subdir"}, + Config{Host: "host", Path: "dir/subdir"}, }, { "sftp://host//dir/subdir", - Config{Host: "host", Dir: "/dir/subdir"}, + Config{Host: "host", Path: "/dir/subdir"}, }, { "sftp://host:10022//dir/subdir", - Config{Host: "host:10022", Dir: "/dir/subdir"}, + Config{Host: "host:10022", Path: "/dir/subdir"}, }, { "sftp://user@host:10022//dir/subdir", - Config{User: "user", Host: "host:10022", Dir: "/dir/subdir"}, + Config{User: "user", Host: "host:10022", Path: "/dir/subdir"}, }, { "sftp://user@host/dir/subdir/../other", - Config{User: "user", Host: "host", Dir: "dir/other"}, + Config{User: "user", Host: "host", Path: "dir/other"}, }, { "sftp://user@host/dir///subdir", - Config{User: "user", Host: "host", Dir: "dir/subdir"}, + Config{User: "user", Host: "host", Path: "dir/subdir"}, }, // second form, user specified sftp:user@host:/dir { "sftp:user@host:/dir/subdir", - Config{User: "user", Host: "host", Dir: "/dir/subdir"}, + Config{User: "user", Host: "host", Path: "/dir/subdir"}, }, { "sftp:host:../dir/subdir", - Config{Host: "host", Dir: "../dir/subdir"}, + Config{Host: "host", Path: "../dir/subdir"}, }, { "sftp:user@host:dir/subdir:suffix", - Config{User: "user", Host: "host", Dir: "dir/subdir:suffix"}, + Config{User: "user", Host: "host", Path: "dir/subdir:suffix"}, }, { "sftp:user@host:dir/subdir/../other", - Config{User: "user", Host: "host", Dir: "dir/other"}, + Config{User: "user", Host: "host", Path: "dir/other"}, }, { "sftp:user@host:dir///subdir", - Config{User: "user", Host: "host", Dir: "dir/subdir"}, + Config{User: "user", Host: "host", Path: "dir/subdir"}, }, } diff --git a/src/restic/backend/sftp/layout_test.go b/src/restic/backend/sftp/layout_test.go new file mode 100644 index 000000000..0159976fe --- /dev/null +++ b/src/restic/backend/sftp/layout_test.go @@ -0,0 +1,91 @@ +package sftp_test + +import ( + "fmt" + "path/filepath" + "restic" + "restic/backend/sftp" + . "restic/test" + "testing" +) + +func TestLayout(t *testing.T) { + if sftpserver == "" { + t.Skip("sftp server binary not available") + } + + path, cleanup := TempDir(t) + defer cleanup() + + var tests = []struct { + filename string + layout string + failureExpected bool + datafiles map[string]bool + }{ + {"repo-layout-local.tar.gz", "", false, map[string]bool{ + "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, + "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, + "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, + }}, + {"repo-layout-cloud.tar.gz", "", false, map[string]bool{ + "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, + "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, + "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, + }}, + {"repo-layout-s3-old.tar.gz", "", false, map[string]bool{ + "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, + "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, + "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, + }}, + } + + for _, test := range tests { + t.Run(test.filename, func(t *testing.T) { + SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename)) + + repo := filepath.Join(path, "repo") + be, err := sftp.Open(sftp.Config{ + Command: fmt.Sprintf("%q -e", sftpserver), + Path: repo, + Layout: test.layout, + }) + if err != nil { + t.Fatal(err) + } + + if be == nil { + t.Fatalf("Open() returned nil but no error") + } + + datafiles := make(map[string]bool) + for id := range be.List(restic.DataFile, nil) { + datafiles[id] = false + } + + if len(datafiles) == 0 { + t.Errorf("List() returned zero data files") + } + + for id := range test.datafiles { + if _, ok := datafiles[id]; !ok { + t.Errorf("datafile with id %v not found", id) + } + + datafiles[id] = true + } + + for id, v := range datafiles { + if !v { + t.Errorf("unexpected id %v found", id) + } + } + + if err = be.Close(); err != nil { + t.Errorf("Close() returned error %v", err) + } + + RemoveAll(t, filepath.Join(path, "repo")) + }) + } +} diff --git a/src/restic/backend/sftp/sftp.go b/src/restic/backend/sftp/sftp.go index 8894cdc85..b6f71b4c3 100644 --- a/src/restic/backend/sftp/sftp.go +++ b/src/restic/backend/sftp/sftp.go @@ -2,13 +2,12 @@ package sftp import ( "bufio" - "crypto/rand" - "encoding/hex" "fmt" "io" "os" "os/exec" "path" + "path/filepath" "restic" "strings" "time" @@ -21,10 +20,6 @@ import ( "github.com/pkg/sftp" ) -const ( - tempfileRandomSuffixLength = 10 -) - // SFTP is a backend in a directory accessed via SFTP. type SFTP struct { c *sftp.Client @@ -32,11 +27,17 @@ type SFTP struct { cmd *exec.Cmd result <-chan error + + backend.Layout + Config } var _ restic.Backend = &SFTP{} +const defaultLayout = "default" + func startClient(program string, args ...string) (*SFTP, error) { + debug.Log("start client %v %v", program, args) // Connect to a remote host and request the sftp subsystem via the 'ssh' // command. This assumes that passwordless login is correctly configured. cmd := exec.Command(program, args...) @@ -89,18 +90,6 @@ func startClient(program string, args ...string) (*SFTP, error) { return &SFTP{c: client, cmd: cmd, result: ch}, nil } -func paths(dir string) []string { - return []string{ - dir, - Join(dir, backend.Paths.Data), - Join(dir, backend.Paths.Snapshots), - Join(dir, backend.Paths.Index), - Join(dir, backend.Paths.Locks), - Join(dir, backend.Paths.Keys), - Join(dir, backend.Paths.Temp), - } -} - // clientError returns an error if the client has exited. Otherwise, nil is // returned immediately. func (r *SFTP) clientError() error { @@ -114,32 +103,70 @@ func (r *SFTP) clientError() error { return nil } -// Open opens an sftp backend. When the command is started via -// exec.Command, it is expected to speak sftp on stdin/stdout. The backend -// is expected at the given path. `dir` must be delimited by forward slashes -// ("/"), which is required by sftp. -func Open(dir string, program string, args ...string) (*SFTP, error) { - debug.Log("open backend with program %v, %v at %v", program, args, dir) - sftp, err := startClient(program, args...) +// Open opens an sftp backend as described by the config by running +// "ssh" with the appropriate arguments (or cfg.Command, if set). +func Open(cfg Config) (*SFTP, error) { + debug.Log("open backend with config %#v", cfg) + + cmd, args, err := buildSSHCommand(cfg) + if err != nil { + return nil, err + } + + sftp, err := startClient(cmd, args...) if err != nil { debug.Log("unable to start program: %v", err) return nil, err } + sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path) + if err != nil { + return nil, err + } + // test if all necessary dirs and files are there - for _, d := range paths(dir) { + for _, d := range sftp.Paths() { if _, err := sftp.c.Lstat(d); err != nil { return nil, errors.Errorf("%s does not exist", d) } } - sftp.p = dir + debug.Log("layout: %v\n", sftp.Layout) + + sftp.Config = cfg + sftp.p = cfg.Path return sftp, nil } -func buildSSHCommand(cfg Config) []string { +// Join combines path components with slashes (according to the sftp spec). +func (r *SFTP) Join(p ...string) string { + return path.Join(p...) +} + +// ReadDir returns the entries for a directory. +func (r *SFTP) ReadDir(dir string) ([]os.FileInfo, error) { + return r.c.ReadDir(dir) +} + +// IsNotExist returns true if the error is caused by a not existing file. +func (r *SFTP) IsNotExist(err error) bool { + statusError, ok := err.(*sftp.StatusError) + if !ok { + return false + } + + return statusError.Error() == `sftp: "No such file" (SSH_FX_NO_SUCH_FILE)` +} + +func buildSSHCommand(cfg Config) (cmd string, args []string, err error) { + if cfg.Command != "" { + return SplitShellArgs(cfg.Command) + } + + cmd = "ssh" + hostport := strings.Split(cfg.Host, ":") - args := []string{hostport[0]} + args = []string{hostport[0]} if len(hostport) > 1 { args = append(args, "-p", hostport[1]) } @@ -149,35 +176,38 @@ func buildSSHCommand(cfg Config) []string { } args = append(args, "-s") args = append(args, "sftp") - return args + return cmd, args, nil } -// OpenWithConfig opens an sftp backend as described by the config by running -// "ssh" with the appropriate arguments. -func OpenWithConfig(cfg Config) (*SFTP, error) { - debug.Log("open with config %v", cfg) - return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...) -} +// Create creates an sftp backend as described by the config by running +// "ssh" with the appropriate arguments (or cfg.Command, if set). +func Create(cfg Config) (*SFTP, error) { + cmd, args, err := buildSSHCommand(cfg) + if err != nil { + return nil, err + } -// Create creates all the necessary files and directories for a new sftp -// backend at dir. Afterwards a new config blob should be created. `dir` must -// be delimited by forward slashes ("/"), which is required by sftp. -func Create(dir string, program string, args ...string) (*SFTP, error) { - debug.Log("%v %v", program, args) - sftp, err := startClient(program, args...) + sftp, err := startClient(cmd, args...) + if err != nil { + debug.Log("unable to start program: %v", err) + return nil, err + } + + sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err } // test if config file already exists - _, err = sftp.c.Lstat(Join(dir, backend.Paths.Config)) + _, err = sftp.c.Lstat(Join(cfg.Path, backend.Paths.Config)) if err == nil { return nil, errors.New("config file already exists") } // create paths for data, refs and temp blobs - for _, d := range paths(dir) { + for _, d := range sftp.Paths() { err = sftp.mkdirAll(d, backend.Modes.Dir) + debug.Log("mkdirAll %v -> %v", d, err) if err != nil { return nil, err } @@ -189,14 +219,7 @@ func Create(dir string, program string, args ...string) (*SFTP, error) { } // open backend - return Open(dir, program, args...) -} - -// CreateWithConfig creates an sftp backend as described by the config by running -// "ssh" with the appropriate arguments. -func CreateWithConfig(cfg Config) (*SFTP, error) { - debug.Log("config %v", cfg) - return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...) + return Open(cfg) } // Location returns this backend's location (the directory name). @@ -204,28 +227,6 @@ func (r *SFTP) Location() string { return r.p } -// Return temp directory in correct directory for this backend. -func (r *SFTP) tempFile() (string, *sftp.File, error) { - // choose random suffix - buf := make([]byte, tempfileRandomSuffixLength) - _, err := io.ReadFull(rand.Reader, buf) - if err != nil { - return "", nil, errors.Errorf("unable to read %d random bytes for tempfile name: %v", - tempfileRandomSuffixLength, err) - } - - // construct tempfile name - name := Join(r.p, backend.Paths.Temp, "temp-"+hex.EncodeToString(buf)) - - // create file in temp dir - f, err := r.c.Create(name) - if err != nil { - return "", nil, errors.Errorf("creating tempfile %q failed: %v", name, err) - } - - return name, f, nil -} - func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error { // check if directory already exists fi, err := r.c.Lstat(dir) @@ -258,9 +259,24 @@ func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error { return r.c.Chmod(dir, mode) } -// Rename temp file to final name according to type and name. -func (r *SFTP) renameFile(oldname string, h restic.Handle) error { - filename := r.filename(h) +// Join joins the given paths and cleans them afterwards. This always uses +// forward slashes, which is required by sftp. +func Join(parts ...string) string { + return path.Clean(path.Join(parts...)) +} + +// Save stores data in the backend at the handle. +func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) { + debug.Log("Save %v", h) + if err := r.clientError(); err != nil { + return err + } + + if err := h.Valid(); err != nil { + return err + } + + filename := r.Filename(h) // create directories if necessary if h.Type == restic.DataFile { @@ -270,14 +286,22 @@ func (r *SFTP) renameFile(oldname string, h restic.Handle) error { } } - // test if new file exists - if _, err := r.c.Lstat(filename); err == nil { - return errors.Errorf("Close(): file %v already exists", filename) + // create new file + f, err := r.c.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY) + if err != nil { + return errors.Wrap(err, "OpenFile") } - err := r.c.Rename(oldname, filename) + // save data + _, err = io.Copy(f, rd) if err != nil { - return errors.Wrap(err, "Rename") + f.Close() + return errors.Wrap(err, "Write") + } + + err = f.Close() + if err != nil { + return errors.Wrap(err, "Close") } // set mode to read-only @@ -290,76 +314,6 @@ func (r *SFTP) renameFile(oldname string, h restic.Handle) error { return errors.Wrap(err, "Chmod") } -// Join joins the given paths and cleans them afterwards. This always uses -// forward slashes, which is required by sftp. -func Join(parts ...string) string { - return path.Clean(path.Join(parts...)) -} - -// Construct path for given restic.Type and name. -func (r *SFTP) filename(h restic.Handle) string { - if h.Type == restic.ConfigFile { - return Join(r.p, "config") - } - - return Join(r.dirname(h), h.Name) -} - -// Construct directory for given backend.Type. -func (r *SFTP) dirname(h restic.Handle) string { - var n string - switch h.Type { - case restic.DataFile: - n = backend.Paths.Data - if len(h.Name) > 2 { - n = Join(n, h.Name[:2]) - } - case restic.SnapshotFile: - n = backend.Paths.Snapshots - case restic.IndexFile: - n = backend.Paths.Index - case restic.LockFile: - n = backend.Paths.Locks - case restic.KeyFile: - n = backend.Paths.Keys - } - return Join(r.p, n) -} - -// Save stores data in the backend at the handle. -func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) { - debug.Log("save to %v", h) - if err := r.clientError(); err != nil { - return err - } - - if err := h.Valid(); err != nil { - return err - } - - filename, tmpfile, err := r.tempFile() - if err != nil { - return err - } - - n, err := io.Copy(tmpfile, rd) - if err != nil { - return errors.Wrap(err, "Write") - } - - debug.Log("saved %v (%d bytes) to %v", h, n, filename) - - err = tmpfile.Close() - if err != nil { - return errors.Wrap(err, "Close") - } - - err = r.renameFile(filename, h) - debug.Log("save %v: rename %v: %v", - h, path.Base(filename), err) - return err -} - // Load returns a reader that yields the contents of the file at h at the // given offset. If length is nonzero, only a portion of the file is // returned. rd must be closed after use. @@ -373,7 +327,7 @@ func (r *SFTP) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, e return nil, errors.New("offset is negative") } - f, err := r.c.Open(r.filename(h)) + f, err := r.c.Open(r.Filename(h)) if err != nil { return nil, err } @@ -404,7 +358,7 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) { return restic.FileInfo{}, err } - fi, err := r.c.Lstat(r.filename(h)) + fi, err := r.c.Lstat(r.Filename(h)) if err != nil { return restic.FileInfo{}, errors.Wrap(err, "Lstat") } @@ -419,7 +373,7 @@ func (r *SFTP) Test(h restic.Handle) (bool, error) { return false, err } - _, err := r.c.Lstat(r.filename(h)) + _, err := r.c.Lstat(r.Filename(h)) if os.IsNotExist(errors.Cause(err)) { return false, nil } @@ -438,71 +392,35 @@ func (r *SFTP) Remove(h restic.Handle) error { return err } - return r.c.Remove(r.filename(h)) + return r.c.Remove(r.Filename(h)) } // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string { - debug.Log("list all %v", t) + debug.Log("List %v", t) + ch := make(chan string) go func() { defer close(ch) - if t == restic.DataFile { - // read first level - basedir := r.dirname(restic.Handle{Type: t}) + walker := r.c.Walk(r.Basedir(t)) + for walker.Step() { + if walker.Err() != nil { + continue + } - list1, err := r.c.ReadDir(basedir) - if err != nil { + if !walker.Stat().Mode().IsRegular() { + continue + } + + select { + case ch <- filepath.Base(walker.Path()): + case <-done: return } - - dirs := make([]string, 0, len(list1)) - for _, d := range list1 { - dirs = append(dirs, d.Name()) - } - - // read files - for _, dir := range dirs { - entries, err := r.c.ReadDir(Join(basedir, dir)) - if err != nil { - continue - } - - items := make([]string, 0, len(entries)) - for _, entry := range entries { - items = append(items, entry.Name()) - } - - for _, file := range items { - select { - case ch <- file: - case <-done: - return - } - } - } - } else { - entries, err := r.c.ReadDir(r.dirname(restic.Handle{Type: t})) - if err != nil { - return - } - - items := make([]string, 0, len(entries)) - for _, entry := range entries { - items = append(items, entry.Name()) - } - - for _, file := range items { - select { - case ch <- file: - case <-done: - return - } - } } }() diff --git a/src/restic/backend/sftp/sftp_backend_test.go b/src/restic/backend/sftp/sftp_backend_test.go index 567b2cf94..1834beee8 100644 --- a/src/restic/backend/sftp/sftp_backend_test.go +++ b/src/restic/backend/sftp/sftp_backend_test.go @@ -1,6 +1,7 @@ package sftp_test import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -33,24 +34,29 @@ func createTempdir() error { return nil } -func init() { - sftpserver := "" - +func findSFTPServerBinary() string { for _, dir := range strings.Split(TestSFTPPath, ":") { testpath := filepath.Join(dir, "sftp-server") _, err := os.Stat(testpath) if !os.IsNotExist(errors.Cause(err)) { - sftpserver = testpath - break + return testpath } } + return "" +} + +var sftpserver = findSFTPServerBinary() + +func init() { if sftpserver == "" { SkipMessage = "sftp server binary not found, skipping tests" return } - args := []string{"-e"} + cfg := sftp.Config{ + Command: fmt.Sprintf("%q -e", sftpserver), + } test.CreateFn = func() (restic.Backend, error) { err := createTempdir() @@ -58,7 +64,9 @@ func init() { return nil, err } - return sftp.Create(tempBackendDir, sftpserver, args...) + cfg.Path = tempBackendDir + + return sftp.Create(cfg) } test.OpenFn = func() (restic.Backend, error) { @@ -66,7 +74,10 @@ func init() { if err != nil { return nil, err } - return sftp.Open(tempBackendDir, sftpserver, args...) + + cfg.Path = tempBackendDir + + return sftp.Open(cfg) } test.CleanupFn = func() error { diff --git a/src/restic/backend/sftp/split.go b/src/restic/backend/sftp/split.go new file mode 100644 index 000000000..dd6f68db9 --- /dev/null +++ b/src/restic/backend/sftp/split.go @@ -0,0 +1,77 @@ +package sftp + +import ( + "restic/errors" + "unicode" +) + +// shellSplitter splits a command string into separater arguments. It supports +// single and double quoted strings. +type shellSplitter struct { + quote rune + lastChar rune +} + +func (s *shellSplitter) isSplitChar(c rune) bool { + // only test for quotes if the last char was not a backslash + if s.lastChar != '\\' { + + // quote ended + if s.quote != 0 && c == s.quote { + s.quote = 0 + return true + } + + // quote starts + if s.quote == 0 && (c == '"' || c == '\'') { + s.quote = c + return true + } + } + + s.lastChar = c + + // within quote + if s.quote != 0 { + return false + } + + // outside quote + return c == '\\' || unicode.IsSpace(c) +} + +// SplitShellArgs returns the list of arguments from a shell command string. +func SplitShellArgs(data string) (cmd string, args []string, err error) { + s := &shellSplitter{} + + // derived from strings.SplitFunc + fieldStart := -1 // Set to -1 when looking for start of field. + for i, rune := range data { + if s.isSplitChar(rune) { + if fieldStart >= 0 { + args = append(args, data[fieldStart:i]) + fieldStart = -1 + } + } else if fieldStart == -1 { + fieldStart = i + } + } + if fieldStart >= 0 { // Last field might end at EOF. + args = append(args, data[fieldStart:]) + } + + switch s.quote { + case '\'': + return "", nil, errors.New("single-quoted string not terminated") + case '"': + return "", nil, errors.New("double-quoted string not terminated") + } + + if len(args) == 0 { + return "", nil, errors.New("command string is empty") + } + + cmd, args = args[0], args[1:] + + return cmd, args, nil +} diff --git a/src/restic/backend/sftp/split_test.go b/src/restic/backend/sftp/split_test.go new file mode 100644 index 000000000..06241b29a --- /dev/null +++ b/src/restic/backend/sftp/split_test.go @@ -0,0 +1,115 @@ +package sftp + +import ( + "reflect" + "testing" +) + +func TestShellSplitter(t *testing.T) { + var tests = []struct { + data string + cmd string + args []string + }{ + { + `foo`, + "foo", []string{}, + }, + { + `'foo'`, + "foo", []string{}, + }, + { + `foo bar baz`, + "foo", []string{"bar", "baz"}, + }, + { + `foo 'bar' baz`, + "foo", []string{"bar", "baz"}, + }, + { + `'bar box' baz`, + "bar box", []string{"baz"}, + }, + { + `"bar 'box'" baz`, + "bar 'box'", []string{"baz"}, + }, + { + `'bar "box"' baz`, + `bar "box"`, []string{"baz"}, + }, + { + `\"bar box baz`, + `"bar`, []string{"box", "baz"}, + }, + { + `"bar/foo/x" "box baz"`, + "bar/foo/x", []string{"box baz"}, + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + cmd, args, err := SplitShellArgs(test.data) + if err != nil { + t.Fatal(err) + } + + if cmd != test.cmd { + t.Fatalf("wrong cmd returned, want:\n %#v\ngot:\n %#v", + test.cmd, cmd) + } + + if !reflect.DeepEqual(args, test.args) { + t.Fatalf("wrong args returned, want:\n %#v\ngot:\n %#v", + test.args, args) + } + }) + } +} + +func TestShellSplitterInvalid(t *testing.T) { + var tests = []struct { + data string + err string + }{ + { + "foo'", + "single-quoted string not terminated", + }, + { + `foo"`, + "double-quoted string not terminated", + }, + { + "foo 'bar", + "single-quoted string not terminated", + }, + { + `foo "bar`, + "double-quoted string not terminated", + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + cmd, args, err := SplitShellArgs(test.data) + if err == nil { + t.Fatalf("expected error not found: %v", test.err) + } + + if err.Error() != test.err { + t.Fatalf("expected error not found, want:\n %q\ngot:\n %q", test.err, err.Error()) + } + + if cmd != "" { + t.Fatalf("splitter returned cmd from invalid data: %v", cmd) + } + + if len(args) > 0 { + t.Fatalf("splitter returned fields from invalid data: %v", args) + } + }) + } +} diff --git a/src/restic/backend/sftp/sshcmd_test.go b/src/restic/backend/sftp/sshcmd_test.go index d98309c67..dea811a35 100644 --- a/src/restic/backend/sftp/sshcmd_test.go +++ b/src/restic/backend/sftp/sshcmd_test.go @@ -1,46 +1,52 @@ package sftp -import "testing" +import ( + "reflect" + "testing" +) var sshcmdTests = []struct { - cfg Config - s []string + cfg Config + cmd string + args []string }{ { - Config{User: "user", Host: "host", Dir: "dir/subdir"}, + Config{User: "user", Host: "host", Path: "dir/subdir"}, + "ssh", []string{"host", "-l", "user", "-s", "sftp"}, }, { - Config{Host: "host", Dir: "dir/subdir"}, + Config{Host: "host", Path: "dir/subdir"}, + "ssh", []string{"host", "-s", "sftp"}, }, { - Config{Host: "host:10022", Dir: "/dir/subdir"}, + Config{Host: "host:10022", Path: "/dir/subdir"}, + "ssh", []string{"host", "-p", "10022", "-s", "sftp"}, }, { - Config{User: "user", Host: "host:10022", Dir: "/dir/subdir"}, + Config{User: "user", Host: "host:10022", Path: "/dir/subdir"}, + "ssh", []string{"host", "-p", "10022", "-l", "user", "-s", "sftp"}, }, } func TestBuildSSHCommand(t *testing.T) { - for i, test := range sshcmdTests { - cmd := buildSSHCommand(test.cfg) - failed := false - if len(cmd) != len(test.s) { - failed = true - } else { - for l := range test.s { - if test.s[l] != cmd[l] { - failed = true - break - } + for _, test := range sshcmdTests { + t.Run("", func(t *testing.T) { + cmd, args, err := buildSSHCommand(test.cfg) + if err != nil { + t.Fatal(err) } - } - if failed { - t.Errorf("test %d: wrong cmd, want:\n %v\ngot:\n %v", - i, test.s, cmd) - } + + if cmd != test.cmd { + t.Fatalf("cmd: want %v, got %v", test.cmd, cmd) + } + + if !reflect.DeepEqual(test.args, args) { + t.Fatalf("wrong args, want:\n %v\ngot:\n %v", test.args, args) + } + }) } } diff --git a/src/restic/backend/test/tests.go b/src/restic/backend/test/tests.go index f2e6b2820..fd32e0826 100644 --- a/src/restic/backend/test/tests.go +++ b/src/restic/backend/test/tests.go @@ -43,7 +43,7 @@ func open(t testing.TB) restic.Backend { if !butInitialized { be, err := CreateFn() if err != nil { - t.Fatalf("Create returned unexpected error: %v", err) + t.Fatalf("Create returned unexpected error: %+v", err) } but = be @@ -54,7 +54,7 @@ func open(t testing.TB) restic.Backend { var err error but, err = OpenFn() if err != nil { - t.Fatalf("Open returned unexpected error: %v", err) + t.Fatalf("Open returned unexpected error: %+v", err) } } @@ -68,7 +68,7 @@ func close(t testing.TB) { err := but.Close() if err != nil { - t.Fatalf("Close returned unexpected error: %v", err) + t.Fatalf("Close returned unexpected error: %+v", err) } but = nil @@ -82,14 +82,14 @@ func TestCreate(t testing.TB) { be, err := CreateFn() if err != nil { - t.Fatalf("Create returned error: %v", err) + t.Fatalf("Create returned error: %+v", err) } butInitialized = true err = be.Close() if err != nil { - t.Fatalf("Close returned error: %v", err) + t.Fatalf("Close returned error: %+v", err) } } @@ -101,12 +101,12 @@ func TestOpen(t testing.TB) { be, err := OpenFn() if err != nil { - t.Fatalf("Open returned error: %v", err) + t.Fatalf("Open returned error: %+v", err) } err = be.Close() if err != nil { - t.Fatalf("Close returned error: %v", err) + t.Fatalf("Close returned error: %+v", err) } } @@ -132,7 +132,7 @@ func TestCreateWithConfig(t testing.TB) { // remove config err = b.Remove(restic.Handle{Type: restic.ConfigFile, Name: ""}) if err != nil { - t.Fatalf("unexpected error removing config: %v", err) + t.Fatalf("unexpected error removing config: %+v", err) } } @@ -162,7 +162,7 @@ func TestConfig(t testing.TB) { err = b.Save(restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString)) if err != nil { - t.Fatalf("Save() error: %v", err) + t.Fatalf("Save() error: %+v", err) } // try accessing the config with different names, should all return the @@ -171,7 +171,7 @@ func TestConfig(t testing.TB) { h := restic.Handle{Type: restic.ConfigFile, Name: name} buf, err := backend.LoadAll(b, h) if err != nil { - t.Fatalf("unable to read config with name %q: %v", name, err) + t.Fatalf("unable to read config with name %q: %+v", name, err) } if string(buf) != testString { @@ -203,7 +203,7 @@ func TestLoad(t testing.TB) { handle := restic.Handle{Type: restic.DataFile, Name: id.String()} err = b.Save(handle, bytes.NewReader(data)) if err != nil { - t.Fatalf("Save() error: %v", err) + t.Fatalf("Save() error: %+v", err) } rd, err := b.Load(handle, 100, -1) @@ -238,13 +238,13 @@ func TestLoad(t testing.TB) { rd, err := b.Load(handle, getlen, int64(o)) if err != nil { - t.Errorf("Load(%d, %d) returned unexpected error: %v", l, o, err) + t.Errorf("Load(%d, %d) returned unexpected error: %+v", l, o, err) continue } buf, err := ioutil.ReadAll(rd) if err != nil { - t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %v", l, o, err) + t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %+v", l, o, err) rd.Close() continue } @@ -269,7 +269,7 @@ func TestLoad(t testing.TB) { err = rd.Close() if err != nil { - t.Errorf("Load(%d, %d) rd.Close() returned unexpected error: %v", l, o, err) + t.Errorf("Load(%d, %d) rd.Close() returned unexpected error: %+v", l, o, err) continue } } @@ -325,7 +325,7 @@ func TestSave(t testing.TB) { err = b.Remove(h) if err != nil { - t.Fatalf("error removing item: %v", err) + t.Fatalf("error removing item: %+v", err) } } @@ -366,7 +366,7 @@ func TestSave(t testing.TB) { err = b.Remove(h) if err != nil { - t.Fatalf("error removing item: %v", err) + t.Fatalf("error removing item: %+v", err) } } @@ -391,13 +391,13 @@ func TestSaveFilenames(t testing.TB) { h := restic.Handle{Name: test.name, Type: restic.DataFile} err := b.Save(h, strings.NewReader(test.data)) if err != nil { - t.Errorf("test %d failed: Save() returned %v", i, err) + t.Errorf("test %d failed: Save() returned %+v", i, err) continue } buf, err := backend.LoadAll(b, h) if err != nil { - t.Errorf("test %d failed: Load() returned %v", i, err) + t.Errorf("test %d failed: Load() returned %+v", i, err) continue } @@ -407,7 +407,7 @@ func TestSaveFilenames(t testing.TB) { err = b.Remove(h) if err != nil { - t.Errorf("test %d failed: Remove() returned %v", i, err) + t.Errorf("test %d failed: Remove() returned %+v", i, err) continue } } @@ -511,7 +511,7 @@ func TestBackend(t testing.TB) { // test that the blob is gone ok, err := b.Test(h) test.OK(t, err) - test.Assert(t, ok == false, "removed blob still present") + test.Assert(t, !ok, "removed blob still present") // create blob err = b.Save(h, strings.NewReader(ts.data)) @@ -553,6 +553,7 @@ func TestBackend(t testing.TB) { found, err := b.Test(h) test.OK(t, err) + test.Assert(t, found, fmt.Sprintf("id %q not found", id)) test.OK(t, b.Remove(h)) @@ -576,7 +577,7 @@ func TestDelete(t testing.TB) { err := be.Delete() if err != nil { - t.Fatalf("error deleting backend: %v", err) + t.Fatalf("error deleting backend: %+v", err) } } @@ -594,6 +595,6 @@ func TestCleanup(t testing.TB) { err := CleanupFn() if err != nil { - t.Fatalf("Cleanup returned error: %v", err) + t.Fatalf("Cleanup returned error: %+v", err) } } diff --git a/src/restic/backend/testdata/repo-layout-cloud.tar.gz b/src/restic/backend/testdata/repo-layout-cloud.tar.gz new file mode 100644 index 000000000..189832589 Binary files /dev/null and b/src/restic/backend/testdata/repo-layout-cloud.tar.gz differ diff --git a/src/restic/backend/testdata/repo-layout-local.tar.gz b/src/restic/backend/testdata/repo-layout-local.tar.gz new file mode 100644 index 000000000..e38deb54b Binary files /dev/null and b/src/restic/backend/testdata/repo-layout-local.tar.gz differ diff --git a/src/restic/backend/testdata/repo-layout-s3-old.tar.gz b/src/restic/backend/testdata/repo-layout-s3-old.tar.gz new file mode 100644 index 000000000..2b7d852cc Binary files /dev/null and b/src/restic/backend/testdata/repo-layout-s3-old.tar.gz differ diff --git a/src/restic/backend/utils_test.go b/src/restic/backend/utils_test.go index 2996cf494..51481ed0b 100644 --- a/src/restic/backend/utils_test.go +++ b/src/restic/backend/utils_test.go @@ -49,8 +49,7 @@ func TestLoadSmallBuffer(t *testing.T) { err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data)) OK(t, err) - buf := make([]byte, len(data)-23) - buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}) + buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}) OK(t, err) if len(buf) != len(data) { @@ -75,8 +74,7 @@ func TestLoadLargeBuffer(t *testing.T) { err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data)) OK(t, err) - buf := make([]byte, len(data)+100) - buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}) + buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}) OK(t, err) if len(buf) != len(data) { diff --git a/src/restic/options/options.go b/src/restic/options/options.go index c5d9ff3e3..55ef59609 100644 --- a/src/restic/options/options.go +++ b/src/restic/options/options.go @@ -3,6 +3,7 @@ package options import ( "reflect" "restic/errors" + "sort" "strconv" "strings" "time" @@ -11,6 +12,85 @@ import ( // Options holds options in the form key=value. type Options map[string]string +var opts []Help + +// Register allows registering options so that they can be listed with List. +func Register(ns string, cfg interface{}) { + opts = appendAllOptions(opts, ns, cfg) +} + +// List returns a list of all registered options (using Register()). +func List() (list []Help) { + list = make([]Help, 0, len(opts)) + for _, opt := range opts { + list = append(list, opt) + } + return list +} + +// appendAllOptions appends all options in cfg to opts, sorted by namespace. +func appendAllOptions(opts []Help, ns string, cfg interface{}) []Help { + for _, opt := range listOptions(cfg) { + opt.Namespace = ns + opts = append(opts, opt) + } + + sort.Sort(helpList(opts)) + return opts +} + +// listOptions returns a list of options of cfg. +func listOptions(cfg interface{}) (opts []Help) { + // resolve indirection if cfg is a pointer + v := reflect.Indirect(reflect.ValueOf(cfg)) + + for i := 0; i < v.NumField(); i++ { + f := v.Type().Field(i) + + h := Help{ + Name: f.Tag.Get("option"), + Text: f.Tag.Get("help"), + } + + if h.Name == "" { + continue + } + + opts = append(opts, h) + } + + return opts +} + +// Help contains information about an option. +type Help struct { + Namespace string + Name string + Text string +} + +type helpList []Help + +// Len is the number of elements in the collection. +func (h helpList) Len() int { + return len(h) +} + +// Less reports whether the element with +// index i should sort before the element with index j. +func (h helpList) Less(i, j int) bool { + if h[i].Namespace == h[j].Namespace { + return h[i].Name < h[j].Name + } + + return h[i].Namespace < h[j].Namespace +} + +// Swap swaps the elements with indexes i and j. +func (h helpList) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + // splitKeyValue splits at the first equals (=) sign. func splitKeyValue(s string) (key string, value string) { data := strings.SplitN(s, "=", 2) diff --git a/src/restic/options/options_test.go b/src/restic/options/options_test.go index a5ab83952..f5d43b9a7 100644 --- a/src/restic/options/options_test.go +++ b/src/restic/options/options_test.go @@ -218,3 +218,95 @@ func TestOptionsApplyInvalid(t *testing.T) { }) } } + +func TestListOptions(t *testing.T) { + var teststruct = struct { + Foo string `option:"foo" help:"bar text help"` + }{} + + var tests = []struct { + cfg interface{} + opts []Help + }{ + { + struct { + Foo string `option:"foo" help:"bar text help"` + }{}, + []Help{ + Help{Name: "foo", Text: "bar text help"}, + }, + }, + { + struct { + Foo string `option:"foo" help:"bar text help"` + Bar string `option:"bar" help:"bar text help"` + }{}, + []Help{ + Help{Name: "foo", Text: "bar text help"}, + Help{Name: "bar", Text: "bar text help"}, + }, + }, + { + struct { + Bar string `option:"bar" help:"bar text help"` + Foo string `option:"foo" help:"bar text help"` + }{}, + []Help{ + Help{Name: "bar", Text: "bar text help"}, + Help{Name: "foo", Text: "bar text help"}, + }, + }, + { + &teststruct, + []Help{ + Help{Name: "foo", Text: "bar text help"}, + }, + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + opts := listOptions(test.cfg) + if !reflect.DeepEqual(opts, test.opts) { + t.Fatalf("wrong opts, want:\n %v\ngot:\n %v", test.opts, opts) + } + }) + } +} + +func TestAppendAllOptions(t *testing.T) { + var tests = []struct { + cfgs map[string]interface{} + opts []Help + }{ + { + map[string]interface{}{ + "local": struct { + Foo string `option:"foo" help:"bar text help"` + }{}, + "sftp": struct { + Foo string `option:"foo" help:"bar text help2"` + Bar string `option:"bar" help:"bar text help"` + }{}, + }, + []Help{ + Help{Namespace: "local", Name: "foo", Text: "bar text help"}, + Help{Namespace: "sftp", Name: "bar", Text: "bar text help"}, + Help{Namespace: "sftp", Name: "foo", Text: "bar text help2"}, + }, + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + var opts []Help + for ns, cfg := range test.cfgs { + opts = appendAllOptions(opts, ns, cfg) + } + + if !reflect.DeepEqual(opts, test.opts) { + t.Fatalf("wrong list, want:\n %v\ngot:\n %v", test.opts, opts) + } + }) + } +} diff --git a/src/restic/repository/testing.go b/src/restic/repository/testing.go index 6f590e13a..a24912257 100644 --- a/src/restic/repository/testing.go +++ b/src/restic/repository/testing.go @@ -19,8 +19,12 @@ var testKDFParams = crypto.KDFParams{ P: 1, } +type logger interface { + Logf(format string, args ...interface{}) +} + // TestUseLowSecurityKDFParameters configures low-security KDF parameters for testing. -func TestUseLowSecurityKDFParameters(t testing.TB) { +func TestUseLowSecurityKDFParameters(t logger) { t.Logf("using low-security KDF parameters for test") KDFParams = &testKDFParams } diff --git a/src/restic/test/helpers.go b/src/restic/test/helpers.go index 4e19000e8..a6dffc0a4 100644 --- a/src/restic/test/helpers.go +++ b/src/restic/test/helpers.go @@ -170,3 +170,21 @@ func RemoveAll(t testing.TB, path string) { ResetReadOnly(t, path) OK(t, os.RemoveAll(path)) } + +// TempDir returns a temporary directory that is removed when cleanup is +// called, except if TestCleanupTempDirs is set to false. +func TempDir(t testing.TB) (path string, cleanup func()) { + tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-") + if err != nil { + t.Fatal(err) + } + + return tempdir, func() { + if !TestCleanupTempDirs { + t.Logf("leaving temporary directory %v used for test", tempdir) + return + } + + RemoveAll(t, tempdir) + } +} diff --git a/src/restic/test/vars.go b/src/restic/test/vars.go index bb9f6b13d..908e4bf6b 100644 --- a/src/restic/test/vars.go +++ b/src/restic/test/vars.go @@ -11,7 +11,7 @@ var ( TestTempDir = getStringVar("RESTIC_TEST_TMPDIR", "") RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true) RunFuseTest = getBoolVar("RESTIC_TEST_FUSE", true) - TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh") + TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh:/usr/libexec") TestWalkerPath = getStringVar("RESTIC_TEST_PATH", ".") BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".") TestS3Server = getStringVar("RESTIC_TEST_S3_SERVER", "")