alexvanin/restic
1
0
Fork 0
forked from TrueCloudLab/restic
Code Pull requests Activity

Merge pull request #966 from restic/unify-repo-layout

Browse source 
WIP: Unify repository layout
This commit is contained in:
Alexander Neumann 2017-05-22 21:15:38 +02:00
commit d1cc87ba28
17 changed files with 200 additions and 201 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
doc/design.rst
View file

@ -76,8 +76,8 @@ identifies the repository, regardless if it is accessed via SFTP or
locally. The field ``chunker_polynomial`` contains a parameter that is
used for splitting large files into smaller chunks (see below).
Filesystem-Based Repositories
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Repository Layout
~~~~~~~~~~~~~~~~~
The ``local`` and ``sftp`` backends are implemented using files and
directories stored in a file system. The directory layout is the same
@ -117,44 +117,20 @@ e.g.:
$ restic -r /tmp/restic-repo init
The local and sftp backends will also accept the repository layout
described in the following section, so that remote repositories mounted
locally e.g. via fuse can be accessed. The layout auto-detection can be
overridden by specifying the option ``-o local.layout=default``, valid
values are ``default``, ``cloud`` and ``s3``. The option for the sftp
backend is named ``sftp.layout``.
The local and sftp backends will auto-detect and accept all layouts described
in the following sections, so that remote repositories mounted locally e.g. via
fuse can be accessed. The layout auto-detection can be overridden by specifying
the option ``-o local.layout=default``, valid values are ``default`` and
``s3legacy``. The option for the sftp backend is named ``sftp.layout``, for the
s3 backend ``s3.layout``.
Object-Storage-Based Repositories
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
S3 Legacy Layout
~~~~~~~~~~~~~~~~
Repositories in a backend based on an object store (e.g. Amazon s3) have
the same basic layout, with the exception that all data pack files are
directly saved in the ``data`` path, without the sub-directories listed
for the filesystem-based backends as listed in the previous section. The
layout looks like this:
::
/config
/data
├── 2159dd48f8a24f33c307b750592773f8b71ff8d11452132a7b2e2a6a01611be1
├── 32ea976bc30771cebad8285cd99120ac8786f9ffd42141d452458089985043a5
├── 59fe4bcde59bd6222eba87795e35a90d82cd2f138a27b6835032b7b58173a426
├── 73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c
[...]
/index
├── c38f5fb68307c6a3e3aa945d556e325dc38f5fb68307c6a3e3aa945d556e325d
└── ca171b1b7394d90d330b265d90f506f9984043b342525f019788f97e745c71fd
/keys
└── b02de829beeb3c01a63e6b25cbd421a98fef144f03b9a02e46eff9e2ca3f0bd7
/locks
/snapshots
└── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec
Unfortunately during development the s3 backend uses slightly different
Unfortunately during development the AWS S3 backend uses slightly different
paths (directory names use singular instead of plural for ``key``,
``lock``, and ``snapshot`` files), for s3 the repository layout looks
like this:
``lock``, and ``snapshot`` files), and the data files are stored directly below
the ``data`` directory. The S3 Legacy repository layout looks like this:
::
@ -174,8 +150,8 @@ like this:
/snapshot
└── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec
The s3 backend understands and accepts both forms, new backends are
always created with the former layout for compatibility reasons.
The S3 backend understands and accepts both forms, new backends are
always created with the default layout for compatibility reasons.
Pack Format
-----------

2
src/cmds/restic/flags_test.go
View file

@ -1,6 +1,7 @@
package main
import (
"io/ioutil"
"testing"
)
@ -13,6 +14,7 @@ func TestFlags(t *testing.T) {
for _, cmd := range cmdRoot.Commands() {
t.Run(cmd.Name(), func(t *testing.T) {
cmd.Flags().SetOutput(ioutil.Discard)
err := cmd.ParseFlags([]string{"--help"})
if err.Error() == "pflag: help requested" {
err = nil

10
src/cmds/restic/local_layout_test.go
View file

@ -12,12 +12,10 @@ func TestRestoreLocalLayout(t *testing.T) {
filename string
layout string
}{
{"repo-layout-cloud.tar.gz", ""},
{"repo-layout-local.tar.gz", ""},
{"repo-layout-s3-old.tar.gz", ""},
{"repo-layout-cloud.tar.gz", "cloud"},
{"repo-layout-local.tar.gz", "default"},
{"repo-layout-s3-old.tar.gz", "s3"},
{"repo-layout-default.tar.gz", ""},
{"repo-layout-s3legacy.tar.gz", ""},
{"repo-layout-default.tar.gz", "default"},
{"repo-layout-s3legacy.tar.gz", "s3legacy"},
}
for _, test := range tests {

73
src/restic/backend/layout.go
View file

@ -85,36 +85,6 @@ func hasBackendFile(fs Filesystem, dir string) (bool, error) {
return false, nil
}
var dataSubdirName = regexp.MustCompile("^[a-fA-F0-9]{2}$")
func hasSubdirBackendFile(fs Filesystem, dir string) (bool, error) {
entries, err := fs.ReadDir(dir)
if err != nil && fs.IsNotExist(errors.Cause(err)) {
return false, nil
}
if err != nil {
return false, errors.Wrap(err, "ReadDir")
}
for _, subdir := range entries {
if !dataSubdirName.MatchString(subdir.Name()) {
continue
}
present, err := hasBackendFile(fs, fs.Join(dir, subdir.Name()))
if err != nil {
return false, err
}
if present {
return true, nil
}
}
return false, nil
}
// ErrLayoutDetectionFailed is returned by DetectLayout() when the layout
// cannot be detected automatically.
var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed")
@ -128,39 +98,19 @@ func DetectLayout(repo Filesystem, dir string) (Layout, error) {
repo = &LocalFilesystem{}
}
// key file in the "keys" dir (DefaultLayout or CloudLayout)
// key file in the "keys" dir (DefaultLayout)
foundKeysFile, err := hasBackendFile(repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
if err != nil {
return nil, err
}
// key file in the "key" dir (S3Layout)
// key file in the "key" dir (S3LegacyLayout)
foundKeyFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
if err != nil {
return nil, err
}
// data file in "data" directory (S3Layout or CloudLayout)
foundDataFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile]))
if err != nil {
return nil, err
}
// data file in subdir of "data" directory (DefaultLayout)
foundDataSubdirFile, err := hasSubdirBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.DataFile]))
if err != nil {
return nil, err
}
if foundKeysFile && foundDataFile && !foundKeyFile && !foundDataSubdirFile {
debug.Log("found cloud layout at %v", dir)
return &CloudLayout{
Path: dir,
Join: repo.Join,
}, nil
}
if foundKeysFile && foundDataSubdirFile && !foundKeyFile && !foundDataFile {
if foundKeysFile && !foundKeyFile {
debug.Log("found default layout at %v", dir)
return &DefaultLayout{
Path: dir,
@ -168,9 +118,9 @@ func DetectLayout(repo Filesystem, dir string) (Layout, error) {
}, nil
}
if foundKeyFile && foundDataFile && !foundKeysFile && !foundDataSubdirFile {
if foundKeyFile && !foundKeysFile {
debug.Log("found s3 layout at %v", dir)
return &S3Layout{
return &S3LegacyLayout{
Path: dir,
Join: repo.Join,
}, nil
@ -190,13 +140,8 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout,
Path: path,
Join: repo.Join,
}
case "cloud":
l = &CloudLayout{
Path: path,
Join: repo.Join,
}
case "s3":
l = &S3Layout{
case "s3legacy":
l = &S3LegacyLayout{
Path: path,
Join: repo.Join,
}
@ -205,7 +150,7 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout,
// use the default layout if auto detection failed
if errors.Cause(err) == ErrLayoutDetectionFailed && defaultLayout != "" {
debug.Log("error: %v, use default layout %v", defaultLayout)
debug.Log("error: %v, use default layout %v", err, defaultLayout)
return ParseLayout(repo, defaultLayout, "", path)
}
@ -214,7 +159,7 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout,
}
debug.Log("layout detected: %v", l)
default:
return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, cloud, s3", layout)
return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, s3legacy", layout)
}
return l, nil

46
src/restic/backend/layout_cloud.go
View file

@ -1,46 +0,0 @@
package backend
import "restic"
// CloudLayout implements the default layout for cloud storage backends, as
// described in the Design document.
type CloudLayout struct {
URL string
Path string
Join func(...string) string
}
var cloudLayoutPaths = defaultLayoutPaths
// Dirname returns the directory path for a given file type and name.
func (l *CloudLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type]) + "/"
}
// Filename returns a path to a file, including its name.
func (l *CloudLayout) Filename(h restic.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
name = "config"
}
return l.URL + l.Join(l.Path, "/", cloudLayoutPaths[h.Type], name)
}
// Paths returns all directory names
func (l *CloudLayout) Paths() (dirs []string) {
for _, p := range cloudLayoutPaths {
dirs = append(dirs, l.URL+l.Join(l.Path, p))
}
return dirs
}
// Basedir returns the base dir name for files of type t.
func (l *CloudLayout) Basedir(t restic.FileType) string {
return l.URL + l.Join(l.Path, cloudLayoutPaths[t])
}

4
src/restic/backend/layout_default.go
View file

@ -24,10 +24,10 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string {
p := defaultLayoutPaths[h.Type]
if h.Type == restic.DataFile && len(h.Name) > 2 {
p = l.Join(p, h.Name[:2])
p = l.Join(p, h.Name[:2]) + "/"
}
return l.Join(l.Path, p)
return l.Join(l.Path, p) + "/"
}
// Filename returns a path to a file, including its name.

45
src/restic/backend/layout_rest.go Normal file
View file

@ -0,0 +1,45 @@
package backend
import "restic"
// RESTLayout implements the default layout for the REST protocol.
type RESTLayout struct {
URL string
Path string
Join func(...string) string
}
var restLayoutPaths = defaultLayoutPaths
// Dirname returns the directory path for a given file type and name.
func (l *RESTLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type]) + "/"
}
// Filename returns a path to a file, including its name.
func (l *RESTLayout) Filename(h restic.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
name = "config"
}
return l.URL + l.Join(l.Path, "/", restLayoutPaths[h.Type], name)
}
// Paths returns all directory names
func (l *RESTLayout) Paths() (dirs []string) {
for _, p := range restLayoutPaths {
dirs = append(dirs, l.URL+l.Join(l.Path, p))
}
return dirs
}
// Basedir returns the base dir name for files of type t.
func (l *RESTLayout) Basedir(t restic.FileType) string {
return l.URL + l.Join(l.Path, restLayoutPaths[t])
}

14
src/restic/backend/layout_s3.go → src/restic/backend/layout_s3legacy.go
View file

@ -2,9 +2,9 @@ package backend
import "restic"
// S3Layout implements the old layout used for s3 cloud storage backends, as
// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as
// described in the Design document.
type S3Layout struct {
type S3LegacyLayout struct {
URL string
Path string
Join func(...string) string
@ -19,7 +19,7 @@ var s3LayoutPaths = map[restic.FileType]string{
}
// join calls Join with the first empty elements removed.
func (l *S3Layout) join(url string, items ...string) string {
func (l *S3LegacyLayout) join(url string, items ...string) string {
for len(items) > 0 && items[0] == "" {
items = items[1:]
}
@ -35,7 +35,7 @@ func (l *S3Layout) join(url string, items ...string) string {
}
// Dirname returns the directory path for a given file type and name.
func (l *S3Layout) Dirname(h restic.Handle) string {
func (l *S3LegacyLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
@ -44,7 +44,7 @@ func (l *S3Layout) Dirname(h restic.Handle) string {
}
// Filename returns a path to a file, including its name.
func (l *S3Layout) Filename(h restic.Handle) string {
func (l *S3LegacyLayout) Filename(h restic.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
@ -55,7 +55,7 @@ func (l *S3Layout) Filename(h restic.Handle) string {
}
// Paths returns all directory names
func (l *S3Layout) Paths() (dirs []string) {
func (l *S3LegacyLayout) Paths() (dirs []string) {
for _, p := range s3LayoutPaths {
dirs = append(dirs, l.Join(l.Path, p))
}
@ -63,6 +63,6 @@ func (l *S3Layout) Paths() (dirs []string) {
}
// Basedir returns the base dir name for type t.
func (l *S3Layout) Basedir(t restic.FileType) string {
func (l *S3LegacyLayout) Basedir(t restic.FileType) string {
return l.Join(l.Path, s3LayoutPaths[t])
}

40
src/restic/backend/layout_test.go
View file

@ -79,7 +79,7 @@ func TestDefaultLayout(t *testing.T) {
}
}
func TestCloudLayout(t *testing.T) {
func TestRESTLayout(t *testing.T) {
path, cleanup := TempDir(t)
defer cleanup()
@ -113,7 +113,7 @@ func TestCloudLayout(t *testing.T) {
},
}
l := &CloudLayout{
l := &RESTLayout{
Path: path,
Join: filepath.Join,
}
@ -147,7 +147,7 @@ func TestCloudLayout(t *testing.T) {
}
}
func TestCloudLayoutURLs(t *testing.T) {
func TestRESTLayoutURLs(t *testing.T) {
var tests = []struct {
l Layout
h restic.Handle
@ -155,55 +155,55 @@ func TestCloudLayoutURLs(t *testing.T) {
dir string
}{
{
&CloudLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
&RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
restic.Handle{Type: restic.DataFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/locks/foobar",
"https://hostname.foo:1234/prefix/repo/locks/",
},
{
&CloudLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3Layout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
&S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.DataFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/lock/foobar",
"https://hostname.foo:1234/prefix/repo/lock/",
},
{
&S3Layout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3Layout{URL: "", Path: "", Join: path.Join},
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.DataFile, Name: "foobar"},
"data/foobar",
"data/",
},
{
&S3Layout{URL: "", Path: "", Join: path.Join},
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
"lock/foobar",
"lock/",
},
{
&S3Layout{URL: "", Path: "/", Join: path.Join},
&S3LegacyLayout{URL: "", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
"/config",
"/",
@ -225,7 +225,7 @@ func TestCloudLayoutURLs(t *testing.T) {
}
}
func TestS3Layout(t *testing.T) {
func TestS3LegacyLayout(t *testing.T) {
path, cleanup := TempDir(t)
defer cleanup()
@ -259,7 +259,7 @@ func TestS3Layout(t *testing.T) {
},
}
l := &S3Layout{
l := &S3LegacyLayout{
Path: path,
Join: filepath.Join,
}
@ -301,9 +301,8 @@ func TestDetectLayout(t *testing.T) {
filename string
want string
}{
{"repo-layout-local.tar.gz", "*backend.DefaultLayout"},
{"repo-layout-cloud.tar.gz", "*backend.CloudLayout"},
{"repo-layout-s3-old.tar.gz", "*backend.S3Layout"},
{"repo-layout-default.tar.gz", "*backend.DefaultLayout"},
{"repo-layout-s3legacy.tar.gz", "*backend.S3LegacyLayout"},
}
var fs = &LocalFilesystem{}
@ -342,12 +341,11 @@ func TestParseLayout(t *testing.T) {
want string
}{
{"default", "", "*backend.DefaultLayout"},
{"cloud", "", "*backend.CloudLayout"},
{"s3", "", "*backend.S3Layout"},
{"", "", "*backend.CloudLayout"},
{"s3legacy", "", "*backend.S3LegacyLayout"},
{"", "", "*backend.DefaultLayout"},
}
SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-cloud.tar.gz"))
SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-default.tar.gz"))
for _, test := range tests {
t.Run(test.layoutName, func(t *testing.T) {

9
src/restic/backend/local/layout_test.go
View file

@ -17,17 +17,12 @@ func TestLayout(t *testing.T) {
failureExpected bool
datafiles map[string]bool
}{
{"repo-layout-local.tar.gz", "", false, map[string]bool{
{"repo-layout-default.tar.gz", "", false, map[string]bool{
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
}},
{"repo-layout-cloud.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
}},
{"repo-layout-s3-old.tar.gz", "", false, map[string]bool{
{"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,

2
src/restic/backend/rest/rest.go
View file

@ -48,7 +48,7 @@ func Open(cfg Config) (restic.Backend, error) {
url: cfg.URL,
connChan: connChan,
client: client,
Layout: &backend.CloudLayout{URL: url, Join: path.Join},
Layout: &backend.RESTLayout{URL: url, Join: path.Join},
}
return be, nil

6
src/restic/backend/s3/config.go
View file

@ -6,6 +6,7 @@ import (
"strings"
"restic/errors"
"restic/options"
)
// Config contains all configuration necessary to connect to an s3 compatible
@ -16,6 +17,11 @@ type Config struct {
KeyID, Secret string
Bucket string
Prefix string
Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"`
}
func init() {
options.Register("s3", Config{})
}
const defaultPrefix = "restic"

87
src/restic/backend/s3/s3.go
View file

@ -8,6 +8,7 @@ import (
"restic"
"strings"
"sync"
"time"
"restic/backend"
"restic/errors"
@ -30,6 +31,8 @@ type s3 struct {
backend.Layout
}
const defaultLayout = "s3legacy"
// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(cfg Config) (restic.Backend, error) {
@ -45,11 +48,17 @@ func Open(cfg Config) (restic.Backend, error) {
bucketname: cfg.Bucket,
prefix: cfg.Prefix,
cacheObjSize: make(map[string]int64),
Layout: &backend.S3Layout{Path: cfg.Prefix, Join: path.Join},
}
client.SetCustomTransport(backend.Transport())
l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
if err != nil {
return nil, err
}
be.Layout = l
be.createConnections()
found, err := client.BucketExists(cfg.Bucket)
@ -76,6 +85,77 @@ func (be *s3) createConnections() {
}
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *s3) IsNotExist(err error) bool {
debug.Log("IsNotExist(%T, %#v)", err, err)
if os.IsNotExist(err) {
return true
}
return false
}
// Join combines path components with slashes.
func (be *s3) Join(p ...string) string {
return path.Join(p...)
}
type fileInfo struct {
name string
size int64
mode os.FileMode
modTime time.Time
isDir bool
}
func (fi fileInfo) Name() string { return fi.name } // base name of the file
func (fi fileInfo) Size() int64 { return fi.size } // length in bytes for regular files; system-dependent for others
func (fi fileInfo) Mode() os.FileMode { return fi.mode } // file mode bits
func (fi fileInfo) ModTime() time.Time { return fi.modTime } // modification time
func (fi fileInfo) IsDir() bool { return fi.isDir } // abbreviation for Mode().IsDir()
func (fi fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil)
// ReadDir returns the entries for a directory.
func (be *s3) ReadDir(dir string) (list []os.FileInfo, err error) {
debug.Log("ReadDir(%v)", dir)
// make sure dir ends with a slash
if dir[len(dir)-1] != '/' {
dir += "/"
}
done := make(chan struct{})
defer close(done)
for obj := range be.client.ListObjects(be.bucketname, dir, false, done) {
if obj.Key == "" {
continue
}
name := strings.TrimPrefix(obj.Key, dir)
if name == "" {
return nil, errors.Errorf("invalid key name %v, removing prefix %v yielded empty string", obj.Key, dir)
}
entry := fileInfo{
name: name,
size: obj.Size,
modTime: obj.LastModified,
}
if name[len(name)-1] == '/' {
entry.isDir = true
entry.mode = os.ModeDir | 0755
entry.name = name[:len(name)-1]
} else {
entry.mode = 0644
}
list = append(list, entry)
}
return list, nil
}
// Location returns this backend's location (the bucket name).
func (be *s3) Location() string {
return be.bucketname
@ -290,6 +370,11 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
prefix := be.Dirname(restic.Handle{Type: t})
// make sure prefix ends with a slash
if prefix[len(prefix)-1] != '/' {
prefix += "/"
}
listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
go func() {

9
src/restic/backend/sftp/layout_test.go
View file

@ -23,17 +23,12 @@ func TestLayout(t *testing.T) {
failureExpected bool
datafiles map[string]bool
}{
{"repo-layout-local.tar.gz", "", false, map[string]bool{
{"repo-layout-default.tar.gz", "", false, map[string]bool{
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
}},
{"repo-layout-cloud.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,
}},
{"repo-layout-s3-old.tar.gz", "", false, map[string]bool{
{"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{
"fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false,
"c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false,
"aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false,

BIN
src/restic/backend/testdata/repo-layout-cloud.tar.gz vendored
View file

Binary file not shown.

0
src/restic/backend/testdata/repo-layout-local.tar.gz → src/restic/backend/testdata/repo-layout-default.tar.gz vendored
View file

0
src/restic/backend/testdata/repo-layout-s3-old.tar.gz → src/restic/backend/testdata/repo-layout-s3legacy.tar.gz vendored
View file