diff --git a/src/restic/backend/layout_s3.go b/src/restic/backend/layout_s3.go index 9db8ae7f3..91273d63a 100644 --- a/src/restic/backend/layout_s3.go +++ b/src/restic/backend/layout_s3.go @@ -18,13 +18,29 @@ var s3LayoutPaths = map[restic.FileType]string{ restic.KeyFile: "key", } +// join calls Join with the first empty elements removed. +func (l *S3Layout) join(url string, items ...string) string { + for len(items) > 0 && items[0] == "" { + items = items[1:] + } + + path := l.Join(items...) + if path == "" || path[0] != '/' { + if url != "" && url[len(url)-1] != '/' { + url += "/" + } + } + + return url + path +} + // Dirname returns the directory path for a given file type and name. func (l *S3Layout) Dirname(h restic.Handle) string { if h.Type == restic.ConfigFile { return l.URL + l.Join(l.Path, "/") } - return l.URL + l.Join(l.Path, "/", s3LayoutPaths[h.Type]) + "/" + return l.join(l.URL, l.Path, s3LayoutPaths[h.Type]) + "/" } // Filename returns a path to a file, including its name. @@ -35,7 +51,7 @@ func (l *S3Layout) Filename(h restic.Handle) string { name = "config" } - return l.URL + l.Join(l.Path, "/", s3LayoutPaths[h.Type], name) + return l.join(l.URL, l.Path, s3LayoutPaths[h.Type], name) } // Paths returns all directory names diff --git a/src/restic/backend/layout_test.go b/src/restic/backend/layout_test.go index 4fee713cb..606da3241 100644 --- a/src/restic/backend/layout_test.go +++ b/src/restic/backend/layout_test.go @@ -190,6 +190,24 @@ func TestCloudLayoutURLs(t *testing.T) { "https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/", }, + { + &S3Layout{URL: "", Path: "", Join: path.Join}, + restic.Handle{Type: restic.DataFile, Name: "foobar"}, + "data/foobar", + "data/", + }, + { + &S3Layout{URL: "", Path: "", Join: path.Join}, + restic.Handle{Type: restic.LockFile, Name: "foobar"}, + "lock/foobar", + "lock/", + }, + { + &S3Layout{URL: "", Path: "/", Join: path.Join}, + restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + "/config", + "/", + }, } for _, test := range tests { diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index f59a92a15..2f6db84f5 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -17,7 +17,7 @@ import ( "restic/debug" ) -const connLimit = 40 +const connLimit = 10 // s3 is a backend which stores the data on an S3 endpoint. type s3 struct { @@ -45,7 +45,7 @@ func Open(cfg Config) (restic.Backend, error) { bucketname: cfg.Bucket, prefix: cfg.Prefix, cacheObjSize: make(map[string]int64), - Layout: &backend.S3Layout{URL: cfg.Endpoint, Join: path.Join}, + Layout: &backend.S3Layout{Path: cfg.Prefix, Join: path.Join}, } tr := &http.Transport{MaxIdleConnsPerHost: connLimit} @@ -88,10 +88,10 @@ func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) { return err } - debug.Log("Save %v", h) - objName := be.Filename(h) + debug.Log("Save %v at %v", h, objName) + // Check key does not already exist _, err = be.client.StatObject(be.bucketname, objName) if err == nil { @@ -128,7 +128,7 @@ func (wr wrapReader) Close() error { // given offset. If length is nonzero, only a portion of the file is // returned. rd must be closed after use. func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v", h, length, offset) + debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) if err := h.Valid(); err != nil { return nil, err } @@ -280,7 +280,7 @@ func (be *s3) Test(h restic.Handle) (bool, error) { func (be *s3) Remove(h restic.Handle) error { objName := be.Filename(h) err := be.client.RemoveObject(be.bucketname, objName) - debug.Log("Remove(%v) -> err %v", h, err) + debug.Log("Remove(%v) at %v -> err %v", h, objName, err) return errors.Wrap(err, "client.RemoveObject") }