Merge pull request #443 from ckemper67/fix-387

Introduced a configurable object path prefix for s3 repositories to address #387

backend/s3/config.go

@@ -3,6 +3,7 @@ package s3
 import (
 	"errors"
 	"net/url"
+	"path"
 	"strings"
 )
 
@@ -13,54 +14,22 @@ type Config struct {
 	UseHTTP       bool
 	KeyID, Secret string
 	Bucket        string
+	Prefix        string
 }
 
+const defaultPrefix = "restic"
+
 // ParseConfig parses the string s and extracts the s3 config. The two
-// supported configuration formats are s3://host/bucketname and
+// supported configuration formats are s3://host/bucketname/prefix and
-// s3:host:bucketname. The host can also be a valid s3 region name.
+// s3:host:bucketname/prefix. The host can also be a valid s3 region
+// name. If no prefix is given the prefix "restic" will be used.
 func ParseConfig(s string) (interface{}, error) {
-	if strings.HasPrefix(s, "s3://") {
+	switch {
-		s = s[5:]
+	case strings.HasPrefix(s, "s3:http"):
-
+		// assume that a URL has been specified, parse it and
-		data := strings.SplitN(s, "/", 2)
+		// use the host as the endpoint and the path as the
-		if len(data) != 2 {
+		// bucket name and prefix
-			return nil, errors.New("s3: invalid format, host/region or bucket name not found")
+		url, err := url.Parse(s[3:])
-		}
-
-		cfg := Config{
-			Endpoint: data[0],
-			Bucket:   data[1],
-		}
-
-		return cfg, nil
-	}
-
-	data := strings.SplitN(s, ":", 2)
-	if len(data) != 2 {
-		return nil, errors.New("s3: invalid format")
-	}
-
-	if data[0] != "s3" {
-		return nil, errors.New(`s3: config does not start with "s3"`)
-	}
-
-	s = data[1]
-
-	cfg := Config{}
-	rest := strings.Split(s, "/")
-	if len(rest) < 2 {
-		return nil, errors.New("s3: region or bucket not found")
-	}
-
-	if len(rest) == 2 {
-		// assume that just a region name and a bucket has been specified, in
-		// the format region/bucket
-		cfg.Endpoint = rest[0]
-		cfg.Bucket = rest[1]
-	} else {
-		// assume that a URL has been specified, parse it and use the path as
-		// the bucket name.
-		url, err := url.Parse(s)
 		if err != nil {
 			return nil, err
 		}
@@ -69,13 +38,35 @@ func ParseConfig(s string) (interface{}, error) {
 			return nil, errors.New("s3: bucket name not found")
 		}
 
-		cfg.Endpoint = url.Host
+		path := strings.SplitN(url.Path[1:], "/", 2)
-		if url.Scheme == "http" {
+		return createConfig(url.Host, path, url.Scheme == "http")
-			cfg.UseHTTP = true
+	case strings.HasPrefix(s, "s3://"):
+		s = s[5:]
+	case strings.HasPrefix(s, "s3:"):
+		s = s[3:]
+	default:
+		return nil, errors.New("s3: invalid format")
 	}
-
+	// use the first entry of the path as the endpoint and the
-		cfg.Bucket = url.Path[1:]
+	// remainder as bucket name and prefix
-	}
+	path := strings.SplitN(s, "/", 3)
-
+	return createConfig(path[0], path[1:], false)
-	return cfg, nil
+}
+
+func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error) {
+	var prefix string
+	switch {
+	case len(p) < 1:
+		return nil, errors.New("s3: invalid format, host/region or bucket name not found")
+	case len(p) == 1 || p[1] == "":
+		prefix = defaultPrefix
+	default:
+		prefix = path.Clean(p[1])
+	}
+	return Config{
+		Endpoint: endpoint,
+		UseHTTP:  useHTTP,
+		Bucket:   p[0],
+		Prefix:   prefix,
+	}, nil
 }

backend/s3/config_test.go

@@ -9,18 +9,75 @@ var configTests = []struct {
 	{"s3://eu-central-1/bucketname", Config{
 		Endpoint: "eu-central-1",
 		Bucket:   "bucketname",
+		Prefix:   "restic",
+	}},
+	{"s3://eu-central-1/bucketname/", Config{
+		Endpoint: "eu-central-1",
+		Bucket:   "bucketname",
+		Prefix:   "restic",
+	}},
+	{"s3://eu-central-1/bucketname/prefix/directory", Config{
+		Endpoint: "eu-central-1",
+		Bucket:   "bucketname",
+		Prefix:   "prefix/directory",
+	}},
+	{"s3://eu-central-1/bucketname/prefix/directory/", Config{
+		Endpoint: "eu-central-1",
+		Bucket:   "bucketname",
+		Prefix:   "prefix/directory",
 	}},
 	{"s3:eu-central-1/foobar", Config{
 		Endpoint: "eu-central-1",
 		Bucket:   "foobar",
+		Prefix:   "restic",
+	}},
+	{"s3:eu-central-1/foobar/", Config{
+		Endpoint: "eu-central-1",
+		Bucket:   "foobar",
+		Prefix:   "restic",
+	}},
+	{"s3:eu-central-1/foobar/prefix/directory", Config{
+		Endpoint: "eu-central-1",
+		Bucket:   "foobar",
+		Prefix:   "prefix/directory",
+	}},
+	{"s3:eu-central-1/foobar/prefix/directory/", Config{
+		Endpoint: "eu-central-1",
+		Bucket:   "foobar",
+		Prefix:   "prefix/directory",
 	}},
 	{"s3:https://hostname:9999/foobar", Config{
 		Endpoint: "hostname:9999",
 		Bucket:   "foobar",
+		Prefix:   "restic",
+	}},
+	{"s3:https://hostname:9999/foobar/", Config{
+		Endpoint: "hostname:9999",
+		Bucket:   "foobar",
+		Prefix:   "restic",
 	}},
 	{"s3:http://hostname:9999/foobar", Config{
 		Endpoint: "hostname:9999",
 		Bucket:   "foobar",
+		Prefix:   "restic",
+		UseHTTP:  true,
+	}},
+	{"s3:http://hostname:9999/foobar/", Config{
+		Endpoint: "hostname:9999",
+		Bucket:   "foobar",
+		Prefix:   "restic",
+		UseHTTP:  true,
+	}},
+	{"s3:http://hostname:9999/bucket/prefix/directory", Config{
+		Endpoint: "hostname:9999",
+		Bucket:   "bucket",
+		Prefix:   "prefix/directory",
+		UseHTTP:  true,
+	}},
+	{"s3:http://hostname:9999/bucket/prefix/directory/", Config{
+		Endpoint: "hostname:9999",
+		Bucket:   "bucket",
+		Prefix:   "prefix/directory",
 		UseHTTP:  true,
 	}},
 }
@@ -29,13 +86,13 @@ func TestParseConfig(t *testing.T) {
 	for i, test := range configTests {
 		cfg, err := ParseConfig(test.s)
 		if err != nil {
-			t.Errorf("test %d failed: %v", i, err)
+			t.Errorf("test %d:%s failed: %v", i, test.s, err)
 			continue
 		}
 
 		if cfg != test.cfg {
-			t.Errorf("test %d: wrong config, want:\n  %v\ngot:\n  %v",
+			t.Errorf("test %d:\ninput:\n  %s\n wrong config, want:\n  %v\ngot:\n  %v",
-				i, test.cfg, cfg)
+				i, test.s, test.cfg, cfg)
 			continue
 		}
 	}

backend/s3/s3.go

@@ -13,20 +13,13 @@ import (
 )
 
 const connLimit = 10
-const backendPrefix = "restic"
-
-func s3path(t backend.Type, name string) string {
-	if t == backend.Config {
-		return backendPrefix + "/" + string(t)
-	}
-	return backendPrefix + "/" + string(t) + "/" + name
-}
 
 // s3 is a backend which stores the data on an S3 endpoint.
 type s3 struct {
 	client     minio.CloudStorageClient
 	connChan   chan struct{}
 	bucketname string
+	prefix     string
 }
 
 // Open opens the S3 backend at bucket and region. The bucket is created if it
@@ -39,7 +32,7 @@ func Open(cfg Config) (backend.Backend, error) {
 		return nil, err
 	}
 
-	be := &s3{client: client, bucketname: cfg.Bucket}
+	be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
 	be.createConnections()
 
 	if err := client.BucketExists(cfg.Bucket); err != nil {
@@ -56,6 +49,20 @@ func Open(cfg Config) (backend.Backend, error) {
 	return be, nil
 }
 
+func (be *s3) s3path(t backend.Type, name string) string {
+	var path string
+
+	if be.prefix != "" {
+		path = be.prefix + "/"
+	}
+	path += string(t)
+
+	if t == backend.Config {
+		return path
+	}
+	return path + "/" + name
+}
+
 func (be *s3) createConnections() {
 	be.connChan = make(chan struct{}, connLimit)
 	for i := 0; i < connLimit; i++ {
@@ -72,7 +79,7 @@ func (be *s3) Location() string {
 // and saves it in p. Load has the same semantics as io.ReaderAt.
 func (be s3) Load(h backend.Handle, p []byte, off int64) (int, error) {
 	debug.Log("s3.Load", "%v, offset %v, len %v", h, off, len(p))
-	path := s3path(h.Type, h.Name)
+	path := be.s3path(h.Type, h.Name)
 	obj, err := be.client.GetObject(be.bucketname, path)
 	if err != nil {
 		debug.Log("s3.GetReader", "  err %v", err)
@@ -101,7 +108,7 @@ func (be s3) Save(h backend.Handle, p []byte) (err error) {
 
 	debug.Log("s3.Save", "%v bytes at %d", len(p), h)
 
-	path := s3path(h.Type, h.Name)
+	path := be.s3path(h.Type, h.Name)
 
 	// Check key does not already exist
 	_, err = be.client.StatObject(be.bucketname, path)
@@ -126,7 +133,7 @@ func (be s3) Save(h backend.Handle, p []byte) (err error) {
 // Stat returns information about a blob.
 func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) {
 	debug.Log("s3.Stat", "%v")
-	path := s3path(h.Type, h.Name)
+	path := be.s3path(h.Type, h.Name)
 	obj, err := be.client.GetObject(be.bucketname, path)
 	if err != nil {
 		debug.Log("s3.Stat", "GetObject() err %v", err)
@@ -145,7 +152,7 @@ func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) {
 // Test returns true if a blob of the given type and name exists in the backend.
 func (be *s3) Test(t backend.Type, name string) (bool, error) {
 	found := false
-	path := s3path(t, name)
+	path := be.s3path(t, name)
 	_, err := be.client.StatObject(be.bucketname, path)
 	if err == nil {
 		found = true
@@ -157,7 +164,7 @@ func (be *s3) Test(t backend.Type, name string) (bool, error) {
 
 // Remove removes the blob with the given name and type.
 func (be *s3) Remove(t backend.Type, name string) error {
-	path := s3path(t, name)
+	path := be.s3path(t, name)
 	err := be.client.RemoveObject(be.bucketname, path)
 	debug.Log("s3.Remove", "%v %v -> err %v", t, name, err)
 	return err
@@ -170,7 +177,7 @@ func (be *s3) List(t backend.Type, done <-chan struct{}) <-chan string {
 	debug.Log("s3.List", "listing %v", t)
 	ch := make(chan string)
 
-	prefix := s3path(t, "")
+	prefix := be.s3path(t, "")
 
 	listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
 

location/location_test.go

@@ -48,30 +48,56 @@ var parseTests = []struct {
 		Config: s3.Config{
 			Endpoint: "eu-central-1",
 			Bucket:   "bucketname",
+			Prefix:   "restic",
 		}},
 	},
 	{"s3://hostname.foo/bucketname", Location{Scheme: "s3",
 		Config: s3.Config{
 			Endpoint: "hostname.foo",
 			Bucket:   "bucketname",
+			Prefix:   "restic",
+		}},
+	},
+	{"s3://hostname.foo/bucketname/prefix/directory", Location{Scheme: "s3",
+		Config: s3.Config{
+			Endpoint: "hostname.foo",
+			Bucket:   "bucketname",
+			Prefix:   "prefix/directory",
 		}},
 	},
 	{"s3:eu-central-1/repo", Location{Scheme: "s3",
 		Config: s3.Config{
 			Endpoint: "eu-central-1",
 			Bucket:   "repo",
+			Prefix:   "restic",
+		}},
+	},
+	{"s3:eu-central-1/repo/prefix/directory", Location{Scheme: "s3",
+		Config: s3.Config{
+			Endpoint: "eu-central-1",
+			Bucket:   "repo",
+			Prefix:   "prefix/directory",
 		}},
 	},
 	{"s3:https://hostname.foo/repo", Location{Scheme: "s3",
 		Config: s3.Config{
 			Endpoint: "hostname.foo",
 			Bucket:   "repo",
+			Prefix:   "restic",
+		}},
+	},
+	{"s3:https://hostname.foo/repo/prefix/directory", Location{Scheme: "s3",
+		Config: s3.Config{
+			Endpoint: "hostname.foo",
+			Bucket:   "repo",
+			Prefix:   "prefix/directory",
 		}},
 	},
 	{"s3:http://hostname.foo/repo", Location{Scheme: "s3",
 		Config: s3.Config{
 			Endpoint: "hostname.foo",
 			Bucket:   "repo",
+			Prefix:   "restic",
 			UseHTTP:  true,
 		}},
 	},