s3: implement open with config

This commit is contained in:
Alexander Neumann 2015-12-28 18:23:02 +01:00
parent f7c9091970
commit 2b0b44c5ce
4 changed files with 72 additions and 44 deletions

View file

@ -2,13 +2,15 @@ package s3
import ( import (
"errors" "errors"
"net/url"
"strings" "strings"
) )
// Config contains all configuration necessary to connect to an s3 compatible // Config contains all configuration necessary to connect to an s3 compatible
// server. // server.
type Config struct { type Config struct {
Host string Region string
URL string
KeyID, Secret string KeyID, Secret string
Bucket string Bucket string
} }
@ -26,15 +28,15 @@ func ParseConfig(s string) (interface{}, error) {
} }
cfg := Config{ cfg := Config{
Host: data[0], Region: data[0],
Bucket: data[1], Bucket: data[1],
} }
return cfg, nil return cfg, nil
} }
data := strings.SplitN(s, ":", 3) data := strings.SplitN(s, ":", 2)
if len(data) != 3 { if len(data) != 2 {
return nil, errors.New("s3: invalid format") return nil, errors.New("s3: invalid format")
} }
@ -42,9 +44,35 @@ func ParseConfig(s string) (interface{}, error) {
return nil, errors.New(`s3: config does not start with "s3"`) return nil, errors.New(`s3: config does not start with "s3"`)
} }
cfg := Config{ s = data[1]
Host: data[1],
Bucket: data[2], cfg := Config{}
rest := strings.Split(s, "/")
if len(rest) < 2 {
return nil, errors.New("s3: region or bucket not found")
}
if len(rest) == 2 {
// assume that just a region name and a bucket has been specified, in
// the format region/bucket
cfg.Region = rest[0]
cfg.Bucket = rest[1]
} else {
// assume that a URL has been specified, parse it and use the path as
// the bucket name.
url, err := url.Parse(s)
if err != nil {
return nil, err
}
if url.Path == "" {
return nil, errors.New("s3: bucket name not found")
}
cfg.Bucket = url.Path[1:]
url.Path = ""
cfg.URL = url.String()
} }
return cfg, nil return cfg, nil

View file

@ -7,11 +7,19 @@ var configTests = []struct {
cfg Config cfg Config
}{ }{
{"s3://eu-central-1/bucketname", Config{ {"s3://eu-central-1/bucketname", Config{
Host: "eu-central-1", Region: "eu-central-1",
Bucket: "bucketname", Bucket: "bucketname",
}}, }},
{"s3:hostname:foobar", Config{ {"s3:eu-central-1/foobar", Config{
Host: "hostname", Region: "eu-central-1",
Bucket: "foobar",
}},
{"s3:https://hostname:9999/foobar", Config{
URL: "https://hostname:9999",
Bucket: "foobar",
}},
{"s3:http://hostname:9999/foobar", Config{
URL: "http://hostname:9999",
Bucket: "foobar", Bucket: "foobar",
}}, }},
} }

View file

@ -4,7 +4,6 @@ import (
"bytes" "bytes"
"errors" "errors"
"io" "io"
"os"
"strings" "strings"
"github.com/minio/minio-go" "github.com/minio/minio-go"
@ -29,45 +28,32 @@ type S3Backend struct {
bucketname string bucketname string
} }
func getConfig(region, bucket string) minio.Config {
config := minio.Config{
AccessKeyID: os.Getenv("AWS_ACCESS_KEY_ID"),
SecretAccessKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
Region: "us-east-1",
}
if !strings.Contains(region, ".") {
// Amazon region name
switch region {
case "us-east-1":
config.Endpoint = "https://s3.amazonaws.com"
default:
config.Endpoint = "https://s3-" + region + ".amazonaws.com"
config.Region = region
}
} else {
// S3 compatible endpoint, use default region "us-east-1"
if strings.Contains(region, "localhost") || strings.Contains(region, "127.0.0.1") {
config.Endpoint = "http://" + region
} else {
config.Endpoint = "https://" + region
}
}
return config
}
// Open opens the S3 backend at bucket and region. The bucket is created if it does not exist yet. // Open opens the S3 backend at bucket and region. The bucket is created if it does not exist yet.
func Open(regionname, bucketname string) (backend.Backend, error) { func Open(cfg Config) (backend.Backend, error) {
s3api, err := minio.New(getConfig(regionname, bucketname)) mcfg := minio.Config{
AccessKeyID: cfg.KeyID,
SecretAccessKey: cfg.Secret,
}
if cfg.URL != "" {
mcfg.Endpoint = cfg.URL
} else {
mcfg.Region = cfg.Region
}
if mcfg.Region == "" {
mcfg.Region = "us-east-1"
}
s3api, err := minio.New(mcfg)
if err != nil { if err != nil {
return nil, err return nil, err
} }
be := &S3Backend{s3api: s3api, bucketname: bucketname} be := &S3Backend{s3api: s3api, bucketname: cfg.Bucket}
be.createConnections() be.createConnections()
err = s3api.MakeBucket(bucketname, "") err = s3api.MakeBucket(cfg.Bucket, "")
if err != nil { if err != nil {
return nil, err return nil, err
} }

View file

@ -1,6 +1,7 @@
package backend_test package backend_test
import ( import (
"os"
"testing" "testing"
"github.com/restic/restic/backend/s3" "github.com/restic/restic/backend/s3"
@ -16,7 +17,12 @@ func TestS3Backend(t *testing.T) {
t.Skip("s3 test server not available") t.Skip("s3 test server not available")
} }
be, err := s3.Open(TestS3Server, "restictestbucket") be, err := s3.Open(s3.Config{
URL: TestS3Server,
Bucket: "restictestbucket",
KeyID: os.Getenv("AWS_ACCESS_KEY_ID"),
Secret: os.Getenv("AWS_SECRET_ACCESS_KEY"),
})
OK(t, err) OK(t, err)
testBackend(be, t) testBackend(be, t)