forked from TrueCloudLab/restic
s3: implement open with config
This commit is contained in:
parent
f7c9091970
commit
2b0b44c5ce
4 changed files with 72 additions and 44 deletions
|
@ -2,13 +2,15 @@ package s3
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config contains all configuration necessary to connect to an s3 compatible
|
// Config contains all configuration necessary to connect to an s3 compatible
|
||||||
// server.
|
// server.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Host string
|
Region string
|
||||||
|
URL string
|
||||||
KeyID, Secret string
|
KeyID, Secret string
|
||||||
Bucket string
|
Bucket string
|
||||||
}
|
}
|
||||||
|
@ -26,15 +28,15 @@ func ParseConfig(s string) (interface{}, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := Config{
|
cfg := Config{
|
||||||
Host: data[0],
|
Region: data[0],
|
||||||
Bucket: data[1],
|
Bucket: data[1],
|
||||||
}
|
}
|
||||||
|
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
data := strings.SplitN(s, ":", 3)
|
data := strings.SplitN(s, ":", 2)
|
||||||
if len(data) != 3 {
|
if len(data) != 2 {
|
||||||
return nil, errors.New("s3: invalid format")
|
return nil, errors.New("s3: invalid format")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -42,9 +44,35 @@ func ParseConfig(s string) (interface{}, error) {
|
||||||
return nil, errors.New(`s3: config does not start with "s3"`)
|
return nil, errors.New(`s3: config does not start with "s3"`)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := Config{
|
s = data[1]
|
||||||
Host: data[1],
|
|
||||||
Bucket: data[2],
|
cfg := Config{}
|
||||||
|
rest := strings.Split(s, "/")
|
||||||
|
if len(rest) < 2 {
|
||||||
|
return nil, errors.New("s3: region or bucket not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(rest) == 2 {
|
||||||
|
// assume that just a region name and a bucket has been specified, in
|
||||||
|
// the format region/bucket
|
||||||
|
cfg.Region = rest[0]
|
||||||
|
cfg.Bucket = rest[1]
|
||||||
|
} else {
|
||||||
|
// assume that a URL has been specified, parse it and use the path as
|
||||||
|
// the bucket name.
|
||||||
|
url, err := url.Parse(s)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if url.Path == "" {
|
||||||
|
return nil, errors.New("s3: bucket name not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.Bucket = url.Path[1:]
|
||||||
|
url.Path = ""
|
||||||
|
|
||||||
|
cfg.URL = url.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
|
|
|
@ -7,11 +7,19 @@ var configTests = []struct {
|
||||||
cfg Config
|
cfg Config
|
||||||
}{
|
}{
|
||||||
{"s3://eu-central-1/bucketname", Config{
|
{"s3://eu-central-1/bucketname", Config{
|
||||||
Host: "eu-central-1",
|
Region: "eu-central-1",
|
||||||
Bucket: "bucketname",
|
Bucket: "bucketname",
|
||||||
}},
|
}},
|
||||||
{"s3:hostname:foobar", Config{
|
{"s3:eu-central-1/foobar", Config{
|
||||||
Host: "hostname",
|
Region: "eu-central-1",
|
||||||
|
Bucket: "foobar",
|
||||||
|
}},
|
||||||
|
{"s3:https://hostname:9999/foobar", Config{
|
||||||
|
URL: "https://hostname:9999",
|
||||||
|
Bucket: "foobar",
|
||||||
|
}},
|
||||||
|
{"s3:http://hostname:9999/foobar", Config{
|
||||||
|
URL: "http://hostname:9999",
|
||||||
Bucket: "foobar",
|
Bucket: "foobar",
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,7 +4,6 @@ import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/minio/minio-go"
|
"github.com/minio/minio-go"
|
||||||
|
@ -29,45 +28,32 @@ type S3Backend struct {
|
||||||
bucketname string
|
bucketname string
|
||||||
}
|
}
|
||||||
|
|
||||||
func getConfig(region, bucket string) minio.Config {
|
|
||||||
config := minio.Config{
|
|
||||||
AccessKeyID: os.Getenv("AWS_ACCESS_KEY_ID"),
|
|
||||||
SecretAccessKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
|
|
||||||
Region: "us-east-1",
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.Contains(region, ".") {
|
|
||||||
// Amazon region name
|
|
||||||
switch region {
|
|
||||||
case "us-east-1":
|
|
||||||
config.Endpoint = "https://s3.amazonaws.com"
|
|
||||||
default:
|
|
||||||
config.Endpoint = "https://s3-" + region + ".amazonaws.com"
|
|
||||||
config.Region = region
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// S3 compatible endpoint, use default region "us-east-1"
|
|
||||||
if strings.Contains(region, "localhost") || strings.Contains(region, "127.0.0.1") {
|
|
||||||
config.Endpoint = "http://" + region
|
|
||||||
} else {
|
|
||||||
config.Endpoint = "https://" + region
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return config
|
|
||||||
}
|
|
||||||
|
|
||||||
// Open opens the S3 backend at bucket and region. The bucket is created if it does not exist yet.
|
// Open opens the S3 backend at bucket and region. The bucket is created if it does not exist yet.
|
||||||
func Open(regionname, bucketname string) (backend.Backend, error) {
|
func Open(cfg Config) (backend.Backend, error) {
|
||||||
s3api, err := minio.New(getConfig(regionname, bucketname))
|
mcfg := minio.Config{
|
||||||
|
AccessKeyID: cfg.KeyID,
|
||||||
|
SecretAccessKey: cfg.Secret,
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.URL != "" {
|
||||||
|
mcfg.Endpoint = cfg.URL
|
||||||
|
} else {
|
||||||
|
mcfg.Region = cfg.Region
|
||||||
|
}
|
||||||
|
|
||||||
|
if mcfg.Region == "" {
|
||||||
|
mcfg.Region = "us-east-1"
|
||||||
|
}
|
||||||
|
|
||||||
|
s3api, err := minio.New(mcfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
be := &S3Backend{s3api: s3api, bucketname: bucketname}
|
be := &S3Backend{s3api: s3api, bucketname: cfg.Bucket}
|
||||||
be.createConnections()
|
be.createConnections()
|
||||||
|
|
||||||
err = s3api.MakeBucket(bucketname, "")
|
err = s3api.MakeBucket(cfg.Bucket, "")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
package backend_test
|
package backend_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/restic/restic/backend/s3"
|
"github.com/restic/restic/backend/s3"
|
||||||
|
@ -16,7 +17,12 @@ func TestS3Backend(t *testing.T) {
|
||||||
t.Skip("s3 test server not available")
|
t.Skip("s3 test server not available")
|
||||||
}
|
}
|
||||||
|
|
||||||
be, err := s3.Open(TestS3Server, "restictestbucket")
|
be, err := s3.Open(s3.Config{
|
||||||
|
URL: TestS3Server,
|
||||||
|
Bucket: "restictestbucket",
|
||||||
|
KeyID: os.Getenv("AWS_ACCESS_KEY_ID"),
|
||||||
|
Secret: os.Getenv("AWS_SECRET_ACCESS_KEY"),
|
||||||
|
})
|
||||||
OK(t, err)
|
OK(t, err)
|
||||||
|
|
||||||
testBackend(be, t)
|
testBackend(be, t)
|
||||||
|
|
Loading…
Reference in a new issue