From 7dcac52f18ba7ea70d31e4de4abfedb170432681 Mon Sep 17 00:00:00 2001 From: Fabio Berchtold Date: Thu, 1 Sep 2016 22:52:40 +0200 Subject: [PATCH] Add v2 signature signing to S3 storage driver (#1800) * Add v2 signature signing to S3 storage driver Closes #1796 Closes #1606 Signed-off-by: Fabio Berchtold * use Logrus for debug logging Signed-off-by: Fabio Berchtold * use 'date' instead of 'x-amz-date' in request header Signed-off-by: Fabio Berchtold * only allow v4 signature signing against AWS S3 Signed-off-by: Fabio Berchtold --- docs/storage-drivers/s3.md | 11 +- registry/storage/driver/s3-aws/s3.go | 81 +++++-- registry/storage/driver/s3-aws/s3_test.go | 14 ++ .../storage/driver/s3-aws/s3_v2_signer.go | 212 ++++++++++++++++++ 4 files changed, 299 insertions(+), 19 deletions(-) create mode 100644 registry/storage/driver/s3-aws/s3_v2_signer.go diff --git a/docs/storage-drivers/s3.md b/docs/storage-drivers/s3.md index 6bcb8a20..30187db4 100644 --- a/docs/storage-drivers/s3.md +++ b/docs/storage-drivers/s3.md @@ -122,8 +122,11 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz Indicates whether the registry uses Version 4 of AWS's authentication. - Generally, you should set this to true. By default, this is - false. + Generally, you should set this to true unless you are using an + S3 compatible provider that does not support v4 signature signing. + If you set this to false then the storage driver will use v2 signature signing. + By default, this is true. + You can not use v2 signing if you are using AWS S3. @@ -228,13 +231,13 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz `secure`: (optional) Whether you would like to transfer data to the bucket over ssl or not. Defaults to true (meaning transferring over ssl) if not specified. Note that while setting this to false will improve performance, it is not recommended due to security concerns. -`v4auth`: (optional) Whether you would like to use aws signature version 4 with your requests. This defaults to false if not specified (note that the eu-central-1 region does not work with version 2 signatures, so the driver will error out if initialized with this region and v4auth set to false) +`v4auth`: (optional) Whether you would like to use aws signature version 4 with your requests. This defaults to true if not specified (note that the eu-central-1 region does not work with version 2 signatures, so the driver will error out if initialized with this region and v4auth set to false) `chunksize`: (optional) The default part size for multipart uploads (performed by WriteStream) to S3. The default is 10 MB. Keep in mind that the minimum part size for S3 is 5MB. Depending on the speed of your connection to S3, a larger chunk size may result in better performance; faster connections will benefit from larger chunk sizes. `rootdirectory`: (optional) The root directory tree in which all registry files will be stored. Defaults to the empty string (bucket root). -`storageclass`: (optional) The storage class applied to each registry file. Defaults to STANDARD. Valid options are STANDARD and REDUCED_REDUNDANCY. +`storageclass`: (optional) The storage class applied to each registry file. Defaults to STANDARD. Valid options are NONE, STANDARD and REDUCED_REDUNDANCY. Use NONE if your S3 compatible provider does not support storage classes. `objectacl`: (optional) The canned object ACL to be applied to each registry object. Defaults to `private`. If you are using a bucket owned by another AWS account, it is recommended that you set this to `bucket-owner-full-control` so that the bucket owner can access your objects. Other valid options are available in the [AWS S3 documentation](http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl). diff --git a/registry/storage/driver/s3-aws/s3.go b/registry/storage/driver/s3-aws/s3.go index 9d2cac68..e808f760 100644 --- a/registry/storage/driver/s3-aws/s3.go +++ b/registry/storage/driver/s3-aws/s3.go @@ -70,6 +70,9 @@ const ( // listMax is the largest amount of objects you can request from S3 in a list call const listMax = 1000 +// noStorageClass defines the value to be used if storage class is not supported by the S3 endpoint +const noStorageClass = "NONE" + // validRegions maps known s3 region identifiers to region descriptors var validRegions = map[string]struct{}{} @@ -86,6 +89,7 @@ type DriverParameters struct { Encrypt bool KeyID string Secure bool + V4Auth bool ChunkSize int64 MultipartCopyChunkSize int64 MultipartCopyMaxConcurrency int64 @@ -238,6 +242,23 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { return nil, fmt.Errorf("The secure parameter should be a boolean") } + v4Bool := true + v4auth := parameters["v4auth"] + switch v4auth := v4auth.(type) { + case string: + b, err := strconv.ParseBool(v4auth) + if err != nil { + return nil, fmt.Errorf("The v4auth parameter should be a boolean") + } + v4Bool = b + case bool: + v4Bool = v4auth + case nil: + // do nothing + default: + return nil, fmt.Errorf("The v4auth parameter should be a boolean") + } + keyID := parameters["keyid"] if keyID == nil { keyID = "" @@ -273,12 +294,16 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { if storageClassParam != nil { storageClassString, ok := storageClassParam.(string) if !ok { - return nil, fmt.Errorf("The storageclass parameter must be one of %v, %v invalid", []string{s3.StorageClassStandard, s3.StorageClassReducedRedundancy}, storageClassParam) + return nil, fmt.Errorf("The storageclass parameter must be one of %v, %v invalid", + []string{s3.StorageClassStandard, s3.StorageClassReducedRedundancy}, storageClassParam) } // All valid storage class parameters are UPPERCASE, so be a bit more flexible here storageClassString = strings.ToUpper(storageClassString) - if storageClassString != s3.StorageClassStandard && storageClassString != s3.StorageClassReducedRedundancy { - return nil, fmt.Errorf("The storageclass parameter must be one of %v, %v invalid", []string{s3.StorageClassStandard, s3.StorageClassReducedRedundancy}, storageClassParam) + if storageClassString != noStorageClass && + storageClassString != s3.StorageClassStandard && + storageClassString != s3.StorageClassReducedRedundancy { + return nil, fmt.Errorf("The storageclass parameter must be one of %v, %v invalid", + []string{noStorageClass, s3.StorageClassStandard, s3.StorageClassReducedRedundancy}, storageClassParam) } storageClass = storageClassString } @@ -311,6 +336,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) { encryptBool, fmt.Sprint(keyID), secureBool, + v4Bool, chunkSize, multipartCopyChunkSize, multipartCopyMaxConcurrency, @@ -356,22 +382,39 @@ func getParameterAsInt64(parameters map[string]interface{}, name string, default // New constructs a new Driver with the given AWS credentials, region, encryption flag, and // bucketName func New(params DriverParameters) (*Driver, error) { + if !params.V4Auth && + (params.RegionEndpoint == "" || + strings.Contains(params.RegionEndpoint, "s3.amazonaws.com")) { + return nil, fmt.Errorf("On Amazon S3 this storage driver can only be used with v4 authentication") + } + awsConfig := aws.NewConfig() - if params.RegionEndpoint != "" { + var creds *credentials.Credentials + if params.RegionEndpoint == "" { + creds = credentials.NewChainCredentials([]credentials.Provider{ + &credentials.StaticProvider{ + Value: credentials.Value{ + AccessKeyID: params.AccessKey, + SecretAccessKey: params.SecretKey, + }, + }, + &credentials.EnvProvider{}, + &credentials.SharedCredentialsProvider{}, + &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(session.New())}, + }) + } else { + creds = credentials.NewChainCredentials([]credentials.Provider{ + &credentials.StaticProvider{ + Value: credentials.Value{ + AccessKeyID: params.AccessKey, + SecretAccessKey: params.SecretKey, + }, + }, + &credentials.EnvProvider{}, + }) awsConfig.WithS3ForcePathStyle(true) awsConfig.WithEndpoint(params.RegionEndpoint) } - creds := credentials.NewChainCredentials([]credentials.Provider{ - &credentials.StaticProvider{ - Value: credentials.Value{ - AccessKeyID: params.AccessKey, - SecretAccessKey: params.SecretKey, - }, - }, - &credentials.EnvProvider{}, - &credentials.SharedCredentialsProvider{}, - &ec2rolecreds.EC2RoleProvider{Client: ec2metadata.New(session.New())}, - }) awsConfig.WithCredentials(creds) awsConfig.WithRegion(params.Region) @@ -385,6 +428,11 @@ func New(params DriverParameters) (*Driver, error) { s3obj := s3.New(session.New(awsConfig)) + // enable S3 compatible signature v2 signing instead + if !params.V4Auth { + setv2Handlers(s3obj) + } + // TODO Currently multipart uploads have no timestamps, so this would be unwise // if you initiated a new s3driver while another one is running on the same bucket. // multis, _, err := bucket.ListMulti("", "") @@ -868,6 +916,9 @@ func (d *driver) getACL() *string { } func (d *driver) getStorageClass() *string { + if d.StorageClass == noStorageClass { + return nil + } return aws.String(d.StorageClass) } diff --git a/registry/storage/driver/s3-aws/s3_test.go b/registry/storage/driver/s3-aws/s3_test.go index a49c21c7..16c579cb 100644 --- a/registry/storage/driver/s3-aws/s3_test.go +++ b/registry/storage/driver/s3-aws/s3_test.go @@ -31,6 +31,7 @@ func init() { encrypt := os.Getenv("S3_ENCRYPT") keyID := os.Getenv("S3_KEY_ID") secure := os.Getenv("S3_SECURE") + v4Auth := os.Getenv("S3_V4_AUTH") region := os.Getenv("AWS_REGION") objectAcl := os.Getenv("S3_OBJECT_ACL") root, err := ioutil.TempDir("", "driver-") @@ -57,6 +58,14 @@ func init() { } } + v4Bool := true + if v4Auth != "" { + v4Bool, err = strconv.ParseBool(v4Auth) + if err != nil { + return nil, err + } + } + parameters := DriverParameters{ accessKey, secretKey, @@ -66,6 +75,7 @@ func init() { encryptBool, keyID, secureBool, + v4Bool, minChunkSize, defaultMultipartCopyChunkSize, defaultMultipartCopyMaxConcurrency, @@ -163,6 +173,10 @@ func TestStorageClass(t *testing.T) { t.Fatalf("unexpected error creating driver with reduced redundancy storage: %v", err) } + if _, err = s3DriverConstructor(rootDir, noStorageClass); err != nil { + t.Fatalf("unexpected error creating driver without storage class: %v", err) + } + standardFilename := "/test-standard" rrFilename := "/test-rr" contents := []byte("contents") diff --git a/registry/storage/driver/s3-aws/s3_v2_signer.go b/registry/storage/driver/s3-aws/s3_v2_signer.go new file mode 100644 index 00000000..6950f1bc --- /dev/null +++ b/registry/storage/driver/s3-aws/s3_v2_signer.go @@ -0,0 +1,212 @@ +package s3 + +// Source: https://github.com/pivotal-golang/s3cli + +// Copyright (c) 2013 Damien Le Berrigaud and Nick Wade + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "net/http" + "net/url" + "sort" + "strings" + "time" + + log "github.com/Sirupsen/logrus" + "github.com/aws/aws-sdk-go/aws/corehandlers" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/s3" +) + +const ( + signatureVersion = "2" + signatureMethod = "HmacSHA1" + timeFormat = "2006-01-02T15:04:05Z" +) + +type signer struct { + // Values that must be populated from the request + Request *http.Request + Time time.Time + Credentials *credentials.Credentials + Query url.Values + stringToSign string + signature string +} + +var s3ParamsToSign = map[string]bool{ + "acl": true, + "location": true, + "logging": true, + "notification": true, + "partNumber": true, + "policy": true, + "requestPayment": true, + "torrent": true, + "uploadId": true, + "uploads": true, + "versionId": true, + "versioning": true, + "versions": true, + "response-content-type": true, + "response-content-language": true, + "response-expires": true, + "response-cache-control": true, + "response-content-disposition": true, + "response-content-encoding": true, + "website": true, + "delete": true, +} + +// setv2Handlers will setup v2 signature signing on the S3 driver +func setv2Handlers(svc *s3.S3) { + svc.Handlers.Build.PushBack(func(r *request.Request) { + parsedURL, err := url.Parse(r.HTTPRequest.URL.String()) + if err != nil { + log.Fatalf("Failed to parse URL: %v", err) + } + r.HTTPRequest.URL.Opaque = parsedURL.Path + }) + + svc.Handlers.Sign.Clear() + svc.Handlers.Sign.PushBack(Sign) + svc.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler) +} + +// Sign requests with signature version 2. +// +// Will sign the requests with the service config's Credentials object +// Signing is skipped if the credentials is the credentials.AnonymousCredentials +// object. +func Sign(req *request.Request) { + // If the request does not need to be signed ignore the signing of the + // request if the AnonymousCredentials object is used. + if req.Config.Credentials == credentials.AnonymousCredentials { + return + } + + v2 := signer{ + Request: req.HTTPRequest, + Time: req.Time, + Credentials: req.Config.Credentials, + } + v2.Sign() +} + +func (v2 *signer) Sign() error { + credValue, err := v2.Credentials.Get() + if err != nil { + return err + } + accessKey := credValue.AccessKeyID + var ( + md5, ctype, date, xamz string + xamzDate bool + sarray []string + ) + + headers := v2.Request.Header + params := v2.Request.URL.Query() + parsedURL, err := url.Parse(v2.Request.URL.String()) + if err != nil { + return err + } + host, canonicalPath := parsedURL.Host, parsedURL.Path + v2.Request.Header["Host"] = []string{host} + v2.Request.Header["date"] = []string{v2.Time.In(time.UTC).Format(time.RFC1123)} + + for k, v := range headers { + k = strings.ToLower(k) + switch k { + case "content-md5": + md5 = v[0] + case "content-type": + ctype = v[0] + case "date": + if !xamzDate { + date = v[0] + } + default: + if strings.HasPrefix(k, "x-amz-") { + vall := strings.Join(v, ",") + sarray = append(sarray, k+":"+vall) + if k == "x-amz-date" { + xamzDate = true + date = "" + } + } + } + } + if len(sarray) > 0 { + sort.StringSlice(sarray).Sort() + xamz = strings.Join(sarray, "\n") + "\n" + } + + expires := false + if v, ok := params["Expires"]; ok { + expires = true + date = v[0] + params["AWSAccessKeyId"] = []string{accessKey} + } + + sarray = sarray[0:0] + for k, v := range params { + if s3ParamsToSign[k] { + for _, vi := range v { + if vi == "" { + sarray = append(sarray, k) + } else { + sarray = append(sarray, k+"="+vi) + } + } + } + } + if len(sarray) > 0 { + sort.StringSlice(sarray).Sort() + canonicalPath = canonicalPath + "?" + strings.Join(sarray, "&") + } + + v2.stringToSign = strings.Join([]string{ + v2.Request.Method, + md5, + ctype, + date, + xamz + canonicalPath, + }, "\n") + hash := hmac.New(sha1.New, []byte(credValue.SecretAccessKey)) + hash.Write([]byte(v2.stringToSign)) + v2.signature = base64.StdEncoding.EncodeToString(hash.Sum(nil)) + + if expires { + params["Signature"] = []string{string(v2.signature)} + } else { + headers["Authorization"] = []string{"AWS " + accessKey + ":" + string(v2.signature)} + } + + log.WithFields(log.Fields{ + "string-to-sign": v2.stringToSign, + "signature": v2.signature, + }).Debugln("request signature") + return nil +}