From 9bd118665470d85de4457294606accd5e80e9dc9 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Wed, 22 Apr 2015 12:32:59 -0700 Subject: [PATCH] Update goamz package dependency Signed-off-by: Stephen J Day --- Godeps/Godeps.json | 6 +- .../src/github.com/AdRoll/goamz/aws/aws.go | 7 +- .../github.com/AdRoll/goamz/aws/regions.go | 34 +++ .../src/github.com/AdRoll/goamz/s3/s3.go | 25 +- .../src/github.com/AdRoll/goamz/s3/s3_test.go | 16 ++ .../AdRoll/goamz/s3/s3test/server.go | 230 ++++++++++++++++-- 6 files changed, 281 insertions(+), 37 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index dc643683d..ab255849d 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -12,15 +12,15 @@ }, { "ImportPath": "github.com/AdRoll/goamz/aws", - "Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103" + "Rev": "cc210f45dcb9889c2769a274522be2bf70edfb99" }, { "ImportPath": "github.com/AdRoll/goamz/cloudfront", - "Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103" + "Rev": "cc210f45dcb9889c2769a274522be2bf70edfb99" }, { "ImportPath": "github.com/AdRoll/goamz/s3", - "Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103" + "Rev": "cc210f45dcb9889c2769a274522be2bf70edfb99" }, { "ImportPath": "github.com/MSOpenTech/azure-sdk-for-go/storage", diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go index 38c9e6566..87c2d6da7 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go @@ -62,6 +62,7 @@ type Region struct { SESEndpoint string IAMEndpoint string ELBEndpoint string + KMSEndpoint string DynamoDBEndpoint string CloudWatchServicepoint ServiceInfo AutoScalingEndpoint string @@ -83,6 +84,7 @@ var Regions = map[string]Region{ USWest2.Name: USWest2, USGovWest.Name: USGovWest, SAEast.Name: SAEast, + CNNorth1.Name: CNNorth1, } // Designates a signer interface suitable for signing AWS requests, params @@ -208,7 +210,10 @@ func (a *Auth) Token() string { return "" } if time.Since(a.expiration) >= -30*time.Second { //in an ideal world this should be zero assuming the instance is synching it's clock - *a, _ = GetAuth("", "", "", time.Time{}) + auth, err := GetAuth("", "", "", time.Time{}) + if err == nil { + *a = auth + } } return a.token } diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go index 4e39069ef..fdc2626b8 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go @@ -13,6 +13,7 @@ var USGovWest = Region{ "", "https://iam.us-gov.amazonaws.com", "https://elasticloadbalancing.us-gov-west-1.amazonaws.com", + "", "https://dynamodb.us-gov-west-1.amazonaws.com", ServiceInfo{"https://monitoring.us-gov-west-1.amazonaws.com", V2Signature}, "https://autoscaling.us-gov-west-1.amazonaws.com", @@ -36,6 +37,7 @@ var USEast = Region{ "https://email.us-east-1.amazonaws.com", "https://iam.amazonaws.com", "https://elasticloadbalancing.us-east-1.amazonaws.com", + "https://kms.us-east-1.amazonaws.com", "https://dynamodb.us-east-1.amazonaws.com", ServiceInfo{"https://monitoring.us-east-1.amazonaws.com", V2Signature}, "https://autoscaling.us-east-1.amazonaws.com", @@ -59,6 +61,7 @@ var USWest = Region{ "", "https://iam.amazonaws.com", "https://elasticloadbalancing.us-west-1.amazonaws.com", + "https://kms.us-west-1.amazonaws.com", "https://dynamodb.us-west-1.amazonaws.com", ServiceInfo{"https://monitoring.us-west-1.amazonaws.com", V2Signature}, "https://autoscaling.us-west-1.amazonaws.com", @@ -82,6 +85,7 @@ var USWest2 = Region{ "https://email.us-west-2.amazonaws.com", "https://iam.amazonaws.com", "https://elasticloadbalancing.us-west-2.amazonaws.com", + "https://kms.us-west-2.amazonaws.com", "https://dynamodb.us-west-2.amazonaws.com", ServiceInfo{"https://monitoring.us-west-2.amazonaws.com", V2Signature}, "https://autoscaling.us-west-2.amazonaws.com", @@ -105,6 +109,7 @@ var EUWest = Region{ "https://email.eu-west-1.amazonaws.com", "https://iam.amazonaws.com", "https://elasticloadbalancing.eu-west-1.amazonaws.com", + "https://kms.eu-west-1.amazonaws.com", "https://dynamodb.eu-west-1.amazonaws.com", ServiceInfo{"https://monitoring.eu-west-1.amazonaws.com", V2Signature}, "https://autoscaling.eu-west-1.amazonaws.com", @@ -128,6 +133,7 @@ var EUCentral = Region{ "", "https://iam.amazonaws.com", "https://elasticloadbalancing.eu-central-1.amazonaws.com", + "https://kms.eu-central-1.amazonaws.com", "https://dynamodb.eu-central-1.amazonaws.com", ServiceInfo{"https://monitoring.eu-central-1.amazonaws.com", V2Signature}, "https://autoscaling.eu-central-1.amazonaws.com", @@ -151,6 +157,7 @@ var APSoutheast = Region{ "", "https://iam.amazonaws.com", "https://elasticloadbalancing.ap-southeast-1.amazonaws.com", + "https://kms.ap-southeast-1.amazonaws.com", "https://dynamodb.ap-southeast-1.amazonaws.com", ServiceInfo{"https://monitoring.ap-southeast-1.amazonaws.com", V2Signature}, "https://autoscaling.ap-southeast-1.amazonaws.com", @@ -174,6 +181,7 @@ var APSoutheast2 = Region{ "", "https://iam.amazonaws.com", "https://elasticloadbalancing.ap-southeast-2.amazonaws.com", + "https://kms.ap-southeast-2.amazonaws.com", "https://dynamodb.ap-southeast-2.amazonaws.com", ServiceInfo{"https://monitoring.ap-southeast-2.amazonaws.com", V2Signature}, "https://autoscaling.ap-southeast-2.amazonaws.com", @@ -197,6 +205,7 @@ var APNortheast = Region{ "", "https://iam.amazonaws.com", "https://elasticloadbalancing.ap-northeast-1.amazonaws.com", + "https://kms.ap-northeast-1.amazonaws.com", "https://dynamodb.ap-northeast-1.amazonaws.com", ServiceInfo{"https://monitoring.ap-northeast-1.amazonaws.com", V2Signature}, "https://autoscaling.ap-northeast-1.amazonaws.com", @@ -220,6 +229,7 @@ var SAEast = Region{ "", "https://iam.amazonaws.com", "https://elasticloadbalancing.sa-east-1.amazonaws.com", + "https://kms.sa-east-1.amazonaws.com", "https://dynamodb.sa-east-1.amazonaws.com", ServiceInfo{"https://monitoring.sa-east-1.amazonaws.com", V2Signature}, "https://autoscaling.sa-east-1.amazonaws.com", @@ -229,3 +239,27 @@ var SAEast = Region{ "https://cloudformation.sa-east-1.amazonaws.com", "https://elasticache.sa-east-1.amazonaws.com", } + +var CNNorth1 = Region{ + "cn-north-1", + "https://ec2.cn-north-1.amazonaws.com.cn", + "https://s3.cn-north-1.amazonaws.com.cn", + "", + true, + true, + "", + "https://sns.cn-north-1.amazonaws.com.cn", + "https://sqs.cn-north-1.amazonaws.com.cn", + "", + "https://iam.cn-north-1.amazonaws.com.cn", + "https://elasticloadbalancing.cn-north-1.amazonaws.com.cn", + "", + "https://dynamodb.cn-north-1.amazonaws.com.cn", + ServiceInfo{"https://monitoring.cn-north-1.amazonaws.com.cn", V4Signature}, + "https://autoscaling.cn-north-1.amazonaws.com.cn", + ServiceInfo{"https://rds.cn-north-1.amazonaws.com.cn", V4Signature}, + "", + "https://sts.cn-north-1.amazonaws.com.cn", + "", + "", +} diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go index 18313c282..69b2e071d 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go @@ -25,6 +25,7 @@ import ( "net/http" "net/http/httputil" "net/url" + "path" "strconv" "strings" "time" @@ -70,9 +71,8 @@ type Options struct { ContentMD5 string ContentDisposition string Range string + StorageClass StorageClass // What else? - //// The following become headers so they are []strings rather than strings... I think - // x-amz-storage-class []string } type CopyOptions struct { @@ -96,7 +96,7 @@ var attempts = aws.AttemptStrategy{ // New creates a new S3. func New(auth aws.Auth, region aws.Region) *S3 { - return &S3{auth, region, 0, 0, 0, aws.V2Signature} + return &S3{auth, region, 0, 0, aws.V2Signature, 0} } // Bucket returns a Bucket with the given name. @@ -164,6 +164,13 @@ const ( BucketOwnerFull = ACL("bucket-owner-full-control") ) +type StorageClass string + +const ( + ReducedRedundancy = StorageClass("REDUCED_REDUNDANCY") + StandardStorage = StorageClass("STANDARD") +) + // PutBucket creates a new bucket. // // See http://goo.gl/ndjnR for details. @@ -401,6 +408,10 @@ func (o Options) addHeaders(headers map[string][]string) { if len(o.ContentDisposition) != 0 { headers["Content-Disposition"] = []string{o.ContentDisposition} } + if len(o.StorageClass) != 0 { + headers["x-amz-storage-class"] = []string{string(o.StorageClass)} + + } for k, v := range o.Meta { headers["x-amz-meta-"+k] = v } @@ -816,8 +827,8 @@ func (b *Bucket) SignedURLWithMethod(method, path string, expires time.Time, par // UploadSignedURL returns a signed URL that allows anyone holding the URL // to upload the object at path. The signature is valid until expires. // contenttype is a string like image/png -// path is the resource name in s3 terminalogy like images/ali.png [obviously exclusing the bucket name itself] -func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time.Time) string { +// name is the resource name in s3 terminology like images/ali.png [obviously excluding the bucket name itself] +func (b *Bucket) UploadSignedURL(name, method, content_type string, expires time.Time) string { expire_date := expires.Unix() if method != "POST" { method = "PUT" @@ -830,7 +841,7 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time tokenData = "x-amz-security-token:" + a.Token() + "\n" } - stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n" + tokenData + "/" + b.Name + "/" + path + stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n" + tokenData + "/" + path.Join(b.Name, name) secretKey := a.SecretKey accessId := a.AccessKey mac := hmac.New(sha1.New, []byte(secretKey)) @@ -844,7 +855,7 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time log.Println("ERROR sining url for S3 upload", err) return "" } - signedurl.Path += path + signedurl.Path = name params := url.Values{} params.Add("AWSAccessKeyId", accessId) params.Add("Expires", strconv.FormatInt(expire_date, 10)) diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go index 87b23ad0c..161bb3af9 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go @@ -230,6 +230,22 @@ func (s *S) TestPutObject(c *check.C) { c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"}) } +func (s *S) TestPutObjectReducedRedundancy(c *check.C) { + testServer.Response(200, nil, "") + + b := s.s3.Bucket("bucket") + err := b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{StorageClass: s3.ReducedRedundancy}) + c.Assert(err, check.IsNil) + + req := testServer.WaitRequest() + c.Assert(req.Method, check.Equals, "PUT") + c.Assert(req.URL.Path, check.Equals, "/bucket/name") + c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""}) + c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"content-type"}) + c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"7"}) + c.Assert(req.Header["X-Amz-Storage-Class"], check.DeepEquals, []string{"REDUCED_REDUNDANCY"}) +} + // PutCopy docs: http://goo.gl/mhEHtA func (s *S) TestPutCopy(c *check.C) { testServer.Response(200, nil, PutCopyResultDump) diff --git a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go index 4dc95eae0..d54a638c5 100644 --- a/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go +++ b/Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "log" + "math/rand" "net" "net/http" "net/url" @@ -51,6 +52,10 @@ type Config struct { // all other regions. // http://docs.amazonwebservices.com/AmazonS3/latest/API/ErrorResponses.html Send409Conflict bool + + // Address on which to listen. By default, a random port is assigned by the + // operating system and the server listens on localhost. + ListenAddress string } func (c *Config) send409Conflict() bool { @@ -72,10 +77,11 @@ type Server struct { } type bucket struct { - name string - acl s3.ACL - ctime time.Time - objects map[string]*object + name string + acl s3.ACL + ctime time.Time + objects map[string]*object + multipartUploads map[string][]*multipartUploadPart } type object struct { @@ -86,6 +92,12 @@ type object struct { data []byte } +type multipartUploadPart struct { + data []byte + etag string + lastModified time.Time +} + // A resource encapsulates the subject of an HTTP request. // The resource referred to may or may not exist // when the request is made. @@ -97,7 +109,13 @@ type resource interface { } func NewServer(config *Config) (*Server, error) { - l, err := net.Listen("tcp", "localhost:0") + listenAddress := "localhost:0" + + if config != nil && config.ListenAddress != "" { + listenAddress = config.ListenAddress + } + + l, err := net.Listen("tcp", listenAddress) if err != nil { return nil, fmt.Errorf("cannot listen on localhost: %v", err) } @@ -217,10 +235,8 @@ var unimplementedBucketResourceNames = map[string]bool{ } var unimplementedObjectResourceNames = map[string]bool{ - "uploadId": true, - "acl": true, - "torrent": true, - "uploads": true, + "acl": true, + "torrent": true, } var pathRegexp = regexp.MustCompile("/(([^/]+)(/(.*))?)?") @@ -420,7 +436,8 @@ func (r bucketResource) put(a *action) interface{} { r.bucket = &bucket{ name: r.name, // TODO default acl - objects: make(map[string]*object), + objects: make(map[string]*object), + multipartUploads: make(map[string][]*multipartUploadPart), } a.srv.buckets[r.name] = r.bucket created = true @@ -615,12 +632,29 @@ func (objr objectResource) put(a *action) interface{} { // TODO x-amz-server-side-encryption // TODO x-amz-storage-class - // TODO is this correct, or should we erase all previous metadata? - obj := objr.object - if obj == nil { - obj = &object{ - name: objr.name, - meta: make(http.Header), + uploadId := a.req.URL.Query().Get("uploadId") + + // Check that the upload ID is valid if this is a multipart upload + if uploadId != "" { + if _, ok := objr.bucket.multipartUploads[uploadId]; !ok { + fatalf(404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.") + } + + partNumberStr := a.req.URL.Query().Get("partNumber") + + if partNumberStr == "" { + fatalf(400, "InvalidRequest", "Missing partNumber parameter") + } + + partNumber, err := strconv.ParseUint(partNumberStr, 10, 32) + + if err != nil { + fatalf(400, "InvalidRequest", "partNumber is not a number") + } + + // Parts are 1-indexed for multipart uploads + if uint(partNumber)-1 != uint(len(objr.bucket.multipartUploads[uploadId])) { + fatalf(400, "InvalidRequest", "Invalid part number") } } @@ -646,26 +680,170 @@ func (objr objectResource) put(a *action) interface{} { fatalf(400, "IncompleteBody", "You did not provide the number of bytes specified by the Content-Length HTTP header") } - // PUT request has been successful - save data and metadata - for key, values := range a.req.Header { - key = http.CanonicalHeaderKey(key) - if metaHeaders[key] || strings.HasPrefix(key, "X-Amz-Meta-") { - obj.meta[key] = values + etag := fmt.Sprintf("\"%x\"", gotHash) + + a.w.Header().Add("ETag", etag) + + if uploadId == "" { + // For traditional uploads + + // TODO is this correct, or should we erase all previous metadata? + obj := objr.object + if obj == nil { + obj = &object{ + name: objr.name, + meta: make(http.Header), + } } + + // PUT request has been successful - save data and metadata + for key, values := range a.req.Header { + key = http.CanonicalHeaderKey(key) + if metaHeaders[key] || strings.HasPrefix(key, "X-Amz-Meta-") { + obj.meta[key] = values + } + } + obj.data = data + obj.checksum = gotHash + obj.mtime = time.Now() + objr.bucket.objects[objr.name] = obj + } else { + // For multipart commit + + parts := objr.bucket.multipartUploads[uploadId] + part := &multipartUploadPart{ + data, + etag, + time.Now(), + } + + objr.bucket.multipartUploads[uploadId] = append(parts, part) } - obj.data = data - obj.checksum = gotHash - obj.mtime = time.Now() - objr.bucket.objects[objr.name] = obj + return nil } func (objr objectResource) delete(a *action) interface{} { - delete(objr.bucket.objects, objr.name) + uploadId := a.req.URL.Query().Get("uploadId") + + if uploadId == "" { + // Traditional object delete + delete(objr.bucket.objects, objr.name) + } else { + // Multipart commit abort + _, ok := objr.bucket.multipartUploads[uploadId] + + if !ok { + fatalf(404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.") + } + + delete(objr.bucket.multipartUploads, uploadId) + } return nil } func (objr objectResource) post(a *action) interface{} { + // Check if we're initializing a multipart upload + if _, ok := a.req.URL.Query()["uploads"]; ok { + type multipartInitResponse struct { + XMLName struct{} `xml:"InitiateMultipartUploadResult"` + Bucket string + Key string + UploadId string + } + + uploadId := strconv.FormatInt(rand.Int63(), 16) + + objr.bucket.multipartUploads[uploadId] = []*multipartUploadPart{} + + return &multipartInitResponse{ + Bucket: objr.bucket.name, + Key: objr.name, + UploadId: uploadId, + } + } + + // Check if we're completing a multipart upload + if uploadId := a.req.URL.Query().Get("uploadId"); uploadId != "" { + type multipartCompleteRequestPart struct { + XMLName struct{} `xml:"Part"` + PartNumber uint + ETag string + } + + type multipartCompleteRequest struct { + XMLName struct{} `xml:"CompleteMultipartUpload"` + Part []multipartCompleteRequestPart + } + + type multipartCompleteResponse struct { + XMLName struct{} `xml:"CompleteMultipartUploadResult"` + Location string + Bucket string + Key string + ETag string + } + + parts, ok := objr.bucket.multipartUploads[uploadId] + + if !ok { + fatalf(404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.") + } + + req := &multipartCompleteRequest{} + + if err := xml.NewDecoder(a.req.Body).Decode(req); err != nil { + fatalf(400, "InvalidRequest", err.Error()) + } + + if len(req.Part) != len(parts) { + fatalf(400, "InvalidRequest", fmt.Sprintf("Number of parts does not match: expected %d, received %d", len(parts), len(req.Part))) + } + + sum := md5.New() + data := &bytes.Buffer{} + w := io.MultiWriter(sum, data) + + for i, p := range parts { + reqPart := req.Part[i] + + if reqPart.PartNumber != uint(1+i) { + fatalf(400, "InvalidRequest", "Bad part number") + } + + if reqPart.ETag != p.etag { + fatalf(400, "InvalidRequest", fmt.Sprintf("Invalid etag for part %d", reqPart.PartNumber)) + } + + w.Write(p.data) + } + + delete(objr.bucket.multipartUploads, uploadId) + + obj := objr.object + + if obj == nil { + obj = &object{ + name: objr.name, + meta: make(http.Header), + } + } + + obj.data = data.Bytes() + obj.checksum = sum.Sum(nil) + obj.mtime = time.Now() + objr.bucket.objects[objr.name] = obj + + objectLocation := fmt.Sprintf("http://%s/%s/%s", a.srv.listener.Addr().String(), objr.bucket.name, objr.name) + + return &multipartCompleteResponse{ + Location: objectLocation, + Bucket: objr.bucket.name, + Key: objr.name, + ETag: uploadId, + } + } + fatalf(400, "MethodNotAllowed", "The specified method is not allowed against this resource") return nil }