Merge pull request #415 from stevvooe/update-goamz

Update goamz package dependency

Godeps/Godeps.json

@@ -12,15 +12,15 @@
 		},
 		{
 			"ImportPath": "github.com/AdRoll/goamz/aws",
-			"Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103"
+			"Rev": "cc210f45dcb9889c2769a274522be2bf70edfb99"
 		},
 		{
 			"ImportPath": "github.com/AdRoll/goamz/cloudfront",
-			"Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103"
+			"Rev": "cc210f45dcb9889c2769a274522be2bf70edfb99"
 		},
 		{
 			"ImportPath": "github.com/AdRoll/goamz/s3",
-			"Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103"
+			"Rev": "cc210f45dcb9889c2769a274522be2bf70edfb99"
 		},
 		{
 			"ImportPath": "github.com/MSOpenTech/azure-sdk-for-go/storage",

Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go

@@ -62,6 +62,7 @@ type Region struct {
 	SESEndpoint            string
 	IAMEndpoint            string
 	ELBEndpoint            string
+	KMSEndpoint            string
 	DynamoDBEndpoint       string
 	CloudWatchServicepoint ServiceInfo
 	AutoScalingEndpoint    string
@@ -83,6 +84,7 @@ var Regions = map[string]Region{
 	USWest2.Name:      USWest2,
 	USGovWest.Name:    USGovWest,
 	SAEast.Name:       SAEast,
+	CNNorth1.Name:     CNNorth1,
 }
 
 // Designates a signer interface suitable for signing AWS requests, params
@@ -208,7 +210,10 @@ func (a *Auth) Token() string {
 		return ""
 	}
 	if time.Since(a.expiration) >= -30*time.Second { //in an ideal world this should be zero assuming the instance is synching it's clock
-		*a, _ = GetAuth("", "", "", time.Time{})
+		auth, err := GetAuth("", "", "", time.Time{})
+		if err == nil {
+			*a = auth
+		}
 	}
 	return a.token
 }

Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go

@@ -13,6 +13,7 @@ var USGovWest = Region{
 	"",
 	"https://iam.us-gov.amazonaws.com",
 	"https://elasticloadbalancing.us-gov-west-1.amazonaws.com",
+	"",
 	"https://dynamodb.us-gov-west-1.amazonaws.com",
 	ServiceInfo{"https://monitoring.us-gov-west-1.amazonaws.com", V2Signature},
 	"https://autoscaling.us-gov-west-1.amazonaws.com",
@@ -36,6 +37,7 @@ var USEast = Region{
 	"https://email.us-east-1.amazonaws.com",
 	"https://iam.amazonaws.com",
 	"https://elasticloadbalancing.us-east-1.amazonaws.com",
+	"https://kms.us-east-1.amazonaws.com",
 	"https://dynamodb.us-east-1.amazonaws.com",
 	ServiceInfo{"https://monitoring.us-east-1.amazonaws.com", V2Signature},
 	"https://autoscaling.us-east-1.amazonaws.com",
@@ -59,6 +61,7 @@ var USWest = Region{
 	"",
 	"https://iam.amazonaws.com",
 	"https://elasticloadbalancing.us-west-1.amazonaws.com",
+	"https://kms.us-west-1.amazonaws.com",
 	"https://dynamodb.us-west-1.amazonaws.com",
 	ServiceInfo{"https://monitoring.us-west-1.amazonaws.com", V2Signature},
 	"https://autoscaling.us-west-1.amazonaws.com",
@@ -82,6 +85,7 @@ var USWest2 = Region{
 	"https://email.us-west-2.amazonaws.com",
 	"https://iam.amazonaws.com",
 	"https://elasticloadbalancing.us-west-2.amazonaws.com",
+	"https://kms.us-west-2.amazonaws.com",
 	"https://dynamodb.us-west-2.amazonaws.com",
 	ServiceInfo{"https://monitoring.us-west-2.amazonaws.com", V2Signature},
 	"https://autoscaling.us-west-2.amazonaws.com",
@@ -105,6 +109,7 @@ var EUWest = Region{
 	"https://email.eu-west-1.amazonaws.com",
 	"https://iam.amazonaws.com",
 	"https://elasticloadbalancing.eu-west-1.amazonaws.com",
+	"https://kms.eu-west-1.amazonaws.com",
 	"https://dynamodb.eu-west-1.amazonaws.com",
 	ServiceInfo{"https://monitoring.eu-west-1.amazonaws.com", V2Signature},
 	"https://autoscaling.eu-west-1.amazonaws.com",
@@ -128,6 +133,7 @@ var EUCentral = Region{
 	"",
 	"https://iam.amazonaws.com",
 	"https://elasticloadbalancing.eu-central-1.amazonaws.com",
+	"https://kms.eu-central-1.amazonaws.com",
 	"https://dynamodb.eu-central-1.amazonaws.com",
 	ServiceInfo{"https://monitoring.eu-central-1.amazonaws.com", V2Signature},
 	"https://autoscaling.eu-central-1.amazonaws.com",
@@ -151,6 +157,7 @@ var APSoutheast = Region{
 	"",
 	"https://iam.amazonaws.com",
 	"https://elasticloadbalancing.ap-southeast-1.amazonaws.com",
+	"https://kms.ap-southeast-1.amazonaws.com",
 	"https://dynamodb.ap-southeast-1.amazonaws.com",
 	ServiceInfo{"https://monitoring.ap-southeast-1.amazonaws.com", V2Signature},
 	"https://autoscaling.ap-southeast-1.amazonaws.com",
@@ -174,6 +181,7 @@ var APSoutheast2 = Region{
 	"",
 	"https://iam.amazonaws.com",
 	"https://elasticloadbalancing.ap-southeast-2.amazonaws.com",
+	"https://kms.ap-southeast-2.amazonaws.com",
 	"https://dynamodb.ap-southeast-2.amazonaws.com",
 	ServiceInfo{"https://monitoring.ap-southeast-2.amazonaws.com", V2Signature},
 	"https://autoscaling.ap-southeast-2.amazonaws.com",
@@ -197,6 +205,7 @@ var APNortheast = Region{
 	"",
 	"https://iam.amazonaws.com",
 	"https://elasticloadbalancing.ap-northeast-1.amazonaws.com",
+	"https://kms.ap-northeast-1.amazonaws.com",
 	"https://dynamodb.ap-northeast-1.amazonaws.com",
 	ServiceInfo{"https://monitoring.ap-northeast-1.amazonaws.com", V2Signature},
 	"https://autoscaling.ap-northeast-1.amazonaws.com",
@@ -220,6 +229,7 @@ var SAEast = Region{
 	"",
 	"https://iam.amazonaws.com",
 	"https://elasticloadbalancing.sa-east-1.amazonaws.com",
+	"https://kms.sa-east-1.amazonaws.com",
 	"https://dynamodb.sa-east-1.amazonaws.com",
 	ServiceInfo{"https://monitoring.sa-east-1.amazonaws.com", V2Signature},
 	"https://autoscaling.sa-east-1.amazonaws.com",
@@ -229,3 +239,27 @@ var SAEast = Region{
 	"https://cloudformation.sa-east-1.amazonaws.com",
 	"https://elasticache.sa-east-1.amazonaws.com",
 }
+
+var CNNorth1 = Region{
+	"cn-north-1",
+	"https://ec2.cn-north-1.amazonaws.com.cn",
+	"https://s3.cn-north-1.amazonaws.com.cn",
+	"",
+	true,
+	true,
+	"",
+	"https://sns.cn-north-1.amazonaws.com.cn",
+	"https://sqs.cn-north-1.amazonaws.com.cn",
+	"",
+	"https://iam.cn-north-1.amazonaws.com.cn",
+	"https://elasticloadbalancing.cn-north-1.amazonaws.com.cn",
+	"",
+	"https://dynamodb.cn-north-1.amazonaws.com.cn",
+	ServiceInfo{"https://monitoring.cn-north-1.amazonaws.com.cn", V4Signature},
+	"https://autoscaling.cn-north-1.amazonaws.com.cn",
+	ServiceInfo{"https://rds.cn-north-1.amazonaws.com.cn", V4Signature},
+	"",
+	"https://sts.cn-north-1.amazonaws.com.cn",
+	"",
+	"",
+}

Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go

@@ -25,6 +25,7 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -70,9 +71,8 @@ type Options struct {
 	ContentMD5           string
 	ContentDisposition   string
 	Range                string
+	StorageClass         StorageClass
 	// What else?
-	//// The following become headers so they are []strings rather than strings... I think
-	// x-amz-storage-class []string
 }
 
 type CopyOptions struct {
@@ -96,7 +96,7 @@ var attempts = aws.AttemptStrategy{
 
 // New creates a new S3.
 func New(auth aws.Auth, region aws.Region) *S3 {
-	return &S3{auth, region, 0, 0, 0, aws.V2Signature}
+	return &S3{auth, region, 0, 0, aws.V2Signature, 0}
 }
 
 // Bucket returns a Bucket with the given name.
@@ -164,6 +164,13 @@ const (
 	BucketOwnerFull   = ACL("bucket-owner-full-control")
 )
 
+type StorageClass string
+
+const (
+	ReducedRedundancy = StorageClass("REDUCED_REDUNDANCY")
+	StandardStorage   = StorageClass("STANDARD")
+)
+
 // PutBucket creates a new bucket.
 //
 // See http://goo.gl/ndjnR for details.
@@ -401,6 +408,10 @@ func (o Options) addHeaders(headers map[string][]string) {
 	if len(o.ContentDisposition) != 0 {
 		headers["Content-Disposition"] = []string{o.ContentDisposition}
 	}
+	if len(o.StorageClass) != 0 {
+		headers["x-amz-storage-class"] = []string{string(o.StorageClass)}
+
+	}
 	for k, v := range o.Meta {
 		headers["x-amz-meta-"+k] = v
 	}
@@ -816,8 +827,8 @@ func (b *Bucket) SignedURLWithMethod(method, path string, expires time.Time, par
 // UploadSignedURL returns a signed URL that allows anyone holding the URL
 // to upload the object at path. The signature is valid until expires.
 // contenttype is a string like image/png
-// path is the resource name in s3 terminalogy like images/ali.png [obviously exclusing the bucket name itself]
+// name is the resource name in s3 terminology like images/ali.png [obviously excluding the bucket name itself]
-func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time.Time) string {
+func (b *Bucket) UploadSignedURL(name, method, content_type string, expires time.Time) string {
 	expire_date := expires.Unix()
 	if method != "POST" {
 		method = "PUT"
@@ -830,7 +841,7 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time
 		tokenData = "x-amz-security-token:" + a.Token() + "\n"
 	}
 
-	stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n" + tokenData + "/" + b.Name + "/" + path
+	stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n" + tokenData + "/" + path.Join(b.Name, name)
 	secretKey := a.SecretKey
 	accessId := a.AccessKey
 	mac := hmac.New(sha1.New, []byte(secretKey))
@@ -844,7 +855,7 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time
 		log.Println("ERROR sining url for S3 upload", err)
 		return ""
 	}
-	signedurl.Path += path
+	signedurl.Path = name
 	params := url.Values{}
 	params.Add("AWSAccessKeyId", accessId)
 	params.Add("Expires", strconv.FormatInt(expire_date, 10))

Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go

@@ -230,6 +230,22 @@ func (s *S) TestPutObject(c *check.C) {
 	c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"})
 }
 
+func (s *S) TestPutObjectReducedRedundancy(c *check.C) {
+	testServer.Response(200, nil, "")
+
+	b := s.s3.Bucket("bucket")
+	err := b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{StorageClass: s3.ReducedRedundancy})
+	c.Assert(err, check.IsNil)
+
+	req := testServer.WaitRequest()
+	c.Assert(req.Method, check.Equals, "PUT")
+	c.Assert(req.URL.Path, check.Equals, "/bucket/name")
+	c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""})
+	c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"content-type"})
+	c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"7"})
+	c.Assert(req.Header["X-Amz-Storage-Class"], check.DeepEquals, []string{"REDUCED_REDUNDANCY"})
+}
+
 // PutCopy docs: http://goo.gl/mhEHtA
 func (s *S) TestPutCopy(c *check.C) {
 	testServer.Response(200, nil, PutCopyResultDump)

Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/url"
@@ -51,6 +52,10 @@ type Config struct {
 	// all other regions.
 	// http://docs.amazonwebservices.com/AmazonS3/latest/API/ErrorResponses.html
 	Send409Conflict bool
+
+	// Address on which to listen. By default, a random port is assigned by the
+	// operating system and the server listens on localhost.
+	ListenAddress string
 }
 
 func (c *Config) send409Conflict() bool {
@@ -72,10 +77,11 @@ type Server struct {
 }
 
 type bucket struct {
-	name    string
+	name             string
-	acl     s3.ACL
+	acl              s3.ACL
-	ctime   time.Time
+	ctime            time.Time
-	objects map[string]*object
+	objects          map[string]*object
+	multipartUploads map[string][]*multipartUploadPart
 }
 
 type object struct {
@@ -86,6 +92,12 @@ type object struct {
 	data     []byte
 }
 
+type multipartUploadPart struct {
+	data         []byte
+	etag         string
+	lastModified time.Time
+}
+
 // A resource encapsulates the subject of an HTTP request.
 // The resource referred to may or may not exist
 // when the request is made.
@@ -97,7 +109,13 @@ type resource interface {
 }
 
 func NewServer(config *Config) (*Server, error) {
-	l, err := net.Listen("tcp", "localhost:0")
+	listenAddress := "localhost:0"
+
+	if config != nil && config.ListenAddress != "" {
+		listenAddress = config.ListenAddress
+	}
+
+	l, err := net.Listen("tcp", listenAddress)
 	if err != nil {
 		return nil, fmt.Errorf("cannot listen on localhost: %v", err)
 	}
@@ -217,10 +235,8 @@ var unimplementedBucketResourceNames = map[string]bool{
 }
 
 var unimplementedObjectResourceNames = map[string]bool{
-	"uploadId": true,
+	"acl":     true,
-	"acl":      true,
+	"torrent": true,
-	"torrent":  true,
-	"uploads":  true,
 }
 
 var pathRegexp = regexp.MustCompile("/(([^/]+)(/(.*))?)?")
@@ -420,7 +436,8 @@ func (r bucketResource) put(a *action) interface{} {
 		r.bucket = &bucket{
 			name: r.name,
 			// TODO default acl
-			objects: make(map[string]*object),
+			objects:          make(map[string]*object),
+			multipartUploads: make(map[string][]*multipartUploadPart),
 		}
 		a.srv.buckets[r.name] = r.bucket
 		created = true
@@ -615,12 +632,29 @@ func (objr objectResource) put(a *action) interface{} {
 	// TODO x-amz-server-side-encryption
 	// TODO x-amz-storage-class
 
-	// TODO is this correct, or should we erase all previous metadata?
+	uploadId := a.req.URL.Query().Get("uploadId")
-	obj := objr.object
+
-	if obj == nil {
+	// Check that the upload ID is valid if this is a multipart upload
-		obj = &object{
+	if uploadId != "" {
-			name: objr.name,
+		if _, ok := objr.bucket.multipartUploads[uploadId]; !ok {
-			meta: make(http.Header),
+			fatalf(404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")
+		}
+
+		partNumberStr := a.req.URL.Query().Get("partNumber")
+
+		if partNumberStr == "" {
+			fatalf(400, "InvalidRequest", "Missing partNumber parameter")
+		}
+
+		partNumber, err := strconv.ParseUint(partNumberStr, 10, 32)
+
+		if err != nil {
+			fatalf(400, "InvalidRequest", "partNumber is not a number")
+		}
+
+		// Parts are 1-indexed for multipart uploads
+		if uint(partNumber)-1 != uint(len(objr.bucket.multipartUploads[uploadId])) {
+			fatalf(400, "InvalidRequest", "Invalid part number")
 		}
 	}
 
@@ -646,26 +680,170 @@ func (objr objectResource) put(a *action) interface{} {
 		fatalf(400, "IncompleteBody", "You did not provide the number of bytes specified by the Content-Length HTTP header")
 	}
 
-	// PUT request has been successful - save data and metadata
+	etag := fmt.Sprintf("\"%x\"", gotHash)
-	for key, values := range a.req.Header {
+
-		key = http.CanonicalHeaderKey(key)
+	a.w.Header().Add("ETag", etag)
-		if metaHeaders[key] || strings.HasPrefix(key, "X-Amz-Meta-") {
+
-			obj.meta[key] = values
+	if uploadId == "" {
+		// For traditional uploads
+
+		// TODO is this correct, or should we erase all previous metadata?
+		obj := objr.object
+		if obj == nil {
+			obj = &object{
+				name: objr.name,
+				meta: make(http.Header),
+			}
 		}
+
+		// PUT request has been successful - save data and metadata
+		for key, values := range a.req.Header {
+			key = http.CanonicalHeaderKey(key)
+			if metaHeaders[key] || strings.HasPrefix(key, "X-Amz-Meta-") {
+				obj.meta[key] = values
+			}
+		}
+		obj.data = data
+		obj.checksum = gotHash
+		obj.mtime = time.Now()
+		objr.bucket.objects[objr.name] = obj
+	} else {
+		// For multipart commit
+
+		parts := objr.bucket.multipartUploads[uploadId]
+		part := &multipartUploadPart{
+			data,
+			etag,
+			time.Now(),
+		}
+
+		objr.bucket.multipartUploads[uploadId] = append(parts, part)
 	}
-	obj.data = data
+
-	obj.checksum = gotHash
-	obj.mtime = time.Now()
-	objr.bucket.objects[objr.name] = obj
 	return nil
 }
 
 func (objr objectResource) delete(a *action) interface{} {
-	delete(objr.bucket.objects, objr.name)
+	uploadId := a.req.URL.Query().Get("uploadId")
+
+	if uploadId == "" {
+		// Traditional object delete
+		delete(objr.bucket.objects, objr.name)
+	} else {
+		// Multipart commit abort
+		_, ok := objr.bucket.multipartUploads[uploadId]
+
+		if !ok {
+			fatalf(404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")
+		}
+
+		delete(objr.bucket.multipartUploads, uploadId)
+	}
 	return nil
 }
 
 func (objr objectResource) post(a *action) interface{} {
+	// Check if we're initializing a multipart upload
+	if _, ok := a.req.URL.Query()["uploads"]; ok {
+		type multipartInitResponse struct {
+			XMLName  struct{} `xml:"InitiateMultipartUploadResult"`
+			Bucket   string
+			Key      string
+			UploadId string
+		}
+
+		uploadId := strconv.FormatInt(rand.Int63(), 16)
+
+		objr.bucket.multipartUploads[uploadId] = []*multipartUploadPart{}
+
+		return &multipartInitResponse{
+			Bucket:   objr.bucket.name,
+			Key:      objr.name,
+			UploadId: uploadId,
+		}
+	}
+
+	// Check if we're completing a multipart upload
+	if uploadId := a.req.URL.Query().Get("uploadId"); uploadId != "" {
+		type multipartCompleteRequestPart struct {
+			XMLName    struct{} `xml:"Part"`
+			PartNumber uint
+			ETag       string
+		}
+
+		type multipartCompleteRequest struct {
+			XMLName struct{} `xml:"CompleteMultipartUpload"`
+			Part    []multipartCompleteRequestPart
+		}
+
+		type multipartCompleteResponse struct {
+			XMLName  struct{} `xml:"CompleteMultipartUploadResult"`
+			Location string
+			Bucket   string
+			Key      string
+			ETag     string
+		}
+
+		parts, ok := objr.bucket.multipartUploads[uploadId]
+
+		if !ok {
+			fatalf(404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")
+		}
+
+		req := &multipartCompleteRequest{}
+
+		if err := xml.NewDecoder(a.req.Body).Decode(req); err != nil {
+			fatalf(400, "InvalidRequest", err.Error())
+		}
+
+		if len(req.Part) != len(parts) {
+			fatalf(400, "InvalidRequest", fmt.Sprintf("Number of parts does not match: expected %d, received %d", len(parts), len(req.Part)))
+		}
+
+		sum := md5.New()
+		data := &bytes.Buffer{}
+		w := io.MultiWriter(sum, data)
+
+		for i, p := range parts {
+			reqPart := req.Part[i]
+
+			if reqPart.PartNumber != uint(1+i) {
+				fatalf(400, "InvalidRequest", "Bad part number")
+			}
+
+			if reqPart.ETag != p.etag {
+				fatalf(400, "InvalidRequest", fmt.Sprintf("Invalid etag for part %d", reqPart.PartNumber))
+			}
+
+			w.Write(p.data)
+		}
+
+		delete(objr.bucket.multipartUploads, uploadId)
+
+		obj := objr.object
+
+		if obj == nil {
+			obj = &object{
+				name: objr.name,
+				meta: make(http.Header),
+			}
+		}
+
+		obj.data = data.Bytes()
+		obj.checksum = sum.Sum(nil)
+		obj.mtime = time.Now()
+		objr.bucket.objects[objr.name] = obj
+
+		objectLocation := fmt.Sprintf("http://%s/%s/%s", a.srv.listener.Addr().String(), objr.bucket.name, objr.name)
+
+		return &multipartCompleteResponse{
+			Location: objectLocation,
+			Bucket:   objr.bucket.name,
+			Key:      objr.name,
+			ETag:     uploadId,
+		}
+	}
+
 	fatalf(400, "MethodNotAllowed", "The specified method is not allowed against this resource")
 	return nil
 }