Merge pull request #1042 from tt/upgrade-amazon-library

Upgrade Amazon library

Godeps/Godeps.json

@@ -7,15 +7,15 @@
 	"Deps": [
 		{
 			"ImportPath": "github.com/AdRoll/goamz/aws",
-			"Rev": "f8c4952d5bc3056c0ca6711a1f56bc88b828d989"
+			"Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1"
 		},
 		{
 			"ImportPath": "github.com/AdRoll/goamz/cloudfront",
-			"Rev": "f8c4952d5bc3056c0ca6711a1f56bc88b828d989"
+			"Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1"
 		},
 		{
 			"ImportPath": "github.com/AdRoll/goamz/s3",
-			"Rev": "f8c4952d5bc3056c0ca6711a1f56bc88b828d989"
+			"Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1"
 		},
 		{
 			"ImportPath": "github.com/Azure/azure-sdk-for-go/storage",

Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go

@@ -51,7 +51,7 @@ type ServiceInfo struct {
 // See http://goo.gl/d8BP1 for more details.
 type Region struct {
 	Name                   string // the canonical name of this region.
-	EC2Endpoint            string
+	EC2Endpoint            ServiceInfo
 	S3Endpoint             string
 	S3BucketEndpoint       string // Not needed by AWS S3. Use ${bucket} for bucket name.
 	S3LocationConstraint   bool   // true if this region requires a LocationConstraint declaration.

Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go

@@ -2,7 +2,7 @@ package aws
 
 var USGovWest = Region{
 	"us-gov-west-1",
-	"https://ec2.us-gov-west-1.amazonaws.com",
+	ServiceInfo{"https://ec2.us-gov-west-1.amazonaws.com", V2Signature},
 	"https://s3-fips-us-gov-west-1.amazonaws.com",
 	"",
 	true,
@@ -26,8 +26,8 @@ var USGovWest = Region{
 
 var USEast = Region{
 	"us-east-1",
-	"https://ec2.us-east-1.amazonaws.com",
+	ServiceInfo{"https://ec2.us-east-1.amazonaws.com", V2Signature},
-	"https://s3.amazonaws.com",
+	"https://s3-external-1.amazonaws.com",
 	"",
 	false,
 	false,
@@ -50,7 +50,7 @@ var USEast = Region{
 
 var USWest = Region{
 	"us-west-1",
-	"https://ec2.us-west-1.amazonaws.com",
+	ServiceInfo{"https://ec2.us-west-1.amazonaws.com", V2Signature},
 	"https://s3-us-west-1.amazonaws.com",
 	"",
 	true,
@@ -74,7 +74,7 @@ var USWest = Region{
 
 var USWest2 = Region{
 	"us-west-2",
-	"https://ec2.us-west-2.amazonaws.com",
+	ServiceInfo{"https://ec2.us-west-2.amazonaws.com", V2Signature},
 	"https://s3-us-west-2.amazonaws.com",
 	"",
 	true,
@@ -98,7 +98,7 @@ var USWest2 = Region{
 
 var EUWest = Region{
 	"eu-west-1",
-	"https://ec2.eu-west-1.amazonaws.com",
+	ServiceInfo{"https://ec2.eu-west-1.amazonaws.com", V2Signature},
 	"https://s3-eu-west-1.amazonaws.com",
 	"",
 	true,
@@ -122,7 +122,7 @@ var EUWest = Region{
 
 var EUCentral = Region{
 	"eu-central-1",
-	"https://ec2.eu-central-1.amazonaws.com",
+	ServiceInfo{"https://ec2.eu-central-1.amazonaws.com", V4Signature},
 	"https://s3-eu-central-1.amazonaws.com",
 	"",
 	true,
@@ -146,7 +146,7 @@ var EUCentral = Region{
 
 var APSoutheast = Region{
 	"ap-southeast-1",
-	"https://ec2.ap-southeast-1.amazonaws.com",
+	ServiceInfo{"https://ec2.ap-southeast-1.amazonaws.com", V2Signature},
 	"https://s3-ap-southeast-1.amazonaws.com",
 	"",
 	true,
@@ -170,7 +170,7 @@ var APSoutheast = Region{
 
 var APSoutheast2 = Region{
 	"ap-southeast-2",
-	"https://ec2.ap-southeast-2.amazonaws.com",
+	ServiceInfo{"https://ec2.ap-southeast-2.amazonaws.com", V2Signature},
 	"https://s3-ap-southeast-2.amazonaws.com",
 	"",
 	true,
@@ -194,7 +194,7 @@ var APSoutheast2 = Region{
 
 var APNortheast = Region{
 	"ap-northeast-1",
-	"https://ec2.ap-northeast-1.amazonaws.com",
+	ServiceInfo{"https://ec2.ap-northeast-1.amazonaws.com", V2Signature},
 	"https://s3-ap-northeast-1.amazonaws.com",
 	"",
 	true,
@@ -218,7 +218,7 @@ var APNortheast = Region{
 
 var SAEast = Region{
 	"sa-east-1",
-	"https://ec2.sa-east-1.amazonaws.com",
+	ServiceInfo{"https://ec2.sa-east-1.amazonaws.com", V2Signature},
 	"https://s3-sa-east-1.amazonaws.com",
 	"",
 	true,
@@ -242,7 +242,7 @@ var SAEast = Region{
 
 var CNNorth1 = Region{
 	"cn-north-1",
-	"https://ec2.cn-north-1.amazonaws.com.cn",
+	ServiceInfo{"https://ec2.cn-north-1.amazonaws.com.cn", V2Signature},
 	"https://s3.cn-north-1.amazonaws.com.cn",
 	"",
 	true,

Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign.go

@@ -15,6 +15,28 @@ import (
 	"time"
 )
 
+// AWS specifies that the parameters in a signed request must
+// be provided in the natural order of the keys. This is distinct
+// from the natural order of the encoded value of key=value.
+// Percent and gocheck.Equals affect the sorting order.
+func EncodeSorted(values url.Values) string {
+	// preallocate the arrays for perfomance
+	keys := make([]string, 0, len(values))
+	sarray := make([]string, 0, len(values))
+	for k, _ := range values {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, k := range keys {
+		for _, v := range values[k] {
+			sarray = append(sarray, Encode(k)+"="+Encode(v))
+		}
+	}
+
+	return strings.Join(sarray, "&")
+}
+
 type V2Signer struct {
 	auth    Auth
 	service ServiceInfo
@@ -38,7 +60,6 @@ func (s *V2Signer) Sign(method, path string, params map[string]string) {
 	if s.auth.Token() != "" {
 		params["SecurityToken"] = s.auth.Token()
 	}
-
 	// AWS specifies that the parameters in a signed request must
 	// be provided in the natural order of the keys. This is distinct
 	// from the natural order of the encoded value of key=value.
@@ -61,6 +82,28 @@ func (s *V2Signer) Sign(method, path string, params map[string]string) {
 	params["Signature"] = string(signature)
 }
 
+func (s *V2Signer) SignRequest(req *http.Request) error {
+	req.ParseForm()
+	req.Form.Set("AWSAccessKeyId", s.auth.AccessKey)
+	req.Form.Set("SignatureVersion", "2")
+	req.Form.Set("SignatureMethod", "HmacSHA256")
+	if s.auth.Token() != "" {
+		req.Form.Set("SecurityToken", s.auth.Token())
+	}
+
+	payload := req.Method + "\n" + req.URL.Host + "\n" + req.URL.Path + "\n" + EncodeSorted(req.Form)
+	hash := hmac.New(sha256.New, []byte(s.auth.SecretKey))
+	hash.Write([]byte(payload))
+	signature := make([]byte, b64.EncodedLen(hash.Size()))
+	b64.Encode(signature, hash.Sum(nil))
+
+	req.Form.Set("Signature", string(signature))
+
+	req.URL.RawQuery = req.Form.Encode()
+
+	return nil
+}
+
 // Common date formats for signing requests
 const (
 	ISO8601BasicFormat      = "20060102T150405Z"
@@ -174,6 +217,11 @@ func (s *V4Signer) Sign(req *http.Request) {
 	return
 }
 
+func (s *V4Signer) SignRequest(req *http.Request) error {
+	s.Sign(req)
+	return nil
+}
+
 /*
 requestTime method will parse the time from the request "x-amz-date" or "date" headers.
 If the "x-amz-date" header is present, that will take priority over the "date" header.

Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi.go

@@ -8,6 +8,7 @@ import (
 	"encoding/xml"
 	"errors"
 	"io"
+	"net/http"
 	"net/url"
 	"sort"
 	"strconv"
@@ -428,6 +429,11 @@ func (m *Multi) Complete(parts []Part) error {
 			payload: bytes.NewReader(data),
 		}
 		var resp completeUploadResp
+		if m.Bucket.Region.Name == "generic" {
+			headers := make(http.Header)
+			headers.Add("Content-Length", strconv.FormatInt(int64(len(data)), 10))
+			req.headers = headers
+		}
 		err := m.Bucket.S3.query(req, &resp)
 		if shouldRetry(err) && attempt.HasNext() {
 			continue

Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi_test.go

@@ -2,11 +2,12 @@ package s3_test
 
 import (
 	"encoding/xml"
-	"github.com/AdRoll/goamz/s3"
-	"gopkg.in/check.v1"
 	"io"
 	"io/ioutil"
 	"strings"
+
+	"github.com/AdRoll/goamz/s3"
+	"gopkg.in/check.v1"
 )
 
 func (s *S) TestInitMulti(c *check.C) {
@@ -438,3 +439,42 @@ func (s *S) TestListMulti(c *check.C) {
 	c.Assert(req.Form["delimiter"], check.DeepEquals, []string{"/"})
 	c.Assert(req.Form["max-uploads"], check.DeepEquals, []string{"1000"})
 }
+
+func (s *S) TestMultiCompleteSupportRadosGW(c *check.C) {
+	testServer.Response(200, nil, InitMultiResultDump)
+	testServer.Response(200, nil, MultiCompleteDump)
+	s.s3.Region.Name = "generic"
+	b := s.s3.Bucket("sample")
+
+	multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
+	c.Assert(err, check.IsNil)
+
+	err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}})
+	c.Assert(err, check.IsNil)
+
+	testServer.WaitRequest()
+	req := testServer.WaitRequest()
+	c.Assert(req.Method, check.Equals, "POST")
+	c.Assert(req.URL.Path, check.Equals, "/sample/multi")
+	c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
+	c.Assert(req.Header["Content-Length"], check.NotNil)
+
+	var payload struct {
+		XMLName xml.Name
+		Part    []struct {
+			PartNumber int
+			ETag       string
+		}
+	}
+
+	dec := xml.NewDecoder(req.Body)
+	err = dec.Decode(&payload)
+	c.Assert(err, check.IsNil)
+
+	c.Assert(payload.XMLName.Local, check.Equals, "CompleteMultipartUpload")
+	c.Assert(len(payload.Part), check.Equals, 2)
+	c.Assert(payload.Part[0].PartNumber, check.Equals, 1)
+	c.Assert(payload.Part[0].ETag, check.Equals, `"ETag1"`)
+	c.Assert(payload.Part[1].PartNumber, check.Equals, 2)
+	c.Assert(payload.Part[1].ETag, check.Equals, `"ETag2"`)
+}

Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go

@@ -1074,11 +1074,14 @@ func (s3 *S3) prepare(req *request) error {
 			return err
 		}
 
-		signpathPatiallyEscaped := partiallyEscapedPath(req.path)
+		signpathPartiallyEscaped := partiallyEscapedPath(req.path)
+		if strings.IndexAny(s3.Region.S3BucketEndpoint, "${bucket}") >= 0 {
+			signpathPartiallyEscaped = "/" + req.bucket + signpathPartiallyEscaped
+		}
 		req.headers["Host"] = []string{u.Host}
 		req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)}
 
-		sign(s3.Auth, req.method, signpathPatiallyEscaped, req.params, req.headers)
+		sign(s3.Auth, req.method, signpathPartiallyEscaped, req.params, req.headers)
 	} else {
 		hreq, err := s3.setupHttpRequest(req)
 		if err != nil {
@@ -1111,7 +1114,9 @@ func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) {
 	if err != nil {
 		return nil, err
 	}
-	u.Opaque = fmt.Sprintf("//%s%s", u.Host, partiallyEscapedPath(u.Path))
+	if s3.Region.Name != "generic" {
+		u.Opaque = fmt.Sprintf("//%s%s", u.Host, partiallyEscapedPath(u.Path))
+	}
 
 	hreq := http.Request{
 		URL:        u,

Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go

@@ -500,3 +500,14 @@ func (s *S) TestLocation(c *check.C) {
 	c.Assert(err, check.IsNil)
 	c.Assert(resultUsWest1, check.Equals, expectedUsWest1)
 }
+
+func (s *S) TestSupportRadosGW(c *check.C) {
+	testServer.Response(200, nil, "content")
+	s.s3.Region.Name = "generic"
+	b := s.s3.Bucket("bucket")
+	_, err := b.Get("rgw")
+
+	req := testServer.WaitRequest()
+	c.Assert(err, check.IsNil)
+	c.Assert(req.RequestURI, check.Equals, "/bucket/rgw")
+}

Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go

@@ -316,7 +316,8 @@ func (nullResource) get(a *action) interface{}    { return notAllowed() }
 func (nullResource) post(a *action) interface{}   { return notAllowed() }
 func (nullResource) delete(a *action) interface{} { return notAllowed() }
 
-const timeFormat = "2006-01-02T15:04:05.000Z07:00"
+const timeFormat = "2006-01-02T15:04:05.000Z"
+const lastModifiedTimeFormat = "Mon, 2 Jan 2006 15:04:05 GMT"
 
 type bucketResource struct {
 	name   string
@@ -418,7 +419,7 @@ func (s orderedObjects) Less(i, j int) bool {
 func (obj *object) s3Key() s3.Key {
 	return s3.Key{
 		Key:          obj.name,
-		LastModified: obj.mtime.Format(timeFormat),
+		LastModified: obj.mtime.UTC().Format(timeFormat),
 		Size:         int64(len(obj.data)),
 		ETag:         fmt.Sprintf(`"%x"`, obj.checksum),
 		// TODO StorageClass
@@ -647,8 +648,8 @@ func (objr objectResource) get(a *action) interface{} {
 	// TODO Connection: close ??
 	// TODO x-amz-request-id
 	h.Set("Content-Length", fmt.Sprint(len(data)))
-	h.Set("ETag", hex.EncodeToString(obj.checksum))
+	h.Set("ETag", "\""+hex.EncodeToString(obj.checksum)+"\"")
-	h.Set("Last-Modified", obj.mtime.Format(time.RFC1123))
+	h.Set("Last-Modified", obj.mtime.Format(lastModifiedTimeFormat))
 
 	if status != http.StatusOK {
 		a.w.WriteHeader(status)
@@ -750,8 +751,45 @@ func (objr objectResource) put(a *action) interface{} {
 				obj.meta[key] = values
 			}
 		}
-		obj.data = data
+
-		obj.checksum = gotHash
+		if copySource := a.req.Header.Get("X-Amz-Copy-Source"); copySource != "" {
+			idx := strings.IndexByte(copySource, '/')
+
+			if idx == -1 {
+				fatalf(400, "InvalidRequest", "Wrongly formatted X-Amz-Copy-Source")
+			}
+
+			sourceBucketName := copySource[0:idx]
+			sourceKey := copySource[1+idx:]
+
+			sourceBucket := a.srv.buckets[sourceBucketName]
+
+			if sourceBucket == nil {
+				fatalf(404, "NoSuchBucket", "The specified source bucket does not exist")
+			}
+
+			sourceObject := sourceBucket.objects[sourceKey]
+
+			if sourceObject == nil {
+				fatalf(404, "NoSuchKey", "The specified source key does not exist")
+			}
+
+			obj.data = make([]byte, len(sourceObject.data))
+			copy(obj.data, sourceObject.data)
+
+			obj.checksum = make([]byte, len(sourceObject.checksum))
+			copy(obj.checksum, sourceObject.checksum)
+
+			obj.meta = make(http.Header, len(sourceObject.meta))
+
+			for k, v := range sourceObject.meta {
+				obj.meta[k] = make([]string, len(v))
+				copy(obj.meta[k], v)
+			}
+		} else {
+			obj.data = data
+			obj.checksum = gotHash
+		}
 		obj.mtime = time.Now()
 		objr.bucket.objects[objr.name] = obj
 	} else {