TrueCloudLab
/
distribution
13
0
Fork 3
Code Issues 2 Pull Requests Packages Projects Releases 1 Wiki Activity

Upgrade Amazon library to revision aa6e716d710a0c7941cb2075cfbb9661f16d21f1 

Signed-off-by: Troels Thomsen <troels@thomsen.io>
pull/1042/head
Troels Thomsen 2015-09-30 00:29:17 +02:00
parent 14a4b4d9d6
commit cca9f74eec
9 changed files with 176 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Godeps/Godeps.json generated
View File

@ -7,15 +7,15 @@
"Deps": [
{
"ImportPath": "github.com/AdRoll/goamz/aws",
"Rev": "f8c4952d5bc3056c0ca6711a1f56bc88b828d989"
"Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1"
},
{
"ImportPath": "github.com/AdRoll/goamz/cloudfront",
"Rev": "f8c4952d5bc3056c0ca6711a1f56bc88b828d989"
"Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1"
},
{
"ImportPath": "github.com/AdRoll/goamz/s3",
"Rev": "f8c4952d5bc3056c0ca6711a1f56bc88b828d989"
"Rev": "aa6e716d710a0c7941cb2075cfbb9661f16d21f1"
},
{
"ImportPath": "github.com/Azure/azure-sdk-for-go/storage",

2
Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go generated vendored
View File

@ -51,7 +51,7 @@ type ServiceInfo struct {
// See http://goo.gl/d8BP1 for more details.
type Region struct {
Name string // the canonical name of this region.
EC2Endpoint string
EC2Endpoint ServiceInfo
S3Endpoint string
S3BucketEndpoint string // Not needed by AWS S3. Use ${bucket} for bucket name.
S3LocationConstraint bool // true if this region requires a LocationConstraint declaration.

24
Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go generated vendored
View File

@ -2,7 +2,7 @@ package aws
var USGovWest = Region{
"us-gov-west-1",
"https://ec2.us-gov-west-1.amazonaws.com",
ServiceInfo{"https://ec2.us-gov-west-1.amazonaws.com", V2Signature},
"https://s3-fips-us-gov-west-1.amazonaws.com",
"",
true,
@ -26,8 +26,8 @@ var USGovWest = Region{
var USEast = Region{
"us-east-1",
"https://ec2.us-east-1.amazonaws.com",
"https://s3.amazonaws.com",
ServiceInfo{"https://ec2.us-east-1.amazonaws.com", V2Signature},
"https://s3-external-1.amazonaws.com",
"",
false,
false,
@ -50,7 +50,7 @@ var USEast = Region{
var USWest = Region{
"us-west-1",
"https://ec2.us-west-1.amazonaws.com",
ServiceInfo{"https://ec2.us-west-1.amazonaws.com", V2Signature},
"https://s3-us-west-1.amazonaws.com",
"",
true,
@ -74,7 +74,7 @@ var USWest = Region{
var USWest2 = Region{
"us-west-2",
"https://ec2.us-west-2.amazonaws.com",
ServiceInfo{"https://ec2.us-west-2.amazonaws.com", V2Signature},
"https://s3-us-west-2.amazonaws.com",
"",
true,
@ -98,7 +98,7 @@ var USWest2 = Region{
var EUWest = Region{
"eu-west-1",
"https://ec2.eu-west-1.amazonaws.com",
ServiceInfo{"https://ec2.eu-west-1.amazonaws.com", V2Signature},
"https://s3-eu-west-1.amazonaws.com",
"",
true,
@ -122,7 +122,7 @@ var EUWest = Region{
var EUCentral = Region{
"eu-central-1",
"https://ec2.eu-central-1.amazonaws.com",
ServiceInfo{"https://ec2.eu-central-1.amazonaws.com", V4Signature},
"https://s3-eu-central-1.amazonaws.com",
"",
true,
@ -146,7 +146,7 @@ var EUCentral = Region{
var APSoutheast = Region{
"ap-southeast-1",
"https://ec2.ap-southeast-1.amazonaws.com",
ServiceInfo{"https://ec2.ap-southeast-1.amazonaws.com", V2Signature},
"https://s3-ap-southeast-1.amazonaws.com",
"",
true,
@ -170,7 +170,7 @@ var APSoutheast = Region{
var APSoutheast2 = Region{
"ap-southeast-2",
"https://ec2.ap-southeast-2.amazonaws.com",
ServiceInfo{"https://ec2.ap-southeast-2.amazonaws.com", V2Signature},
"https://s3-ap-southeast-2.amazonaws.com",
"",
true,
@ -194,7 +194,7 @@ var APSoutheast2 = Region{
var APNortheast = Region{
"ap-northeast-1",
"https://ec2.ap-northeast-1.amazonaws.com",
ServiceInfo{"https://ec2.ap-northeast-1.amazonaws.com", V2Signature},
"https://s3-ap-northeast-1.amazonaws.com",
"",
true,
@ -218,7 +218,7 @@ var APNortheast = Region{
var SAEast = Region{
"sa-east-1",
"https://ec2.sa-east-1.amazonaws.com",
ServiceInfo{"https://ec2.sa-east-1.amazonaws.com", V2Signature},
"https://s3-sa-east-1.amazonaws.com",
"",
true,
@ -242,7 +242,7 @@ var SAEast = Region{
var CNNorth1 = Region{
"cn-north-1",
"https://ec2.cn-north-1.amazonaws.com.cn",
ServiceInfo{"https://ec2.cn-north-1.amazonaws.com.cn", V2Signature},
"https://s3.cn-north-1.amazonaws.com.cn",
"",
true,

50
Godeps/_workspace/src/github.com/AdRoll/goamz/aws/sign.go generated vendored
View File

@ -15,6 +15,28 @@ import (
"time"
)
// AWS specifies that the parameters in a signed request must
// be provided in the natural order of the keys. This is distinct
// from the natural order of the encoded value of key=value.
// Percent and gocheck.Equals affect the sorting order.
func EncodeSorted(values url.Values) string {
// preallocate the arrays for perfomance
keys := make([]string, 0, len(values))
sarray := make([]string, 0, len(values))
for k, _ := range values {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
for _, v := range values[k] {
sarray = append(sarray, Encode(k)+"="+Encode(v))
}
}
return strings.Join(sarray, "&")
}
type V2Signer struct {
auth Auth
service ServiceInfo
@ -38,7 +60,6 @@ func (s *V2Signer) Sign(method, path string, params map[string]string) {
if s.auth.Token() != "" {
params["SecurityToken"] = s.auth.Token()
}
// AWS specifies that the parameters in a signed request must
// be provided in the natural order of the keys. This is distinct
// from the natural order of the encoded value of key=value.
@ -61,6 +82,28 @@ func (s *V2Signer) Sign(method, path string, params map[string]string) {
params["Signature"] = string(signature)
}
func (s *V2Signer) SignRequest(req *http.Request) error {
req.ParseForm()
req.Form.Set("AWSAccessKeyId", s.auth.AccessKey)
req.Form.Set("SignatureVersion", "2")
req.Form.Set("SignatureMethod", "HmacSHA256")
if s.auth.Token() != "" {
req.Form.Set("SecurityToken", s.auth.Token())
}
payload := req.Method + "\n" + req.URL.Host + "\n" + req.URL.Path + "\n" + EncodeSorted(req.Form)
hash := hmac.New(sha256.New, []byte(s.auth.SecretKey))
hash.Write([]byte(payload))
signature := make([]byte, b64.EncodedLen(hash.Size()))
b64.Encode(signature, hash.Sum(nil))
req.Form.Set("Signature", string(signature))
req.URL.RawQuery = req.Form.Encode()
return nil
}
// Common date formats for signing requests
const (
ISO8601BasicFormat = "20060102T150405Z"
@ -174,6 +217,11 @@ func (s *V4Signer) Sign(req *http.Request) {
return
}
func (s *V4Signer) SignRequest(req *http.Request) error {
s.Sign(req)
return nil
}
/*
requestTime method will parse the time from the request "x-amz-date" or "date" headers.
If the "x-amz-date" header is present, that will take priority over the "date" header.

6
Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi.go generated vendored
View File

@ -8,6 +8,7 @@ import (
"encoding/xml"
"errors"
"io"
"net/http"
"net/url"
"sort"
"strconv"
@ -428,6 +429,11 @@ func (m *Multi) Complete(parts []Part) error {
payload: bytes.NewReader(data),
}
var resp completeUploadResp
if m.Bucket.Region.Name == "generic" {
headers := make(http.Header)
headers.Add("Content-Length", strconv.FormatInt(int64(len(data)), 10))
req.headers = headers
}
err := m.Bucket.S3.query(req, &resp)
if shouldRetry(err) && attempt.HasNext() {
continue

44
Godeps/_workspace/src/github.com/AdRoll/goamz/s3/multi_test.go generated vendored
View File

@ -2,11 +2,12 @@ package s3_test
import (
"encoding/xml"
"github.com/AdRoll/goamz/s3"
"gopkg.in/check.v1"
"io"
"io/ioutil"
"strings"
"github.com/AdRoll/goamz/s3"
"gopkg.in/check.v1"
)
func (s *S) TestInitMulti(c *check.C) {
@ -438,3 +439,42 @@ func (s *S) TestListMulti(c *check.C) {
c.Assert(req.Form["delimiter"], check.DeepEquals, []string{"/"})
c.Assert(req.Form["max-uploads"], check.DeepEquals, []string{"1000"})
}
func (s *S) TestMultiCompleteSupportRadosGW(c *check.C) {
testServer.Response(200, nil, InitMultiResultDump)
testServer.Response(200, nil, MultiCompleteDump)
s.s3.Region.Name = "generic"
b := s.s3.Bucket("sample")
multi, err := b.InitMulti("multi", "text/plain", s3.Private, s3.Options{})
c.Assert(err, check.IsNil)
err = multi.Complete([]s3.Part{{2, `"ETag2"`, 32}, {1, `"ETag1"`, 64}})
c.Assert(err, check.IsNil)
testServer.WaitRequest()
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "POST")
c.Assert(req.URL.Path, check.Equals, "/sample/multi")
c.Assert(req.Form.Get("uploadId"), check.Matches, "JNbR_[A-Za-z0-9.]+QQ--")
c.Assert(req.Header["Content-Length"], check.NotNil)
var payload struct {
XMLName xml.Name
Part []struct {
PartNumber int
ETag string
}
}
dec := xml.NewDecoder(req.Body)
err = dec.Decode(&payload)
c.Assert(err, check.IsNil)
c.Assert(payload.XMLName.Local, check.Equals, "CompleteMultipartUpload")
c.Assert(len(payload.Part), check.Equals, 2)
c.Assert(payload.Part[0].PartNumber, check.Equals, 1)
c.Assert(payload.Part[0].ETag, check.Equals, `"ETag1"`)
c.Assert(payload.Part[1].PartNumber, check.Equals, 2)
c.Assert(payload.Part[1].ETag, check.Equals, `"ETag2"`)
}

11
Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go generated vendored
View File

@ -1074,11 +1074,14 @@ func (s3 *S3) prepare(req *request) error {
return err
}
signpathPatiallyEscaped := partiallyEscapedPath(req.path)
signpathPartiallyEscaped := partiallyEscapedPath(req.path)
if strings.IndexAny(s3.Region.S3BucketEndpoint, "${bucket}") >= 0 {
signpathPartiallyEscaped = "/" + req.bucket + signpathPartiallyEscaped
}
req.headers["Host"] = []string{u.Host}
req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)}
sign(s3.Auth, req.method, signpathPatiallyEscaped, req.params, req.headers)
sign(s3.Auth, req.method, signpathPartiallyEscaped, req.params, req.headers)
} else {
hreq, err := s3.setupHttpRequest(req)
if err != nil {
@ -1111,7 +1114,9 @@ func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) {
if err != nil {
return nil, err
}
u.Opaque = fmt.Sprintf("//%s%s", u.Host, partiallyEscapedPath(u.Path))
if s3.Region.Name != "generic" {
u.Opaque = fmt.Sprintf("//%s%s", u.Host, partiallyEscapedPath(u.Path))
}
hreq := http.Request{
URL: u,

11
Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go generated vendored
View File

@ -500,3 +500,14 @@ func (s *S) TestLocation(c *check.C) {
c.Assert(err, check.IsNil)
c.Assert(resultUsWest1, check.Equals, expectedUsWest1)
}
func (s *S) TestSupportRadosGW(c *check.C) {
testServer.Response(200, nil, "content")
s.s3.Region.Name = "generic"
b := s.s3.Bucket("bucket")
_, err := b.Get("rgw")
req := testServer.WaitRequest()
c.Assert(err, check.IsNil)
c.Assert(req.RequestURI, check.Equals, "/bucket/rgw")
}

50
Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go generated vendored
View File

@ -316,7 +316,8 @@ func (nullResource) get(a *action) interface{} { return notAllowed() }
func (nullResource) post(a *action) interface{} { return notAllowed() }
func (nullResource) delete(a *action) interface{} { return notAllowed() }
const timeFormat = "2006-01-02T15:04:05.000Z07:00"
const timeFormat = "2006-01-02T15:04:05.000Z"
const lastModifiedTimeFormat = "Mon, 2 Jan 2006 15:04:05 GMT"
type bucketResource struct {
name string
@ -418,7 +419,7 @@ func (s orderedObjects) Less(i, j int) bool {
func (obj *object) s3Key() s3.Key {
return s3.Key{
Key: obj.name,
LastModified: obj.mtime.Format(timeFormat),
LastModified: obj.mtime.UTC().Format(timeFormat),
Size: int64(len(obj.data)),
ETag: fmt.Sprintf(`"%x"`, obj.checksum),
// TODO StorageClass
@ -647,8 +648,8 @@ func (objr objectResource) get(a *action) interface{} {
// TODO Connection: close ??
// TODO x-amz-request-id
h.Set("Content-Length", fmt.Sprint(len(data)))
h.Set("ETag", hex.EncodeToString(obj.checksum))
h.Set("Last-Modified", obj.mtime.Format(time.RFC1123))
h.Set("ETag", "\""+hex.EncodeToString(obj.checksum)+"\"")
h.Set("Last-Modified", obj.mtime.Format(lastModifiedTimeFormat))
if status != http.StatusOK {
a.w.WriteHeader(status)
@ -750,8 +751,45 @@ func (objr objectResource) put(a *action) interface{} {
obj.meta[key] = values
}
}
obj.data = data
obj.checksum = gotHash
if copySource := a.req.Header.Get("X-Amz-Copy-Source"); copySource != "" {
idx := strings.IndexByte(copySource, '/')
if idx == -1 {
fatalf(400, "InvalidRequest", "Wrongly formatted X-Amz-Copy-Source")
}
sourceBucketName := copySource[0:idx]
sourceKey := copySource[1+idx:]
sourceBucket := a.srv.buckets[sourceBucketName]
if sourceBucket == nil {
fatalf(404, "NoSuchBucket", "The specified source bucket does not exist")
}
sourceObject := sourceBucket.objects[sourceKey]
if sourceObject == nil {
fatalf(404, "NoSuchKey", "The specified source key does not exist")
}
obj.data = make([]byte, len(sourceObject.data))
copy(obj.data, sourceObject.data)
obj.checksum = make([]byte, len(sourceObject.checksum))
copy(obj.checksum, sourceObject.checksum)
obj.meta = make(http.Header, len(sourceObject.meta))
for k, v := range sourceObject.meta {
obj.meta[k] = make([]string, len(v))
copy(obj.meta[k], v)
}
} else {
obj.data = data
obj.checksum = gotHash
}
obj.mtime = time.Now()
objr.bucket.objects[objr.name] = obj
} else {