alexvanin/distribution
1
0
Fork 0
forked from TrueCloudLab/distribution
Code Pull requests Activity

Merge pull request #415 from stevvooe/update-goamz

Browse source 
Update goamz package dependency
This commit is contained in:
Stephen Day 2015-04-22 13:06:46 -07:00
commit cbfce39acb
6 changed files with 281 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Godeps/Godeps.json generated
View file

@ -12,15 +12,15 @@
},
{
"ImportPath": "github.com/AdRoll/goamz/aws",
"Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103"
"Rev": "cc210f45dcb9889c2769a274522be2bf70edfb99"
},
{
"ImportPath": "github.com/AdRoll/goamz/cloudfront",
"Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103"
"Rev": "cc210f45dcb9889c2769a274522be2bf70edfb99"
},
{
"ImportPath": "github.com/AdRoll/goamz/s3",
"Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103"
"Rev": "cc210f45dcb9889c2769a274522be2bf70edfb99"
},
{
"ImportPath": "github.com/MSOpenTech/azure-sdk-for-go/storage",

7
Godeps/_workspace/src/github.com/AdRoll/goamz/aws/aws.go generated vendored
View file

@ -62,6 +62,7 @@ type Region struct {
SESEndpoint string
IAMEndpoint string
ELBEndpoint string
KMSEndpoint string
DynamoDBEndpoint string
CloudWatchServicepoint ServiceInfo
AutoScalingEndpoint string
@ -83,6 +84,7 @@ var Regions = map[string]Region{
USWest2.Name: USWest2,
USGovWest.Name: USGovWest,
SAEast.Name: SAEast,
CNNorth1.Name: CNNorth1,
}
// Designates a signer interface suitable for signing AWS requests, params
@ -208,7 +210,10 @@ func (a *Auth) Token() string {
return ""
}
if time.Since(a.expiration) >= -30*time.Second { //in an ideal world this should be zero assuming the instance is synching it's clock
*a, _ = GetAuth("", "", "", time.Time{})
auth, err := GetAuth("", "", "", time.Time{})
if err == nil {
*a = auth
}
}
return a.token
}

34
Godeps/_workspace/src/github.com/AdRoll/goamz/aws/regions.go generated vendored
View file

@ -13,6 +13,7 @@ var USGovWest = Region{
"",
"https://iam.us-gov.amazonaws.com",
"https://elasticloadbalancing.us-gov-west-1.amazonaws.com",
"",
"https://dynamodb.us-gov-west-1.amazonaws.com",
ServiceInfo{"https://monitoring.us-gov-west-1.amazonaws.com", V2Signature},
"https://autoscaling.us-gov-west-1.amazonaws.com",
@ -36,6 +37,7 @@ var USEast = Region{
"https://email.us-east-1.amazonaws.com",
"https://iam.amazonaws.com",
"https://elasticloadbalancing.us-east-1.amazonaws.com",
"https://kms.us-east-1.amazonaws.com",
"https://dynamodb.us-east-1.amazonaws.com",
ServiceInfo{"https://monitoring.us-east-1.amazonaws.com", V2Signature},
"https://autoscaling.us-east-1.amazonaws.com",
@ -59,6 +61,7 @@ var USWest = Region{
"",
"https://iam.amazonaws.com",
"https://elasticloadbalancing.us-west-1.amazonaws.com",
"https://kms.us-west-1.amazonaws.com",
"https://dynamodb.us-west-1.amazonaws.com",
ServiceInfo{"https://monitoring.us-west-1.amazonaws.com", V2Signature},
"https://autoscaling.us-west-1.amazonaws.com",
@ -82,6 +85,7 @@ var USWest2 = Region{
"https://email.us-west-2.amazonaws.com",
"https://iam.amazonaws.com",
"https://elasticloadbalancing.us-west-2.amazonaws.com",
"https://kms.us-west-2.amazonaws.com",
"https://dynamodb.us-west-2.amazonaws.com",
ServiceInfo{"https://monitoring.us-west-2.amazonaws.com", V2Signature},
"https://autoscaling.us-west-2.amazonaws.com",
@ -105,6 +109,7 @@ var EUWest = Region{
"https://email.eu-west-1.amazonaws.com",
"https://iam.amazonaws.com",
"https://elasticloadbalancing.eu-west-1.amazonaws.com",
"https://kms.eu-west-1.amazonaws.com",
"https://dynamodb.eu-west-1.amazonaws.com",
ServiceInfo{"https://monitoring.eu-west-1.amazonaws.com", V2Signature},
"https://autoscaling.eu-west-1.amazonaws.com",
@ -128,6 +133,7 @@ var EUCentral = Region{
"",
"https://iam.amazonaws.com",
"https://elasticloadbalancing.eu-central-1.amazonaws.com",
"https://kms.eu-central-1.amazonaws.com",
"https://dynamodb.eu-central-1.amazonaws.com",
ServiceInfo{"https://monitoring.eu-central-1.amazonaws.com", V2Signature},
"https://autoscaling.eu-central-1.amazonaws.com",
@ -151,6 +157,7 @@ var APSoutheast = Region{
"",
"https://iam.amazonaws.com",
"https://elasticloadbalancing.ap-southeast-1.amazonaws.com",
"https://kms.ap-southeast-1.amazonaws.com",
"https://dynamodb.ap-southeast-1.amazonaws.com",
ServiceInfo{"https://monitoring.ap-southeast-1.amazonaws.com", V2Signature},
"https://autoscaling.ap-southeast-1.amazonaws.com",
@ -174,6 +181,7 @@ var APSoutheast2 = Region{
"",
"https://iam.amazonaws.com",
"https://elasticloadbalancing.ap-southeast-2.amazonaws.com",
"https://kms.ap-southeast-2.amazonaws.com",
"https://dynamodb.ap-southeast-2.amazonaws.com",
ServiceInfo{"https://monitoring.ap-southeast-2.amazonaws.com", V2Signature},
"https://autoscaling.ap-southeast-2.amazonaws.com",
@ -197,6 +205,7 @@ var APNortheast = Region{
"",
"https://iam.amazonaws.com",
"https://elasticloadbalancing.ap-northeast-1.amazonaws.com",
"https://kms.ap-northeast-1.amazonaws.com",
"https://dynamodb.ap-northeast-1.amazonaws.com",
ServiceInfo{"https://monitoring.ap-northeast-1.amazonaws.com", V2Signature},
"https://autoscaling.ap-northeast-1.amazonaws.com",
@ -220,6 +229,7 @@ var SAEast = Region{
"",
"https://iam.amazonaws.com",
"https://elasticloadbalancing.sa-east-1.amazonaws.com",
"https://kms.sa-east-1.amazonaws.com",
"https://dynamodb.sa-east-1.amazonaws.com",
ServiceInfo{"https://monitoring.sa-east-1.amazonaws.com", V2Signature},
"https://autoscaling.sa-east-1.amazonaws.com",
@ -229,3 +239,27 @@ var SAEast = Region{
"https://cloudformation.sa-east-1.amazonaws.com",
"https://elasticache.sa-east-1.amazonaws.com",
}
var CNNorth1 = Region{
"cn-north-1",
"https://ec2.cn-north-1.amazonaws.com.cn",
"https://s3.cn-north-1.amazonaws.com.cn",
"",
true,
true,
"",
"https://sns.cn-north-1.amazonaws.com.cn",
"https://sqs.cn-north-1.amazonaws.com.cn",
"",
"https://iam.cn-north-1.amazonaws.com.cn",
"https://elasticloadbalancing.cn-north-1.amazonaws.com.cn",
"",
"https://dynamodb.cn-north-1.amazonaws.com.cn",
ServiceInfo{"https://monitoring.cn-north-1.amazonaws.com.cn", V4Signature},
"https://autoscaling.cn-north-1.amazonaws.com.cn",
ServiceInfo{"https://rds.cn-north-1.amazonaws.com.cn", V4Signature},
"",
"https://sts.cn-north-1.amazonaws.com.cn",
"",
"",
}

25
Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3.go generated vendored
View file

@ -25,6 +25,7 @@ import (
"net/http"
"net/http/httputil"
"net/url"
"path"
"strconv"
"strings"
"time"
@ -70,9 +71,8 @@ type Options struct {
ContentMD5 string
ContentDisposition string
Range string
StorageClass StorageClass
// What else?
//// The following become headers so they are []strings rather than strings... I think
// x-amz-storage-class []string
}
type CopyOptions struct {
@ -96,7 +96,7 @@ var attempts = aws.AttemptStrategy{
// New creates a new S3.
func New(auth aws.Auth, region aws.Region) *S3 {
return &S3{auth, region, 0, 0, 0, aws.V2Signature}
return &S3{auth, region, 0, 0, aws.V2Signature, 0}
}
// Bucket returns a Bucket with the given name.
@ -164,6 +164,13 @@ const (
BucketOwnerFull = ACL("bucket-owner-full-control")
)
type StorageClass string
const (
ReducedRedundancy = StorageClass("REDUCED_REDUNDANCY")
StandardStorage = StorageClass("STANDARD")
)
// PutBucket creates a new bucket.
//
// See http://goo.gl/ndjnR for details.
@ -401,6 +408,10 @@ func (o Options) addHeaders(headers map[string][]string) {
if len(o.ContentDisposition) != 0 {
headers["Content-Disposition"] = []string{o.ContentDisposition}
}
if len(o.StorageClass) != 0 {
headers["x-amz-storage-class"] = []string{string(o.StorageClass)}
}
for k, v := range o.Meta {
headers["x-amz-meta-"+k] = v
}
@ -816,8 +827,8 @@ func (b *Bucket) SignedURLWithMethod(method, path string, expires time.Time, par
// UploadSignedURL returns a signed URL that allows anyone holding the URL
// to upload the object at path. The signature is valid until expires.
// contenttype is a string like image/png
// path is the resource name in s3 terminalogy like images/ali.png [obviously exclusing the bucket name itself]
func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time.Time) string {
// name is the resource name in s3 terminology like images/ali.png [obviously excluding the bucket name itself]
func (b *Bucket) UploadSignedURL(name, method, content_type string, expires time.Time) string {
expire_date := expires.Unix()
if method != "POST" {
method = "PUT"
@ -830,7 +841,7 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time
tokenData = "x-amz-security-token:" + a.Token() + "\n"
}
stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n" + tokenData + "/" + b.Name + "/" + path
stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n" + tokenData + "/" + path.Join(b.Name, name)
secretKey := a.SecretKey
accessId := a.AccessKey
mac := hmac.New(sha1.New, []byte(secretKey))
@ -844,7 +855,7 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time
log.Println("ERROR sining url for S3 upload", err)
return ""
}
signedurl.Path += path
signedurl.Path = name
params := url.Values{}
params.Add("AWSAccessKeyId", accessId)
params.Add("Expires", strconv.FormatInt(expire_date, 10))

16
Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3_test.go generated vendored
View file

@ -230,6 +230,22 @@ func (s *S) TestPutObject(c *check.C) {
c.Assert(req.Header["X-Amz-Acl"], check.DeepEquals, []string{"private"})
}
func (s *S) TestPutObjectReducedRedundancy(c *check.C) {
testServer.Response(200, nil, "")
b := s.s3.Bucket("bucket")
err := b.Put("name", []byte("content"), "content-type", s3.Private, s3.Options{StorageClass: s3.ReducedRedundancy})
c.Assert(err, check.IsNil)
req := testServer.WaitRequest()
c.Assert(req.Method, check.Equals, "PUT")
c.Assert(req.URL.Path, check.Equals, "/bucket/name")
c.Assert(req.Header["Date"], check.Not(check.DeepEquals), []string{""})
c.Assert(req.Header["Content-Type"], check.DeepEquals, []string{"content-type"})
c.Assert(req.Header["Content-Length"], check.DeepEquals, []string{"7"})
c.Assert(req.Header["X-Amz-Storage-Class"], check.DeepEquals, []string{"REDUCED_REDUNDANCY"})
}
// PutCopy docs: http://goo.gl/mhEHtA
func (s *S) TestPutCopy(c *check.C) {
testServer.Response(200, nil, PutCopyResultDump)

230
Godeps/_workspace/src/github.com/AdRoll/goamz/s3/s3test/server.go generated vendored
View file

@ -11,6 +11,7 @@ import (
"io"
"io/ioutil"
"log"
"math/rand"
"net"
"net/http"
"net/url"
@ -51,6 +52,10 @@ type Config struct {
// all other regions.
// http://docs.amazonwebservices.com/AmazonS3/latest/API/ErrorResponses.html
Send409Conflict bool
// Address on which to listen. By default, a random port is assigned by the
// operating system and the server listens on localhost.
ListenAddress string
}
func (c *Config) send409Conflict() bool {
@ -72,10 +77,11 @@ type Server struct {
}
type bucket struct {
name string
acl s3.ACL
ctime time.Time
objects map[string]*object
name string
acl s3.ACL
ctime time.Time
objects map[string]*object
multipartUploads map[string][]*multipartUploadPart
}
type object struct {
@ -86,6 +92,12 @@ type object struct {
data []byte
}
type multipartUploadPart struct {
data []byte
etag string
lastModified time.Time
}
// A resource encapsulates the subject of an HTTP request.
// The resource referred to may or may not exist
// when the request is made.
@ -97,7 +109,13 @@ type resource interface {
}
func NewServer(config *Config) (*Server, error) {
l, err := net.Listen("tcp", "localhost:0")
listenAddress := "localhost:0"
if config != nil && config.ListenAddress != "" {
listenAddress = config.ListenAddress
}
l, err := net.Listen("tcp", listenAddress)
if err != nil {
return nil, fmt.Errorf("cannot listen on localhost: %v", err)
}
@ -217,10 +235,8 @@ var unimplementedBucketResourceNames = map[string]bool{
}
var unimplementedObjectResourceNames = map[string]bool{
"uploadId": true,
"acl": true,
"torrent": true,
"uploads": true,
"acl": true,
"torrent": true,
}
var pathRegexp = regexp.MustCompile("/(([^/]+)(/(.*))?)?")
@ -420,7 +436,8 @@ func (r bucketResource) put(a *action) interface{} {
r.bucket = &bucket{
name: r.name,
// TODO default acl
objects: make(map[string]*object),
objects: make(map[string]*object),
multipartUploads: make(map[string][]*multipartUploadPart),
}
a.srv.buckets[r.name] = r.bucket
created = true
@ -615,12 +632,29 @@ func (objr objectResource) put(a *action) interface{} {
// TODO x-amz-server-side-encryption
// TODO x-amz-storage-class
// TODO is this correct, or should we erase all previous metadata?
obj := objr.object
if obj == nil {
obj = &object{
name: objr.name,
meta: make(http.Header),
uploadId := a.req.URL.Query().Get("uploadId")
// Check that the upload ID is valid if this is a multipart upload
if uploadId != "" {
if _, ok := objr.bucket.multipartUploads[uploadId]; !ok {
fatalf(404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")
}
partNumberStr := a.req.URL.Query().Get("partNumber")
if partNumberStr == "" {
fatalf(400, "InvalidRequest", "Missing partNumber parameter")
}
partNumber, err := strconv.ParseUint(partNumberStr, 10, 32)
if err != nil {
fatalf(400, "InvalidRequest", "partNumber is not a number")
}
// Parts are 1-indexed for multipart uploads
if uint(partNumber)-1 != uint(len(objr.bucket.multipartUploads[uploadId])) {
fatalf(400, "InvalidRequest", "Invalid part number")
}
}
@ -646,26 +680,170 @@ func (objr objectResource) put(a *action) interface{} {
fatalf(400, "IncompleteBody", "You did not provide the number of bytes specified by the Content-Length HTTP header")
}
// PUT request has been successful - save data and metadata
for key, values := range a.req.Header {
key = http.CanonicalHeaderKey(key)
if metaHeaders[key] || strings.HasPrefix(key, "X-Amz-Meta-") {
obj.meta[key] = values
etag := fmt.Sprintf("\"%x\"", gotHash)
a.w.Header().Add("ETag", etag)
if uploadId == "" {
// For traditional uploads
// TODO is this correct, or should we erase all previous metadata?
obj := objr.object
if obj == nil {
obj = &object{
name: objr.name,
meta: make(http.Header),
}
}
// PUT request has been successful - save data and metadata
for key, values := range a.req.Header {
key = http.CanonicalHeaderKey(key)
if metaHeaders[key] || strings.HasPrefix(key, "X-Amz-Meta-") {
obj.meta[key] = values
}
}
obj.data = data
obj.checksum = gotHash
obj.mtime = time.Now()
objr.bucket.objects[objr.name] = obj
} else {
// For multipart commit
parts := objr.bucket.multipartUploads[uploadId]
part := &multipartUploadPart{
data,
etag,
time.Now(),
}
objr.bucket.multipartUploads[uploadId] = append(parts, part)
}
obj.data = data
obj.checksum = gotHash
obj.mtime = time.Now()
objr.bucket.objects[objr.name] = obj
return nil
}
func (objr objectResource) delete(a *action) interface{} {
delete(objr.bucket.objects, objr.name)
uploadId := a.req.URL.Query().Get("uploadId")
if uploadId == "" {
// Traditional object delete
delete(objr.bucket.objects, objr.name)
} else {
// Multipart commit abort
_, ok := objr.bucket.multipartUploads[uploadId]
if !ok {
fatalf(404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")
}
delete(objr.bucket.multipartUploads, uploadId)
}
return nil
}
func (objr objectResource) post(a *action) interface{} {
// Check if we're initializing a multipart upload
if _, ok := a.req.URL.Query()["uploads"]; ok {
type multipartInitResponse struct {
XMLName struct{} `xml:"InitiateMultipartUploadResult"`
Bucket string
Key string
UploadId string
}
uploadId := strconv.FormatInt(rand.Int63(), 16)
objr.bucket.multipartUploads[uploadId] = []*multipartUploadPart{}
return &multipartInitResponse{
Bucket: objr.bucket.name,
Key: objr.name,
UploadId: uploadId,
}
}
// Check if we're completing a multipart upload
if uploadId := a.req.URL.Query().Get("uploadId"); uploadId != "" {
type multipartCompleteRequestPart struct {
XMLName struct{} `xml:"Part"`
PartNumber uint
ETag string
}
type multipartCompleteRequest struct {
XMLName struct{} `xml:"CompleteMultipartUpload"`
Part []multipartCompleteRequestPart
}
type multipartCompleteResponse struct {
XMLName struct{} `xml:"CompleteMultipartUploadResult"`
Location string
Bucket string
Key string
ETag string
}
parts, ok := objr.bucket.multipartUploads[uploadId]
if !ok {
fatalf(404, "NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")
}
req := &multipartCompleteRequest{}
if err := xml.NewDecoder(a.req.Body).Decode(req); err != nil {
fatalf(400, "InvalidRequest", err.Error())
}
if len(req.Part) != len(parts) {
fatalf(400, "InvalidRequest", fmt.Sprintf("Number of parts does not match: expected %d, received %d", len(parts), len(req.Part)))
}
sum := md5.New()
data := &bytes.Buffer{}
w := io.MultiWriter(sum, data)
for i, p := range parts {
reqPart := req.Part[i]
if reqPart.PartNumber != uint(1+i) {
fatalf(400, "InvalidRequest", "Bad part number")
}
if reqPart.ETag != p.etag {
fatalf(400, "InvalidRequest", fmt.Sprintf("Invalid etag for part %d", reqPart.PartNumber))
}
w.Write(p.data)
}
delete(objr.bucket.multipartUploads, uploadId)
obj := objr.object
if obj == nil {
obj = &object{
name: objr.name,
meta: make(http.Header),
}
}
obj.data = data.Bytes()
obj.checksum = sum.Sum(nil)
obj.mtime = time.Now()
objr.bucket.objects[objr.name] = obj
objectLocation := fmt.Sprintf("http://%s/%s/%s", a.srv.listener.Addr().String(), objr.bucket.name, objr.name)
return &multipartCompleteResponse{
Location: objectLocation,
Bucket: objr.bucket.name,
Key: objr.name,
ETag: uploadId,
}
}
fatalf(400, "MethodNotAllowed", "The specified method is not allowed against this resource")
return nil
}