From e1ab3443db301fa2a12c9f95451154a39bdc2fb9 Mon Sep 17 00:00:00 2001 From: Brian Bland Date: Fri, 16 Jan 2015 13:56:37 -0800 Subject: [PATCH] Updates Godeps to a compiling/stable state Includes newer docker/libtrust and crowdmob/goamz --- Godeps/Godeps.json | 8 +- .../src/github.com/crowdmob/goamz/aws/aws.go | 16 +- .../src/github.com/crowdmob/goamz/aws/sign.go | 46 +++++- .../src/github.com/crowdmob/goamz/s3/multi.go | 2 +- .../src/github.com/crowdmob/goamz/s3/s3.go | 110 +++++++++----- .../github.com/docker/libtrust/jsonsign.go | 137 ++++++++++++++---- 6 files changed, 239 insertions(+), 80 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index ab700617..c578c78c 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -30,15 +30,15 @@ }, { "ImportPath": "github.com/crowdmob/goamz/aws", - "Rev": "cd22d9897beff6f3de22cec4bdb7d46b9e2dee67" + "Rev": "962cedbbde5e1af59fb0b4ab681c848e61676941" }, { "ImportPath": "github.com/crowdmob/goamz/cloudfront", - "Rev": "cd22d9897beff6f3de22cec4bdb7d46b9e2dee67" + "Rev": "962cedbbde5e1af59fb0b4ab681c848e61676941" }, { "ImportPath": "github.com/crowdmob/goamz/s3", - "Rev": "cd22d9897beff6f3de22cec4bdb7d46b9e2dee67" + "Rev": "962cedbbde5e1af59fb0b4ab681c848e61676941" }, { "ImportPath": "github.com/docker/docker/pkg/tarsum", @@ -52,7 +52,7 @@ }, { "ImportPath": "github.com/docker/libtrust", - "Rev": "a9625ce37e2dc5fed2e51eec2d39c39e4ac4c1df" + "Rev": "c54fbb67c1f1e68d7d6f8d2ad7c9360404616a41" }, { "ImportPath": "github.com/gorilla/context", diff --git a/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/aws.go b/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/aws.go index 89be74b5..38c9e656 100644 --- a/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/aws.go +++ b/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/aws.go @@ -518,7 +518,7 @@ func dialTimeout(network, addr string) (net.Conn, error) { return net.DialTimeout(network, addr, time.Duration(2*time.Second)) } -func InstanceRegion() string { +func AvailabilityZone() string { transport := http.Transport{Dial: dialTimeout} client := http.Client{ Transport: &transport, @@ -532,13 +532,21 @@ func InstanceRegion() string { if err != nil { return "unknown" } else { - b := string(body) - region := b[:len(b)-1] - return region + return string(body) } } } +func InstanceRegion() string { + az := AvailabilityZone() + if az == "unknown" { + return az + } else { + region := az[:len(az)-1] + return region + } +} + func InstanceId() string { transport := http.Transport{Dial: dialTimeout} client := http.Client{ diff --git a/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/sign.go b/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/sign.go index 4aeb3c38..20bcf011 100644 --- a/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/sign.go +++ b/Godeps/_workspace/src/github.com/crowdmob/goamz/aws/sign.go @@ -151,16 +151,37 @@ Any changes to the request after signing the request will invalidate the signatu */ func (s *V4Signer) Sign(req *http.Request) { req.Header.Set("host", req.Host) // host header must be included as a signed header - payloadHash := s.payloadHash(req) - if s.IncludeXAmzContentSha256 { - req.Header.Set("x-amz-content-sha256", payloadHash) // x-amz-content-sha256 contains the payload hash + t := s.requestTime(req) // Get request time + + payloadHash := "" + + if _, ok := req.Form["X-Amz-Expires"]; ok { + // We are authenticating the the request by using query params + // (also known as pre-signing a url, http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html) + payloadHash = "UNSIGNED-PAYLOAD" + req.Header.Del("x-amz-date") + + req.Form["X-Amz-SignedHeaders"] = []string{s.signedHeaders(req.Header)} + req.Form["X-Amz-Algorithm"] = []string{"AWS4-HMAC-SHA256"} + req.Form["X-Amz-Credential"] = []string{s.auth.AccessKey + "/" + s.credentialScope(t)} + req.Form["X-Amz-Date"] = []string{t.Format(ISO8601BasicFormat)} + req.URL.RawQuery = req.Form.Encode() + } else { + payloadHash = s.payloadHash(req) + if s.IncludeXAmzContentSha256 { + req.Header.Set("x-amz-content-sha256", payloadHash) // x-amz-content-sha256 contains the payload hash + } } - t := s.requestTime(req) // Get request time creq := s.canonicalRequest(req, payloadHash) // Build canonical request sts := s.stringToSign(t, creq) // Build string to sign signature := s.signature(t, sts) // Calculate the AWS Signature Version 4 auth := s.authorization(req.Header, t, signature) // Create Authorization header value - req.Header.Set("Authorization", auth) // Add Authorization header to request + + if _, ok := req.Form["X-Amz-Expires"]; ok { + req.Form["X-Amz-Signature"] = []string{signature} + } else { + req.Header.Set("Authorization", auth) // Add Authorization header to request + } return } @@ -266,8 +287,20 @@ func (s *V4Signer) canonicalQueryString(u *url.URL) string { } func (s *V4Signer) canonicalHeaders(h http.Header) string { - i, a := 0, make([]string, len(h)) + i, a, lowerCase := 0, make([]string, len(h)), make(map[string][]string) + for k, v := range h { + lowerCase[strings.ToLower(k)] = v + } + + var keys []string + for k := range lowerCase { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + v := lowerCase[k] for j, w := range v { v[j] = strings.Trim(w, " ") } @@ -275,7 +308,6 @@ func (s *V4Signer) canonicalHeaders(h http.Header) string { a[i] = strings.ToLower(k) + ":" + strings.Join(v, ",") i++ } - sort.Strings(a) return strings.Join(a, "\n") } diff --git a/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/multi.go b/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/multi.go index 6799ca51..80f17cbe 100644 --- a/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/multi.go +++ b/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/multi.go @@ -165,7 +165,7 @@ func (m *Multi) PutPartCopy(n int, options CopyOptions, source string) (*CopyObj params: params, } resp := &CopyObjectResult{} - err := m.Bucket.S3.query(req, resp) + err = m.Bucket.S3.query(req, resp) if shouldRetry(err) && attempt.HasNext() { continue } diff --git a/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3.go b/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3.go index b51f4078..d89be8b9 100644 --- a/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3.go +++ b/Godeps/_workspace/src/github.com/crowdmob/goamz/s3/s3.go @@ -40,6 +40,7 @@ type S3 struct { aws.Region ConnectTimeout time.Duration ReadTimeout time.Duration + Signature int private byte // Reserve the right of using private data. } @@ -95,7 +96,7 @@ var attempts = aws.AttemptStrategy{ // New creates a new S3. func New(auth aws.Auth, region aws.Region) *S3 { - return &S3{auth, region, 0, 0, 0} + return &S3{auth, region, 0, 0, 0, aws.V2Signature} } // Bucket returns a Bucket with the given name. @@ -772,15 +773,26 @@ func (b *Bucket) SignedURL(path string, expires time.Time) string { // SignedURLWithArgs returns a signed URL that allows anyone holding the URL // to retrieve the object at path. The signature is valid until expires. func (b *Bucket) SignedURLWithArgs(path string, expires time.Time, params url.Values, headers http.Header) string { + return b.SignedURLWithMethod("GET", path, expires, params, headers) +} + +// SignedURLWithMethod returns a signed URL that allows anyone holding the URL +// to either retrieve the object at path or make a HEAD request against it. The signature is valid until expires. +func (b *Bucket) SignedURLWithMethod(method, path string, expires time.Time, params url.Values, headers http.Header) string { var uv = url.Values{} if params != nil { uv = params } - uv.Set("Expires", strconv.FormatInt(expires.Unix(), 10)) + if b.S3.Signature == aws.V2Signature { + uv.Set("Expires", strconv.FormatInt(expires.Unix(), 10)) + } else { + uv.Set("X-Amz-Expires", strconv.FormatInt(expires.Unix()-time.Now().Unix(), 10)) + } req := &request{ + method: method, bucket: b.Name, path: path, params: uv, @@ -810,9 +822,15 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time if method != "POST" { method = "PUT" } - stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n/" + b.Name + "/" + path - fmt.Println("String to sign:\n", stringToSign) + a := b.S3.Auth + tokenData := "" + + if a.Token() != "" { + tokenData = "x-amz-security-token:" + a.Token() + "\n" + } + + stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n" + tokenData + "/" + b.Name + "/" + path secretKey := a.SecretKey accessId := a.AccessKey mac := hmac.New(sha1.New, []byte(secretKey)) @@ -832,7 +850,7 @@ func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time params.Add("Expires", strconv.FormatInt(expire_date, 10)) params.Add("Signature", signature) if a.Token() != "" { - params.Add("token", a.Token()) + params.Add("x-amz-security-token", a.Token()) } signedurl.RawQuery = params.Encode() @@ -924,7 +942,10 @@ func (s3 *S3) queryV4Sign(req *request, resp interface{}) error { req.headers = map[string][]string{} } - s3.setBaseURL(req) + err := s3.setBaseURL(req) + if err != nil { + return err + } hreq, err := s3.setupHttpRequest(req) if err != nil { @@ -992,57 +1013,79 @@ func partiallyEscapedPath(path string) string { // prepare sets up req to be delivered to S3. func (s3 *S3) prepare(req *request) error { - var signpath = req.path + // Copy so they can be mutated without affecting on retries. + params := make(url.Values) + headers := make(http.Header) + for k, v := range req.params { + params[k] = v + } + for k, v := range req.headers { + headers[k] = v + } + req.params = params + req.headers = headers if !req.prepared { req.prepared = true if req.method == "" { req.method = "GET" } - // Copy so they can be mutated without affecting on retries. - params := make(url.Values) - headers := make(http.Header) - for k, v := range req.params { - params[k] = v - } - for k, v := range req.headers { - headers[k] = v - } - req.params = params - req.headers = headers + if !strings.HasPrefix(req.path, "/") { req.path = "/" + req.path } - signpath = req.path err := s3.setBaseURL(req) if err != nil { return err } - if req.bucket != "" { - signpath = "/" + req.bucket + signpath - } } - // Always sign again as it's not clear how far the - // server has handled a previous attempt. - u, err := url.Parse(req.baseurl) - if err != nil { - return fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) - } - - signpathPatiallyEscaped := partiallyEscapedPath(signpath) - req.headers["Host"] = []string{u.Host} - req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)} if s3.Auth.Token() != "" { req.headers["X-Amz-Security-Token"] = []string{s3.Auth.Token()} } - sign(s3.Auth, req.method, signpathPatiallyEscaped, req.params, req.headers) + + if s3.Signature == aws.V2Signature { + // Always sign again as it's not clear how far the + // server has handled a previous attempt. + u, err := url.Parse(req.baseurl) + if err != nil { + return err + } + + signpathPatiallyEscaped := partiallyEscapedPath(req.path) + req.headers["Host"] = []string{u.Host} + req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)} + + sign(s3.Auth, req.method, signpathPatiallyEscaped, req.params, req.headers) + } else { + hreq, err := s3.setupHttpRequest(req) + if err != nil { + return err + } + + hreq.Host = hreq.URL.Host + signer := aws.NewV4Signer(s3.Auth, "s3", s3.Region) + signer.IncludeXAmzContentSha256 = true + signer.Sign(hreq) + + req.payload = hreq.Body + if _, ok := headers["Content-Length"]; ok { + req.headers["Content-Length"] = headers["Content-Length"] + } + } return nil } // Prepares an *http.Request for doHttpRequest func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) { + // Copy so that signing the http request will not mutate it + headers := make(http.Header) + for k, v := range req.headers { + headers[k] = v + } + req.headers = headers + u, err := req.url() if err != nil { return nil, err @@ -1056,6 +1099,7 @@ func (s3 *S3) setupHttpRequest(req *request) (*http.Request, error) { ProtoMinor: 1, Close: true, Header: req.headers, + Form: req.params, } if v, ok := req.headers["Content-Length"]; ok { diff --git a/Godeps/_workspace/src/github.com/docker/libtrust/jsonsign.go b/Godeps/_workspace/src/github.com/docker/libtrust/jsonsign.go index 8d84f6dd..cb2ca9a7 100644 --- a/Godeps/_workspace/src/github.com/docker/libtrust/jsonsign.go +++ b/Godeps/_workspace/src/github.com/docker/libtrust/jsonsign.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "sort" "time" "unicode" ) @@ -31,9 +32,25 @@ type jsHeader struct { } type jsSignature struct { - Header *jsHeader `json:"header"` - Signature string `json:"signature"` - Protected string `json:"protected,omitempty"` + Header jsHeader `json:"header"` + Signature string `json:"signature"` + Protected string `json:"protected,omitempty"` +} + +type jsSignaturesSorted []jsSignature + +func (jsbkid jsSignaturesSorted) Swap(i, j int) { jsbkid[i], jsbkid[j] = jsbkid[j], jsbkid[i] } +func (jsbkid jsSignaturesSorted) Len() int { return len(jsbkid) } + +func (jsbkid jsSignaturesSorted) Less(i, j int) bool { + ki, kj := jsbkid[i].Header.JWK.KeyID(), jsbkid[j].Header.JWK.KeyID() + si, sj := jsbkid[i].Signature, jsbkid[j].Signature + + if ki == kj { + return si < sj + } + + return ki < kj } type signKey struct { @@ -44,7 +61,7 @@ type signKey struct { // JSONSignature represents a signature of a json object. type JSONSignature struct { payload string - signatures []*jsSignature + signatures []jsSignature indent string formatLength int formatTail []byte @@ -52,7 +69,7 @@ type JSONSignature struct { func newJSONSignature() *JSONSignature { return &JSONSignature{ - signatures: make([]*jsSignature, 0, 1), + signatures: make([]jsSignature, 0, 1), } } @@ -99,17 +116,14 @@ func (js *JSONSignature) Sign(key PrivateKey) error { return err } - header := &jsHeader{ - JWK: key.PublicKey(), - Algorithm: algorithm, - } - sig := &jsSignature{ - Header: header, + js.signatures = append(js.signatures, jsSignature{ + Header: jsHeader{ + JWK: key.PublicKey(), + Algorithm: algorithm, + }, Signature: joseBase64UrlEncode(sigBytes), Protected: protected, - } - - js.signatures = append(js.signatures, sig) + }) return nil } @@ -136,7 +150,7 @@ func (js *JSONSignature) SignWithChain(key PrivateKey, chain []*x509.Certificate return err } - header := &jsHeader{ + header := jsHeader{ Chain: make([]string, len(chain)), Algorithm: algorithm, } @@ -145,13 +159,11 @@ func (js *JSONSignature) SignWithChain(key PrivateKey, chain []*x509.Certificate header.Chain[i] = base64.StdEncoding.EncodeToString(cert.Raw) } - sig := &jsSignature{ + js.signatures = append(js.signatures, jsSignature{ Header: header, Signature: joseBase64UrlEncode(sigBytes), Protected: protected, - } - - js.signatures = append(js.signatures, sig) + }) return nil } @@ -272,6 +284,9 @@ func (js *JSONSignature) JWS() ([]byte, error) { if len(js.signatures) == 0 { return nil, errors.New("missing signature") } + + sort.Sort(jsSignaturesSorted(js.signatures)) + jsonMap := map[string]interface{}{ "payload": js.payload, "signatures": js.signatures, @@ -301,16 +316,16 @@ type jsParsedHeader struct { } type jsParsedSignature struct { - Header *jsParsedHeader `json:"header"` - Signature string `json:"signature"` - Protected string `json:"protected"` + Header jsParsedHeader `json:"header"` + Signature string `json:"signature"` + Protected string `json:"protected"` } // ParseJWS parses a JWS serialized JSON object into a Json Signature. func ParseJWS(content []byte) (*JSONSignature, error) { type jsParsed struct { - Payload string `json:"payload"` - Signatures []*jsParsedSignature `json:"signatures"` + Payload string `json:"payload"` + Signatures []jsParsedSignature `json:"signatures"` } parsed := &jsParsed{} err := json.Unmarshal(content, parsed) @@ -329,9 +344,9 @@ func ParseJWS(content []byte) (*JSONSignature, error) { if err != nil { return nil, err } - js.signatures = make([]*jsSignature, len(parsed.Signatures)) + js.signatures = make([]jsSignature, len(parsed.Signatures)) for i, signature := range parsed.Signatures { - header := &jsHeader{ + header := jsHeader{ Algorithm: signature.Header.Algorithm, } if signature.Header.Chain != nil { @@ -344,7 +359,7 @@ func ParseJWS(content []byte) (*JSONSignature, error) { } header.JWK = publicKey } - js.signatures[i] = &jsSignature{ + js.signatures[i] = jsSignature{ Header: header, Signature: signature.Signature, Protected: signature.Protected, @@ -356,7 +371,11 @@ func ParseJWS(content []byte) (*JSONSignature, error) { // NewJSONSignature returns a new unsigned JWS from a json byte array. // JSONSignature will need to be signed before serializing or storing. -func NewJSONSignature(content []byte) (*JSONSignature, error) { +// Optionally, one or more signatures can be provided as byte buffers, +// containing serialized JWS signatures, to assemble a fully signed JWS +// package. It is the callers responsibility to ensure uniqueness of the +// provided signatures. +func NewJSONSignature(content []byte, signatures ...[]byte) (*JSONSignature, error) { var dataMap map[string]interface{} err := json.Unmarshal(content, &dataMap) if err != nil { @@ -380,6 +399,40 @@ func NewJSONSignature(content []byte) (*JSONSignature, error) { js.formatLength = lastRuneIndex + 1 js.formatTail = content[js.formatLength:] + if len(signatures) > 0 { + for _, signature := range signatures { + var parsedJSig jsParsedSignature + + if err := json.Unmarshal(signature, &parsedJSig); err != nil { + return nil, err + } + + // TODO(stevvooe): A lot of the code below is repeated in + // ParseJWS. It will require more refactoring to fix that. + jsig := jsSignature{ + Header: jsHeader{ + Algorithm: parsedJSig.Header.Algorithm, + }, + Signature: parsedJSig.Signature, + Protected: parsedJSig.Protected, + } + + if parsedJSig.Header.Chain != nil { + jsig.Header.Chain = parsedJSig.Header.Chain + } + + if parsedJSig.Header.JWK != nil { + publicKey, err := UnmarshalPublicKeyJWK([]byte(parsedJSig.Header.JWK)) + if err != nil { + return nil, err + } + jsig.Header.JWK = publicKey + } + + js.signatures = append(js.signatures, jsig) + } + } + return js, nil } @@ -455,7 +508,7 @@ func ParsePrettySignature(content []byte, signatureKey string) (*JSONSignature, } js := newJSONSignature() - js.signatures = make([]*jsSignature, len(signatureBlocks)) + js.signatures = make([]jsSignature, len(signatureBlocks)) for i, signatureBlock := range signatureBlocks { protectedBytes, err := joseBase64UrlDecode(signatureBlock.Protected) @@ -491,7 +544,7 @@ func ParsePrettySignature(content []byte, signatureKey string) (*JSONSignature, return nil, errors.New("conflicting format tail") } - header := &jsHeader{ + header := jsHeader{ Algorithm: signatureBlock.Header.Algorithm, Chain: signatureBlock.Header.Chain, } @@ -502,7 +555,7 @@ func ParsePrettySignature(content []byte, signatureKey string) (*JSONSignature, } header.JWK = publicKey } - js.signatures[i] = &jsSignature{ + js.signatures[i] = jsSignature{ Header: header, Signature: signatureBlock.Signature, Protected: signatureBlock.Protected, @@ -532,6 +585,8 @@ func (js *JSONSignature) PrettySignature(signatureKey string) ([]byte, error) { } payload = payload[:js.formatLength] + sort.Sort(jsSignaturesSorted(js.signatures)) + var marshalled []byte var marshallErr error if js.indent != "" { @@ -565,6 +620,26 @@ func (js *JSONSignature) PrettySignature(signatureKey string) ([]byte, error) { return buf.Bytes(), nil } +// Signatures provides the signatures on this JWS as opaque blobs, sorted by +// keyID. These blobs can be stored and reassembled with payloads. Internally, +// they are simply marshaled json web signatures but implementations should +// not rely on this. +func (js *JSONSignature) Signatures() ([][]byte, error) { + sort.Sort(jsSignaturesSorted(js.signatures)) + + var sb [][]byte + for _, jsig := range js.signatures { + p, err := json.Marshal(jsig) + if err != nil { + return nil, err + } + + sb = append(sb, p) + } + + return sb, nil +} + // Merge combines the signatures from one or more other signatures into the // method receiver. If the payloads differ for any argument, an error will be // returned and the receiver will not be modified.