From 861454e4998fa3af9fe19f107f2851969dd42756 Mon Sep 17 00:00:00 2001 From: Roman Loginov Date: Mon, 13 Nov 2023 11:01:47 +0300 Subject: [PATCH] [#218] Add check content sha256 header The X-Amz-Content-Sha256 header check is done only for unencrypted payload. Signed-off-by: Roman Loginov --- CHANGELOG.md | 1 + api/auth/center.go | 40 ++++++++++++++ api/auth/center_test.go | 46 ++++++++++++++++ api/handler/multipart_upload.go | 9 ++-- api/handler/multipart_upload_test.go | 78 ++++++++++++++++++++++++++++ api/handler/put.go | 15 +++--- api/handler/put_test.go | 57 ++++++++++++++++++++ api/layer/layer.go | 21 ++++---- api/layer/multipart_upload.go | 26 ++++++++-- api/layer/object.go | 15 ++++++ 10 files changed, 282 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 866787a6..0cba0c8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ This document outlines major changes between releases. - Add `X-Amz-Version-Id` header after complete multipart upload (#227) - Add handling of `X-Amz-Copy-Source-Server-Side-Encryption-Customer-*` headers during copy (#217) - Add new `logger.destination` config param (#236) +- Add `X-Amz-Content-Sha256` header validation (#218) ### Changed - Update prometheus to v1.15.0 (#94) diff --git a/api/auth/center.go b/api/auth/center.go index a06885b8..c131424a 100644 --- a/api/auth/center.go +++ b/api/auth/center.go @@ -76,10 +76,27 @@ const ( AmzSignedHeaders = "X-Amz-SignedHeaders" AmzExpires = "X-Amz-Expires" AmzDate = "X-Amz-Date" + AmzContentSHA256 = "X-Amz-Content-Sha256" AuthorizationHdr = "Authorization" ContentTypeHdr = "Content-Type" + + UnsignedPayload = "UNSIGNED-PAYLOAD" + StreamingUnsignedPayloadTrailer = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" + StreamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + StreamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" + StreamingContentECDSASHA256 = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD" + StreamingContentECDSASHA256Trailer = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER" ) +var ContentSHA256HeaderStandardValue = map[string]struct{}{ + UnsignedPayload: {}, + StreamingUnsignedPayloadTrailer: {}, + StreamingContentSHA256: {}, + StreamingContentSHA256Trailer: {}, + StreamingContentECDSASHA256: {}, + StreamingContentECDSASHA256Trailer: {}, +} + // ErrNoAuthorizationHeader is returned for unauthenticated requests. var ErrNoAuthorizationHeader = errors.New("no authorization header") @@ -134,6 +151,11 @@ func (a *AuthHeader) getAddress() (oid.Address, error) { return addr, nil } +func IsStandardContentSHA256(key string) bool { + _, ok := ContentSHA256HeaderStandardValue[key] + return ok +} + func (c *center) Authenticate(r *http.Request) (*Box, error) { var ( err error @@ -197,6 +219,10 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) { return nil, fmt.Errorf("get box: %w", err) } + if err = checkFormatHashContentSHA256(r.Header.Get(AmzContentSHA256)); err != nil { + return nil, err + } + clonedRequest := cloneRequest(r, authHdr) if err = c.checkSign(authHdr, box, clonedRequest, signatureDateTime); err != nil { return nil, err @@ -213,6 +239,20 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) { return result, nil } +func checkFormatHashContentSHA256(hash string) error { + if !IsStandardContentSHA256(hash) { + hashBinary, err := hex.DecodeString(hash) + if err != nil { + return apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch) + } + if len(hashBinary) != sha256.Size && len(hash) != 0 { + return apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch) + } + } + + return nil +} + func (c center) checkAccessKeyID(accessKeyID string) error { if len(c.allowedAccessKeyIDPrefixes) == 0 { return nil diff --git a/api/auth/center_test.go b/api/auth/center_test.go index 15069a22..3097325f 100644 --- a/api/auth/center_test.go +++ b/api/auth/center_test.go @@ -99,3 +99,49 @@ func TestSignature(t *testing.T) { signature := signStr(secret, "s3", "us-east-1", signTime, strToSign) require.Equal(t, "dfbe886241d9e369cf4b329ca0f15eb27306c97aa1022cc0bb5a914c4ef87634", signature) } + +func TestCheckFormatContentSHA256(t *testing.T) { + defaultErr := errors.GetAPIError(errors.ErrContentSHA256Mismatch) + + for _, tc := range []struct { + name string + hash string + error error + }{ + { + name: "invalid hash format: length and character", + hash: "invalid-hash", + error: defaultErr, + }, + { + name: "invalid hash format: length (63 characters)", + hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f7", + error: defaultErr, + }, + { + name: "invalid hash format: character", + hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f7s", + error: defaultErr, + }, + { + name: "unsigned payload", + hash: "UNSIGNED-PAYLOAD", + error: nil, + }, + { + name: "no hash", + hash: "", + error: nil, + }, + { + name: "correct hash format", + hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73", + error: nil, + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := checkFormatHashContentSHA256(tc.hash) + require.Equal(t, tc.error, err) + }) + } +} diff --git a/api/handler/multipart_upload.go b/api/handler/multipart_upload.go index de095c9d..25e2ce60 100644 --- a/api/handler/multipart_upload.go +++ b/api/handler/multipart_upload.go @@ -242,10 +242,11 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) { Bkt: bktInfo, Key: reqInfo.ObjectName, }, - PartNumber: partNumber, - Size: size, - Reader: body, - ContentMD5: r.Header.Get(api.ContentMD5), + PartNumber: partNumber, + Size: size, + Reader: body, + ContentMD5: r.Header.Get(api.ContentMD5), + ContentSHA256Hash: r.Header.Get(api.AmzContentSha256), } p.Info.Encryption, err = formEncryptionParams(r) diff --git a/api/handler/multipart_upload_test.go b/api/handler/multipart_upload_test.go index 4e3d6d80..a6dd9bc1 100644 --- a/api/handler/multipart_upload_test.go +++ b/api/handler/multipart_upload_test.go @@ -314,6 +314,84 @@ func TestMultipartUploadEnabledMD5(t *testing.T) { require.Equal(t, data.Quote(hex.EncodeToString(completeMD5Sum[:])+"-2"), resp.ETag) } +func TestUploadPartCheckContentSHA256(t *testing.T) { + hc := prepareHandlerContext(t) + + bktName, objName := "bucket-1", "object-1" + createTestBucket(hc, bktName) + partSize := 5 * 1024 * 1024 + + for _, tc := range []struct { + name string + hash string + content []byte + error bool + }{ + { + name: "invalid hash value", + hash: "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8", + content: []byte("content"), + error: true, + }, + { + name: "correct hash for empty payload", + hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + content: []byte(""), + error: false, + }, + { + name: "unsigned payload", + hash: "UNSIGNED-PAYLOAD", + content: []byte("content"), + error: false, + }, + { + name: "correct hash", + hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73", + content: []byte("content"), + error: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + multipartUpload := createMultipartUpload(hc, bktName, objName, map[string]string{}) + + etag1, data1 := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 1, partSize) + + query := make(url.Values) + query.Set(uploadIDQuery, multipartUpload.UploadID) + query.Set(partNumberQuery, strconv.Itoa(2)) + + w, r := prepareTestRequestWithQuery(hc, bktName, objName, query, tc.content) + r.Header.Set(api.AmzContentSha256, tc.hash) + hc.Handler().UploadPartHandler(w, r) + if tc.error { + assertS3Error(t, w, s3Errors.GetAPIError(s3Errors.ErrContentSHA256Mismatch)) + + list := listParts(hc, bktName, objName, multipartUpload.UploadID, "0", http.StatusOK) + require.Len(t, list.Parts, 1) + + w := completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1}) + assertStatus(t, w, http.StatusOK) + + data, _ := getObject(hc, bktName, objName) + equalDataSlices(t, data1, data) + return + } + assertStatus(t, w, http.StatusOK) + + list := listParts(hc, bktName, objName, multipartUpload.UploadID, "0", http.StatusOK) + require.Len(t, list.Parts, 2) + + etag2 := w.Header().Get(api.ETag) + w = completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1, etag2}) + assertStatus(t, w, http.StatusOK) + + data, _ := getObject(hc, bktName, objName) + equalDataSlices(t, append(data1, tc.content...), data) + }) + } +} + func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse { return uploadPartCopyBase(hc, bktName, objName, false, uploadID, num, srcObj, start, end) } diff --git a/api/handler/put.go b/api/handler/put.go index 4c923e9c..3158a878 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -238,13 +238,14 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { } params := &layer.PutObjectParams{ - BktInfo: bktInfo, - Object: reqInfo.ObjectName, - Reader: body, - Size: size, - Header: metadata, - Encryption: encryptionParams, - ContentMD5: r.Header.Get(api.ContentMD5), + BktInfo: bktInfo, + Object: reqInfo.ObjectName, + Reader: body, + Size: size, + Header: metadata, + Encryption: encryptionParams, + ContentMD5: r.Header.Get(api.ContentMD5), + ContentSHA256Hash: r.Header.Get(api.AmzContentSha256), } params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, bktInfo.LocationConstraint) diff --git a/api/handler/put_test.go b/api/handler/put_test.go index 0f360aca..6f3a9c0e 100644 --- a/api/handler/put_test.go +++ b/api/handler/put_test.go @@ -208,6 +208,63 @@ func TestPutObjectWithEnabledMD5(t *testing.T) { require.Equal(t, data.Quote(hex.EncodeToString(md5Hash.Sum(nil))), w.Header().Get(api.ETag)) } +func TestPutObjectCheckContentSHA256(t *testing.T) { + hc := prepareHandlerContext(t) + + bktName, objName := "bucket-for-put", "object-for-put" + createTestBucket(hc, bktName) + + for _, tc := range []struct { + name string + hash string + content []byte + error bool + }{ + { + name: "invalid hash value", + hash: "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8", + content: []byte("content"), + error: true, + }, + { + name: "correct hash for empty payload", + hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + content: []byte(""), + error: false, + }, + { + name: "unsigned payload", + hash: "UNSIGNED-PAYLOAD", + content: []byte("content"), + error: false, + }, + { + name: "correct hash", + hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73", + content: []byte("content"), + error: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + w, r := prepareTestPayloadRequest(hc, bktName, objName, bytes.NewReader(tc.content)) + r.Header.Set("X-Amz-Content-Sha256", tc.hash) + hc.Handler().PutObjectHandler(w, r) + + if tc.error { + assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)) + + w, r := prepareTestRequest(hc, bktName, objName, nil) + hc.Handler().GetObjectHandler(w, r) + + assertStatus(t, w, http.StatusNotFound) + return + } + + assertStatus(t, w, http.StatusOK) + }) + } +} + func TestPutObjectWithStreamBodyAWSExample(t *testing.T) { hc := prepareHandlerContext(t) diff --git a/api/layer/layer.go b/api/layer/layer.go index 126523a7..2b592dd6 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -112,16 +112,17 @@ type ( // PutObjectParams stores object put request parameters. PutObjectParams struct { - BktInfo *data.BucketInfo - Object string - Size uint64 - Reader io.Reader - Header map[string]string - Lock *data.ObjectLock - Encryption encryption.Params - CopiesNumbers []uint32 - CompleteMD5Hash string - ContentMD5 string + BktInfo *data.BucketInfo + Object string + Size uint64 + Reader io.Reader + Header map[string]string + Lock *data.ObjectLock + Encryption encryption.Params + CopiesNumbers []uint32 + CompleteMD5Hash string + ContentMD5 string + ContentSHA256Hash string } PutCombinedObjectParams struct { diff --git a/api/layer/multipart_upload.go b/api/layer/multipart_upload.go index a6762c92..052fae62 100644 --- a/api/layer/multipart_upload.go +++ b/api/layer/multipart_upload.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption" @@ -66,11 +67,12 @@ type ( } UploadPartParams struct { - Info *UploadInfoParams - PartNumber int - Size uint64 - Reader io.Reader - ContentMD5 string + Info *UploadInfoParams + PartNumber int + Size uint64 + Reader io.Reader + ContentMD5 string + ContentSHA256Hash string } UploadCopyParams struct { @@ -260,6 +262,20 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf size = decSize } + if !p.Info.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) { + contentHashBytes, err := hex.DecodeString(p.ContentSHA256Hash) + if err != nil { + return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch) + } + if !bytes.Equal(contentHashBytes, hash) { + err = n.objectDelete(ctx, bktInfo, id) + if err != nil { + n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id)) + } + return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch) + } + } + n.reqLogger(ctx).Debug(logs.UploadPart, zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber), zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id)) diff --git a/api/layer/object.go b/api/layer/object.go index e4059c69..08dc79f7 100644 --- a/api/layer/object.go +++ b/api/layer/object.go @@ -19,6 +19,7 @@ import ( "sync" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" @@ -316,6 +317,20 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend } } + if !p.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) { + contentHashBytes, err := hex.DecodeString(p.ContentSHA256Hash) + if err != nil { + return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch) + } + if !bytes.Equal(contentHashBytes, hash) { + err = n.objectDelete(ctx, p.BktInfo, id) + if err != nil { + n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id)) + } + return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch) + } + } + n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id)) newVersion := &data.NodeVersion{