[#218] Add check content sha256 header

The X-Amz-Content-Sha256 header check is done only for unencrypted payload.

Signed-off-by: Roman Loginov <r.loginov@yadro.com>
This commit is contained in:
Roman Loginov 2023-11-13 11:01:47 +03:00 committed by Alexey Vanin
parent b28ecef43b
commit 861454e499
10 changed files with 282 additions and 26 deletions

View file

@ -38,6 +38,7 @@ This document outlines major changes between releases.
- Add `X-Amz-Version-Id` header after complete multipart upload (#227) - Add `X-Amz-Version-Id` header after complete multipart upload (#227)
- Add handling of `X-Amz-Copy-Source-Server-Side-Encryption-Customer-*` headers during copy (#217) - Add handling of `X-Amz-Copy-Source-Server-Side-Encryption-Customer-*` headers during copy (#217)
- Add new `logger.destination` config param (#236) - Add new `logger.destination` config param (#236)
- Add `X-Amz-Content-Sha256` header validation (#218)
### Changed ### Changed
- Update prometheus to v1.15.0 (#94) - Update prometheus to v1.15.0 (#94)

View file

@ -76,10 +76,27 @@ const (
AmzSignedHeaders = "X-Amz-SignedHeaders" AmzSignedHeaders = "X-Amz-SignedHeaders"
AmzExpires = "X-Amz-Expires" AmzExpires = "X-Amz-Expires"
AmzDate = "X-Amz-Date" AmzDate = "X-Amz-Date"
AmzContentSHA256 = "X-Amz-Content-Sha256"
AuthorizationHdr = "Authorization" AuthorizationHdr = "Authorization"
ContentTypeHdr = "Content-Type" ContentTypeHdr = "Content-Type"
UnsignedPayload = "UNSIGNED-PAYLOAD"
StreamingUnsignedPayloadTrailer = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
StreamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
StreamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
StreamingContentECDSASHA256 = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
StreamingContentECDSASHA256Trailer = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
) )
var ContentSHA256HeaderStandardValue = map[string]struct{}{
UnsignedPayload: {},
StreamingUnsignedPayloadTrailer: {},
StreamingContentSHA256: {},
StreamingContentSHA256Trailer: {},
StreamingContentECDSASHA256: {},
StreamingContentECDSASHA256Trailer: {},
}
// ErrNoAuthorizationHeader is returned for unauthenticated requests. // ErrNoAuthorizationHeader is returned for unauthenticated requests.
var ErrNoAuthorizationHeader = errors.New("no authorization header") var ErrNoAuthorizationHeader = errors.New("no authorization header")
@ -134,6 +151,11 @@ func (a *AuthHeader) getAddress() (oid.Address, error) {
return addr, nil return addr, nil
} }
func IsStandardContentSHA256(key string) bool {
_, ok := ContentSHA256HeaderStandardValue[key]
return ok
}
func (c *center) Authenticate(r *http.Request) (*Box, error) { func (c *center) Authenticate(r *http.Request) (*Box, error) {
var ( var (
err error err error
@ -197,6 +219,10 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) {
return nil, fmt.Errorf("get box: %w", err) return nil, fmt.Errorf("get box: %w", err)
} }
if err = checkFormatHashContentSHA256(r.Header.Get(AmzContentSHA256)); err != nil {
return nil, err
}
clonedRequest := cloneRequest(r, authHdr) clonedRequest := cloneRequest(r, authHdr)
if err = c.checkSign(authHdr, box, clonedRequest, signatureDateTime); err != nil { if err = c.checkSign(authHdr, box, clonedRequest, signatureDateTime); err != nil {
return nil, err return nil, err
@ -213,6 +239,20 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) {
return result, nil return result, nil
} }
func checkFormatHashContentSHA256(hash string) error {
if !IsStandardContentSHA256(hash) {
hashBinary, err := hex.DecodeString(hash)
if err != nil {
return apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
}
if len(hashBinary) != sha256.Size && len(hash) != 0 {
return apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
}
}
return nil
}
func (c center) checkAccessKeyID(accessKeyID string) error { func (c center) checkAccessKeyID(accessKeyID string) error {
if len(c.allowedAccessKeyIDPrefixes) == 0 { if len(c.allowedAccessKeyIDPrefixes) == 0 {
return nil return nil

View file

@ -99,3 +99,49 @@ func TestSignature(t *testing.T) {
signature := signStr(secret, "s3", "us-east-1", signTime, strToSign) signature := signStr(secret, "s3", "us-east-1", signTime, strToSign)
require.Equal(t, "dfbe886241d9e369cf4b329ca0f15eb27306c97aa1022cc0bb5a914c4ef87634", signature) require.Equal(t, "dfbe886241d9e369cf4b329ca0f15eb27306c97aa1022cc0bb5a914c4ef87634", signature)
} }
func TestCheckFormatContentSHA256(t *testing.T) {
defaultErr := errors.GetAPIError(errors.ErrContentSHA256Mismatch)
for _, tc := range []struct {
name string
hash string
error error
}{
{
name: "invalid hash format: length and character",
hash: "invalid-hash",
error: defaultErr,
},
{
name: "invalid hash format: length (63 characters)",
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f7",
error: defaultErr,
},
{
name: "invalid hash format: character",
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f7s",
error: defaultErr,
},
{
name: "unsigned payload",
hash: "UNSIGNED-PAYLOAD",
error: nil,
},
{
name: "no hash",
hash: "",
error: nil,
},
{
name: "correct hash format",
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
error: nil,
},
} {
t.Run(tc.name, func(t *testing.T) {
err := checkFormatHashContentSHA256(tc.hash)
require.Equal(t, tc.error, err)
})
}
}

View file

@ -242,10 +242,11 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
Bkt: bktInfo, Bkt: bktInfo,
Key: reqInfo.ObjectName, Key: reqInfo.ObjectName,
}, },
PartNumber: partNumber, PartNumber: partNumber,
Size: size, Size: size,
Reader: body, Reader: body,
ContentMD5: r.Header.Get(api.ContentMD5), ContentMD5: r.Header.Get(api.ContentMD5),
ContentSHA256Hash: r.Header.Get(api.AmzContentSha256),
} }
p.Info.Encryption, err = formEncryptionParams(r) p.Info.Encryption, err = formEncryptionParams(r)

View file

@ -314,6 +314,84 @@ func TestMultipartUploadEnabledMD5(t *testing.T) {
require.Equal(t, data.Quote(hex.EncodeToString(completeMD5Sum[:])+"-2"), resp.ETag) require.Equal(t, data.Quote(hex.EncodeToString(completeMD5Sum[:])+"-2"), resp.ETag)
} }
func TestUploadPartCheckContentSHA256(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "bucket-1", "object-1"
createTestBucket(hc, bktName)
partSize := 5 * 1024 * 1024
for _, tc := range []struct {
name string
hash string
content []byte
error bool
}{
{
name: "invalid hash value",
hash: "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
content: []byte("content"),
error: true,
},
{
name: "correct hash for empty payload",
hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
content: []byte(""),
error: false,
},
{
name: "unsigned payload",
hash: "UNSIGNED-PAYLOAD",
content: []byte("content"),
error: false,
},
{
name: "correct hash",
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
content: []byte("content"),
error: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
multipartUpload := createMultipartUpload(hc, bktName, objName, map[string]string{})
etag1, data1 := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 1, partSize)
query := make(url.Values)
query.Set(uploadIDQuery, multipartUpload.UploadID)
query.Set(partNumberQuery, strconv.Itoa(2))
w, r := prepareTestRequestWithQuery(hc, bktName, objName, query, tc.content)
r.Header.Set(api.AmzContentSha256, tc.hash)
hc.Handler().UploadPartHandler(w, r)
if tc.error {
assertS3Error(t, w, s3Errors.GetAPIError(s3Errors.ErrContentSHA256Mismatch))
list := listParts(hc, bktName, objName, multipartUpload.UploadID, "0", http.StatusOK)
require.Len(t, list.Parts, 1)
w := completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1})
assertStatus(t, w, http.StatusOK)
data, _ := getObject(hc, bktName, objName)
equalDataSlices(t, data1, data)
return
}
assertStatus(t, w, http.StatusOK)
list := listParts(hc, bktName, objName, multipartUpload.UploadID, "0", http.StatusOK)
require.Len(t, list.Parts, 2)
etag2 := w.Header().Get(api.ETag)
w = completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1, etag2})
assertStatus(t, w, http.StatusOK)
data, _ := getObject(hc, bktName, objName)
equalDataSlices(t, append(data1, tc.content...), data)
})
}
}
func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse { func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
return uploadPartCopyBase(hc, bktName, objName, false, uploadID, num, srcObj, start, end) return uploadPartCopyBase(hc, bktName, objName, false, uploadID, num, srcObj, start, end)
} }

View file

@ -238,13 +238,14 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
} }
params := &layer.PutObjectParams{ params := &layer.PutObjectParams{
BktInfo: bktInfo, BktInfo: bktInfo,
Object: reqInfo.ObjectName, Object: reqInfo.ObjectName,
Reader: body, Reader: body,
Size: size, Size: size,
Header: metadata, Header: metadata,
Encryption: encryptionParams, Encryption: encryptionParams,
ContentMD5: r.Header.Get(api.ContentMD5), ContentMD5: r.Header.Get(api.ContentMD5),
ContentSHA256Hash: r.Header.Get(api.AmzContentSha256),
} }
params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, bktInfo.LocationConstraint) params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, bktInfo.LocationConstraint)

View file

@ -208,6 +208,63 @@ func TestPutObjectWithEnabledMD5(t *testing.T) {
require.Equal(t, data.Quote(hex.EncodeToString(md5Hash.Sum(nil))), w.Header().Get(api.ETag)) require.Equal(t, data.Quote(hex.EncodeToString(md5Hash.Sum(nil))), w.Header().Get(api.ETag))
} }
func TestPutObjectCheckContentSHA256(t *testing.T) {
hc := prepareHandlerContext(t)
bktName, objName := "bucket-for-put", "object-for-put"
createTestBucket(hc, bktName)
for _, tc := range []struct {
name string
hash string
content []byte
error bool
}{
{
name: "invalid hash value",
hash: "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
content: []byte("content"),
error: true,
},
{
name: "correct hash for empty payload",
hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
content: []byte(""),
error: false,
},
{
name: "unsigned payload",
hash: "UNSIGNED-PAYLOAD",
content: []byte("content"),
error: false,
},
{
name: "correct hash",
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
content: []byte("content"),
error: false,
},
} {
t.Run(tc.name, func(t *testing.T) {
w, r := prepareTestPayloadRequest(hc, bktName, objName, bytes.NewReader(tc.content))
r.Header.Set("X-Amz-Content-Sha256", tc.hash)
hc.Handler().PutObjectHandler(w, r)
if tc.error {
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch))
w, r := prepareTestRequest(hc, bktName, objName, nil)
hc.Handler().GetObjectHandler(w, r)
assertStatus(t, w, http.StatusNotFound)
return
}
assertStatus(t, w, http.StatusOK)
})
}
}
func TestPutObjectWithStreamBodyAWSExample(t *testing.T) { func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
hc := prepareHandlerContext(t) hc := prepareHandlerContext(t)

View file

@ -112,16 +112,17 @@ type (
// PutObjectParams stores object put request parameters. // PutObjectParams stores object put request parameters.
PutObjectParams struct { PutObjectParams struct {
BktInfo *data.BucketInfo BktInfo *data.BucketInfo
Object string Object string
Size uint64 Size uint64
Reader io.Reader Reader io.Reader
Header map[string]string Header map[string]string
Lock *data.ObjectLock Lock *data.ObjectLock
Encryption encryption.Params Encryption encryption.Params
CopiesNumbers []uint32 CopiesNumbers []uint32
CompleteMD5Hash string CompleteMD5Hash string
ContentMD5 string ContentMD5 string
ContentSHA256Hash string
} }
PutCombinedObjectParams struct { PutCombinedObjectParams struct {

View file

@ -15,6 +15,7 @@ import (
"strings" "strings"
"time" "time"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
@ -66,11 +67,12 @@ type (
} }
UploadPartParams struct { UploadPartParams struct {
Info *UploadInfoParams Info *UploadInfoParams
PartNumber int PartNumber int
Size uint64 Size uint64
Reader io.Reader Reader io.Reader
ContentMD5 string ContentMD5 string
ContentSHA256Hash string
} }
UploadCopyParams struct { UploadCopyParams struct {
@ -260,6 +262,20 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
size = decSize size = decSize
} }
if !p.Info.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) {
contentHashBytes, err := hex.DecodeString(p.ContentSHA256Hash)
if err != nil {
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
}
if !bytes.Equal(contentHashBytes, hash) {
err = n.objectDelete(ctx, bktInfo, id)
if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
}
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
}
}
n.reqLogger(ctx).Debug(logs.UploadPart, n.reqLogger(ctx).Debug(logs.UploadPart,
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber), zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id)) zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))

View file

@ -19,6 +19,7 @@ import (
"sync" "sync"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
@ -316,6 +317,20 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
} }
} }
if !p.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) {
contentHashBytes, err := hex.DecodeString(p.ContentSHA256Hash)
if err != nil {
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
}
if !bytes.Equal(contentHashBytes, hash) {
err = n.objectDelete(ctx, p.BktInfo, id)
if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
}
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
}
}
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id)) n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
newVersion := &data.NodeVersion{ newVersion := &data.NodeVersion{