forked from TrueCloudLab/frostfs-s3-gw
[#218] Add check content sha256 header
The X-Amz-Content-Sha256 header check is done only for unencrypted payload. Signed-off-by: Roman Loginov <r.loginov@yadro.com>
This commit is contained in:
parent
b28ecef43b
commit
861454e499
10 changed files with 282 additions and 26 deletions
|
@ -38,6 +38,7 @@ This document outlines major changes between releases.
|
|||
- Add `X-Amz-Version-Id` header after complete multipart upload (#227)
|
||||
- Add handling of `X-Amz-Copy-Source-Server-Side-Encryption-Customer-*` headers during copy (#217)
|
||||
- Add new `logger.destination` config param (#236)
|
||||
- Add `X-Amz-Content-Sha256` header validation (#218)
|
||||
|
||||
### Changed
|
||||
- Update prometheus to v1.15.0 (#94)
|
||||
|
|
|
@ -76,10 +76,27 @@ const (
|
|||
AmzSignedHeaders = "X-Amz-SignedHeaders"
|
||||
AmzExpires = "X-Amz-Expires"
|
||||
AmzDate = "X-Amz-Date"
|
||||
AmzContentSHA256 = "X-Amz-Content-Sha256"
|
||||
AuthorizationHdr = "Authorization"
|
||||
ContentTypeHdr = "Content-Type"
|
||||
|
||||
UnsignedPayload = "UNSIGNED-PAYLOAD"
|
||||
StreamingUnsignedPayloadTrailer = "STREAMING-UNSIGNED-PAYLOAD-TRAILER"
|
||||
StreamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
|
||||
StreamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"
|
||||
StreamingContentECDSASHA256 = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"
|
||||
StreamingContentECDSASHA256Trailer = "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER"
|
||||
)
|
||||
|
||||
var ContentSHA256HeaderStandardValue = map[string]struct{}{
|
||||
UnsignedPayload: {},
|
||||
StreamingUnsignedPayloadTrailer: {},
|
||||
StreamingContentSHA256: {},
|
||||
StreamingContentSHA256Trailer: {},
|
||||
StreamingContentECDSASHA256: {},
|
||||
StreamingContentECDSASHA256Trailer: {},
|
||||
}
|
||||
|
||||
// ErrNoAuthorizationHeader is returned for unauthenticated requests.
|
||||
var ErrNoAuthorizationHeader = errors.New("no authorization header")
|
||||
|
||||
|
@ -134,6 +151,11 @@ func (a *AuthHeader) getAddress() (oid.Address, error) {
|
|||
return addr, nil
|
||||
}
|
||||
|
||||
func IsStandardContentSHA256(key string) bool {
|
||||
_, ok := ContentSHA256HeaderStandardValue[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (c *center) Authenticate(r *http.Request) (*Box, error) {
|
||||
var (
|
||||
err error
|
||||
|
@ -197,6 +219,10 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) {
|
|||
return nil, fmt.Errorf("get box: %w", err)
|
||||
}
|
||||
|
||||
if err = checkFormatHashContentSHA256(r.Header.Get(AmzContentSHA256)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clonedRequest := cloneRequest(r, authHdr)
|
||||
if err = c.checkSign(authHdr, box, clonedRequest, signatureDateTime); err != nil {
|
||||
return nil, err
|
||||
|
@ -213,6 +239,20 @@ func (c *center) Authenticate(r *http.Request) (*Box, error) {
|
|||
return result, nil
|
||||
}
|
||||
|
||||
func checkFormatHashContentSHA256(hash string) error {
|
||||
if !IsStandardContentSHA256(hash) {
|
||||
hashBinary, err := hex.DecodeString(hash)
|
||||
if err != nil {
|
||||
return apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
||||
}
|
||||
if len(hashBinary) != sha256.Size && len(hash) != 0 {
|
||||
return apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c center) checkAccessKeyID(accessKeyID string) error {
|
||||
if len(c.allowedAccessKeyIDPrefixes) == 0 {
|
||||
return nil
|
||||
|
|
|
@ -99,3 +99,49 @@ func TestSignature(t *testing.T) {
|
|||
signature := signStr(secret, "s3", "us-east-1", signTime, strToSign)
|
||||
require.Equal(t, "dfbe886241d9e369cf4b329ca0f15eb27306c97aa1022cc0bb5a914c4ef87634", signature)
|
||||
}
|
||||
|
||||
func TestCheckFormatContentSHA256(t *testing.T) {
|
||||
defaultErr := errors.GetAPIError(errors.ErrContentSHA256Mismatch)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
hash string
|
||||
error error
|
||||
}{
|
||||
{
|
||||
name: "invalid hash format: length and character",
|
||||
hash: "invalid-hash",
|
||||
error: defaultErr,
|
||||
},
|
||||
{
|
||||
name: "invalid hash format: length (63 characters)",
|
||||
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f7",
|
||||
error: defaultErr,
|
||||
},
|
||||
{
|
||||
name: "invalid hash format: character",
|
||||
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f7s",
|
||||
error: defaultErr,
|
||||
},
|
||||
{
|
||||
name: "unsigned payload",
|
||||
hash: "UNSIGNED-PAYLOAD",
|
||||
error: nil,
|
||||
},
|
||||
{
|
||||
name: "no hash",
|
||||
hash: "",
|
||||
error: nil,
|
||||
},
|
||||
{
|
||||
name: "correct hash format",
|
||||
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
|
||||
error: nil,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := checkFormatHashContentSHA256(tc.hash)
|
||||
require.Equal(t, tc.error, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
@ -242,10 +242,11 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
|
|||
Bkt: bktInfo,
|
||||
Key: reqInfo.ObjectName,
|
||||
},
|
||||
PartNumber: partNumber,
|
||||
Size: size,
|
||||
Reader: body,
|
||||
ContentMD5: r.Header.Get(api.ContentMD5),
|
||||
PartNumber: partNumber,
|
||||
Size: size,
|
||||
Reader: body,
|
||||
ContentMD5: r.Header.Get(api.ContentMD5),
|
||||
ContentSHA256Hash: r.Header.Get(api.AmzContentSha256),
|
||||
}
|
||||
|
||||
p.Info.Encryption, err = formEncryptionParams(r)
|
||||
|
|
|
@ -314,6 +314,84 @@ func TestMultipartUploadEnabledMD5(t *testing.T) {
|
|||
require.Equal(t, data.Quote(hex.EncodeToString(completeMD5Sum[:])+"-2"), resp.ETag)
|
||||
}
|
||||
|
||||
func TestUploadPartCheckContentSHA256(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-1", "object-1"
|
||||
createTestBucket(hc, bktName)
|
||||
partSize := 5 * 1024 * 1024
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
hash string
|
||||
content []byte
|
||||
error bool
|
||||
}{
|
||||
{
|
||||
name: "invalid hash value",
|
||||
hash: "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
|
||||
content: []byte("content"),
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "correct hash for empty payload",
|
||||
hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
content: []byte(""),
|
||||
error: false,
|
||||
},
|
||||
{
|
||||
name: "unsigned payload",
|
||||
hash: "UNSIGNED-PAYLOAD",
|
||||
content: []byte("content"),
|
||||
error: false,
|
||||
},
|
||||
{
|
||||
name: "correct hash",
|
||||
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
|
||||
content: []byte("content"),
|
||||
error: false,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
multipartUpload := createMultipartUpload(hc, bktName, objName, map[string]string{})
|
||||
|
||||
etag1, data1 := uploadPart(hc, bktName, objName, multipartUpload.UploadID, 1, partSize)
|
||||
|
||||
query := make(url.Values)
|
||||
query.Set(uploadIDQuery, multipartUpload.UploadID)
|
||||
query.Set(partNumberQuery, strconv.Itoa(2))
|
||||
|
||||
w, r := prepareTestRequestWithQuery(hc, bktName, objName, query, tc.content)
|
||||
r.Header.Set(api.AmzContentSha256, tc.hash)
|
||||
hc.Handler().UploadPartHandler(w, r)
|
||||
if tc.error {
|
||||
assertS3Error(t, w, s3Errors.GetAPIError(s3Errors.ErrContentSHA256Mismatch))
|
||||
|
||||
list := listParts(hc, bktName, objName, multipartUpload.UploadID, "0", http.StatusOK)
|
||||
require.Len(t, list.Parts, 1)
|
||||
|
||||
w := completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1})
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
data, _ := getObject(hc, bktName, objName)
|
||||
equalDataSlices(t, data1, data)
|
||||
return
|
||||
}
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
list := listParts(hc, bktName, objName, multipartUpload.UploadID, "0", http.StatusOK)
|
||||
require.Len(t, list.Parts, 2)
|
||||
|
||||
etag2 := w.Header().Get(api.ETag)
|
||||
w = completeMultipartUploadBase(hc, bktName, objName, multipartUpload.UploadID, []string{etag1, etag2})
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
|
||||
data, _ := getObject(hc, bktName, objName)
|
||||
equalDataSlices(t, append(data1, tc.content...), data)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func uploadPartCopy(hc *handlerContext, bktName, objName, uploadID string, num int, srcObj string, start, end int) *UploadPartCopyResponse {
|
||||
return uploadPartCopyBase(hc, bktName, objName, false, uploadID, num, srcObj, start, end)
|
||||
}
|
||||
|
|
|
@ -238,13 +238,14 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
|||
}
|
||||
|
||||
params := &layer.PutObjectParams{
|
||||
BktInfo: bktInfo,
|
||||
Object: reqInfo.ObjectName,
|
||||
Reader: body,
|
||||
Size: size,
|
||||
Header: metadata,
|
||||
Encryption: encryptionParams,
|
||||
ContentMD5: r.Header.Get(api.ContentMD5),
|
||||
BktInfo: bktInfo,
|
||||
Object: reqInfo.ObjectName,
|
||||
Reader: body,
|
||||
Size: size,
|
||||
Header: metadata,
|
||||
Encryption: encryptionParams,
|
||||
ContentMD5: r.Header.Get(api.ContentMD5),
|
||||
ContentSHA256Hash: r.Header.Get(api.AmzContentSha256),
|
||||
}
|
||||
|
||||
params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, bktInfo.LocationConstraint)
|
||||
|
|
|
@ -208,6 +208,63 @@ func TestPutObjectWithEnabledMD5(t *testing.T) {
|
|||
require.Equal(t, data.Quote(hex.EncodeToString(md5Hash.Sum(nil))), w.Header().Get(api.ETag))
|
||||
}
|
||||
|
||||
func TestPutObjectCheckContentSHA256(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
bktName, objName := "bucket-for-put", "object-for-put"
|
||||
createTestBucket(hc, bktName)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
hash string
|
||||
content []byte
|
||||
error bool
|
||||
}{
|
||||
{
|
||||
name: "invalid hash value",
|
||||
hash: "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
|
||||
content: []byte("content"),
|
||||
error: true,
|
||||
},
|
||||
{
|
||||
name: "correct hash for empty payload",
|
||||
hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
content: []byte(""),
|
||||
error: false,
|
||||
},
|
||||
{
|
||||
name: "unsigned payload",
|
||||
hash: "UNSIGNED-PAYLOAD",
|
||||
content: []byte("content"),
|
||||
error: false,
|
||||
},
|
||||
{
|
||||
name: "correct hash",
|
||||
hash: "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
|
||||
content: []byte("content"),
|
||||
error: false,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w, r := prepareTestPayloadRequest(hc, bktName, objName, bytes.NewReader(tc.content))
|
||||
r.Header.Set("X-Amz-Content-Sha256", tc.hash)
|
||||
hc.Handler().PutObjectHandler(w, r)
|
||||
|
||||
if tc.error {
|
||||
assertS3Error(t, w, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch))
|
||||
|
||||
w, r := prepareTestRequest(hc, bktName, objName, nil)
|
||||
hc.Handler().GetObjectHandler(w, r)
|
||||
|
||||
assertStatus(t, w, http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
assertStatus(t, w, http.StatusOK)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutObjectWithStreamBodyAWSExample(t *testing.T) {
|
||||
hc := prepareHandlerContext(t)
|
||||
|
||||
|
|
|
@ -112,16 +112,17 @@ type (
|
|||
|
||||
// PutObjectParams stores object put request parameters.
|
||||
PutObjectParams struct {
|
||||
BktInfo *data.BucketInfo
|
||||
Object string
|
||||
Size uint64
|
||||
Reader io.Reader
|
||||
Header map[string]string
|
||||
Lock *data.ObjectLock
|
||||
Encryption encryption.Params
|
||||
CopiesNumbers []uint32
|
||||
CompleteMD5Hash string
|
||||
ContentMD5 string
|
||||
BktInfo *data.BucketInfo
|
||||
Object string
|
||||
Size uint64
|
||||
Reader io.Reader
|
||||
Header map[string]string
|
||||
Lock *data.ObjectLock
|
||||
Encryption encryption.Params
|
||||
CopiesNumbers []uint32
|
||||
CompleteMD5Hash string
|
||||
ContentMD5 string
|
||||
ContentSHA256Hash string
|
||||
}
|
||||
|
||||
PutCombinedObjectParams struct {
|
||||
|
|
|
@ -15,6 +15,7 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
s3errors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/encryption"
|
||||
|
@ -66,11 +67,12 @@ type (
|
|||
}
|
||||
|
||||
UploadPartParams struct {
|
||||
Info *UploadInfoParams
|
||||
PartNumber int
|
||||
Size uint64
|
||||
Reader io.Reader
|
||||
ContentMD5 string
|
||||
Info *UploadInfoParams
|
||||
PartNumber int
|
||||
Size uint64
|
||||
Reader io.Reader
|
||||
ContentMD5 string
|
||||
ContentSHA256Hash string
|
||||
}
|
||||
|
||||
UploadCopyParams struct {
|
||||
|
@ -260,6 +262,20 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
|
|||
size = decSize
|
||||
}
|
||||
|
||||
if !p.Info.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) {
|
||||
contentHashBytes, err := hex.DecodeString(p.ContentSHA256Hash)
|
||||
if err != nil {
|
||||
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
|
||||
}
|
||||
if !bytes.Equal(contentHashBytes, hash) {
|
||||
err = n.objectDelete(ctx, bktInfo, id)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
|
||||
}
|
||||
return nil, s3errors.GetAPIError(s3errors.ErrContentSHA256Mismatch)
|
||||
}
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Debug(logs.UploadPart,
|
||||
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
|
||||
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", id))
|
||||
|
|
|
@ -19,6 +19,7 @@ import (
|
|||
"sync"
|
||||
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/auth"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
|
||||
apiErrors "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
|
||||
|
@ -316,6 +317,20 @@ func (n *layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
|
|||
}
|
||||
}
|
||||
|
||||
if !p.Encryption.Enabled() && len(p.ContentSHA256Hash) > 0 && !auth.IsStandardContentSHA256(p.ContentSHA256Hash) {
|
||||
contentHashBytes, err := hex.DecodeString(p.ContentSHA256Hash)
|
||||
if err != nil {
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
||||
}
|
||||
if !bytes.Equal(contentHashBytes, hash) {
|
||||
err = n.objectDelete(ctx, p.BktInfo, id)
|
||||
if err != nil {
|
||||
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
||||
}
|
||||
return nil, apiErrors.GetAPIError(apiErrors.ErrContentSHA256Mismatch)
|
||||
}
|
||||
}
|
||||
|
||||
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", id))
|
||||
|
||||
newVersion := &data.NodeVersion{
|
||||
|
|
Loading…
Reference in a new issue