[#540] Add md5 S3Tests compatability #546

Merged
alexvanin merged 1 commit from pogpp/frostfs-s3-gw:bugfix/540_md5_header_check into master 2024-11-13 12:33:41 +00:00
6 changed files with 53 additions and 9 deletions

View file

@ -246,7 +246,7 @@ func (h *handler) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
} }
if err = checkOwner(bktInfo, reqInfo.User); err != nil { if err = checkOwner(bktInfo, reqInfo.User); err != nil {
h.logAndSendError(w, "request owner id does not match bucket owner id", reqInfo, err) h.logAndSendError(ctx, w, "request owner id does not match bucket owner id", reqInfo, err)
return return
} }

View file

@ -48,6 +48,25 @@ func TestSimpleGetEncrypted(t *testing.T) {
require.Equal(t, content, string(response)) require.Equal(t, content, string(response))
} }
func TestMD5HeaderBadOrEmpty(t *testing.T) {
tc := prepareHandlerContext(t)
bktName, objName := "bucket-for-sse-c", "object-to-encrypt"
createTestBucket(tc, bktName)
content := "content"
headers := map[string]string{
api.ContentMD5: "",
}
putEncryptedObjectWithHeadersErr(t, tc, bktName, objName, content, headers, errors.ErrInvalidDigest)
headers = map[string]string{
api.ContentMD5: "YWJjMTIzIT8kKiYoKSctPUB+",
}
putEncryptedObjectWithHeadersErr(t, tc, bktName, objName, content, headers, errors.ErrBadDigest)
dkirillov marked this conversation as resolved Outdated

Why don't we have the same test for regular PutObject?

Why don't we have the same test for regular `PutObject`?
}
func TestGetEncryptedRange(t *testing.T) { func TestGetEncryptedRange(t *testing.T) {
tc := prepareHandlerContext(t) tc := prepareHandlerContext(t)
@ -360,6 +379,15 @@ func putEncryptedObject(t *testing.T, tc *handlerContext, bktName, objName, cont
assertStatus(t, w, http.StatusOK) assertStatus(t, w, http.StatusOK)
} }
func putEncryptedObjectWithHeadersErr(t *testing.T, tc *handlerContext, bktName, objName, content string, headers map[string]string, code errors.ErrorCode) {
body := bytes.NewReader([]byte(content))
w, r := prepareTestPayloadRequest(tc, bktName, objName, body)
setHeaders(r, headers)
tc.Handler().PutObjectHandler(w, r)
assertS3Error(t, w, errors.GetAPIError(code))
}
func getEncryptedObject(hc *handlerContext, bktName, objName string) ([]byte, http.Header) { func getEncryptedObject(hc *handlerContext, bktName, objName string) ([]byte, http.Header) {
w, r := prepareTestRequest(hc, bktName, objName, nil) w, r := prepareTestRequest(hc, bktName, objName, nil)
setEncryptHeaders(r) setEncryptHeaders(r)

View file

@ -251,7 +251,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
Reader: body, Reader: body,
Header: metadata, Header: metadata,
Encryption: encryptionParams, Encryption: encryptionParams,
ContentMD5: r.Header.Get(api.ContentMD5), ContentMD5: getMD5Header(r),
ContentSHA256Hash: r.Header.Get(api.AmzContentSha256), ContentSHA256Hash: r.Header.Get(api.AmzContentSha256),
} }
@ -1038,3 +1038,13 @@ func (h *handler) parseLocationConstraint(r *http.Request) (*createBucketParams,
} }
return params, nil return params, nil
} }
func getMD5Header(r *http.Request) *string {
var md5Hdr *string
if len(r.Header.Values(api.ContentMD5)) != 0 {
hdr := r.Header.Get(api.ContentMD5)
md5Hdr = &hdr
}
return md5Hdr
}

View file

@ -284,6 +284,12 @@ func TestPutObjectWithInvalidContentMD5(t *testing.T) {
w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content)) w, r := prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString([]byte("invalid"))) r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString([]byte("invalid")))
tc.Handler().PutObjectHandler(w, r) tc.Handler().PutObjectHandler(w, r)
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrBadDigest))
content = []byte("content")
w, r = prepareTestPayloadRequest(tc, bktName, objName, bytes.NewReader(content))
r.Header.Set(api.ContentMD5, base64.StdEncoding.EncodeToString([]byte("")))
tc.Handler().PutObjectHandler(w, r)
assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidDigest)) assertS3Error(t, w, apierr.GetAPIError(apierr.ErrInvalidDigest))
checkNotFound(t, tc, bktName, objName, emptyVersion) checkNotFound(t, tc, bktName, objName, emptyVersion)
@ -498,9 +504,6 @@ func getEmptyChunkedRequest(ctx context.Context, t *testing.T, bktName, objName
AWSAccessKeyID := "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh" AWSAccessKeyID := "48c1K4PLVb7SvmV3PjDKEuXaMh8yZMXZ8Wx9msrkKcYw06dZeaxeiPe8vyFm2WsoeVaNt7UWEjNsVkagDs8oX4XXh"
AWSSecretAccessKey := "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0" AWSSecretAccessKey := "09260955b4eb0279dc017ba20a1ddac909cbd226c86cbb2d868e55534c8e64b0"
//awsCreds := credentials.NewStaticCredentials(AWSAccessKeyID, AWSSecretAccessKey, "")
//signer := v4.NewSigner(awsCreds)
reqBody := bytes.NewBufferString("0;chunk-signature=311a7142c8f3a07972c3aca65c36484b513a8fee48ab7178c7225388f2ae9894\r\n\r\n") reqBody := bytes.NewBufferString("0;chunk-signature=311a7142c8f3a07972c3aca65c36484b513a8fee48ab7178c7225388f2ae9894\r\n\r\n")

Do we even need this? Check git blame and remove unused code.

Do we even need this? Check git blame and remove unused code.
req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, reqBody) req, err := http.NewRequest("PUT", "http://localhost:8084/"+bktName+"/"+objName, reqBody)

View file

@ -111,7 +111,7 @@ type (
Encryption encryption.Params Encryption encryption.Params
CopiesNumbers []uint32 CopiesNumbers []uint32
CompleteMD5Hash string CompleteMD5Hash string
ContentMD5 string ContentMD5 *string
ContentSHA256Hash string ContentSHA256Hash string
} }

View file

@ -286,8 +286,11 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
return nil, err return nil, err
} }
if !p.Encryption.Enabled() && len(p.ContentMD5) > 0 { if !p.Encryption.Enabled() && p.ContentMD5 != nil {
headerMd5Hash, err := base64.StdEncoding.DecodeString(p.ContentMD5) if len(*p.ContentMD5) == 0 {

Looks good, I would write

if !p.Encryption.Enabled() && p.ContentMD5 != nil  {
	if len(*p.ContentMD5) == 0 {
		return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
	}
	headerMd5Hash, err := base64.StdEncoding.DecodeString(*p.ContentMD5)
	...
}
Looks good, I would write ```go if !p.Encryption.Enabled() && p.ContentMD5 != nil { if len(*p.ContentMD5) == 0 { return nil, apierr.GetAPIError(apierr.ErrInvalidDigest) } headerMd5Hash, err := base64.StdEncoding.DecodeString(*p.ContentMD5) ... } ```
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
}
headerMd5Hash, err := base64.StdEncoding.DecodeString(*p.ContentMD5)
if err != nil { if err != nil {
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest) return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
} }
@ -296,7 +299,7 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
if err != nil { if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID)) n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
} }
dkirillov marked this conversation as resolved Outdated

I suppose we should distinguish cases where Contet-Md5 isn't set and set but empty. Error should be occurred in the second case

I suppose we should distinguish cases where `Contet-Md5` isn't set and set but empty. Error should be occurred in the second case
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest) return nil, apierr.GetAPIError(apierr.ErrBadDigest)
} }
} }