From 910331cd42372aa810188036538db2c53d935a2d Mon Sep 17 00:00:00 2001
From: Pavel Pogodaev
Date: Fri, 15 Nov 2024 13:20:18 +0300
Subject: [PATCH] [#543] Add md5 sse-c S3Tests compatability
Signed-off-by: Pavel Pogodaev
---
api/handler/encryption_test.go | 8 ++++++++
api/handler/multipart_upload_test.go | 4 ++++
api/layer/multipart_upload.go | 18 ++++++++++++------
api/layer/object.go | 4 +---
4 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/api/handler/encryption_test.go b/api/handler/encryption_test.go
index 8efdc45..94a06de 100644
--- a/api/handler/encryption_test.go
+++ b/api/handler/encryption_test.go
@@ -46,6 +46,10 @@ func TestSimpleGetEncrypted(t *testing.T) {
response, _ := getEncryptedObject(tc, bktName, objName)
require.Equal(t, content, string(response))
+
+ result := listVersions(t, tc, bktName)
+ require.Len(t, result.Version, 1)
+ require.Equal(t, uint64(len(content)), result.Version[0].Size)
}
func TestMD5HeaderBadOrEmpty(t *testing.T) {
@@ -369,6 +373,10 @@ func TestMultipartEncrypted(t *testing.T) {
part2Range := getEncryptedObjectRange(t, hc, bktName, objName, len(part1), len(part1)+len(part2)-1)
require.Equal(t, part2[0:], part2Range)
+
+ result := listVersions(t, hc, bktName)
+ require.Len(t, result.Version, 1)
+ require.EqualValues(t, uint64(partSize+5), result.Version[0].Size)
}
func putEncryptedObject(t *testing.T, tc *handlerContext, bktName, objName, content string) {
diff --git a/api/handler/multipart_upload_test.go b/api/handler/multipart_upload_test.go
index 2831b3e..38f0e56 100644
--- a/api/handler/multipart_upload_test.go
+++ b/api/handler/multipart_upload_test.go
@@ -382,6 +382,10 @@ func TestMultipartUploadSize(t *testing.T) {
attr := getObjectAttributes(hc, newBucket, newObjName, objectParts)
require.Equal(t, 1, attr.ObjectParts.PartsCount)
require.Equal(t, srcObjInfo.Headers[layer.AttributeDecryptedSize], strconv.Itoa(attr.ObjectParts.Parts[0].Size))
+
+ result := listVersions(t, hc, bktName)
+ require.Len(t, result.Version, 1)
+ require.EqualValues(t, objLen, result.Version[0].Size)
})
}
diff --git a/api/layer/multipart_upload.go b/api/layer/multipart_upload.go
index ed7612c..1a23161 100644
--- a/api/layer/multipart_upload.go
+++ b/api/layer/multipart_upload.go
@@ -224,8 +224,12 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
}
decSize := p.Size
+ md5Hash := md5.New()
if p.Info.Encryption.Enabled() {
- r, encSize, err := encryptionReader(p.Reader, p.Size, p.Info.Encryption.Key())
+ rr := wrapReader(p.Reader, 64*1024, func(buf []byte) {
+ md5Hash.Write(buf)
+ })
+ r, encSize, err := encryptionReader(rr, p.Size, p.Info.Encryption.Key())
if err != nil {
return nil, fmt.Errorf("failed to create ecnrypted reader: %w", err)
}
@@ -246,7 +250,12 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
if err != nil {
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
}
- if hex.EncodeToString(hashBytes) != hex.EncodeToString(createdObj.MD5Sum) {
+
+ match := hex.EncodeToString(hashBytes) == hex.EncodeToString(createdObj.MD5Sum)
+ if p.Info.Encryption.Enabled() {
+ match = bytes.Equal(hashBytes, md5Hash.Sum(nil))
+ }
+ if !match {
prm := frostfs.PrmObjectDelete{
Object: createdObj.ID,
Container: bktInfo.CID,
@@ -388,7 +397,6 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
}
var multipartObjetSize uint64
- var encMultipartObjectSize uint64
parts := make([]*data.PartInfoExtended, 0, len(p.Parts))
var completedPartsHeader strings.Builder
@@ -407,11 +415,10 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
multipartObjetSize += partInfo.Size // even if encryption is enabled size is actual (decrypted)
if encInfo.Enabled {
- encPartSize, err := sio.EncryptedSize(partInfo.Size)
+ _, err := sio.EncryptedSize(partInfo.Size)
if err != nil {
return nil, nil, fmt.Errorf("compute encrypted size: %w", err)
}
- encMultipartObjectSize += encPartSize
}
partInfoStr := partInfo.ToHeaderString()
@@ -449,7 +456,6 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
initMetadata[AttributeHMACKey] = encInfo.HMACKey
initMetadata[AttributeHMACSalt] = encInfo.HMACSalt
initMetadata[AttributeDecryptedSize] = strconv.FormatUint(multipartObjetSize, 10)
- multipartObjetSize = encMultipartObjectSize
}
partsData, err := json.Marshal(parts)
diff --git a/api/layer/object.go b/api/layer/object.go
index 8f4d64f..555a0cb 100644
--- a/api/layer/object.go
+++ b/api/layer/object.go
@@ -243,11 +243,9 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
return nil, fmt.Errorf("add encryption header: %w", err)
}
- var encSize uint64
- if r, encSize, err = encryptionReader(p.Reader, size, p.Encryption.Key()); err != nil {
+ if r, _, err = encryptionReader(p.Reader, size, p.Encryption.Key()); err != nil {
return nil, fmt.Errorf("create encrypter: %w", err)
}
- p.Size = &encSize
}
if r != nil {