pogpp/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Pull requests Activity

[#543] Add md5 sse-c S3Tests compatability

Browse source 
Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>
This commit is contained in:
Pavel Pogodaev 2024-11-15 13:20:18 +03:00
parent a12fea8a5b
commit dda6a17e8b
4 changed files with 24 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
api/handler/encryption_test.go
View file

@ -46,6 +46,10 @@ func TestSimpleGetEncrypted(t *testing.T) {
response, _ := getEncryptedObject(tc, bktName, objName) response, _ := getEncryptedObject(tc, bktName, objName)
require.Equal(t, content, string(response)) require.Equal(t, content, string(response))
result := listVersions(t, tc, bktName)
require.Len(t, result.Version, 1)
require.Equal(t, uint64(len(content)), result.Version[0].Size)
} }
func TestMD5HeaderBadOrEmpty(t *testing.T) { func TestMD5HeaderBadOrEmpty(t *testing.T) {
@ -369,6 +373,10 @@ func TestMultipartEncrypted(t *testing.T) {
part2Range := getEncryptedObjectRange(t, hc, bktName, objName, len(part1), len(part1)+len(part2)-1) part2Range := getEncryptedObjectRange(t, hc, bktName, objName, len(part1), len(part1)+len(part2)-1)
require.Equal(t, part2[0:], part2Range) require.Equal(t, part2[0:], part2Range)
result := listVersions(t, hc, bktName)
require.Len(t, result.Version, 1)
require.EqualValues(t, uint64(partSize+5), result.Version[0].Size)
} }
func putEncryptedObject(t *testing.T, tc *handlerContext, bktName, objName, content string) { func putEncryptedObject(t *testing.T, tc *handlerContext, bktName, objName, content string) {

4
api/handler/multipart_upload_test.go
View file

@ -382,6 +382,10 @@ func TestMultipartUploadSize(t *testing.T) {
attr := getObjectAttributes(hc, newBucket, newObjName, objectParts) attr := getObjectAttributes(hc, newBucket, newObjName, objectParts)
require.Equal(t, 1, attr.ObjectParts.PartsCount) require.Equal(t, 1, attr.ObjectParts.PartsCount)
require.Equal(t, srcObjInfo.Headers[layer.AttributeDecryptedSize], strconv.Itoa(attr.ObjectParts.Parts[0].Size)) require.Equal(t, srcObjInfo.Headers[layer.AttributeDecryptedSize], strconv.Itoa(attr.ObjectParts.Parts[0].Size))
result := listVersions(t, hc, bktName)
require.Len(t, result.Version, 1)
require.EqualValues(t, objLen, result.Version[0].Size)
}) })
} }

14
api/layer/multipart_upload.go
View file

@ -224,8 +224,12 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
} }
decSize := p.Size decSize := p.Size
md5Hash := md5.New()
if p.Info.Encryption.Enabled() { if p.Info.Encryption.Enabled() {
r, encSize, err := encryptionReader(p.Reader, p.Size, p.Info.Encryption.Key()) rr := wrapReader(p.Reader, 64*1024, func(buf []byte) {
md5Hash.Write(buf)
})
r, encSize, err := encryptionReader(rr, p.Size, p.Info.Encryption.Key())
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create ecnrypted reader: %w", err) return nil, fmt.Errorf("failed to create ecnrypted reader: %w", err)
} }
@ -246,7 +250,12 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
if err != nil { if err != nil {
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest) return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
} }
if hex.EncodeToString(hashBytes) != hex.EncodeToString(createdObj.MD5Sum) {
match := hex.EncodeToString(hashBytes) == hex.EncodeToString(createdObj.MD5Sum)
if p.Info.Encryption.Enabled() {
match = hex.EncodeToString(hashBytes) == hex.EncodeToString(md5Hash.Sum(nil))
}
if !match {
prm := frostfs.PrmObjectDelete{ prm := frostfs.PrmObjectDelete{
Object: createdObj.ID, Object: createdObj.ID,
Container: bktInfo.CID, Container: bktInfo.CID,
@ -449,7 +458,6 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
initMetadata[AttributeHMACKey] = encInfo.HMACKey initMetadata[AttributeHMACKey] = encInfo.HMACKey
initMetadata[AttributeHMACSalt] = encInfo.HMACSalt initMetadata[AttributeHMACSalt] = encInfo.HMACSalt
initMetadata[AttributeDecryptedSize] = strconv.FormatUint(multipartObjetSize, 10) initMetadata[AttributeDecryptedSize] = strconv.FormatUint(multipartObjetSize, 10)
multipartObjetSize = encMultipartObjectSize
} }
partsData, err := json.Marshal(parts) partsData, err := json.Marshal(parts)

4
api/layer/object.go
View file

@ -243,11 +243,9 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
return nil, fmt.Errorf("add encryption header: %w", err) return nil, fmt.Errorf("add encryption header: %w", err)
} }
var encSize uint64 if r, _, err = encryptionReader(p.Reader, size, p.Encryption.Key()); err != nil {
if r, encSize, err = encryptionReader(p.Reader, size, p.Encryption.Key()); err != nil {
return nil, fmt.Errorf("create encrypter: %w", err) return nil, fmt.Errorf("create encrypter: %w", err)
} }
p.Size = &encSize
} }
if r != nil { if r != nil {