[#543] Add md5 sse-c S3Tests compatability

Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>

api/handler/attributes.go

@@ -73,7 +73,7 @@ func (h *handler) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Requ
 	ctx := r.Context()
 	reqInfo := middleware.GetReqInfo(ctx)
 
-	params, err := parseGetObjectAttributeArgs(r)
+	params, err := parseGetObjectAttributeArgs(r, h.reqLogger(ctx))
 	if err != nil {
 		h.logAndSendError(ctx, w, "invalid request", reqInfo, err)
 		return
@@ -145,7 +145,7 @@ func writeAttributesHeaders(h http.Header, info *data.ExtendedObjectInfo, isBuck
 	// x-amz-request-charged
 }
 
-func parseGetObjectAttributeArgs(r *http.Request) (*GetObjectAttributesArgs, error) {
+func parseGetObjectAttributeArgs(r *http.Request, log *zap.Logger) (*GetObjectAttributesArgs, error) {
 	res := &GetObjectAttributesArgs{
 		VersionID: r.URL.Query().Get(api.QueryVersionID),
 	}
@@ -178,8 +178,8 @@ func parseGetObjectAttributeArgs(r *http.Request) (*GetObjectAttributesArgs, err
 		}
 	}
 
-	res.Conditional, err = parseConditionalHeaders(r.Header)
+	res.Conditional = parseConditionalHeaders(r.Header, log)
-	return res, err
+	return res, nil
 }
 
 func encodeToObjectAttributesResponse(info *data.ObjectInfo, p *GetObjectAttributesArgs, md5Enabled bool) (*GetObjectAttributesResponse, error) {

api/handler/encryption_test.go

@@ -46,6 +46,10 @@ func TestSimpleGetEncrypted(t *testing.T) {
 
 	response, _ := getEncryptedObject(tc, bktName, objName)
 	require.Equal(t, content, string(response))
+
+	result := listVersions(t, tc, bktName)
+	require.Len(t, result.Version, 1)
+	require.Equal(t, uint64(len(content)), result.Version[0].Size)
 }
 
 func TestMD5HeaderBadOrEmpty(t *testing.T) {
@@ -369,6 +373,10 @@ func TestMultipartEncrypted(t *testing.T) {
 
 	part2Range := getEncryptedObjectRange(t, hc, bktName, objName, len(part1), len(part1)+len(part2)-1)
 	require.Equal(t, part2[0:], part2Range)
+
+	result := listVersions(t, hc, bktName)
+	require.Len(t, result.Version, 1)
+	require.EqualValues(t, uint64(partSize+5), result.Version[0].Size)
 }
 
 func putEncryptedObject(t *testing.T, tc *handlerContext, bktName, objName, content string) {

api/handler/get.go

@@ -13,6 +13,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
 	"go.uber.org/zap"
 )
 
@@ -154,11 +155,7 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
 		reqInfo = middleware.GetReqInfo(ctx)
 	)
 
-	conditional, err := parseConditionalHeaders(r.Header)
+	conditional := parseConditionalHeaders(r.Header, h.reqLogger(ctx))
-	if err != nil {
-		h.logAndSendError(ctx, w, "could not parse request params", reqInfo, err)
-		return
-	}
 
 	bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
 	if err != nil {
@@ -290,21 +287,24 @@ func checkPreconditions(info *data.ObjectInfo, args *conditionalArgs, md5Enabled
 	return nil
 }
 
-func parseConditionalHeaders(headers http.Header) (*conditionalArgs, error) {
+func parseConditionalHeaders(headers http.Header, log *zap.Logger) *conditionalArgs {
-	var err error
 	args := &conditionalArgs{
 		IfMatch:     data.UnQuote(headers.Get(api.IfMatch)),
 		IfNoneMatch: data.UnQuote(headers.Get(api.IfNoneMatch)),
 	}
 
-	if args.IfModifiedSince, err = parseHTTPTime(headers.Get(api.IfModifiedSince)); err != nil {
+	if httpTime, err := parseHTTPTime(headers.Get(api.IfModifiedSince)); err == nil {
-		return nil, err
+		args.IfModifiedSince = httpTime
+	} else {
+		log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfModifiedSince, headers.Get(api.IfModifiedSince)), zap.Error(err))
 	}
-	if args.IfUnmodifiedSince, err = parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err != nil {
+	if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
-		return nil, err
+		args.IfUnmodifiedSince = httpTime
+	} else {
+		log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err))
 	}
 
-	return args, nil
+	return args
 }
 
 func parseHTTPTime(data string) (*time.Time, error) {

api/handler/head.go

@@ -36,11 +36,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	conditional, err := parseConditionalHeaders(r.Header)
+	conditional := parseConditionalHeaders(r.Header, h.reqLogger(ctx))
-	if err != nil {
-		h.logAndSendError(ctx, w, "could not parse request params", reqInfo, err)
-		return
-	}
 
 	p := &layer.HeadObjectParams{
 		BktInfo:   bktInfo,

api/handler/head_test.go

@@ -68,6 +68,16 @@ func TestConditionalHead(t *testing.T) {
 		api.IfModifiedSince: zeroTime.UTC().Format(http.TimeFormat),
 	}
 	headObject(t, tc, bktName, objName, headers, http.StatusNotModified)
+
+	headers = map[string]string{
+		api.IfUnmodifiedSince: zeroTime.UTC().Format(time.RFC3339), // invalid format, header is ignored
+	}
+	headObject(t, tc, bktName, objName, headers, http.StatusOK)
+
+	headers = map[string]string{
+		api.IfModifiedSince: objInfo.Created.Add(time.Minute).Format(time.RFC3339), // invalid format, header is ignored
+	}
+	headObject(t, tc, bktName, objName, headers, http.StatusOK)
 }
 
 func headObject(t *testing.T, tc *handlerContext, bktName, objName string, headers map[string]string, status int) {

api/handler/multipart_upload_test.go

@@ -382,6 +382,10 @@ func TestMultipartUploadSize(t *testing.T) {
 		attr := getObjectAttributes(hc, newBucket, newObjName, objectParts)
 		require.Equal(t, 1, attr.ObjectParts.PartsCount)
 		require.Equal(t, srcObjInfo.Headers[layer.AttributeDecryptedSize], strconv.Itoa(attr.ObjectParts.Parts[0].Size))
+
+		result := listVersions(t, hc, bktName)
+		require.Len(t, result.Version, 1)
+		require.EqualValues(t, objLen, result.Version[0].Size)
 	})
 }
 

api/handler/patch.go

@@ -12,6 +12,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
 	"go.uber.org/zap"
 )
 
@@ -31,11 +32,7 @@ func (h *handler) PatchObjectHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	conditional, err := parsePatchConditionalHeaders(r.Header)
+	conditional := parsePatchConditionalHeaders(r.Header, h.reqLogger(ctx))
-	if err != nil {
-		h.logAndSendError(ctx, w, "could not parse conditional headers", reqInfo, err)
-		return
-	}
 
 	bktInfo, err := h.getBucketAndCheckOwner(r, reqInfo.BucketName)
 	if err != nil {
@@ -137,17 +134,18 @@ func (h *handler) PatchObjectHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func parsePatchConditionalHeaders(headers http.Header) (*conditionalArgs, error) {
+func parsePatchConditionalHeaders(headers http.Header, log *zap.Logger) *conditionalArgs {
-	var err error
 	args := &conditionalArgs{
 		IfMatch: data.UnQuote(headers.Get(api.IfMatch)),
 	}
 
-	if args.IfUnmodifiedSince, err = parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err != nil {
+	if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
-		return nil, err
+		args.IfUnmodifiedSince = httpTime
+	} else {
+		log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err))
 	}
 
-	return args, nil
+	return args
 }
 
 func parsePatchByteRange(rangeStr string, objSize uint64) (*layer.RangeParams, error) {

api/handler/patch_test.go

@@ -60,6 +60,13 @@ func TestPatch(t *testing.T) {
 				api.IfMatch:           etag,
 			},
 		},
+		{
+			name: "If-Unmodified-Since invalid format, header is ignored",
+			rng:  "bytes 0-2/*",
+			headers: map[string]string{
+				api.IfUnmodifiedSince: created.Add(-24 * time.Hour).Format(time.RFC3339),
+			},
+		},
 		{
 			name: "invalid range syntax",
 			rng:  "bytes 0-2",

api/layer/multipart_upload.go

@@ -224,8 +224,12 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
 	}
 
 	decSize := p.Size
+	md5Hash := md5.New()
 	if p.Info.Encryption.Enabled() {
-		r, encSize, err := encryptionReader(p.Reader, p.Size, p.Info.Encryption.Key())
+		rr := wrapReader(p.Reader, 64*1024, func(buf []byte) {
+			md5Hash.Write(buf)
+		})
+		r, encSize, err := encryptionReader(rr, p.Size, p.Info.Encryption.Key())
 		if err != nil {
 			return nil, fmt.Errorf("failed to create ecnrypted reader: %w", err)
 		}
@@ -246,7 +250,12 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
 		if err != nil {
 			return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
 		}
-		if hex.EncodeToString(hashBytes) != hex.EncodeToString(createdObj.MD5Sum) {
+
+		match := bytes.Equal(hashBytes, createdObj.MD5Sum)
+		if p.Info.Encryption.Enabled() {
+			match = bytes.Equal(hashBytes, md5Hash.Sum(nil))
+		}
+		if !match {
 			prm := frostfs.PrmObjectDelete{
 				Object:    createdObj.ID,
 				Container: bktInfo.CID,
@@ -388,7 +397,6 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
 	}
 
 	var multipartObjetSize uint64
-	var encMultipartObjectSize uint64
 	parts := make([]*data.PartInfoExtended, 0, len(p.Parts))
 
 	var completedPartsHeader strings.Builder
@@ -407,11 +415,10 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
 		multipartObjetSize += partInfo.Size // even if encryption is enabled size is actual (decrypted)
 
 		if encInfo.Enabled {
-			encPartSize, err := sio.EncryptedSize(partInfo.Size)
+			_, err := sio.EncryptedSize(partInfo.Size)
 			if err != nil {
 				return nil, nil, fmt.Errorf("compute encrypted size: %w", err)
 			}
-			encMultipartObjectSize += encPartSize
 		}
 
 		partInfoStr := partInfo.ToHeaderString()
@@ -449,7 +456,6 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
 		initMetadata[AttributeHMACKey] = encInfo.HMACKey
 		initMetadata[AttributeHMACSalt] = encInfo.HMACSalt
 		initMetadata[AttributeDecryptedSize] = strconv.FormatUint(multipartObjetSize, 10)
-		multipartObjetSize = encMultipartObjectSize
 	}
 
 	partsData, err := json.Marshal(parts)

api/layer/object.go

@@ -243,11 +243,9 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
 			return nil, fmt.Errorf("add encryption header: %w", err)
 		}
 
-		var encSize uint64
+		if r, _, err = encryptionReader(p.Reader, size, p.Encryption.Key()); err != nil {
-		if r, encSize, err = encryptionReader(p.Reader, size, p.Encryption.Key()); err != nil {
 			return nil, fmt.Errorf("create encrypter: %w", err)
 		}
-		p.Size = &encSize
 	}
 
 	if r != nil {

internal/logs/logs.go

@@ -177,4 +177,5 @@ const (
 	MultinetConfigWontBeUpdated                          = "multinet config won't be updated"
 	MultinetDialSuccess                                  = "multinet dial successful"
 	MultinetDialFail                                     = "multinet dial failed"
+	FailedToParseHTTPTime                                = "failed to parse http time, header is ignored"
 )