From c3ad6d2fafe11634d6d2569a4fbdd5383fa4642e Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Wed, 17 Aug 2022 14:18:36 +0300 Subject: [PATCH] [#637] Add header to override CopiesNumber Signed-off-by: Denis Kirillov --- api/data/tree.go | 13 +++---- api/handler/copy.go | 21 +++++++---- api/handler/multipart_upload.go | 22 +++++++----- api/handler/put.go | 22 +++++++++++- api/handler/put_test.go | 22 ++++++++++++ api/layer/layer.go | 34 ++++++++++-------- api/layer/multipart_upload.go | 62 ++++++++++++++++----------------- 7 files changed, 127 insertions(+), 69 deletions(-) diff --git a/api/data/tree.go b/api/data/tree.go index 60ecf4cd..9ae6af32 100644 --- a/api/data/tree.go +++ b/api/data/tree.go @@ -67,12 +67,13 @@ type ObjectTaggingInfo struct { type MultipartInfo struct { // ID is node id in tree service. // It's ignored when creating a new multipart upload. - ID uint64 - Key string - UploadID string - Owner user.ID - Created time.Time - Meta map[string]string + ID uint64 + Key string + UploadID string + Owner user.ID + Created time.Time + Meta map[string]string + CopiesNumber uint32 } // PartInfo is upload information about part. diff --git a/api/handler/copy.go b/api/handler/copy.go index 9df58ccf..14bc73a5 100644 --- a/api/handler/copy.go +++ b/api/handler/copy.go @@ -121,14 +121,21 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { metadata[api.ContentType] = contentType } + copiesNumber, err := getCopiesNumberOrDefault(metadata, h.cfg.CopiesNumber) + if err != nil { + h.logAndSendError(w, "invalid copies number", reqInfo, err) + return + } + params := &layer.CopyObjectParams{ - SrcObject: objInfo, - ScrBktInfo: p.BktInfo, - DstBktInfo: dstBktInfo, - DstObject: reqInfo.ObjectName, - SrcSize: objInfo.Size, - Header: metadata, - Encryption: encryptionParams, + SrcObject: objInfo, + ScrBktInfo: p.BktInfo, + DstBktInfo: dstBktInfo, + DstObject: reqInfo.ObjectName, + SrcSize: objInfo.Size, + Header: metadata, + Encryption: encryptionParams, + CopiesNuber: copiesNumber, } settings, err := h.obj.GetBucketSettings(r.Context(), dstBktInfo) diff --git a/api/handler/multipart_upload.go b/api/handler/multipart_upload.go index 89c906e4..1dca5e45 100644 --- a/api/handler/multipart_upload.go +++ b/api/handler/multipart_upload.go @@ -148,6 +148,12 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re p.Header[api.ContentType] = contentType } + p.CopiesNumber, err = getCopiesNumberOrDefault(p.Header, h.cfg.CopiesNumber) + if err != nil { + h.logAndSendError(w, "invalid copies number", reqInfo, err) + return + } + if err = h.obj.CreateMultipartUpload(r.Context(), p); err != nil { h.logAndSendError(w, "could create multipart upload", reqInfo, err, additional...) return @@ -215,10 +221,9 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) { Bkt: bktInfo, Key: reqInfo.ObjectName, }, - PartNumber: partNumber, - Size: r.ContentLength, - Reader: r.Body, - CopiesNumber: h.cfg.CopiesNumber, + PartNumber: partNumber, + Size: r.ContentLength, + Reader: r.Body, } p.Info.Encryption, err = h.formEncryptionParams(r.Header) @@ -316,11 +321,10 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) { Bkt: bktInfo, Key: reqInfo.ObjectName, }, - SrcObjInfo: srcInfo, - SrcBktInfo: srcBktInfo, - PartNumber: partNumber, - Range: srcRange, - CopiesNumber: h.cfg.CopiesNumber, + SrcObjInfo: srcInfo, + SrcBktInfo: srcBktInfo, + PartNumber: partNumber, + Range: srcRange, } p.Info.Encryption, err = h.formEncryptionParams(r.Header) diff --git a/api/handler/put.go b/api/handler/put.go index 71eba1b4..6ae31b86 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -212,6 +212,12 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { metadata[api.Expires] = expires } + copiesNumber, err := getCopiesNumberOrDefault(metadata, h.cfg.CopiesNumber) + if err != nil { + h.logAndSendError(w, "invalid copies number", reqInfo, err) + return + } + encryption, err := h.formEncryptionParams(r.Header) if err != nil { h.logAndSendError(w, "invalid sse headers", reqInfo, err) @@ -225,7 +231,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { Size: r.ContentLength, Header: metadata, Encryption: encryption, - CopiesNumber: h.cfg.CopiesNumber, + CopiesNumber: copiesNumber, } settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo) @@ -299,6 +305,20 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) { api.WriteSuccessResponseHeadersOnly(w) } +func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber uint32) (uint32, error) { + copiesNumberStr, ok := metadata[layer.AttributeNeofsCopiesNumber] + if !ok { + return defaultCopiesNumber, nil + } + + copiesNumber, err := strconv.ParseUint(copiesNumberStr, 10, 32) + if err != nil { + return 0, fmt.Errorf("pasrse copies number: %w", err) + } + + return uint32(copiesNumber), nil +} + func (h handler) formEncryptionParams(header http.Header) (enc encryption.Params, err error) { sseCustomerAlgorithm := header.Get(api.AmzServerSideEncryptionCustomerAlgorithm) sseCustomerKey := header.Get(api.AmzServerSideEncryptionCustomerKey) diff --git a/api/handler/put_test.go b/api/handler/put_test.go index dbe02e37..c0309a8c 100644 --- a/api/handler/put_test.go +++ b/api/handler/put_test.go @@ -4,10 +4,12 @@ import ( "encoding/json" "mime/multipart" "net/http" + "strings" "testing" "time" "github.com/nspcc-dev/neofs-s3-gw/api" + "github.com/nspcc-dev/neofs-s3-gw/api/layer" "github.com/stretchr/testify/require" ) @@ -104,3 +106,23 @@ func TestEmptyPostPolicy(t *testing.T) { _, err := checkPostPolicy(r, reqInfo, metadata) require.NoError(t, err) } + +func TestPutObjectOverrideCopiesNumber(t *testing.T) { + tc := prepareHandlerContext(t) + + bktName, objName := "bucket-for-copies-number", "object-for-copies-number" + bktInfo := createTestBucket(tc.Context(), t, tc, bktName) + + w, r := prepareTestRequest(t, bktName, objName, nil) + r.Header.Set(api.MetadataPrefix+strings.ToUpper(layer.AttributeNeofsCopiesNumber), "1") + tc.Handler().PutObjectHandler(w, r) + + p := &layer.HeadObjectParams{ + BktInfo: bktInfo, + Object: objName, + } + + objInfo, err := tc.Layer().GetObjectInfo(tc.Context(), p) + require.NoError(t, err) + require.Equal(t, "1", objInfo.Headers[layer.AttributeNeofsCopiesNumber]) +} diff --git a/api/layer/layer.go b/api/layer/layer.go index 45820098..c69d47a5 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -141,15 +141,16 @@ type ( // CopyObjectParams stores object copy request parameters. CopyObjectParams struct { - SrcObject *data.ObjectInfo - ScrBktInfo *data.BucketInfo - DstBktInfo *data.BucketInfo - DstObject string - SrcSize int64 - Header map[string]string - Range *RangeParams - Lock *data.ObjectLock - Encryption encryption.Params + SrcObject *data.ObjectInfo + ScrBktInfo *data.BucketInfo + DstBktInfo *data.BucketInfo + DstObject string + SrcSize int64 + Header map[string]string + Range *RangeParams + Lock *data.ObjectLock + Encryption encryption.Params + CopiesNuber uint32 } // CreateBucketParams stores bucket create request parameters. CreateBucketParams struct { @@ -264,6 +265,8 @@ const ( AttributeDecryptedSize = api.NeoFSSystemMetadataPrefix + "Decrypted-Size" AttributeHMACSalt = api.NeoFSSystemMetadataPrefix + "HMAC-Salt" AttributeHMACKey = api.NeoFSSystemMetadataPrefix + "HMAC-Key" + + AttributeNeofsCopiesNumber = "neofs-copies-number" // such formate to match X-Amz-Meta-Neofs-Copies-Number header ) func (t *VersionedObject) String() string { @@ -519,12 +522,13 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.Obje }() return n.PutObject(ctx, &PutObjectParams{ - BktInfo: p.DstBktInfo, - Object: p.DstObject, - Size: p.SrcSize, - Reader: pr, - Header: p.Header, - Encryption: p.Encryption, + BktInfo: p.DstBktInfo, + Object: p.DstObject, + Size: p.SrcSize, + Reader: pr, + Header: p.Header, + Encryption: p.Encryption, + CopiesNumber: p.CopiesNuber, }) } diff --git a/api/layer/multipart_upload.go b/api/layer/multipart_upload.go index 4737a298..2d749a64 100644 --- a/api/layer/multipart_upload.go +++ b/api/layer/multipart_upload.go @@ -45,9 +45,10 @@ type ( } CreateMultipartParams struct { - Info *UploadInfoParams - Header map[string]string - Data *UploadData + Info *UploadInfoParams + Header map[string]string + Data *UploadData + CopiesNumber uint32 } UploadData struct { @@ -56,20 +57,18 @@ type ( } UploadPartParams struct { - Info *UploadInfoParams - PartNumber int - Size int64 - Reader io.Reader - CopiesNumber uint32 + Info *UploadInfoParams + PartNumber int + Size int64 + Reader io.Reader } UploadCopyParams struct { - Info *UploadInfoParams - SrcObjInfo *data.ObjectInfo - SrcBktInfo *data.BucketInfo - PartNumber int - Range *RangeParams - CopiesNumber uint32 + Info *UploadInfoParams + SrcObjInfo *data.ObjectInfo + SrcBktInfo *data.BucketInfo + PartNumber int + Range *RangeParams } CompleteMultipartParams struct { @@ -141,11 +140,12 @@ func (n *layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar } info := &data.MultipartInfo{ - Key: p.Info.Key, - UploadID: p.Info.UploadID, - Owner: n.Owner(ctx), - Created: time.Now(), - Meta: make(map[string]string, metaSize), + Key: p.Info.Key, + UploadID: p.Info.UploadID, + Owner: n.Owner(ctx), + Created: time.Now(), + Meta: make(map[string]string, metaSize), + CopiesNumber: p.CopiesNumber, } for key, val := range p.Header { @@ -205,7 +205,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf Creator: bktInfo.Owner, Attributes: make([][2]string, 2), Payload: p.Reader, - CopiesNumber: p.CopiesNumber, + CopiesNumber: multipartInfo.CopiesNumber, } decSize := p.Size @@ -301,11 +301,10 @@ func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data. }() params := &UploadPartParams{ - Info: p.Info, - PartNumber: p.PartNumber, - Size: size, - Reader: pr, - CopiesNumber: p.CopiesNumber, + Info: p.Info, + PartNumber: p.PartNumber, + Size: size, + Reader: pr, } return n.uploadPart(ctx, multipartInfo, params) @@ -435,12 +434,13 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar r.prm.bktInfo = p.Info.Bkt obj, err := n.PutObject(ctx, &PutObjectParams{ - BktInfo: p.Info.Bkt, - Object: p.Info.Key, - Reader: r, - Header: initMetadata, - Size: multipartObjetSize, - Encryption: p.Info.Encryption, + BktInfo: p.Info.Bkt, + Object: p.Info.Key, + Reader: r, + Header: initMetadata, + Size: multipartObjetSize, + Encryption: p.Info.Encryption, + CopiesNumber: multipartInfo.CopiesNumber, }) if err != nil { n.log.Error("could not put a completed object (multipart upload)",