dkirillov/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#637] Add header to override CopiesNumber

Browse source 
Signed-off-by: Denis Kirillov <denis@nspcc.ru>
This commit is contained in:
Denis Kirillov 2022-08-17 14:18:36 +03:00 committed by Kirillov Denis
parent d2c68589b5
commit c3ad6d2faf
7 changed files with 127 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
api/data/tree.go
View file

@ -73,6 +73,7 @@ type MultipartInfo struct {
Owner user.ID
Created time.Time
Meta map[string]string
CopiesNumber uint32
}
// PartInfo is upload information about part.

7
api/handler/copy.go
View file

@ -121,6 +121,12 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
metadata[api.ContentType] = contentType
}
copiesNumber, err := getCopiesNumberOrDefault(metadata, h.cfg.CopiesNumber)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
return
}
params := &layer.CopyObjectParams{
SrcObject: objInfo,
ScrBktInfo: p.BktInfo,
@ -129,6 +135,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
SrcSize: objInfo.Size,
Header: metadata,
Encryption: encryptionParams,
CopiesNuber: copiesNumber,
}
settings, err := h.obj.GetBucketSettings(r.Context(), dstBktInfo)

8
api/handler/multipart_upload.go
View file

@ -148,6 +148,12 @@ func (h *handler) CreateMultipartUploadHandler(w http.ResponseWriter, r *http.Re
p.Header[api.ContentType] = contentType
}
p.CopiesNumber, err = getCopiesNumberOrDefault(p.Header, h.cfg.CopiesNumber)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
return
}
if err = h.obj.CreateMultipartUpload(r.Context(), p); err != nil {
h.logAndSendError(w, "could create multipart upload", reqInfo, err, additional...)
return
@ -218,7 +224,6 @@ func (h *handler) UploadPartHandler(w http.ResponseWriter, r *http.Request) {
PartNumber: partNumber,
Size: r.ContentLength,
Reader: r.Body,
CopiesNumber: h.cfg.CopiesNumber,
}
p.Info.Encryption, err = h.formEncryptionParams(r.Header)
@ -320,7 +325,6 @@ func (h *handler) UploadPartCopy(w http.ResponseWriter, r *http.Request) {
SrcBktInfo: srcBktInfo,
PartNumber: partNumber,
Range: srcRange,
CopiesNumber: h.cfg.CopiesNumber,
}
p.Info.Encryption, err = h.formEncryptionParams(r.Header)

22
api/handler/put.go
View file

@ -212,6 +212,12 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
metadata[api.Expires] = expires
}
copiesNumber, err := getCopiesNumberOrDefault(metadata, h.cfg.CopiesNumber)
if err != nil {
h.logAndSendError(w, "invalid copies number", reqInfo, err)
return
}
encryption, err := h.formEncryptionParams(r.Header)
if err != nil {
h.logAndSendError(w, "invalid sse headers", reqInfo, err)
@ -225,7 +231,7 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
Size: r.ContentLength,
Header: metadata,
Encryption: encryption,
CopiesNumber: h.cfg.CopiesNumber,
CopiesNumber: copiesNumber,
}
settings, err := h.obj.GetBucketSettings(r.Context(), bktInfo)
@ -299,6 +305,20 @@ func (h *handler) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
api.WriteSuccessResponseHeadersOnly(w)
}
func getCopiesNumberOrDefault(metadata map[string]string, defaultCopiesNumber uint32) (uint32, error) {
copiesNumberStr, ok := metadata[layer.AttributeNeofsCopiesNumber]
if !ok {
return defaultCopiesNumber, nil
}
copiesNumber, err := strconv.ParseUint(copiesNumberStr, 10, 32)
if err != nil {
return 0, fmt.Errorf("pasrse copies number: %w", err)
}
return uint32(copiesNumber), nil
}
func (h handler) formEncryptionParams(header http.Header) (enc encryption.Params, err error) {
sseCustomerAlgorithm := header.Get(api.AmzServerSideEncryptionCustomerAlgorithm)
sseCustomerKey := header.Get(api.AmzServerSideEncryptionCustomerKey)

22
api/handler/put_test.go
View file

@ -4,10 +4,12 @@ import (
"encoding/json"
"mime/multipart"
"net/http"
"strings"
"testing"
"time"
"github.com/nspcc-dev/neofs-s3-gw/api"
"github.com/nspcc-dev/neofs-s3-gw/api/layer"
"github.com/stretchr/testify/require"
)
@ -104,3 +106,23 @@ func TestEmptyPostPolicy(t *testing.T) {
_, err := checkPostPolicy(r, reqInfo, metadata)
require.NoError(t, err)
}
func TestPutObjectOverrideCopiesNumber(t *testing.T) {
tc := prepareHandlerContext(t)
bktName, objName := "bucket-for-copies-number", "object-for-copies-number"
bktInfo := createTestBucket(tc.Context(), t, tc, bktName)
w, r := prepareTestRequest(t, bktName, objName, nil)
r.Header.Set(api.MetadataPrefix+strings.ToUpper(layer.AttributeNeofsCopiesNumber), "1")
tc.Handler().PutObjectHandler(w, r)
p := &layer.HeadObjectParams{
BktInfo: bktInfo,
Object: objName,
}
objInfo, err := tc.Layer().GetObjectInfo(tc.Context(), p)
require.NoError(t, err)
require.Equal(t, "1", objInfo.Headers[layer.AttributeNeofsCopiesNumber])
}

4
api/layer/layer.go
View file

@ -150,6 +150,7 @@ type (
Range *RangeParams
Lock *data.ObjectLock
Encryption encryption.Params
CopiesNuber uint32
}
// CreateBucketParams stores bucket create request parameters.
CreateBucketParams struct {
@ -264,6 +265,8 @@ const (
AttributeDecryptedSize = api.NeoFSSystemMetadataPrefix + "Decrypted-Size"
AttributeHMACSalt = api.NeoFSSystemMetadataPrefix + "HMAC-Salt"
AttributeHMACKey = api.NeoFSSystemMetadataPrefix + "HMAC-Key"
AttributeNeofsCopiesNumber = "neofs-copies-number" // such formate to match X-Amz-Meta-Neofs-Copies-Number header
)
func (t *VersionedObject) String() string {
@ -525,6 +528,7 @@ func (n *layer) CopyObject(ctx context.Context, p *CopyObjectParams) (*data.Obje
Reader: pr,
Header: p.Header,
Encryption: p.Encryption,
CopiesNumber: p.CopiesNuber,
})
}

8
api/layer/multipart_upload.go
View file

@ -48,6 +48,7 @@ type (
Info *UploadInfoParams
Header map[string]string
Data *UploadData
CopiesNumber uint32
}
UploadData struct {
@ -60,7 +61,6 @@ type (
PartNumber int
Size int64
Reader io.Reader
CopiesNumber uint32
}
UploadCopyParams struct {
@ -69,7 +69,6 @@ type (
SrcBktInfo *data.BucketInfo
PartNumber int
Range *RangeParams
CopiesNumber uint32
}
CompleteMultipartParams struct {
@ -146,6 +145,7 @@ func (n *layer) CreateMultipartUpload(ctx context.Context, p *CreateMultipartPar
Owner: n.Owner(ctx),
Created: time.Now(),
Meta: make(map[string]string, metaSize),
CopiesNumber: p.CopiesNumber,
}
for key, val := range p.Header {
@ -205,7 +205,7 @@ func (n *layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
Creator: bktInfo.Owner,
Attributes: make([][2]string, 2),
Payload: p.Reader,
CopiesNumber: p.CopiesNumber,
CopiesNumber: multipartInfo.CopiesNumber,
}
decSize := p.Size
@ -305,7 +305,6 @@ func (n *layer) UploadPartCopy(ctx context.Context, p *UploadCopyParams) (*data.
PartNumber: p.PartNumber,
Size: size,
Reader: pr,
CopiesNumber: p.CopiesNumber,
}
return n.uploadPart(ctx, multipartInfo, params)
@ -441,6 +440,7 @@ func (n *layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
Header: initMetadata,
Size: multipartObjetSize,
Encryption: p.Info.Encryption,
CopiesNumber: multipartInfo.CopiesNumber,
})
if err != nil {
n.log.Error("could not put a completed object (multipart upload)",