From c0c4bdb3662857f9f79aeb014b3cdb84df524261 Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Thu, 27 Feb 2025 12:19:49 +0300 Subject: [PATCH] [#650] Add Copies Numbers for PostObject operation Signed-off-by: Pavel Pogodaev --- api/handler/put.go | 6 ++++ api/layer/cors_test.go | 46 +++++++++++++++++++++++++++ api/layer/lifecycle.go | 2 +- api/layer/lifecycle_test.go | 48 ++++++++++++++++++++++++++++ api/layer/tree_mock.go | 14 +++++++-- api/layer/versioning_test.go | 61 ++++++++++++++++++++++++++++++++++++ 6 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 api/layer/cors_test.go diff --git a/api/handler/put.go b/api/handler/put.go index b70b05ca..4d50b217 100644 --- a/api/handler/put.go +++ b/api/handler/put.go @@ -591,6 +591,12 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) { Header: metadata, } + params.CopiesNumbers, err = h.pickCopiesNumbers(metadata, reqInfo.Namespace, bktInfo.LocationConstraint) + if err != nil { + h.logAndSendError(ctx, w, "invalid copies number", reqInfo, err) + return + } + extendedObjInfo, err := h.obj.PutObject(ctx, params) if err != nil { h.logAndSendError(ctx, w, "could not upload object", reqInfo, err) diff --git a/api/layer/cors_test.go b/api/layer/cors_test.go new file mode 100644 index 00000000..d2e96a95 --- /dev/null +++ b/api/layer/cors_test.go @@ -0,0 +1,46 @@ +package layer + +import ( + "encoding/xml" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCorsCopiesNumber(t *testing.T) { + tc := prepareCORSContext(t) + body := ` + + + GET + http://www.example.com + Authorization + x-amz-* + + +` + + copies := []uint32{2, 0} + + err := tc.layer.PutBucketCORS(tc.ctx, &PutCORSParams{ + BktInfo: tc.bktInfo, + Reader: strings.NewReader(body), + CopiesNumbers: copies, + UserAgent: "", + NewDecoder: NewXMLDecoder, + }) + require.NoError(t, err) + + objs := tc.testFrostFS.Objects() + require.Len(t, objs, 1) + + require.EqualValues(t, copies, tc.testFrostFS.CopiesNumbers(addrFromObject(objs[0]).EncodeToString())) +} + +func NewXMLDecoder(r io.Reader, _ string) *xml.Decoder { + dec := xml.NewDecoder(r) + + return dec +} diff --git a/api/layer/lifecycle.go b/api/layer/lifecycle.go index 2e558715..d7772971 100644 --- a/api/layer/lifecycle.go +++ b/api/layer/lifecycle.go @@ -41,13 +41,13 @@ func (n *Layer) PutBucketLifecycleConfiguration(ctx context.Context, p *PutBucke var lifecycleBkt *data.BucketInfo if n.lifecycleCnrInfo == nil { lifecycleBkt = p.BktInfo - prm.CopiesNumber = p.CopiesNumbers } else { lifecycleBkt = n.lifecycleCnrInfo prm.PrmAuth.PrivateKey = &n.gateKey.PrivateKey } prm.Container = lifecycleBkt.CID + prm.CopiesNumber = p.CopiesNumbers createdObj, err := n.objectPutAndHash(ctx, prm, lifecycleBkt) if err != nil { diff --git a/api/layer/lifecycle_test.go b/api/layer/lifecycle_test.go index b45c7fe7..8a7db451 100644 --- a/api/layer/lifecycle_test.go +++ b/api/layer/lifecycle_test.go @@ -8,6 +8,8 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data" apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors" frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/stretchr/testify/require" ) @@ -56,6 +58,52 @@ func TestBucketLifecycle(t *testing.T) { require.Equal(t, apierr.GetAPIError(apierr.ErrNoSuchLifecycleConfiguration), frosterr.UnwrapErr(err)) } +func TestLifecycleCopiesNumber(t *testing.T) { + tc := prepareCORSContext(t) + + lifecycle := &data.LifecycleConfiguration{ + XMLName: xml.Name{ + Space: `http://s3.amazonaws.com/doc/2006-03-01/`, + Local: "LifecycleConfiguration", + }, + Rules: []data.LifecycleRule{ + { + Status: data.LifecycleStatusEnabled, + Expiration: &data.LifecycleExpiration{ + Days: ptr(21), + }, + }, + }, + } + copies := []uint32{2, 0} + + err := tc.layer.PutBucketLifecycleConfiguration(tc.ctx, &PutBucketLifecycleParams{ + BktInfo: tc.bktInfo, + LifecycleCfg: lifecycle, + CopiesNumbers: copies, + }) + require.NoError(t, err) + + cfg, err := tc.layer.GetBucketLifecycleConfiguration(tc.ctx, tc.bktInfo) + require.NoError(t, err) + require.Equal(t, *lifecycle, *cfg) + + objs := tc.testFrostFS.Objects() + require.Len(t, objs, 1) + + require.EqualValues(t, copies, tc.testFrostFS.CopiesNumbers(addrFromObject(objs[0]).EncodeToString())) +} + func ptr[T any](t T) *T { return &t } +func addrFromObject(obj *object.Object) oid.Address { + var addr oid.Address + cnrID, _ := obj.ContainerID() + objID, _ := obj.ID() + + addr.SetContainer(cnrID) + addr.SetObject(objID) + + return addr +} diff --git a/api/layer/tree_mock.go b/api/layer/tree_mock.go index 4d27da44..ae1aa0ac 100644 --- a/api/layer/tree_mock.go +++ b/api/layer/tree_mock.go @@ -138,8 +138,18 @@ func (t *TreeServiceMock) GetAllBucketCORS(ctx context.Context, bktInfo *data.Bu return []oid.Address{cors}, nil } -func (t *TreeServiceMock) DeleteBucketCORS(context.Context, *data.BucketInfo) ([]oid.Address, error) { - panic("implement me") +func (t *TreeServiceMock) DeleteBucketCORS(_ context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) { + systemMap, ok := t.system[bktInfo.CID.EncodeToString()] + if !ok { + return []oid.Address{}, tree.ErrNodeNotFound + } + + _, ok = systemMap["cors"] + if !ok { + return []oid.Address{}, tree.ErrNodeNotFound + } + + return []oid.Address{}, nil } func (t *TreeServiceMock) GetVersions(_ context.Context, bktInfo *data.BucketInfo, objectName string) ([]*data.NodeVersion, error) { diff --git a/api/layer/versioning_test.go b/api/layer/versioning_test.go index 51d35cd6..e525eca3 100644 --- a/api/layer/versioning_test.go +++ b/api/layer/versioning_test.go @@ -193,6 +193,67 @@ func prepareContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext { } } +func prepareCORSContext(t *testing.T, cachesConfig ...*CachesConfig) *testContext { + logger := zaptest.NewLogger(t) + + key, err := keys.NewPrivateKey() + require.NoError(t, err) + + bearerToken := bearertest.Token() + require.NoError(t, bearerToken.Sign(key.PrivateKey)) + + ctx := middleware.SetBox(context.Background(), &middleware.Box{AccessBox: &accessbox.Box{ + Gate: &accessbox.GateData{ + BearerToken: &bearerToken, + GateKey: key.PublicKey(), + }, + }}) + tp := NewTestFrostFS(key) + + bktName := "testbucket1" + res, err := tp.CreateContainer(ctx, frostfs.PrmContainerCreate{ + Name: bktName, + Policy: getPlacementPolicy(), + }) + require.NoError(t, err) + + config := DefaultCachesConfigs(logger) + if len(cachesConfig) != 0 { + config = cachesConfig[0] + } + + var owner user.ID + user.IDFromKey(&owner, key.PrivateKey.PublicKey) + + corsCnrInfo := &data.BucketInfo{Name: "CORS"} + lifecycleCnrInfo := &data.BucketInfo{Name: "Lifecycle"} + + layerCfg := &Config{ + Cache: NewCache(config), + AnonKey: AnonymousKey{Key: key}, + TreeService: NewTreeService(), + Features: &FeatureSettingsMock{}, + GateOwner: owner, + CORSCnrInfo: corsCnrInfo, + LifecycleCnrInfo: lifecycleCnrInfo, + GateKey: key, + } + + return &testContext{ + ctx: ctx, + layer: NewLayer(logger, tp, layerCfg), + bktInfo: &data.BucketInfo{ + Name: bktName, + Owner: owner, + CID: res.ContainerID, + HomomorphicHashDisabled: res.HomomorphicHashDisabled, + }, + obj: "obj1", + t: t, + testFrostFS: tp, + } +} + func TestSimpleVersioning(t *testing.T) { tc := prepareContext(t) err := tc.layer.PutBucketSettings(tc.ctx, &PutSettingsParams{