diff --git a/api/handler/acl_test.go b/api/handler/acl_test.go index 14b3cf4..fd2f86e 100644 --- a/api/handler/acl_test.go +++ b/api/handler/acl_test.go @@ -26,7 +26,7 @@ func TestPutObjectACLErrorAPE(t *testing.T) { hc := prepareHandlerContext(t) bktName, objName := "bucket-for-acl-ape", "object" - info := createBucket(hc, bktName) + info := createBucket(hc, bucketPrm{bktName: bktName}) putObjectWithHeadersAssertS3Error(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPublic}, apierr.ErrAccessControlListNotSupported) putObjectWithHeaders(hc, bktName, objName, map[string]string{api.AmzACL: basicACLPrivate}) // only `private` canned acl is allowed, that is actually ignored @@ -43,7 +43,7 @@ func TestCreateObjectACLErrorAPE(t *testing.T) { hc := prepareHandlerContext(t) bktName, objName, objNameCopy := "bucket-for-acl-ape", "object", "copy" - createBucket(hc, bktName) + createBucket(hc, bucketPrm{bktName: bktName}) putObject(hc, bktName, objName) copyObject(hc, bktName, objName, objNameCopy, CopyMeta{Headers: map[string]string{api.AmzACL: basicACLPublic}}, http.StatusBadRequest) @@ -57,7 +57,7 @@ func TestBucketACLAPE(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-for-acl-ape" - info := createBucket(hc, bktName) + info := createBucket(hc, bucketPrm{bktName: bktName}) aclBody := &AccessControlPolicy{} putBucketACLAssertS3Error(hc, bktName, info.Box, nil, aclBody, apierr.ErrAccessControlListNotSupported) @@ -297,30 +297,30 @@ type createBucketInfo struct { Key *keys.PrivateKey } -func createBucket(hc *handlerContext, bktName string) *createBucketInfo { - box, key := createAccessBox(hc.t) +func createBucket(hc *handlerContext, prm bucketPrm) *createBucketInfo { + prm.box, prm.key = createAccessBox(hc.t) - w := createBucketBase(hc, bktName, box) + w := createBucketBase(hc, prm) assertStatus(hc.t, w, http.StatusOK) - bktInfo, err := hc.Layer().GetBucketInfo(hc.Context(), bktName) + bktInfo, err := hc.Layer().GetBucketInfo(hc.Context(), prm.bktName) require.NoError(hc.t, err) return &createBucketInfo{ BktInfo: bktInfo, - Box: box, - Key: key, + Box: prm.box, + Key: prm.key, } } -func createBucketAssertS3Error(hc *handlerContext, bktName string, box *accessbox.Box, code apierr.ErrorCode) { - w := createBucketBase(hc, bktName, box) +func createBucketAssertS3Error(hc *handlerContext, prm bucketPrm, code apierr.ErrorCode) { + w := createBucketBase(hc, prm) assertS3Error(hc.t, w, apierr.GetAPIError(code)) } -func createBucketBase(hc *handlerContext, bktName string, box *accessbox.Box) *httptest.ResponseRecorder { - w, r := prepareTestRequest(hc, bktName, "", nil) - ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box}) +func createBucketBase(hc *handlerContext, prm bucketPrm) *httptest.ResponseRecorder { + w, r := prepareTestFullRequest(hc, prm.bktName, "", nil, prm.body) + ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: prm.box}) r = r.WithContext(ctx) hc.Handler().CreateBucketHandler(w, r) return w diff --git a/api/handler/bucket_list_test.go b/api/handler/bucket_list_test.go new file mode 100644 index 0000000..24e26e7 --- /dev/null +++ b/api/handler/bucket_list_test.go @@ -0,0 +1,157 @@ +package handler + +import ( + "encoding/xml" + "net/http" + "net/http/httptest" + "net/url" + "sort" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "github.com/stretchr/testify/require" +) + +func TestHandler_ListBucketsHandler(t *testing.T) { + const defaultConstraint = "default" + + type bktProp struct { + name string + region string + cnrID cid.ID + } + + region := "us-west-1" + hc := prepareHandlerContext(t) + hc.config.putLocationConstraint(region) + + props := map[string]bktProp{ + "first": {name: "first"}, + "regional": {name: "regional", region: "us-west-1"}, + "third": {name: "third"}, + } + for i, bkt := range props { + var body createBucketParams + if bkt.region != "" { + body.LocationConstraint = bkt.region + } + info := createBucket(hc, bucketPrm{bktName: bkt.name, body: body}) + bkt.cnrID = info.BktInfo.CID + props[i] = bkt + } + + for _, tt := range []struct { + title string + query url.Values + expected []bktProp + expectedToken string + }{ + { + title: "without params", + query: url.Values{}, + expected: []bktProp{ + {name: "first", region: defaultConstraint}, + {name: "regional", region: "us-west-1"}, + {name: "third", region: defaultConstraint}, + }, + }, + { + title: "with prefix", + query: url.Values{"prefix": []string{"thi"}}, + expected: []bktProp{{name: "third", region: defaultConstraint}}, + }, + { + title: "wrong prefix", + query: url.Values{"prefix": []string{"sdh"}}, + expected: []bktProp{}, + }, + { + title: "with bucket region", + query: url.Values{"bucket-region": []string{region}}, + expected: []bktProp{{name: "regional", region: "us-west-1"}}, + }, + { + title: "with default bucket region", + query: url.Values{"bucket-region": []string{"default"}}, + expected: []bktProp{ + {"first", defaultConstraint, cid.ID{}}, + {"third", defaultConstraint, cid.ID{}}, + }, + }, + { + title: "wrong bucket region", + query: url.Values{"bucket-region": []string{"sjdfjl"}}, + expected: []bktProp{}, + }, + } { + t.Run(tt.title, func(t *testing.T) { + params := bucketPrm{query: tt.query} + resp := listBuckets(hc, params) + require.Len(t, resp.Buckets.Buckets, len(tt.expected)) + require.Equal(t, params.query.Get("prefix"), resp.Prefix) + if len(resp.Buckets.Buckets) > 0 { + t.Log(resp.Buckets.Buckets[0].Name) + } + respProps := make([]bktProp, 0, len(resp.Buckets.Buckets)) + for _, bkt := range resp.Buckets.Buckets { + respProps = append(respProps, bktProp{name: bkt.Name, region: bkt.BucketRegion}) + } + sort.Slice(respProps, func(i, j int) bool { + return respProps[i].name < respProps[j].name + }) + require.Equal(t, tt.expected, respProps) + }) + } + + t.Run("pagination", func(t *testing.T) { + cids := make(map[string]struct{}, len(props)) + for _, bkt := range props { + cids[bkt.cnrID.String()] = struct{}{} + } + t.Run("happy path", func(t *testing.T) { + params := bucketPrm{query: url.Values{"max-buckets": []string{"1"}}} + resp := listBuckets(hc, params) + require.Len(t, resp.Buckets.Buckets, 1) + require.NotEmpty(t, resp.ContinuationToken) + + params.query.Set("continuation-token", resp.ContinuationToken) + resp = listBuckets(hc, params) + require.Len(t, resp.Buckets.Buckets, 1) + require.NotEmpty(t, resp.ContinuationToken) + + params.query.Set("continuation-token", resp.ContinuationToken) + resp = listBuckets(hc, params) + require.Len(t, resp.Buckets.Buckets, 1) + require.Empty(t, resp.ContinuationToken) + }) + + t.Run("wrong continuation-token", func(t *testing.T) { + params := bucketPrm{query: url.Values{"max-buckets": []string{"1"}}} + params.query.Set("continuation-token", "CebuVwfRpdMqi9dvgV2SUNbrkfteGtudchKKhNabXUu9") + resp := listBuckets(hc, params) + require.Len(t, resp.Buckets.Buckets, 0) + require.Empty(t, resp.ContinuationToken) + }) + }) +} + +func listBuckets(hc *handlerContext, prm bucketPrm) ListBucketsResponse { + prm.box, prm.key = createAccessBox(hc.t) + w := listBucketsBase(hc, prm) + assertStatus(hc.t, w, http.StatusOK) + var resp ListBucketsResponse + err := xml.NewDecoder(w.Body).Decode(&resp) + require.NoError(hc.t, err) + + return resp +} + +func listBucketsBase(hc *handlerContext, prm bucketPrm) *httptest.ResponseRecorder { + w, r := prepareTestFullRequest(hc, "", "", prm.query, nil) + ctx := middleware.SetBox(r.Context(), &middleware.Box{AccessBox: prm.box}) + r = r.WithContext(ctx) + hc.Handler().ListBucketsHandler(w, r) + + return w +} diff --git a/api/handler/handlers_test.go b/api/handler/handlers_test.go index 8c886a1..b5ca3dc 100644 --- a/api/handler/handlers_test.go +++ b/api/handler/handlers_test.go @@ -23,6 +23,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" @@ -74,6 +75,7 @@ func (hc *handlerContextBase) Context() context.Context { type configMock struct { defaultPolicy netmap.PlacementPolicy + placementPolicies map[string]netmap.PlacementPolicy copiesNumbers map[string][]uint32 defaultCopiesNumbers []uint32 bypassContentEncodingInChunks bool @@ -85,8 +87,9 @@ func (c *configMock) DefaultPlacementPolicy(_ string) netmap.PlacementPolicy { return c.defaultPolicy } -func (c *configMock) PlacementPolicy(_, _ string) (netmap.PlacementPolicy, bool) { - return netmap.PlacementPolicy{}, false +func (c *configMock) PlacementPolicy(_, constraint string) (netmap.PlacementPolicy, bool) { + policy, ok := c.placementPolicies[constraint] + return policy, ok } func (c *configMock) CopiesNumbers(_, locationConstraint string) ([]uint32, bool) { @@ -146,6 +149,10 @@ func (c *configMock) TLSTerminationHeader() string { return c.tlsTerminationHeader } +func (c *configMock) putLocationConstraint(constraint string) { + c.placementPolicies[constraint] = c.defaultPolicy +} + func prepareHandlerContext(t *testing.T) *handlerContext { hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(zap.NewExample())) require.NoError(t, err) @@ -212,7 +219,8 @@ func prepareHandlerContextBase(cacheCfg *layer.CachesConfig) (*handlerContextBas } cfg := &configMock{ - defaultPolicy: pp, + defaultPolicy: pp, + placementPolicies: make(map[string]netmap.PlacementPolicy), } h := &handler{ log: log, @@ -394,8 +402,16 @@ func (f *frostfsidMock) GetUserKey(account, user string) (string, error) { return hex.EncodeToString(res.Bytes()), nil } +type bucketPrm struct { + bktName string + query url.Values + box *accessbox.Box + key *keys.PrivateKey + body any +} + func createTestBucket(hc *handlerContext, bktName string) *data.BucketInfo { - info := createBucket(hc, bktName) + info := createBucket(hc, bucketPrm{bktName: bktName}) return info.BktInfo } diff --git a/api/handler/lifecycle_test.go b/api/handler/lifecycle_test.go index 69382ab..27b71ab 100644 --- a/api/handler/lifecycle_test.go +++ b/api/handler/lifecycle_test.go @@ -24,7 +24,7 @@ func TestPutBucketLifecycleConfiguration(t *testing.T) { hc := prepareHandlerContextWithMinCache(t) bktName := "bucket-lifecycle" - createBucket(hc, bktName) + createBucket(hc, bucketPrm{bktName: bktName}) for _, tc := range []struct { name string @@ -429,7 +429,7 @@ func TestPutBucketLifecycleIDGeneration(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-lifecycle-id" - createBucket(hc, bktName) + createBucket(hc, bucketPrm{bktName: bktName}) lifecycle := &data.LifecycleConfiguration{ Rules: []data.LifecycleRule{ @@ -459,7 +459,7 @@ func TestPutBucketLifecycleInvalidMD5(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-lifecycle-md5" - createBucket(hc, bktName) + createBucket(hc, bucketPrm{bktName: bktName}) lifecycle := &data.LifecycleConfiguration{ Rules: []data.LifecycleRule{ @@ -491,7 +491,7 @@ func TestPutBucketLifecycleInvalidXML(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bucket-lifecycle-invalid-xml" - createBucket(hc, bktName) + createBucket(hc, bucketPrm{bktName: bktName}) cfg := &data.CORSConfiguration{} body, err := xml.Marshal(cfg) diff --git a/api/handler/put_test.go b/api/handler/put_test.go index 53968a6..f34d1f0 100644 --- a/api/handler/put_test.go +++ b/api/handler/put_test.go @@ -546,11 +546,11 @@ func TestCreateBucket(t *testing.T) { hc := prepareHandlerContext(t) bktName := "bkt-name" - info := createBucket(hc, bktName) - createBucketAssertS3Error(hc, bktName, info.Box, apierr.ErrBucketAlreadyOwnedByYou) + info := createBucket(hc, bucketPrm{bktName: bktName}) + createBucketAssertS3Error(hc, bucketPrm{bktName: bktName, box: info.Box}, apierr.ErrBucketAlreadyOwnedByYou) box2, _ := createAccessBox(t) - createBucketAssertS3Error(hc, bktName, box2, apierr.ErrBucketAlreadyExists) + createBucketAssertS3Error(hc, bucketPrm{bktName: bktName, box: box2}, apierr.ErrBucketAlreadyExists) } func TestCreateBucketWithoutPermissions(t *testing.T) { @@ -560,7 +560,7 @@ func TestCreateBucketWithoutPermissions(t *testing.T) { hc.h.ape.(*apeMock).err = errors.New("no permissions") box, _ := createAccessBox(t) - createBucketAssertS3Error(hc, bktName, box, apierr.ErrInternalError) + createBucketAssertS3Error(hc, bucketPrm{bktName: bktName, box: box}, apierr.ErrInternalError) _, err := hc.tp.ContainerID(bktName) require.Errorf(t, err, "container exists after failed creation, but shouldn't") diff --git a/api/layer/frostfs_mock.go b/api/layer/frostfs_mock.go index ab983c9..09a9642 100644 --- a/api/layer/frostfs_mock.go +++ b/api/layer/frostfs_mock.go @@ -74,11 +74,46 @@ func (k *FeatureSettingsMock) FormContainerZone(ns string) string { var _ frostfs.FrostFS = (*TestFrostFS)(nil) +type containerKV struct { + key string + Container *container.Container +} + +type containers []containerKV + +func (c *containers) Get(key string) (*container.Container, bool) { + for _, info := range *c { + if info.key == key { + return info.Container, true + } + } + + return nil, false +} + +func (c *containers) Set(key string, value *container.Container) { + for _, info := range *c { + if info.key == key { + info.Container = value + return + } + } + *c = append(*c, containerKV{key: key, Container: value}) +} + +func (c *containers) Delete(key string) { + for i, info := range *c { + if info.key == key { + *c = append((*c)[:i], (*c)[i+1:]...) + } + } +} + type TestFrostFS struct { objects map[string]*object.Object objectErrors map[string]error objectPutErrors map[string]error - containers map[string]*container.Container + containers containers chains map[string][]chain.Chain currentEpoch uint64 key *keys.PrivateKey @@ -90,7 +125,7 @@ func NewTestFrostFS(key *keys.PrivateKey) *TestFrostFS { objects: make(map[string]*object.Object), objectErrors: make(map[string]error), objectPutErrors: make(map[string]error), - containers: make(map[string]*container.Container), + containers: make([]containerKV, 0, 10), chains: make(map[string][]chain.Chain), key: key, } @@ -141,17 +176,17 @@ func (t *TestFrostFS) AddObject(key string, obj *object.Object) { } func (t *TestFrostFS) ContainerID(name string) (cid.ID, error) { - for id, cnr := range t.containers { - if container.Name(*cnr) == name { + for _, cnr := range t.containers { + if container.Name(*cnr.Container) == name { var cnrID cid.ID - return cnrID, cnrID.DecodeString(id) + return cnrID, cnrID.DecodeString(cnr.key) } } return cid.ID{}, fmt.Errorf("not found") } func (t *TestFrostFS) SetContainer(cnrID cid.ID, cnr *container.Container) { - t.containers[cnrID.EncodeToString()] = cnr + t.containers.Set(cnrID.EncodeToString(), cnr) } func (t *TestFrostFS) CreateContainer(_ context.Context, prm frostfs.PrmContainerCreate) (*frostfs.ContainerCreateResult, error) { @@ -186,22 +221,22 @@ func (t *TestFrostFS) CreateContainer(_ context.Context, prm frostfs.PrmContaine var id cid.ID id.SetSHA256(sha256.Sum256(b)) - t.containers[id.EncodeToString()] = &cnr + t.containers.Set(id.EncodeToString(), &cnr) t.chains[id.EncodeToString()] = []chain.Chain{} return &frostfs.ContainerCreateResult{ContainerID: id}, nil } func (t *TestFrostFS) DeleteContainer(_ context.Context, cnrID cid.ID, _ *session.Container) error { - delete(t.containers, cnrID.EncodeToString()) + t.containers.Delete(cnrID.EncodeToString()) return nil } func (t *TestFrostFS) Container(_ context.Context, prm frostfs.PrmContainer) (*container.Container, error) { - for k, v := range t.containers { - if k == prm.ContainerID.EncodeToString() { - return v, nil + for _, v := range t.containers { + if v.key == prm.ContainerID.EncodeToString() { + return v.Container, nil } } @@ -210,9 +245,9 @@ func (t *TestFrostFS) Container(_ context.Context, prm frostfs.PrmContainer) (*c func (t *TestFrostFS) UserContainers(context.Context, frostfs.PrmUserContainers) ([]cid.ID, error) { var res []cid.ID - for k := range t.containers { + for _, info := range t.containers { var idCnr cid.ID - if err := idCnr.DecodeString(k); err != nil { + if err := idCnr.DecodeString(info.key); err != nil { return nil, err } res = append(res, idCnr) @@ -527,7 +562,7 @@ func (t *TestFrostFS) AddContainerPolicyChain(_ context.Context, prm frostfs.Prm } func (t *TestFrostFS) checkAccess(cnrID cid.ID, owner user.ID) bool { - cnr, ok := t.containers[cnrID.EncodeToString()] + cnr, ok := t.containers.Get(cnrID.EncodeToString()) if !ok { return false }