[#641] Rework CORS bucket behaviour

Signed-off-by: Marina Biryukova <m.biryukova@yadro.com>

api/data/info.go

@@ -2,6 +2,7 @@ package data
 
 import (
 	"encoding/xml"
+	"fmt"
 	"strings"
 	"time"
 
@@ -20,6 +21,8 @@ const (
 	VersioningUnversioned = "Unversioned"
 	VersioningEnabled     = "Enabled"
 	VersioningSuspended   = "Suspended"
+
+	corsFilePathTemplate = "/%s.cors"
 )
 
 type (
@@ -103,6 +106,11 @@ func (b *BucketInfo) CORSObjectName() string {
 	return b.CID.EncodeToString() + bktCORSConfigurationObject
 }
 
+// CORSObjectFilePath returns a FilePath for a bucket CORS configuration file.
+func (b *BucketInfo) CORSObjectFilePath() string {
+	return fmt.Sprintf(corsFilePathTemplate, b.CID)
+}
+
 func (b *BucketInfo) LifecycleConfigurationObjectName() string {
 	return b.CID.EncodeToString() + bktLifecycleConfigurationObject
 }

api/handler/bucket_list_test.go

@@ -17,7 +17,7 @@ func TestHandler_ListBucketsHandler(t *testing.T) {
 	const defaultConstraint = "default"
 
 	region := "us-west-1"
-	hc := prepareHandlerContext(t)
+	hc := prepareWithoutCORSHandlerContext(t)
 	hc.config.putLocationConstraint(region)
 
 	props := []Bucket{

api/handler/cors_test.go

@@ -1,12 +1,19 @@
 package handler
 
 import (
+	"encoding/xml"
 	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
+	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
 	"github.com/stretchr/testify/require"
 )
 
@@ -37,26 +44,14 @@ func TestCORSOriginWildcard(t *testing.T) {
 	hc.Handler().CreateBucketHandler(w, r)
 	assertStatus(t, w, http.StatusOK)
 
-	w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
+	putBucketCORS(hc, bktName, body)
-	ctx = middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
-	r = r.WithContext(ctx)
-	hc.Handler().PutBucketCorsHandler(w, r)
-	assertStatus(t, w, http.StatusOK)
 
-	w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
+	getBucketCORS(hc, bktName)
-	hc.Handler().GetBucketCorsHandler(w, r)
-	assertStatus(t, w, http.StatusOK)
 
 	hc.config.useDefaultXMLNS = true
-	w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(bodyNoXmlns))
+	putBucketCORS(hc, bktName, bodyNoXmlns)
-	ctx = middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
-	r = r.WithContext(ctx)
-	hc.Handler().PutBucketCorsHandler(w, r)
-	assertStatus(t, w, http.StatusOK)
 
-	w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
+	getBucketCORS(hc, bktName)
-	hc.Handler().GetBucketCorsHandler(w, r)
-	assertStatus(t, w, http.StatusOK)
 }
 
 func TestPreflight(t *testing.T) {
@@ -170,11 +165,7 @@ func TestPreflightWildcardOrigin(t *testing.T) {
 	hc.Handler().CreateBucketHandler(w, r)
 	assertStatus(t, w, http.StatusOK)
 
-	w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
+	putBucketCORS(hc, bktName, body)
-	ctx = middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box})
-	r = r.WithContext(ctx)
-	hc.Handler().PutBucketCorsHandler(w, r)
-	assertStatus(t, w, http.StatusOK)
 
 	for _, tc := range []struct {
 		name           string
@@ -236,3 +227,183 @@ func TestPreflightWildcardOrigin(t *testing.T) {
 		})
 	}
 }
+
+func TestDeleteAllCORSVersions(t *testing.T) {
+	body := `
+<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+	<CORSRule>
+		<AllowedMethod>GET</AllowedMethod>
+		<AllowedMethod>PUT</AllowedMethod>
+		<AllowedOrigin>*</AllowedOrigin>
+		<AllowedHeader>*</AllowedHeader>
+	</CORSRule>
+</CORSConfiguration>
+`
+	hc := prepareHandlerContext(t)
+
+	bktName := "bucket-delete-all-cors-version"
+	createBucket(hc, bktName)
+	require.Len(t, hc.tp.Objects(), 0)
+
+	for range 5 {
+		putBucketCORS(hc, bktName, body)
+	}
+
+	require.Len(t, hc.tp.Objects(), 5)
+
+	deleteBucketCORS(hc, bktName)
+	require.Len(t, hc.tp.Objects(), 0)
+}
+
+func TestGetLatestCORSVersion(t *testing.T) {
+	bodyTree := `
+<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+	<CORSRule>
+		<AllowedMethod>GET</AllowedMethod>
+		<AllowedMethod>PUT</AllowedMethod>
+		<AllowedOrigin>*</AllowedOrigin>
+		<AllowedHeader>*</AllowedHeader>
+	</CORSRule>
+</CORSConfiguration>
+`
+	body := `
+	<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+		<CORSRule>
+			<AllowedMethod>DELETE</AllowedMethod>
+			<AllowedOrigin>*</AllowedOrigin>
+			<AllowedHeader>*</AllowedHeader>
+		</CORSRule>
+	</CORSConfiguration>
+	`
+	hc := prepareHandlerContextWithMinCache(t)
+
+	bktName := "bucket-get-latest-cors"
+	info := createBucket(hc, bktName)
+
+	addCORSToTree(hc, bodyTree, info.BktInfo, info.BktInfo.CID)
+
+	w := getBucketCORS(hc, bktName)
+	requireEqualCORS(hc.t, bodyTree, w.Body.String())
+
+	hc.tp.AddCORSObject(info.BktInfo, hc.corsCnrID, body)
+
+	w = getBucketCORS(hc, bktName)
+	requireEqualCORS(hc.t, body, w.Body.String())
+
+	hc.tp.AddCORSObject(info.BktInfo, hc.corsCnrID, bodyTree)
+	w = getBucketCORS(hc, bktName)
+	requireEqualCORS(hc.t, bodyTree, w.Body.String())
+}
+
+func TestDeleteTreeCORSVersions(t *testing.T) {
+	body := `
+<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+	<CORSRule>
+		<AllowedMethod>GET</AllowedMethod>
+		<AllowedMethod>PUT</AllowedMethod>
+		<AllowedOrigin>*</AllowedOrigin>
+		<AllowedHeader>*</AllowedHeader>
+	</CORSRule>
+</CORSConfiguration>
+`
+	hc := prepareHandlerContext(t)
+
+	bktName := "bucket-delete-tree-cors-versions"
+	info := createBucket(hc, bktName)
+
+	addCORSToTree(hc, body, info.BktInfo, info.BktInfo.CID)
+	addCORSToTree(hc, body, info.BktInfo, hc.corsCnrID)
+	require.Len(t, hc.tp.Objects(), 2)
+
+	putBucketCORS(hc, bktName, body)
+	require.Len(t, hc.tp.Objects(), 1)
+
+	addCORSToTree(hc, body, info.BktInfo, info.BktInfo.CID)
+	addCORSToTree(hc, body, info.BktInfo, hc.corsCnrID)
+	require.Len(t, hc.tp.Objects(), 3)
+
+	deleteBucketCORS(hc, bktName)
+	require.Len(t, hc.tp.Objects(), 0)
+}
+
+func TestDeleteCORSInDeleteBucket(t *testing.T) {
+	body := `
+<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+	<CORSRule>
+		<AllowedMethod>GET</AllowedMethod>
+		<AllowedMethod>PUT</AllowedMethod>
+		<AllowedOrigin>*</AllowedOrigin>
+		<AllowedHeader>*</AllowedHeader>
+	</CORSRule>
+</CORSConfiguration>
+`
+
+	hc := prepareHandlerContext(t)
+
+	bktName := "bucket-delete-cors-in-delete-bucket"
+	info := createBucket(hc, bktName)
+
+	addCORSToTree(hc, body, info.BktInfo, hc.corsCnrID)
+	addCORSToTree(hc, body, info.BktInfo, info.BktInfo.CID)
+	hc.tp.AddCORSObject(info.BktInfo, hc.corsCnrID, body)
+	require.Len(t, hc.tp.Objects(), 3)
+
+	hc.owner = info.BktInfo.Owner
+	deleteBucket(t, hc, bktName, http.StatusNoContent)
+	require.Len(t, hc.tp.Objects(), 1) // CORS object in bucket container is not deleted
+}
+
+func addCORSToTree(hc *handlerContext, cors string, bkt *data.BucketInfo, corsCnrID cid.ID) {
+	var addr oid.Address
+	addr.SetContainer(corsCnrID)
+	addr.SetObject(oidtest.ID())
+
+	var obj object.Object
+	obj.SetPayload([]byte(cors))
+	obj.SetPayloadSize(uint64(len(cors)))
+
+	hc.tp.SetObject(addr, &obj)
+
+	meta := make(map[string]string)
+	meta["FileName"] = "bucket-cors"
+	meta["OID"] = addr.Object().EncodeToString()
+	meta["CID"] = addr.Container().EncodeToString()
+
+	_, err := hc.treeMock.AddNode(hc.context, bkt, "system", 0, meta)
+	require.NoError(hc.t, err)
+}
+
+func requireEqualCORS(t *testing.T, expected string, actual string) {
+	expectedCORS := &data.CORSConfiguration{}
+	err := xml.NewDecoder(strings.NewReader(expected)).Decode(expectedCORS)
+	require.NoError(t, err)
+
+	actualCORS := &data.CORSConfiguration{}
+	err = xml.NewDecoder(strings.NewReader(actual)).Decode(actualCORS)
+	require.NoError(t, err)
+
+	require.Equal(t, expectedCORS, actualCORS)
+}
+
+func putBucketCORS(hc *handlerContext, bktName string, body string) {
+	w, r := prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
+	box, _ := createAccessBox(hc.t)
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box}))
+	hc.Handler().PutBucketCorsHandler(w, r)
+	assertStatus(hc.t, w, http.StatusOK)
+}
+
+func deleteBucketCORS(hc *handlerContext, bktName string) {
+	w, r := prepareTestPayloadRequest(hc, bktName, "", nil)
+	box, _ := createAccessBox(hc.t)
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: box}))
+	hc.Handler().DeleteBucketCorsHandler(w, r)
+	assertStatus(hc.t, w, http.StatusNoContent)
+}
+
+func getBucketCORS(hc *handlerContext, bktName string) *httptest.ResponseRecorder {
+	w, r := prepareTestPayloadRequest(hc, bktName, "", nil)
+	hc.Handler().GetBucketCorsHandler(w, r)
+	assertStatus(hc.t, w, http.StatusOK)
+	return w
+}

api/handler/handlers_test.go

@@ -23,7 +23,9 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/resolver"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
+	bearertest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer/test"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
@@ -51,6 +53,7 @@ type handlerContextBase struct {
 	tree      *tree.Tree
 	context   context.Context
 	config    *configMock
+	corsCnrID cid.ID
 
 	layerFeatures *layer.FeatureSettingsMock
 	treeMock      *tree.ServiceClientMemory
@@ -162,9 +165,27 @@ func (c *configMock) putLocationConstraint(constraint string) {
 	c.placementPolicies[constraint] = c.defaultPolicy
 }
 
+type handlerConfig struct {
+	cacheCfg    *layer.CachesConfig
+	withoutCORS bool
+}
+
 func prepareHandlerContext(t *testing.T) *handlerContext {
 	log := zaptest.NewLogger(t)
-	hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(log), log)
+	hc, err := prepareHandlerContextBase(&handlerConfig{cacheCfg: layer.DefaultCachesConfigs(log)}, log)
+	require.NoError(t, err)
+	return &handlerContext{
+		handlerContextBase: hc,
+		t:                  t,
+	}
+}
+
+func prepareWithoutCORSHandlerContext(t *testing.T) *handlerContext {
+	log := zaptest.NewLogger(t)
+	hc, err := prepareHandlerContextBase(&handlerConfig{
+		cacheCfg:    layer.DefaultCachesConfigs(log),
+		withoutCORS: true,
+	}, log)
 	require.NoError(t, err)
 	return &handlerContext{
 		handlerContextBase: hc,
@@ -174,7 +195,7 @@ func prepareHandlerContext(t *testing.T) *handlerContext {
 
 func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
 	log := zaptest.NewLogger(t)
-	hc, err := prepareHandlerContextBase(getMinCacheConfig(log), log)
+	hc, err := prepareHandlerContextBase(&handlerConfig{cacheCfg: getMinCacheConfig(log)}, log)
 	require.NoError(t, err)
 	return &handlerContext{
 		handlerContextBase: hc,
@@ -182,7 +203,7 @@ func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
 	}
 }
 
-func prepareHandlerContextBase(cacheCfg *layer.CachesConfig, log *zap.Logger) (*handlerContextBase, error) {
+func prepareHandlerContextBase(config *handlerConfig, log *zap.Logger) (*handlerContextBase, error) {
 	key, err := keys.NewPrivateKey()
 	if err != nil {
 		return nil, err
@@ -213,15 +234,23 @@ func prepareHandlerContextBase(cacheCfg *layer.CachesConfig, log *zap.Logger) (*
 	}
 
 	layerCfg := &layer.Config{
-		Cache:       layer.NewCache(cacheCfg),
+		Cache:       layer.NewCache(config.cacheCfg),
 		AnonKey:     layer.AnonymousKey{Key: key},
 		Resolver:    testResolver,
 		TreeService: treeMock,
 		Features:    features,
 		GateOwner:   owner,
+		GateKey:     key,
 		WorkerPool:  pool,
 	}
 
+	if !config.withoutCORS {
+		layerCfg.CORSCnrInfo, err = createCORSContainer(key, tp)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	var pp netmap.PlacementPolicy
 	err = pp.DecodeString("REP 1")
 	if err != nil {
@@ -245,7 +274,7 @@ func prepareHandlerContextBase(cacheCfg *layer.CachesConfig, log *zap.Logger) (*
 		return nil, err
 	}
 
-	return &handlerContextBase{
+	hc := &handlerContextBase{
 		owner:   owner,
 		h:       h,
 		tp:      tp,
@@ -256,6 +285,44 @@ func prepareHandlerContextBase(cacheCfg *layer.CachesConfig, log *zap.Logger) (*
 		layerFeatures: features,
 		treeMock:      memCli,
 		cache:         layerCfg.Cache,
+	}
+
+	if layerCfg.CORSCnrInfo != nil {
+		hc.corsCnrID = layerCfg.CORSCnrInfo.CID
+	}
+
+	return hc, nil
+}
+
+func createCORSContainer(key *keys.PrivateKey, tp *layer.TestFrostFS) (*data.BucketInfo, error) {
+	bearerToken := bearertest.Token()
+	err := bearerToken.Sign(key.PrivateKey)
+	if err != nil {
+		return nil, err
+	}
+
+	bktName := "cors"
+	res, err := tp.CreateContainer(middleware.SetBox(context.Background(), &middleware.Box{AccessBox: &accessbox.Box{
+		Gate: &accessbox.GateData{
+			BearerToken: &bearerToken,
+			GateKey:     key.PublicKey(),
+		},
+	}}), frostfs.PrmContainerCreate{
+		Name:   bktName,
+		Policy: getPlacementPolicy(),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var owner user.ID
+	user.IDFromKey(&owner, key.PrivateKey.PublicKey)
+
+	return &data.BucketInfo{
+		Name:                    bktName,
+		Owner:                   owner,
+		CID:                     res.ContainerID,
+		HomomorphicHashDisabled: res.HomomorphicHashDisabled,
 	}, nil
 }
 

api/handler/object_list_test.go

@@ -105,7 +105,7 @@ func TestListObjectsVersionsSkipLogTaggingNodesError(t *testing.T) {
 	loggerCore, observedLog := observer.New(zap.DebugLevel)
 	log := zap.New(loggerCore)
 
-	hcBase, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(log), log)
+	hcBase, err := prepareHandlerContextBase(&handlerConfig{cacheCfg: layer.DefaultCachesConfigs(log)}, log)
 	require.NoError(t, err)
 	hc := &handlerContext{
 		handlerContextBase: hcBase,
@@ -178,7 +178,7 @@ func TestListObjectsContextCanceled(t *testing.T) {
 	layerCfg.SessionList.Lifetime = time.Hour
 	layerCfg.SessionList.Size = 1
 
-	hcBase, err := prepareHandlerContextBase(layerCfg, log)
+	hcBase, err := prepareHandlerContextBase(&handlerConfig{cacheCfg: layerCfg}, log)
 	require.NoError(t, err)
 	hc := &handlerContext{
 		handlerContextBase: hcBase,

api/layer/cors.go

@@ -46,41 +46,36 @@ func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
 	}
 
 	prm := frostfs.PrmObjectCreate{
+		Container:    n.corsCnrInfo.CID,
 		Payload:      &buf,
-		Filepath:     p.BktInfo.CORSObjectName(),
+		Filepath:     p.BktInfo.CORSObjectFilePath(),
 		CreationTime: TimeNow(ctx),
+		CopiesNumber: p.CopiesNumbers,
+		PrmAuth: frostfs.PrmAuth{
+			PrivateKey: &n.gateKey.PrivateKey,
+		},
 	}
 
-	var corsBkt *data.BucketInfo
+	_, err := n.objectPutAndHash(ctx, prm, n.corsCnrInfo)
-	if n.corsCnrInfo == nil {
-		corsBkt = p.BktInfo
-		prm.CopiesNumber = p.CopiesNumbers
-	} else {
-		corsBkt = n.corsCnrInfo
-		prm.PrmAuth.PrivateKey = &n.gateKey.PrivateKey
-	}
-
-	prm.Container = corsBkt.CID
-
-	createdObj, err := n.objectPutAndHash(ctx, prm, corsBkt)
 	if err != nil {
 		return fmt.Errorf("put cors object: %w", err)
 	}
 
-	objsToDelete, err := n.treeService.PutBucketCORS(ctx, p.BktInfo, newAddress(corsBkt.CID, createdObj.ID))
+	n.cache.PutCORS(n.BearerOwner(ctx), p.BktInfo, cors)
-	objToDeleteNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
+
-	if err != nil && !objToDeleteNotFound {
+	objs, err := n.treeService.DeleteBucketCORS(ctx, p.BktInfo)
-		return err
+	objNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
+	if err != nil && !objNotFound {
+		n.reqLogger(ctx).Error(logs.CouldntDeleteBucketCORS, zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
+		return nil
 	}
 
-	if !objToDeleteNotFound {
+	if !objNotFound {
-		for _, addr := range objsToDelete {
+		for _, addr := range objs {
 			n.deleteCORSObject(ctx, p.BktInfo, addr)
 		}
 	}
 
-	n.cache.PutCORS(n.BearerOwner(ctx), p.BktInfo, cors)
-
 	return nil
 }
 
@@ -117,10 +112,22 @@ func (n *Layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo)
 	ctx, span := tracing.StartSpanFromContext(ctx, "layer.DeleteBucketCORS")
 	defer span.End()
 
+	corsVersions, err := n.getCORSVersions(ctx, bktInfo)
+	if err != nil {
+		return fmt.Errorf("get cors versions: %w", err)
+	}
+
+	sortedVersions := corsVersions.GetSorted()
+	for _, version := range sortedVersions {
+		if err = n.objectDeleteWithAuth(ctx, n.corsCnrInfo, version.ObjID, frostfs.PrmAuth{PrivateKey: &n.gateKey.PrivateKey}); err != nil {
+			return fmt.Errorf("delete cors object '%s': %w", version.VersionID(), err)
+		}
+	}
+
 	objs, err := n.treeService.DeleteBucketCORS(ctx, bktInfo)
 	objNotFound := errors.Is(err, tree.ErrNoNodeToRemove)
 	if err != nil && !objNotFound {
-		return err
+		return fmt.Errorf("delete cors from tree: %w", err)
 	}
 
 	if !objNotFound {
@@ -134,6 +141,22 @@ func (n *Layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo)
 	return nil
 }
 
+func (n *Layer) deleteCORSVersions(ctx context.Context, bktInfo *data.BucketInfo) {
+	corsVersions, err := n.getCORSVersions(ctx, bktInfo)
+	if err != nil {
+		n.reqLogger(ctx).Error(logs.CouldntGetCORSObjectVersions, zap.Error(err), logs.TagField(logs.TagExternalStorage))
+		return
+	}
+
+	var addr oid.Address
+	addr.SetContainer(n.corsCnrInfo.CID)
+	sortedVersions := corsVersions.GetSorted()
+	for _, version := range sortedVersions {
+		addr.SetObject(version.ObjID)
+		n.deleteCORSObject(ctx, bktInfo, addr)
+	}
+}
+
 func checkCORS(cors *data.CORSConfiguration) error {
 	for _, r := range cors.CORSRules {
 		for _, m := range r.AllowedMethods {

api/layer/frostfs_mock.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
 	v2container "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/container"
@@ -175,6 +176,34 @@ func (t *TestFrostFS) SetContainer(cnrID cid.ID, cnr *container.Container) {
 	t.containers[cnrID.EncodeToString()] = cnr
 }
 
+func (t *TestFrostFS) SetObject(addr oid.Address, obj *object.Object) {
+	t.objects[addr.EncodeToString()] = obj
+}
+
+func (t *TestFrostFS) AddCORSObject(bkt *data.BucketInfo, corsCnrID cid.ID, cors string) {
+	a := object.NewAttribute()
+	a.SetKey(object.AttributeFilePath)
+	a.SetValue(bkt.CORSObjectFilePath())
+
+	var owner user.ID
+	user.IDFromKey(&owner, t.key.PrivateKey.PublicKey)
+
+	objID := oidtest.ID()
+
+	obj := object.New()
+	obj.SetContainerID(corsCnrID)
+	obj.SetID(objID)
+	obj.SetPayloadSize(uint64(len(cors)))
+	obj.SetPayload([]byte(cors))
+	obj.SetAttributes(*a)
+	obj.SetCreationEpoch(t.currentEpoch)
+	obj.SetOwnerID(owner)
+	t.currentEpoch++
+
+	addr := newAddress(corsCnrID, objID)
+	t.objects[addr.EncodeToString()] = obj
+}
+
 func (t *TestFrostFS) CreateContainer(_ context.Context, prm frostfs.PrmContainerCreate) (*frostfs.ContainerCreateResult, error) {
 	var cnr container.Container
 	cnr.Init()

api/layer/layer.go

@@ -907,9 +907,9 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
 
 	n.cache.DeleteBucket(p.BktInfo)
 
-	corsObj, err := n.treeService.GetBucketCORS(ctx, p.BktInfo)
+	corsObjs, err := n.treeService.GetAllBucketCORS(ctx, p.BktInfo)
 	if err != nil {
-		n.reqLogger(ctx).Error(logs.GetBucketCorsFromTree, zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
+		n.reqLogger(ctx).Error(logs.GetAllBucketCorsFromTree, zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
 	}
 
 	lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo)
@@ -922,9 +922,13 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
 		return fmt.Errorf("delete container: %w", err)
 	}
 
+	for _, corsObj := range corsObjs {
 		if !corsObj.Container().Equals(p.BktInfo.CID) && !corsObj.Container().Equals(cid.ID{}) {
 			n.deleteCORSObject(ctx, p.BktInfo, corsObj)
 		}
+	}
+
+	n.deleteCORSVersions(ctx, p.BktInfo)
 
 	if treeErr == nil && !lifecycleObj.Container().Equals(p.BktInfo.CID) {
 		n.deleteLifecycleObject(ctx, p.BktInfo, lifecycleObj)

api/layer/system_object.go

@@ -15,9 +15,11 @@ import (
 	apierr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/errors"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/tree"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/crdt"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
+	apiobject "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/api/object"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
 )
 
@@ -175,6 +177,23 @@ func (n *Layer) getCORS(ctx context.Context, bkt *data.BucketInfo, decoder func(
 		return cors, nil
 	}
 
+	corsVersions, err := n.getCORSVersions(ctx, bkt)
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		prmAuth  frostfs.PrmAuth
+		objID    oid.ID
+		corsBkt  = bkt
+		lastCORS = corsVersions.GetLast()
+	)
+
+	if lastCORS != nil {
+		prmAuth.PrivateKey = &n.gateKey.PrivateKey
+		corsBkt = n.corsCnrInfo
+		objID = lastCORS.ObjID
+	} else {
 		addr, err := n.treeService.GetBucketCORS(ctx, bkt)
 		objNotFound := errors.Is(err, tree.ErrNodeNotFound)
 		if err != nil && !objNotFound {
@@ -185,14 +204,15 @@ func (n *Layer) getCORS(ctx context.Context, bkt *data.BucketInfo, decoder func(
 			return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrNoSuchCORSConfiguration), err.Error())
 		}
 
-	var prmAuth frostfs.PrmAuth
-	corsBkt := bkt
 		if !addr.Container().Equals(bkt.CID) && !addr.Container().Equals(cid.ID{}) {
 			corsBkt = &data.BucketInfo{CID: addr.Container()}
 			prmAuth.PrivateKey = &n.gateKey.PrivateKey
 		}
 
-	obj, err := n.objectGetWithAuth(ctx, corsBkt, addr.Object(), prmAuth)
+		objID = addr.Object()
+	}
+
+	obj, err := n.objectGetWithAuth(ctx, corsBkt, objID, prmAuth)
 	if err != nil {
 		return nil, fmt.Errorf("get cors object: %w", err)
 	}
@@ -207,6 +227,56 @@ func (n *Layer) getCORS(ctx context.Context, bkt *data.BucketInfo, decoder func(
 	return cors, nil
 }
 
+func (n *Layer) getCORSVersions(ctx context.Context, bkt *data.BucketInfo) (*crdt.ObjectVersions, error) {
+	corsVersions, err := n.frostFS.SearchObjects(ctx, frostfs.PrmObjectSearch{
+		Container:      n.corsCnrInfo.CID,
+		ExactAttribute: [2]string{object.AttributeFilePath, bkt.CORSObjectFilePath()},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("search cors objects: %w", err)
+	}
+
+	versions := crdt.NewObjectVersions(bkt.CORSObjectFilePath())
+	versions.SetLessFunc(func(ov1, ov2 *crdt.ObjectVersion) bool {
+		versionID1, versionID2 := ov1.VersionID(), ov2.VersionID()
+		timestamp1, timestamp2 := ov1.Headers[object.AttributeTimestamp], ov2.Headers[object.AttributeTimestamp]
+
+		if ov1.CreationEpoch != ov2.CreationEpoch {
+			return ov1.CreationEpoch < ov2.CreationEpoch
+		}
+
+		if len(timestamp1) > 0 && len(timestamp2) > 0 && timestamp1 != timestamp2 {
+			unixTime1, err := strconv.ParseInt(timestamp1, 10, 64)
+			if err != nil {
+				return versionID1 < versionID2
+			}
+
+			unixTime2, err := strconv.ParseInt(timestamp2, 10, 64)
+			if err != nil {
+				return versionID1 < versionID2
+			}
+
+			return unixTime1 < unixTime2
+		}
+
+		return versionID1 < versionID2
+	})
+
+	for _, id := range corsVersions {
+		objVersion, err := n.frostFS.HeadObject(ctx, frostfs.PrmObjectHead{
+			Container: n.corsCnrInfo.CID,
+			Object:    id,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("head cors object '%s': %w", id.EncodeToString(), err)
+		}
+
+		versions.AppendVersion(crdt.NewObjectVersion(objVersion))
+	}
+
+	return versions, nil
+}
+
 func lockObjectKey(objVersion *data.ObjectVersion) string {
 	// todo reconsider forming name since versionID can be "null" or ""
 	return ".lock." + objVersion.BktInfo.CID.EncodeToString() + "." + objVersion.ObjectName + "." + objVersion.VersionID
@@ -274,7 +344,7 @@ func (n *Layer) attributesFromLock(ctx context.Context, lock *data.ObjectLock) (
 
 	if expEpoch != 0 {
 		result = append(result, [2]string{
-			object.SysAttributeExpEpoch, strconv.FormatUint(expEpoch, 10),
+			apiobject.SysAttributeExpEpoch, strconv.FormatUint(expEpoch, 10),
 		})
 	}
 

api/layer/tree/tree_service.go

@@ -18,19 +18,19 @@ type Service interface {
 	// If tree node is not found returns ErrNodeNotFound error.
 	GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error)
 
-	// GetBucketCORS gets an object id that corresponds to object with bucket CORS.
+	// GetBucketCORS gets an object address that corresponds to object with bucket CORS.
 	//
-	// If object id is not found returns ErrNodeNotFound error.
+	// If object is not found returns ErrNodeNotFound error.
 	GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.Address, error)
 
-	// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in FrostFS.
+	// GetAllBucketCORS gets all object addresses that corresponds to objects with bucket CORS.
 	//
-	// If object ids to remove is not found returns ErrNoNodeToRemove error.
+	// If objects are not found returns ErrNodeNotFound error.
-	PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error)
+	GetAllBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error)
 
-	// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in FrostFS.
+	// DeleteBucketCORS removes a node from a system tree and returns object addresses which must be deleted in FrostFS.
 	//
-	// If object ids to remove is not found returns ErrNoNodeToRemove error.
+	// If objects to remove are not found returns ErrNoNodeToRemove error.
 	DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error)
 
 	GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error)

api/layer/tree_mock.go

@@ -115,12 +115,12 @@ func (t *TreeServiceMock) GetSettingsNode(_ context.Context, bktInfo *data.Bucke
 func (t *TreeServiceMock) GetBucketCORS(_ context.Context, bktInfo *data.BucketInfo) (oid.Address, error) {
 	systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
 	if !ok {
-		return oid.Address{}, nil
+		return oid.Address{}, tree.ErrNodeNotFound
 	}
 
 	node, ok := systemMap["cors"]
 	if !ok {
-		return oid.Address{}, nil
+		return oid.Address{}, tree.ErrNodeNotFound
 	}
 
 	var addr oid.Address
@@ -129,19 +129,13 @@ func (t *TreeServiceMock) GetBucketCORS(_ context.Context, bktInfo *data.BucketI
 	return addr, nil
 }
 
-func (t *TreeServiceMock) PutBucketCORS(_ context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
+func (t *TreeServiceMock) GetAllBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
-	systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
+	cors, err := t.GetBucketCORS(ctx, bktInfo)
-	if !ok {
+	if err != nil {
-		systemMap = make(map[string]*data.BaseNodeVersion)
+		return nil, err
 	}
 
-	systemMap["cors"] = &data.BaseNodeVersion{
+	return []oid.Address{cors}, nil
-		OID: addr.Object(),
-	}
-
-	t.system[bktInfo.CID.EncodeToString()] = systemMap
-
-	return nil, tree.ErrNoNodeToRemove
 }
 
 func (t *TreeServiceMock) DeleteBucketCORS(context.Context, *data.BucketInfo) ([]oid.Address, error) {

cmd/s3-gw/app.go

@@ -281,13 +281,10 @@ func (a *App) initLayer(ctx context.Context) {
 	var gateOwner user.ID
 	user.IDFromKey(&gateOwner, a.key.PrivateKey.PublicKey)
 
-	var corsCnrInfo *data.BucketInfo
+	corsCnrInfo, err := a.fetchContainerInfo(ctx, cfgContainersCORS)
-	if a.config().IsSet(cfgContainersCORS) {
-		corsCnrInfo, err = a.fetchContainerInfo(ctx, cfgContainersCORS)
 	if err != nil {
 		a.log.Fatal(logs.CouldNotFetchCORSContainerInfo, zap.Error(err), logs.TagField(logs.TagApp))
 	}
-	}
 
 	var lifecycleCnrInfo *data.BucketInfo
 	if a.config().IsSet(cfgContainersLifecycle) {

docs/configuration.md

@@ -863,7 +863,7 @@ containers:
 
 | Parameter   | Type     | SIGHUP reload | Default value | Description                                                                                                             |
 |-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------------------------------------|
-| `cors`      | `string` | no            |               | Container name for CORS configurations. If not set, container of the bucket is used.                                    |
+| `cors`      | `string` | no            |               | Container name for CORS configurations.                                                                                 |
 | `lifecycle` | `string` | no            |               | Container name for lifecycle configurations. If not set, container of the bucket is used.                               |
 | `accessbox` | `string` | no            |               | Container name to lookup accessbox if custom aws credentials is used. If not set, custom credentials are not supported. |
 

internal/frostfs/crdt/gset.go

@@ -17,6 +17,7 @@ type ObjectVersions struct {
 	objects  []*ObjectVersion
 	addList  []string
 	isSorted bool
+	less     func(*ObjectVersion, *ObjectVersion) bool
 }
 
 type ObjectVersion struct {
@@ -30,7 +31,7 @@ func (o *ObjectVersion) VersionID() string {
 }
 
 func NewObjectVersions(name string) *ObjectVersions {
-	return &ObjectVersions{name: name}
+	return &ObjectVersions{name: name, less: less}
 }
 
 func NewObjectVersion(obj *object.Object) *ObjectVersion {
@@ -86,6 +87,15 @@ func (v *ObjectVersions) GetLast() *ObjectVersion {
 	return v.objects[len(v.objects)-1]
 }
 
+func (v *ObjectVersions) GetSorted() []*ObjectVersion {
+	v.sort()
+	return v.objects
+}
+
+func (v *ObjectVersions) SetLessFunc(less func(*ObjectVersion, *ObjectVersion) bool) {
+	v.less = less
+}
+
 func splitVersions(header string) []string {
 	if len(header) == 0 {
 		return nil
@@ -97,7 +107,7 @@ func splitVersions(header string) []string {
 func (v *ObjectVersions) sort() {
 	if !v.isSorted {
 		sort.Slice(v.objects, func(i, j int) bool {
-			return less(v.objects[i], v.objects[j])
+			return v.less(v.objects[i], v.objects[j])
 		})
 		v.isSorted = true
 	}

internal/logs/logs.go

@@ -180,6 +180,7 @@ const (
 	CouldNotFetchObjectMeta                        = "could not fetch object meta"
 	FailedToDeleteObject                           = "failed to delete object"
 	CouldntDeleteLifecycleObject                   = "couldn't delete lifecycle configuration object"
+	CouldntGetCORSObjectVersions                   = "couldn't get cors object versions"
 )
 
 // External blockchain.
@@ -199,8 +200,8 @@ const (
 	ObjectTaggingNodeHasMultipleIDs  = "object tagging node has multiple ids"
 	BucketTaggingNodeHasMultipleIDs  = "bucket tagging node has multiple ids"
 	BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
-	BucketCORSNodeHasMultipleIDs     = "bucket cors node has multiple ids"
+	GetAllBucketCorsFromTree         = "get all bucket cors from tree"
-	GetBucketCorsFromTree            = "get bucket cors from tree"
+	CouldntDeleteBucketCORS          = "couldn't delete bucket cors"
 )
 
 // Authmate.

pkg/service/tree/tree.go

@@ -570,47 +570,25 @@ func (c *Tree) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid
 	return getTreeNodeAddress(node.Latest())
 }
 
-func (c *Tree) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, addr oid.Address) ([]oid.Address, error) {
+func (c *Tree) GetAllBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {
-	ctx, span := tracing.StartSpanFromContext(ctx, "tree.PutBucketCORS")
+	ctx, span := tracing.StartSpanFromContext(ctx, "tree.GetAllBucketCORS")
 	defer span.End()
 
-	multiNode, err := c.getSystemNode(ctx, bktInfo, corsFilename)
+	node, err := c.getSystemNode(ctx, bktInfo, corsFilename)
-	isErrNotFound := errors.Is(err, tree.ErrNodeNotFound)
+	if err != nil {
-	if err != nil && !isErrNotFound {
-		return nil, fmt.Errorf("couldn't get node: %w", err)
-	}
-
-	meta := make(map[string]string)
-	meta[FileNameKey] = corsFilename
-	meta[oidKV] = addr.Object().EncodeToString()
-	meta[cidKV] = addr.Container().EncodeToString()
-
-	if isErrNotFound {
-		if _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, meta); err != nil {
 		return nil, err
 	}
-		return nil, tree.ErrNoNodeToRemove
-	}
 
-	latest := multiNode.Latest()
+	addrs := make([]oid.Address, 0, len(node.nodes))
-	ind := latest.GetLatestNodeIndex()
+	for _, corsNode := range node.nodes {
-	if latest.IsSplit() {
+		addr, err := getTreeNodeAddress(corsNode)
-		c.reqLogger(ctx).Error(logs.BucketCORSNodeHasMultipleIDs, logs.TagField(logs.TagExternalStorageTree))
-	}
-
-	if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
-		return nil, fmt.Errorf("move cors node: %w", err)
-	}
-
-	objToDelete := make([]oid.Address, 1, len(multiNode.nodes))
-	objToDelete[0], err = getTreeNodeAddress(latest)
 		if err != nil {
-		return nil, fmt.Errorf("parse object addr of latest cors node in tree: %w", err)
+			return nil, err
+		}
+		addrs = append(addrs, addr)
 	}
 
-	objToDelete = append(objToDelete, c.cleanOldNodes(ctx, multiNode.Old(), bktInfo)...)
+	return addrs, nil
-
-	return objToDelete, nil
 }
 
 func (c *Tree) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.Address, error) {