Release v0.30.4

Signed-off-by: Alex Vanin <a.vanin@yadro.com>

CHANGELOG.md

@@ -4,6 +4,32 @@ This document outlines major changes between releases.
 
 ## [Unreleased]
 
+## [0.30.4] - 2024-09-03
+
+### Added
+- Fuzzing tests (#480)
+
+## [0.30.3] - 2024-08-27
+
+### Fixed
+- Empty listing when multipart upload contains more than 1000 parts (#471)
+
+## [0.30.2] - 2024-08-20
+
+### Fixed
+- Error counting in pool component before connection switch (#468)
+
+### Added
+- Log of endpoint address during tree pool errors (#468)
+
+## [0.30.1] - 2024-07-25
+
+### Fixed
+- Redundant system node removal in tree service (#437)
+
+### Added
+- Log details on SDK Pool health status change (#439)
+
 ## [0.30.0] - Kangshung -2024-07-19
 
 ### Fixed
@@ -233,4 +259,8 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs
 [0.29.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.1...v0.29.2
 [0.29.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.2...v0.29.3
 [0.30.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.3...v0.30.0
-[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.0...master
+[0.30.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.0...v0.30.1
+[0.30.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.1...v0.30.2
+[0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.2...v0.30.3
+[0.30.3]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.3...v0.30.4
+[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.30.4...master

Makefile

@@ -23,6 +23,12 @@ OUTPUT_LINT_DIR ?= $(shell pwd)/bin
 LINT_DIR = $(OUTPUT_LINT_DIR)/golangci-lint-$(LINT_VERSION)-v$(TRUECLOUDLAB_LINT_VERSION)
 TMP_DIR := .cache
 
+# Variables for fuzzing
+FUZZ_NGFUZZ_DIR ?= ""
+FUZZ_TIMEOUT ?= 30
+FUZZ_FUNCTIONS ?= "all"
+FUZZ_AUX ?= ""
+
 .PHONY: all $(BINS) $(BINDIR) dep docker/ test cover format image image-push dirty-image lint docker/lint pre-commit unpre-commit version clean protoc
 
 # .deb package versioning
@@ -76,6 +82,34 @@ cover:
 	@go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic
 	@go tool cover -html=coverage.txt -o coverage.html
 
+# Run fuzzing
+CLANG := $(shell which clang-17 2>/dev/null)
+.PHONY: check-clang all
+check-clang:
+ifeq ($(CLANG),)
+	@echo "clang-17 is not installed. Please install it before proceeding - https://apt.llvm.org/llvm.sh "
+	@exit 1
+endif
+
+.PHONY: check-ngfuzz all
+check-ngfuzz:
+	@if [ -z "$(FUZZ_NGFUZZ_DIR)" ]; then \
+		echo "Please set a variable FUZZ_NGFUZZ_DIR to specify path to the ngfuzz"; \
+		exit 1; \
+	fi
+
+.PHONY: install-fuzzing-deps
+install-fuzzing-deps: check-clang check-ngfuzz
+
+.PHONY: fuzz
+fuzz: install-fuzzing-deps  
+	@START_PATH=$$(pwd); \
+	ROOT_PATH=$$(realpath --relative-to=$(FUZZ_NGFUZZ_DIR) $$START_PATH) ; \
+	cd $(FUZZ_NGFUZZ_DIR) && \
+	./ngfuzz -clean && \
+	./ngfuzz -fuzz $(FUZZ_FUNCTIONS) -rootdir $$ROOT_PATH -timeout $(FUZZ_TIMEOUT) $(FUZZ_AUX) && \
+	./ngfuzz -report
+
 # Reformat code
 format:
 	@echo "⇒ Processing gofmt check"

README.md

@@ -93,6 +93,24 @@ HTTP/1.1 200 OK
 
 Also, you can configure domains using `.env` variables or `yaml` file.
 
+## Fuzzing   
+To run fuzzing tests use the following command:  
+
+```shell
+$ make fuzz
+```   
+
+This command will install dependencies for the fuzzing process and run existing fuzzing tests.   
+
+You can also use the following arguments:
+
+```
+FUZZ_TIMEOUT - time to run each fuzzing test (default 30) 
+FUZZ_FUNCTIONS - fuzzing tests that will be started (default "all")
+FUZZ_AUX - additional parameters for the fuzzer (for example, "-debug")
+FUZZ_NGFUZZ_DIR - path to ngfuzz tool
+````
+
 ## Documentation
 
 - [Configuration](./docs/configuration.md)

VERSION

@@ -1 +1 @@
-v0.30.0
+v0.30.4

api/auth/center_fuzz_test.go

@@ -0,0 +1,88 @@
+//go:build gofuzz
+// +build gofuzz
+
+package auth
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	utils "github.com/trailofbits/go-fuzz-utils"
+)
+
+const (
+	fuzzSuccessExitCode = 0
+	fuzzFailExitCode    = -1
+)
+
+func InitFuzzAuthenticate() {
+}
+
+func DoFuzzAuthenticate(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	var accessKeyAddr oid.Address
+	err = tp.Fill(accessKeyAddr)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	accessKeyID := strings.ReplaceAll(accessKeyAddr.String(), "/", "0")
+	secretKey, err := tp.GetString()
+	awsCreds := credentials.NewStaticCredentials(accessKeyID, secretKey, "")
+
+	reqData := RequestData{
+		Method:   "GET",
+		Endpoint: "http://localhost:8084",
+		Bucket:   "my-bucket",
+		Object:   "@obj/name",
+	}
+	presignData := PresignData{
+		Service:  "s3",
+		Region:   "spb",
+		Lifetime: 10 * time.Minute,
+		SignTime: time.Now().UTC(),
+	}
+
+	req, err := PresignRequest(awsCreds, reqData, presignData)
+	if req == nil {
+		return fuzzFailExitCode
+	}
+
+	expBox := &accessbox.Box{
+		Gate: &accessbox.GateData{
+			SecretKey: secretKey,
+		},
+	}
+
+	mock := newTokensFrostfsMock()
+	mock.addBox(accessKeyAddr, expBox)
+
+	c := &Center{
+		cli:     mock,
+		reg:     NewRegexpMatcher(authorizationFieldRegexp),
+		postReg: NewRegexpMatcher(postPolicyCredentialRegexp),
+	}
+
+	_, _ = c.Authenticate(req)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzAuthenticate(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzAuthenticate(data)
+	})
+}

api/handler/handler_fuzz_test.go

@@ -0,0 +1,998 @@
+//go:build gofuzz
+// +build gofuzz
+
+package handler
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/xml"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	tt "testing" // read https://github.com/AdamKorcz/go-118-fuzz-build?tab=readme-ov-file#workflow
+
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/data"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
+	engineiam "git.frostfs.info/TrueCloudLab/policy-engine/iam"
+	utils "github.com/trailofbits/go-fuzz-utils"
+	"go.uber.org/zap/zaptest"
+)
+
+var (
+	fuzzBktName string
+	fuzzBox     *accessbox.Box
+	fuzzHc      *handlerContextBase
+	fuzzt       *tt.T
+)
+
+const (
+	fuzzSuccessExitCode = 0
+	fuzzFailExitCode    = -1
+)
+
+func createTestBucketAndInitContext() {
+	fuzzt = new(tt.T)
+
+	log := zaptest.NewLogger(fuzzt)
+	var err error
+	fuzzHc, err = prepareHandlerContextBase(layer.DefaultCachesConfigs(log))
+	if err != nil {
+		panic(err)
+	}
+
+	fuzzBktName = "bucket"
+	fuzzBox, _ = createAccessBox(fuzzt)
+
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPut, defaultURL, nil)
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	fuzzHc.Handler().CreateBucketHandler(w, r)
+}
+
+func prepareStrings(tp *utils.TypeProvider, count int) ([]string, error) {
+	array := make([]string, count)
+	var err error
+
+	for i := 0; i < count; i++ {
+		err = tp.Reset()
+		if err != nil {
+			return nil, err
+		}
+
+		array[i], err = tp.GetString()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return array, nil
+}
+
+func addMD5Header(tp *utils.TypeProvider, r *http.Request, rawBody []byte) error {
+	if len(rawBody) == 0 {
+		return nil
+	}
+
+	rand, err := tp.GetBool()
+	if err != nil {
+		return err
+	}
+
+	if rand == true {
+		var dst []byte
+		base64.StdEncoding.Encode(dst, rawBody)
+		hash := md5.Sum(dst)
+		r.Header.Set("Content-Md5", hex.EncodeToString(hash[:]))
+	}
+
+	return nil
+}
+
+func generateParams(tp *utils.TypeProvider, input string, params []string) (string, error) {
+	input += "?"
+
+	count, err := tp.GetInt()
+	if err != nil {
+		return "", err
+	}
+	count = count % len(params)
+	if count < 0 {
+		count += len(params)
+	}
+
+	for i := 0; i < count; i++ {
+		position, err := tp.GetInt()
+		if err != nil {
+			return "", err
+		}
+		position = position % len(params)
+		if position < 0 {
+			position += len(params)
+		}
+
+		v, err := tp.GetString()
+		if err != nil {
+			return "", err
+		}
+		input += params[position] + "=" + v + "&"
+	}
+
+	return input, nil
+}
+
+func generateHeaders(tp *utils.TypeProvider, r *http.Request, params []string) error {
+	count, err := tp.GetInt()
+	if err != nil {
+		return err
+	}
+	count = count % len(params)
+	if count < 0 {
+		count += len(params)
+	}
+
+	for i := 0; i < count; i++ {
+		position, err := tp.GetInt()
+		if err != nil {
+			return err
+		}
+		position = position % len(params)
+		if position < 0 {
+			position += len(params)
+		}
+
+		v, err := tp.GetString()
+		if err != nil {
+			return err
+		}
+
+		r.Header.Set(params[position], v)
+
+	}
+
+	return nil
+}
+
+func InitFuzzCreateBucketHandler() {
+	fuzzt = new(tt.T)
+
+	log := zaptest.NewLogger(fuzzt)
+	var err error
+	fuzzHc, err = prepareHandlerContextBase(layer.DefaultCachesConfigs(log))
+	if err != nil {
+		panic(err)
+	}
+
+	fuzzBox, _ = createAccessBox(fuzzt)
+}
+
+func DoFuzzCreateBucketHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	strings, err := prepareStrings(tp, 4)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	bktName := strings[0]
+	body := strings[1]
+
+	bodyXml, err := xml.Marshal(body)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPut, defaultURL, bytes.NewReader(bodyXml))
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: bktName, Object: ""}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"x-amz-acl", "x-amz-bucket-object-lock-enabled", "x-amz-grant-full-control", "x-amz-grant-read", "x-amz-grant-read-acp", "x-amz-grant-write", "x-amz-grant-write-acp", "x-amz-object-ownership"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().CreateBucketHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzCreateBucketHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzCreateBucketHandler(data)
+	})
+}
+
+func InitFuzzPutBucketCorsHandler() {
+	createTestBucketAndInitContext()
+}
+
+func DoFuzzPutBucketCorsHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	var cors data.CORSConfiguration
+	err = tp.Fill(&cors)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	bodyXml, err := xml.Marshal(cors)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPut, defaultURL+"?cors", bytes.NewReader(bodyXml))
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().PutBucketCorsHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzPutBucketCorsHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzPutBucketCorsHandler(data)
+	})
+}
+
+func InitFuzzPutBucketPolicyHandler() {
+	createTestBucketAndInitContext()
+}
+
+func FuzzPutBucketPolicyHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzPutBucketPolicyHandler(data)
+	})
+}
+
+func DoFuzzPutBucketPolicyHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	var policy engineiam.Policy
+	err = tp.Fill(&policy)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	bodyXml, err := xml.Marshal(policy)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPut, defaultURL+"?policy", bytes.NewReader(bodyXml))
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-confirm-remove-self-bucket-access"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	err = addMD5Header(tp, r, bodyXml)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().PutBucketPolicyHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func InitFuzzDeleteMultipleObjectsHandler() {
+	createTestBucketAndInitContext()
+}
+
+func FuzzDeleteMultipleObjectsHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzDeleteMultipleObjectsHandler(data)
+	})
+}
+
+func DoFuzzDeleteMultipleObjectsHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	var body DeleteObjectsRequest
+	err = tp.Fill(&body)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	bodyXml, err := xml.Marshal(body)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, defaultURL+"?delete", bytes.NewReader(bodyXml))
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bypass-governance-retention", "x-amz-mfa"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	err = addMD5Header(tp, r, bodyXml)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().DeleteMultipleObjectsHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func InitFuzzPostObject() {
+	createTestBucketAndInitContext()
+}
+
+func FuzzPostObject(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzPostObject(data)
+	})
+}
+
+func postObject(tp *utils.TypeProvider) ([]byte, string, error) {
+	strings, err := prepareStrings(tp, 2)
+	if err != nil {
+		return nil, "", err
+	}
+
+	bodyXml, err := xml.Marshal(strings[0])
+	if err != nil {
+		return nil, "", err
+	}
+	objName := strings[1]
+
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPost, defaultURL, bytes.NewReader(bodyXml))
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"X-Amz-Grant-Read", "X-Amz-Grant-Full-Control", "X-Amz-Grant-Write", "X-Amz-Acl", "x-amz-expected-bucket-owner"})
+	if err != nil {
+		return nil, "", err
+	}
+
+	var file multipart.Form
+	err = tp.Fill(&file)
+	if err != nil {
+		return nil, "", err
+	}
+
+	r.MultipartForm = &file
+
+	fuzzHc.Handler().PostObject(w, r)
+
+	return bodyXml, objName, nil
+}
+
+func DoFuzzPostObject(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	_, _, err = postObject(tp)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	return fuzzSuccessExitCode
+}
+
+func InitFuzzDeleteBucketHandler() {
+	createTestBucketAndInitContext()
+}
+
+func FuzzDeleteBucketHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzDeleteBucketHandler(data)
+	})
+}
+
+func DoFuzzDeleteBucketHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodDelete, defaultURL, nil)
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().DeleteBucketHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func InitFuzzDeleteBucketCorsHandler() {
+	createTestBucketAndInitContext()
+}
+
+func FuzzDeleteBucketCorsHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzDeleteBucketCorsHandler(data)
+	})
+}
+
+func DoFuzzDeleteBucketCorsHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodDelete, defaultURL+"?cors", nil)
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().DeleteBucketCorsHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func InitFuzzDeleteBucketPolicyHandler() {
+	createTestBucketAndInitContext()
+}
+
+func FuzzDeleteBucketPolicyHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzDeleteBucketPolicyHandler(data)
+	})
+}
+
+func DoFuzzDeleteBucketPolicyHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodDelete, defaultURL+"?policy", nil)
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().DeleteBucketPolicyHandler(w, r)
+
+	return fuzzFailExitCode
+}
+
+func InitFuzzCopyObjectHandler() {
+	createTestBucketAndInitContext()
+}
+
+func FuzzCopyObjectHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzCopyObjectHandler(data)
+	})
+}
+
+func DoFuzzCopyObjectHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+	var r *http.Request
+
+	key, err := tp.GetString()
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	params, err := generateParams(tp, key, []string{"versionId"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	r = httptest.NewRequest(http.MethodPut, defaultURL+params, nil)
+	if r != nil {
+		return fuzzFailExitCode
+	}
+
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"x-amz-acl", "x-amz-checksum-algorithm", "x-amz-copy-source", "x-amz-copy-source-if-match", "x-amz-copy-source-if-match", "x-amz-copy-source-if-unmodified-since", "x-amz-copy-source-if-modified-since", "x-amz-copy-source-if-none-match", "x-amz-copy-source-if-modified-since", "x-amz-copy-source-if-none-match", "x-amz-copy-source-if-none-match", "x-amz-copy-source-if-modified-since", "x-amz-copy-source-if-unmodified-since", "x-amz-copy-source-if-match", "x-amz-copy-source-if-unmodified-since", "x-amz-copy-source-server-side-encryption-customer-algorithm", "x-amz-copy-source-server-side-encryption-customer-key", "x-amz-copy-source-server-side-encryption-customer-key-MD5", "x-amz-expected-bucket-owner", "x-amz-grant-full-control", "x-amz-grant-read", "x-amz-grant-read-acp", "x-amz-grant-write-acp", "x-amz-metadata-directive", "x-amz-website-redirect-location", "x-amz-object-lock-legal-hold", "x-amz-object-lock-mode", "x-amz-object-lock-retain-until-date", "x-amz-request-payer", "x-amz-server-side-encryption", "x-amz-server-side-encryption-aws-kms-key-id", "x-amz-server-side-encryption-bucket-key-enabled", "x-amz-server-side-encryption-context", "x-amz-server-side-encryption-customer-algorithm", "x-amz-server-side-encryption-customer-key", "x-amz-server-side-encryption-customer-key-MD5", "x-amz-source-expected-bucket-owner", "x-amz-storage-class", "x-amz-tagging", "x-amz-tagging-directive", "x-amz-website-redirect-location"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().CopyObjectHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func InitFuzzDeleteObjectHandler() {
+	createTestBucketAndInitContext()
+}
+
+func FuzzDeleteObjectHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzDeleteObjectHandler(data)
+	})
+}
+
+func DoFuzzDeleteObjectHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	_, objName, err := postObject(tp)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+	var r *http.Request
+
+	params, err := generateParams(tp, objName, []string{"versionId"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	r = httptest.NewRequest(http.MethodDelete, defaultURL+params, nil)
+	if r != nil {
+		return fuzzFailExitCode
+	}
+
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bypass-governance-retention", "x-amz-mfa"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().DeleteObjectHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func InitFuzzGetObjectHandler() {
+	createTestBucketAndInitContext()
+}
+
+func FuzzGetObjectHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzGetObjectHandler(data)
+	})
+}
+
+func DoFuzzGetObjectHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	_, objName, err := postObject(tp)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	params, err := generateParams(tp, objName, []string{"versionId", "partNumber", "Range", "response-content-type", "response-content-language", "response-expires", "response-cache-control", "response-content-disposition", "response-content-encoding"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+
+	r := httptest.NewRequest(http.MethodGet, defaultURL+params, nil)
+	if r != nil {
+		return fuzzFailExitCode
+	}
+
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "If-Match", "If-None-Match", "If-Modified-Since", "If-Unmodified-Since", "x-amz-server-side-encryption-customer-algorithm", "x-amz-server-side-encryption-customer-key", "x-amz-server-side-encryption-customer-key-MD5", "Range"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().GetObjectHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func InitFuzzPutObjectHandler() {
+	createTestBucketAndInitContext()
+}
+
+func DoFuzzPutObjectHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	objName, err := tp.GetString()
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	body, err := tp.GetBytes()
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+
+	r := httptest.NewRequest(http.MethodPut, defaultURL+objName, bytes.NewReader(body))
+	if r != nil {
+		return fuzzFailExitCode
+	}
+
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "X-Amz-Grant-Read", "X-Amz-Grant-Full-Control", "X-Amz-Grant-Write", "X-Amz-Acl", "X-Amz-Tagging", "Content-Type", "Cache-Control", "Expires", "Content-Language", "Content-Encoding", "x-amz-server-side-encryption-customer-algorithm", "x-amz-server-side-encryption-customer-key", "x-amz-server-side-encryption-customer-key-MD5", "X-Amz-Content-Sha256", "X-Amz-Object-Lock-Legal-Hold", "X-Amz-Object-Lock-Mode", "X-Amz-Object-Lock-Retain-Until-Date", "X-Amz-Bypass-Governance-Retention", "X-Amz-Meta-*"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	err = addMD5Header(tp, r, body)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().PutObjectHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzPutObjectHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzPutObjectHandler(data)
+	})
+}
+
+func InitFuzzPutObjectLegalHoldHandler() {
+	createTestBucketAndInitContext()
+}
+
+func DoFuzzPutObjectLegalHoldHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	_, objName, err := postObject(tp)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	var hold data.LegalHold
+	err = tp.Fill(&hold)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	rawBody, err := xml.Marshal(hold)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+
+	r := httptest.NewRequest(http.MethodPut, defaultURL+objName+"?legal-hold", bytes.NewReader(rawBody))
+	if r != nil {
+		return fuzzFailExitCode
+	}
+
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = addMD5Header(tp, r, rawBody)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().PutObjectLegalHoldHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzPutObjectLegalHoldHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzPutObjectLegalHoldHandler(data)
+	})
+}
+
+func InitFuzzPutBucketObjectLockConfigHandler() {
+	createTestBucketAndInitContext()
+}
+
+func DoFuzzPutBucketObjectLockConfigHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	var hold data.ObjectLockConfiguration
+	err = tp.Fill(&hold)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	rawBody, err := xml.Marshal(&hold)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+
+	r := httptest.NewRequest(http.MethodPut, defaultURL+"?object-lock", bytes.NewReader(rawBody))
+	if r != nil {
+		return fuzzFailExitCode
+	}
+
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = addMD5Header(tp, r, rawBody)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bucket-object-lock-token"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().PutBucketObjectLockConfigHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzPutBucketObjectLockConfigHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzPutBucketObjectLockConfigHandler(data)
+	})
+}
+
+func InitFuzzPutObjectRetentionHandler() {
+	createTestBucketAndInitContext()
+}
+
+func DoFuzzPutObjectRetentionHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	_, objName, err := postObject(tp)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	var retention data.Retention
+	err = tp.Fill(&retention)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	rawBody, err := xml.Marshal(retention)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+
+	r := httptest.NewRequest(http.MethodPut, defaultURL+objName+"?retention", bytes.NewReader(rawBody))
+	if r != nil {
+		return fuzzFailExitCode
+	}
+
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: objName}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = addMD5Header(tp, r, rawBody)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-bypass-governance-retention"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().PutObjectRetentionHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzPutObjectRetentionHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzPutObjectRetentionHandler(data)
+	})
+}
+
+func InitFuzzPutBucketAclHandler() {
+	createTestBucketAndInitContext()
+}
+
+func DoFuzzPutBucketAclHandler(input []byte) int {
+	// FUZZER INIT
+	if len(input) < 100 {
+		return fuzzFailExitCode
+	}
+
+	tp, err := utils.NewTypeProvider(input)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	var policy AccessControlPolicy
+	err = tp.Fill(&policy)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	rawBody, err := xml.Marshal(policy)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest(http.MethodPut, defaultURL+"?acl", bytes.NewReader(rawBody))
+	reqInfo := middleware.NewReqInfo(w, r, middleware.ObjectRequest{Bucket: fuzzBktName, Object: ""}, "")
+	r = r.WithContext(middleware.SetReqInfo(fuzzHc.Context(), reqInfo))
+	r = r.WithContext(middleware.SetBox(r.Context(), &middleware.Box{AccessBox: fuzzBox}))
+
+	err = addMD5Header(tp, r, rawBody)
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	err = generateHeaders(tp, r, []string{"x-amz-expected-bucket-owner", "x-amz-acl", "x-amz-expected-bucket-owner", "x-amz-grant-full-control", "x-amz-grant-read", "x-amz-grant-read-acp", "x-amz-grant-write", "x-amz-grant-write-acp"})
+	if err != nil {
+		return fuzzFailExitCode
+	}
+
+	fuzzHc.Handler().PutBucketACLHandler(w, r)
+
+	return fuzzSuccessExitCode
+}
+
+func FuzzPutBucketAclHandler(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		DoFuzzPutBucketAclHandler(data)
+	})
+}

api/handler/handlers_test.go

@@ -37,8 +37,12 @@ import (
 )
 
 type handlerContext struct {
+	*handlerContextBase
+	t *testing.T
+}
+
+type handlerContextBase struct {
 	owner   user.ID
-	t       *testing.T
 	h       *handler
 	tp      *layer.TestFrostFS
 	tree    *tree.Tree
@@ -50,19 +54,19 @@ type handlerContext struct {
 	cache         *layer.Cache
 }
 
-func (hc *handlerContext) Handler() *handler {
+func (hc *handlerContextBase) Handler() *handler {
 	return hc.h
 }
 
-func (hc *handlerContext) MockedPool() *layer.TestFrostFS {
+func (hc *handlerContextBase) MockedPool() *layer.TestFrostFS {
 	return hc.tp
 }
 
-func (hc *handlerContext) Layer() *layer.Layer {
+func (hc *handlerContextBase) Layer() *layer.Layer {
 	return hc.h.obj
 }
 
-func (hc *handlerContext) Context() context.Context {
+func (hc *handlerContextBase) Context() context.Context {
 	return hc.context
 }
 
@@ -136,16 +140,28 @@ func (c *configMock) RetryStrategy() RetryStrategy {
 }
 
 func prepareHandlerContext(t *testing.T) *handlerContext {
-	return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(zap.NewExample()))
+	hc, err := prepareHandlerContextBase(layer.DefaultCachesConfigs(zap.NewExample()))
+	require.NoError(t, err)
+	return &handlerContext{
+		handlerContextBase: hc,
+		t:                  t,
+	}
 }
 
 func prepareHandlerContextWithMinCache(t *testing.T) *handlerContext {
-	return prepareHandlerContextBase(t, getMinCacheConfig(zap.NewExample()))
+	hc, err := prepareHandlerContextBase(getMinCacheConfig(zap.NewExample()))
+	require.NoError(t, err)
+	return &handlerContext{
+		handlerContextBase: hc,
+		t:                  t,
+	}
 }
 
-func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *handlerContext {
+func prepareHandlerContextBase(cacheCfg *layer.CachesConfig) (*handlerContextBase, error) {
 	key, err := keys.NewPrivateKey()
-	require.NoError(t, err)
+	if err != nil {
+		return nil, err
+	}
 
 	l := zap.NewExample()
 	tp := layer.NewTestFrostFS(key)
@@ -159,7 +175,9 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand
 	user.IDFromKey(&owner, key.PrivateKey.PublicKey)
 
 	memCli, err := tree.NewTreeServiceClientMemory()
-	require.NoError(t, err)
+	if err != nil {
+		return nil, err
+	}
 
 	treeMock := tree.NewTree(memCli, zap.NewExample())
 
@@ -176,7 +194,9 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand
 
 	var pp netmap.PlacementPolicy
 	err = pp.DecodeString("REP 1")
-	require.NoError(t, err)
+	if err != nil {
+		return nil, err
+	}
 
 	cfg := &configMock{
 		defaultPolicy: pp,
@@ -189,19 +209,23 @@ func prepareHandlerContextBase(t *testing.T, cacheCfg *layer.CachesConfig) *hand
 		frostfsid: newFrostfsIDMock(),
 	}
 
-	return &handlerContext{
+	accessBox, err := newTestAccessBox(key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &handlerContextBase{
 		owner:   owner,
-		t:       t,
 		h:       h,
 		tp:      tp,
 		tree:    treeMock,
-		context: middleware.SetBox(context.Background(), &middleware.Box{AccessBox: newTestAccessBox(t, key)}),
+		context: middleware.SetBox(context.Background(), &middleware.Box{AccessBox: accessBox}),
 		config:  cfg,
 
 		layerFeatures: features,
 		treeMock:      memCli,
 		cache:         layerCfg.Cache,
-	}
+	}, nil
 }
 
 func getMinCacheConfig(logger *zap.Logger) *layer.CachesConfig {

api/handler/head_test.go

@@ -119,21 +119,25 @@ func TestIsAvailableToResolve(t *testing.T) {
 	}
 }
 
-func newTestAccessBox(t *testing.T, key *keys.PrivateKey) *accessbox.Box {
+func newTestAccessBox(key *keys.PrivateKey) (*accessbox.Box, error) {
 	var err error
 	if key == nil {
 		key, err = keys.NewPrivateKey()
-		require.NoError(t, err)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	var btoken bearer.Token
 	btoken.SetImpersonate(true)
 	err = btoken.Sign(key.PrivateKey)
-	require.NoError(t, err)
+	if err != nil {
+		return nil, err
+	}
 
 	return &accessbox.Box{
 		Gate: &accessbox.GateData{
 			BearerToken: &btoken,
 		},
-	}
+	}, nil
 }

api/handler/object_list_test.go

@@ -94,7 +94,7 @@ func TestListObjectsWithOldTreeNodes(t *testing.T) {
 }
 
 func makeAllTreeObjectsOld(hc *handlerContext, bktInfo *data.BucketInfo) {
-	nodes, err := hc.treeMock.GetSubTree(hc.Context(), bktInfo, "version", []uint64{0}, 0)
+	nodes, err := hc.treeMock.GetSubTree(hc.Context(), bktInfo, "version", []uint64{0}, 0, true)
 	require.NoError(hc.t, err)
 
 	for _, node := range nodes {
@@ -142,7 +142,12 @@ func TestListObjectsContextCanceled(t *testing.T) {
 	layerCfg.SessionList.Lifetime = time.Hour
 	layerCfg.SessionList.Size = 1
 
-	hc := prepareHandlerContextBase(t, layerCfg)
+	hcBase, err := prepareHandlerContextBase(layerCfg)
+	require.NoError(t, err)
+	hc := &handlerContext{
+		handlerContextBase: hcBase,
+		t:                  t,
+	}
 
 	bktName := "bucket-versioning-enabled"
 	bktInfo := createTestBucket(hc, bktName)

api/layer/cors.go

@@ -49,17 +49,19 @@ func (n *Layer) PutBucketCORS(ctx context.Context, p *PutCORSParams) error {
 		return fmt.Errorf("put system object: %w", err)
 	}
 
-	objIDToDelete, err := n.treeService.PutBucketCORS(ctx, p.BktInfo, objID)
+	objIDsToDelete, err := n.treeService.PutBucketCORS(ctx, p.BktInfo, objID)
 	objIDToDeleteNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
 	if err != nil && !objIDToDeleteNotFound {
 		return err
 	}
 
 	if !objIDToDeleteNotFound {
-		if err = n.objectDelete(ctx, p.BktInfo, objIDToDelete); err != nil {
-			n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
-				zap.String("cnrID", p.BktInfo.CID.EncodeToString()),
-				zap.String("objID", objIDToDelete.EncodeToString()))
+		for _, id := range objIDsToDelete {
+			if err = n.objectDelete(ctx, p.BktInfo, id); err != nil {
+				n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
+					zap.String("cnrID", p.BktInfo.CID.EncodeToString()),
+					zap.String("objID", id.EncodeToString()))
+			}
 		}
 	}
 
@@ -81,14 +83,16 @@ func (n *Layer) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (*d
 }
 
 func (n *Layer) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) error {
-	objID, err := n.treeService.DeleteBucketCORS(ctx, bktInfo)
+	objIDs, err := n.treeService.DeleteBucketCORS(ctx, bktInfo)
 	objIDNotFound := errorsStd.Is(err, ErrNoNodeToRemove)
 	if err != nil && !objIDNotFound {
 		return err
 	}
 	if !objIDNotFound {
-		if err = n.objectDelete(ctx, bktInfo, objID); err != nil {
-			return err
+		for _, id := range objIDs {
+			if err = n.objectDelete(ctx, bktInfo, id); err != nil {
+				return err
+			}
 		}
 	}
 

api/layer/tree_mock.go

@@ -124,7 +124,7 @@ func (t *TreeServiceMock) GetBucketCORS(_ context.Context, bktInfo *data.BucketI
 	return node.OID, nil
 }
 
-func (t *TreeServiceMock) PutBucketCORS(_ context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
+func (t *TreeServiceMock) PutBucketCORS(_ context.Context, bktInfo *data.BucketInfo, objID oid.ID) ([]oid.ID, error) {
 	systemMap, ok := t.system[bktInfo.CID.EncodeToString()]
 	if !ok {
 		systemMap = make(map[string]*data.BaseNodeVersion)
@@ -136,10 +136,10 @@ func (t *TreeServiceMock) PutBucketCORS(_ context.Context, bktInfo *data.BucketI
 
 	t.system[bktInfo.CID.EncodeToString()] = systemMap
 
-	return oid.ID{}, ErrNoNodeToRemove
+	return nil, ErrNoNodeToRemove
 }
 
-func (t *TreeServiceMock) DeleteBucketCORS(context.Context, *data.BucketInfo) (oid.ID, error) {
+func (t *TreeServiceMock) DeleteBucketCORS(context.Context, *data.BucketInfo) ([]oid.ID, error) {
 	panic("implement me")
 }
 

api/layer/tree_service.go

@@ -25,13 +25,13 @@ type TreeService interface {
 
 	// PutBucketCORS puts a node to a system tree and returns objectID of a previous cors config which must be deleted in FrostFS.
 	//
-	// If object id to remove is not found returns ErrNoNodeToRemove error.
-	PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error)
+	// If object ids to remove is not found returns ErrNoNodeToRemove error.
+	PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) ([]oid.ID, error)
 
 	// DeleteBucketCORS removes a node from a system tree and returns objID which must be deleted in FrostFS.
 	//
-	// If object id to remove is not found returns ErrNoNodeToRemove error.
-	DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error)
+	// If object ids to remove is not found returns ErrNoNodeToRemove error.
+	DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.ID, error)
 
 	GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error)
 	PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion, tagSet map[string]string) error

go.mod

@@ -3,10 +3,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-s3-gw
 go 1.21
 
 require (
-	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164
+	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e
 	git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e
 	git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6
-	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36
+	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240820094724-848446231082
 	git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a
 	git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
 	github.com/aws/aws-sdk-go v1.44.6
@@ -24,6 +24,7 @@ require (
 	github.com/spf13/viper v1.15.0
 	github.com/ssgreg/journald v1.0.0
 	github.com/stretchr/testify v1.9.0
+	github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4
 	github.com/urfave/cli/v2 v2.3.0
 	go.opentelemetry.io/otel v1.16.0
 	go.opentelemetry.io/otel/trace v1.16.0

go.sum

@@ -36,16 +36,16 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164 h1:XxvwQKJT/f16qS3df5PBQPRYKkhy0/A7zH6644QpKD0=
-git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240716113920-f517e3949164/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o=
+git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e h1:gEWT+70E/RvGkxtSv+PlyUN2vtJVymhQa1mypvrXukM=
+git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240726072425-3dfa2f4fd65e/go.mod h1:OBDSr+DqV1z4VDouoX3YMleNc4DPBVBWTG3WDT2PK1o=
 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e h1:kcBqZBiFIUBATUqEuvVigtkJJWQ2Gug/eYXn967o3M4=
 git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240621131249-49e5270f673e/go.mod h1:F/fe1OoIDKr5Bz99q4sriuHDuf3aZefZy9ZsCqEtgxc=
 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSVCB8JNSfPG7Uk4r2oSk=
 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36 h1:MV/vKJWLQT34RRbXYvkNKFYGNjL5bRNuCQMXkbC7fLI=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240718141740-ce8270568d36/go.mod h1:vluJ/+yQMcq8ZIZZSA7Te+JKClr0lgtRErjICvb8wto=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240820094724-848446231082 h1:FfjbUcouQkmvzsz7/cq0/5cxP2ivGjIwUXQPHliIYxU=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240820094724-848446231082/go.mod h1:DlJmgV4/qkFkx2ab+YWznlMijiF2yZHnrJswJOB7XGs=
 git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
 git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240611102930-ac965e8d176a h1:Bk1fB4cQASPKgAVGCdlBOEp5ohZfDxqK6fZM8eP+Emo=
@@ -321,6 +321,8 @@ github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8
 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
 github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs=
 github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM=
+github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4 h1:GpfJ7OdNjS7BFTVwNCUI9L4aCJOFRbr5fdHqjdhoYE8=
+github.com/trailofbits/go-fuzz-utils v0.0.0-20230413173806-58c38daa3cb4/go.mod h1:f3jBhpWvuZmue0HZK52GzRHJOYHYSILs/c8+K2S/J+o=
 github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
 github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=

internal/frostfs/frostfs.go

@@ -237,8 +237,12 @@ func (x *FrostFS) CreateObject(ctx context.Context, prm layer.PrmObjectCreate) (
 		prmPut.UseKey(prm.PrivateKey)
 	}
 
-	idObj, err := x.pool.PutObject(ctx, prmPut)
-	return idObj, handleObjectError("save object via connection pool", err)
+	res, err := x.pool.PutObject(ctx, prmPut)
+	if err = handleObjectError("save object via connection pool", err); err != nil {
+		return oid.ID{}, err
+	}
+
+	return res.ObjectID, nil
 }
 
 // wraps io.ReadCloser and transforms Read errors related to access violation

internal/frostfs/services/pool_wrapper.go

@@ -102,14 +102,18 @@ func (w *PoolWrapper) GetNodes(ctx context.Context, prm *tree.GetNodesParams) ([
 	return res, nil
 }
 
-func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32) ([]tree.NodeResponse, error) {
+func (w *PoolWrapper) GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]tree.NodeResponse, error) {
+	order := treepool.NoneOrder
+	if sort {
+		order = treepool.AscendingOrder
+	}
 	poolPrm := treepool.GetSubTreeParams{
 		CID:         bktInfo.CID,
 		TreeID:      treeID,
 		RootID:      rootID,
 		Depth:       depth,
 		BearerToken: getBearer(ctx, bktInfo),
-		Order:       treepool.AscendingOrder,
+		Order:       order,
 	}
 	if len(rootID) == 1 && rootID[0] == 0 {
 		// storage node interprets 'nil' value as []uint64{0}

internal/logs/logs.go

@@ -142,11 +142,13 @@ const (
 	CouldntCacheSubject                                  = "couldn't cache subject info"
 	UserGroupsListIsEmpty                                = "user groups list is empty, subject not found"
 	CouldntCacheUserKey                                  = "couldn't cache user key"
-	FoundSeveralBucketCorsNodes                          = "found several bucket cors nodes, latest be used"
-	FoundSeveralObjectTaggingNodes                       = "found several object tagging nodes, latest be used"
-	FoundSeveralBucketTaggingNodes                       = "found several bucket tagging nodes, latest be used"
-	FoundSeveralBucketSettingsNodes                      = "found several bucket settings nodes, latest be used"
+	ObjectTaggingNodeHasMultipleIDs                      = "object tagging node has multiple ids"
+	BucketTaggingNodeHasMultipleIDs                      = "bucket tagging node has multiple ids"
+	BucketSettingsNodeHasMultipleIDs                     = "bucket settings node has multiple ids"
+	BucketCORSNodeHasMultipleIDs                         = "bucket cors node has multiple ids"
+	SystemNodeHasMultipleIDs                             = "system node has multiple ids"
+	FailedToRemoveOldSystemNode                          = "failed to remove old system node"
 	UnexpectedMultiNodeIDsInSubTreeMultiParts            = "unexpected multi node ids in sub tree multi parts"
-	FoundSeveralSystemNodes                              = "found several system nodes, latest be used"
+	FoundSeveralSystemNodes                              = "found several system nodes"
 	FailedToParsePartInfo                                = "failed to parse part info"
 )

pkg/service/tree/tree.go

@@ -32,7 +32,7 @@ type (
 	// Each method must return ErrNodeNotFound or ErrNodeAccessDenied if relevant.
 	ServiceClient interface {
 		GetNodes(ctx context.Context, p *GetNodesParams) ([]NodeResponse, error)
-		GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32) ([]NodeResponse, error)
+		GetSubTree(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]NodeResponse, error)
 		GetSubTreeStream(ctx context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32) (SubTreeStream, error)
 		AddNode(ctx context.Context, bktInfo *data.BucketInfo, treeID string, parent uint64, meta map[string]string) (uint64, error)
 		AddNodeByPath(ctx context.Context, bktInfo *data.BucketInfo, treeID string, path []string, meta map[string]string) (uint64, error)
@@ -53,6 +53,11 @@ type (
 		Meta      map[string]string
 	}
 
+	multiSystemNode struct {
+		// the first element is latest
+		nodes []*treeNode
+	}
+
 	GetNodesParams struct {
 		BktInfo    *data.BucketInfo
 		TreeID     string
@@ -268,6 +273,45 @@ func newNodeVersionFromTreeNode(log *zap.Logger, filePath string, treeNode *tree
 	return version, nil
 }
 
+func newMultiNode(nodes []NodeResponse) (*multiSystemNode, error) {
+	var (
+		err          error
+		index        int
+		maxTimestamp uint64
+	)
+
+	if len(nodes) == 0 {
+		return nil, errors.New("multi node must have at least one node")
+	}
+
+	treeNodes := make([]*treeNode, len(nodes))
+
+	for i, node := range nodes {
+		if treeNodes[i], err = newTreeNode(node); err != nil {
+			return nil, fmt.Errorf("parse system node response: %w", err)
+		}
+
+		if timestamp := getMaxTimestamp(node); timestamp > maxTimestamp {
+			index = i
+			maxTimestamp = timestamp
+		}
+	}
+
+	treeNodes[0], treeNodes[index] = treeNodes[index], treeNodes[0]
+
+	return &multiSystemNode{
+		nodes: treeNodes,
+	}, nil
+}
+
+func (m *multiSystemNode) Latest() *treeNode {
+	return m.nodes[0]
+}
+
+func (m *multiSystemNode) Old() []*treeNode {
+	return m.nodes[1:]
+}
+
 func newMultipartInfoFromTreeNode(log *zap.Logger, filePath string, treeNode *treeNode) (*data.MultipartInfo, error) {
 	uploadID, _ := treeNode.Get(uploadIDKV)
 	if uploadID == "" {
@@ -394,11 +438,13 @@ func newPartInfo(node NodeResponse) (*data.PartInfo, error) {
 }
 
 func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*data.BucketSettings, error) {
-	node, err := c.getSystemNode(ctx, bktInfo, []string{settingsFileName})
+	multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't get node: %w", err)
 	}
 
+	node := multiNode.Latest()
+
 	settings := &data.BucketSettings{Versioning: data.VersioningUnversioned}
 	if versioningValue, ok := node.Get(versioningKV); ok {
 		settings.Versioning = versioningValue
@@ -422,7 +468,7 @@ func (c *Tree) GetSettingsNode(ctx context.Context, bktInfo *data.BucketInfo) (*
 }
 
 func (c *Tree) PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, settings *data.BucketSettings) error {
-	node, err := c.getSystemNode(ctx, bktInfo, []string{settingsFileName})
+	multiNode, err := c.getSystemNode(ctx, bktInfo, settingsFileName)
 	isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
 	if err != nil && !isErrNotFound {
 		return fmt.Errorf("couldn't get node: %w", err)
@@ -435,28 +481,35 @@ func (c *Tree) PutSettingsNode(ctx context.Context, bktInfo *data.BucketInfo, se
 		return err
 	}
 
-	ind := node.GetLatestNodeIndex()
-	if node.IsSplit() {
-		c.reqLogger(ctx).Warn(logs.FoundSeveralBucketSettingsNodes)
+	latest := multiNode.Latest()
+	ind := latest.GetLatestNodeIndex()
+	if latest.IsSplit() {
+		c.reqLogger(ctx).Error(logs.BucketSettingsNodeHasMultipleIDs, zap.Uint64s("ids", latest.ID))
 	}
 
-	return c.service.MoveNode(ctx, bktInfo, systemTree, node.ID[ind], 0, meta)
+	if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
+		return fmt.Errorf("move settings node: %w", err)
+	}
+
+	c.cleanOldNodes(ctx, multiNode.Old(), bktInfo)
+
+	return nil
 }
 
 func (c *Tree) GetBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
-	node, err := c.getSystemNode(ctx, bktInfo, []string{corsFilename})
+	node, err := c.getSystemNode(ctx, bktInfo, corsFilename)
 	if err != nil {
 		return oid.ID{}, err
 	}
 
-	return node.ObjID, nil
+	return node.Latest().ObjID, nil
 }
 
-func (c *Tree) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) (oid.ID, error) {
-	node, err := c.getSystemNode(ctx, bktInfo, []string{corsFilename})
+func (c *Tree) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objID oid.ID) ([]oid.ID, error) {
+	multiNode, err := c.getSystemNode(ctx, bktInfo, corsFilename)
 	isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
 	if err != nil && !isErrNotFound {
-		return oid.ID{}, fmt.Errorf("couldn't get node: %w", err)
+		return nil, fmt.Errorf("couldn't get node: %w", err)
 	}
 
 	meta := make(map[string]string)
@@ -465,35 +518,64 @@ func (c *Tree) PutBucketCORS(ctx context.Context, bktInfo *data.BucketInfo, objI
 
 	if isErrNotFound {
 		if _, err = c.service.AddNode(ctx, bktInfo, systemTree, 0, meta); err != nil {
-			return oid.ID{}, err
+			return nil, err
 		}
-		return oid.ID{}, layer.ErrNoNodeToRemove
+		return nil, layer.ErrNoNodeToRemove
 	}
 
-	ind := node.GetLatestNodeIndex()
-	if node.IsSplit() {
-		c.reqLogger(ctx).Warn(logs.FoundSeveralBucketCorsNodes)
+	latest := multiNode.Latest()
+	ind := latest.GetLatestNodeIndex()
+	if latest.IsSplit() {
+		c.reqLogger(ctx).Error(logs.BucketCORSNodeHasMultipleIDs)
 	}
 
-	return node.ObjID, c.service.MoveNode(ctx, bktInfo, systemTree, node.ID[ind], 0, meta)
+	if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, meta); err != nil {
+		return nil, fmt.Errorf("move cors node: %w", err)
+	}
+
+	objToDelete := make([]oid.ID, 1, len(multiNode.nodes))
+	objToDelete[0] = latest.ObjID
+
+	objToDelete = append(objToDelete, c.cleanOldNodes(ctx, multiNode.Old(), bktInfo)...)
+
+	return objToDelete, nil
 }
 
-func (c *Tree) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) (oid.ID, error) {
-	node, err := c.getSystemNode(ctx, bktInfo, []string{corsFilename})
-	if err != nil && !errors.Is(err, layer.ErrNodeNotFound) {
-		return oid.ID{}, err
+func (c *Tree) DeleteBucketCORS(ctx context.Context, bktInfo *data.BucketInfo) ([]oid.ID, error) {
+	multiNode, err := c.getSystemNode(ctx, bktInfo, corsFilename)
+	isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
+	if err != nil && !isErrNotFound {
+		return nil, err
 	}
 
-	if node != nil {
+	if isErrNotFound {
+		return nil, layer.ErrNoNodeToRemove
+	}
+
+	objToDelete := c.cleanOldNodes(ctx, multiNode.nodes, bktInfo)
+	if len(objToDelete) != len(multiNode.nodes) {
+		return nil, fmt.Errorf("clean old cors nodes: %w", err)
+	}
+
+	return objToDelete, nil
+}
+
+func (c *Tree) cleanOldNodes(ctx context.Context, nodes []*treeNode, bktInfo *data.BucketInfo) []oid.ID {
+	res := make([]oid.ID, 0, len(nodes))
+
+	for _, node := range nodes {
 		ind := node.GetLatestNodeIndex()
 		if node.IsSplit() {
-			c.reqLogger(ctx).Warn(logs.FoundSeveralBucketCorsNodes)
+			c.reqLogger(ctx).Error(logs.SystemNodeHasMultipleIDs, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64s("ids", node.ID))
+		}
+		if err := c.service.RemoveNode(ctx, bktInfo, systemTree, node.ID[ind]); err != nil {
+			c.reqLogger(ctx).Warn(logs.FailedToRemoveOldSystemNode, zap.String("FileName", node.Meta[FileNameKey]), zap.Uint64("id", node.ID[ind]))
+		} else {
+			res = append(res, node.ObjID)
 		}
-
-		return node.ObjID, c.service.RemoveNode(ctx, bktInfo, systemTree, node.ID[ind])
 	}
 
-	return oid.ID{}, layer.ErrNoNodeToRemove
+	return res
 }
 
 func (c *Tree) GetObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, error) {
@@ -541,7 +623,7 @@ func (c *Tree) PutObjectTagging(ctx context.Context, bktInfo *data.BucketInfo, o
 
 	ind := tagNode.GetLatestNodeIndex()
 	if tagNode.IsSplit() {
-		c.reqLogger(ctx).Warn(logs.FoundSeveralObjectTaggingNodes)
+		c.reqLogger(ctx).Error(logs.ObjectTaggingNodeHasMultipleIDs)
 	}
 
 	return c.service.MoveNode(ctx, bktInfo, versionTree, tagNode.ID[ind], objVersion.ID, treeTagSet)
@@ -552,14 +634,14 @@ func (c *Tree) DeleteObjectTagging(ctx context.Context, bktInfo *data.BucketInfo
 }
 
 func (c *Tree) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (map[string]string, error) {
-	node, err := c.getSystemNode(ctx, bktInfo, []string{bucketTaggingFilename})
+	multiNode, err := c.getSystemNode(ctx, bktInfo, bucketTaggingFilename)
 	if err != nil {
 		return nil, err
 	}
 
 	tags := make(map[string]string)
 
-	for key, val := range node.Meta {
+	for key, val := range multiNode.Latest().Meta {
 		if strings.HasPrefix(key, userDefinedTagPrefix) {
 			tags[strings.TrimPrefix(key, userDefinedTagPrefix)] = val
 		}
@@ -569,7 +651,7 @@ func (c *Tree) GetBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) (
 }
 
 func (c *Tree) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, tagSet map[string]string) error {
-	node, err := c.getSystemNode(ctx, bktInfo, []string{bucketTaggingFilename})
+	multiNode, err := c.getSystemNode(ctx, bktInfo, bucketTaggingFilename)
 	isErrNotFound := errors.Is(err, layer.ErrNodeNotFound)
 	if err != nil && !isErrNotFound {
 		return fmt.Errorf("couldn't get node: %w", err)
@@ -587,12 +669,19 @@ func (c *Tree) PutBucketTagging(ctx context.Context, bktInfo *data.BucketInfo, t
 		return err
 	}
 
-	ind := node.GetLatestNodeIndex()
-	if node.IsSplit() {
-		c.reqLogger(ctx).Warn(logs.FoundSeveralBucketTaggingNodes)
+	latest := multiNode.Latest()
+	ind := latest.GetLatestNodeIndex()
+	if latest.IsSplit() {
+		c.reqLogger(ctx).Error(logs.BucketTaggingNodeHasMultipleIDs, zap.Uint64s("ids", latest.ID))
 	}
 
-	return c.service.MoveNode(ctx, bktInfo, systemTree, node.ID[ind], 0, treeTagSet)
+	if err = c.service.MoveNode(ctx, bktInfo, systemTree, latest.ID[ind], 0, treeTagSet); err != nil {
+		return fmt.Errorf("move bucket tagging node: %w", err)
+	}
+
+	c.cleanOldNodes(ctx, multiNode.Old(), bktInfo)
+
+	return nil
 }
 
 func (c *Tree) DeleteBucketTagging(ctx context.Context, bktInfo *data.BucketInfo) error {
@@ -610,11 +699,13 @@ func (c *Tree) getTreeNode(ctx context.Context, bktInfo *data.BucketInfo, nodeID
 }
 
 func (c *Tree) getTreeNodes(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, keys ...string) (map[string]*treeNode, error) {
-	subtree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, []uint64{nodeID}, 2)
+	subtree, err := c.service.GetSubTree(ctx, bktInfo, versionTree, []uint64{nodeID}, 2, false)
 	if err != nil {
 		return nil, err
 	}
 
+	// consider using map[string][]*treeNode
+	// to be able to remove unused node, that can be added during split
 	treeNodes := make(map[string]*treeNode, len(keys))
 
 	for _, s := range subtree {
@@ -689,26 +780,6 @@ func getLatestVersionNode(nodes []NodeResponse) (NodeResponse, error) {
 	return nodes[targetIndexNode], nil
 }
 
-func getLatestNode(nodes []NodeResponse) NodeResponse {
-	if len(nodes) == 0 {
-		return nil
-	}
-
-	var (
-		index        int
-		maxTimestamp uint64
-	)
-
-	for i, node := range nodes {
-		if timestamp := getMaxTimestamp(node); timestamp > maxTimestamp {
-			index = i
-			maxTimestamp = timestamp
-		}
-	}
-
-	return nodes[index]
-}
-
 func getMaxTimestamp(node NodeResponse) uint64 {
 	var maxTimestamp uint64
 
@@ -1022,7 +1093,7 @@ func (c *Tree) getSubTreeByPrefix(ctx context.Context, bktInfo *data.BucketInfo,
 		return nil, "", err
 	}
 
-	subTree, err := c.service.GetSubTree(ctx, bktInfo, treeID, rootID, 2)
+	subTree, err := c.service.GetSubTree(ctx, bktInfo, treeID, rootID, 2, false)
 	if err != nil {
 		if errors.Is(err, layer.ErrNodeNotFound) {
 			return nil, "", nil
@@ -1180,7 +1251,11 @@ func (c *Tree) GetMultipartUploadsByPrefix(ctx context.Context, bktInfo *data.Bu
 }
 
 func (c *Tree) getSubTreeMultipartUploads(ctx context.Context, bktInfo *data.BucketInfo, nodeID []uint64, parentFilePath string) ([]*data.MultipartInfo, error) {
-	subTree, err := c.service.GetSubTree(ctx, bktInfo, systemTree, nodeID, maxGetSubTreeDepth)
+	// sorting in getSubTree leads to skipping nodes that doesn't have FileName attribute
+	// so when we are only interested in multipart nodes, we can set this flag
+	// (despite we sort multiparts in above layer anyway)
+	// to skip its children (parts) that don't have FileName
+	subTree, err := c.service.GetSubTree(ctx, bktInfo, systemTree, nodeID, maxGetSubTreeDepth, true)
 	if err != nil {
 		return nil, err
 	}
@@ -1269,7 +1344,7 @@ func (c *Tree) GetMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo,
 }
 
 func (c *Tree) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, info *data.PartInfo) (oldObjIDToDelete oid.ID, err error) {
-	parts, err := c.service.GetSubTree(ctx, bktInfo, systemTree, []uint64{multipartNodeID}, 2)
+	parts, err := c.service.GetSubTree(ctx, bktInfo, systemTree, []uint64{multipartNodeID}, 2, false)
 	if err != nil {
 		return oid.ID{}, err
 	}
@@ -1319,7 +1394,7 @@ func (c *Tree) AddPart(ctx context.Context, bktInfo *data.BucketInfo, multipartN
 }
 
 func (c *Tree) GetParts(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) ([]*data.PartInfo, error) {
-	parts, err := c.service.GetSubTree(ctx, bktInfo, systemTree, []uint64{multipartNodeID}, 2)
+	parts, err := c.service.GetSubTree(ctx, bktInfo, systemTree, []uint64{multipartNodeID}, 2, false)
 	if err != nil {
 		return nil, err
 	}
@@ -1558,11 +1633,11 @@ func metaFromMultipart(info *data.MultipartInfo, fileName string) map[string]str
 	return info.Meta
 }
 
-func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, path []string) (*treeNode, error) {
+func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, name string) (*multiSystemNode, error) {
 	p := &GetNodesParams{
 		BktInfo:    bktInfo,
 		TreeID:     systemTree,
-		Path:       path,
+		Path:       []string{name},
 		LatestOnly: false,
 		AllAttrs:   true,
 	}
@@ -1577,10 +1652,10 @@ func (c *Tree) getSystemNode(ctx context.Context, bktInfo *data.BucketInfo, path
 		return nil, layer.ErrNodeNotFound
 	}
 	if len(nodes) != 1 {
-		c.reqLogger(ctx).Warn(logs.FoundSeveralSystemNodes, zap.String("path", strings.Join(path, "/")))
+		c.reqLogger(ctx).Warn(logs.FoundSeveralSystemNodes, zap.String("name", name))
 	}
 
-	return newTreeNode(getLatestNode(nodes))
+	return newMultiNode(nodes)
 }
 
 func filterMultipartNodes(nodes []NodeResponse) []NodeResponse {

pkg/service/tree/tree_client_in_memory.go

@@ -234,7 +234,7 @@ func (c *ServiceClientMemory) GetNodes(_ context.Context, p *GetNodesParams) ([]
 	return res2, nil
 }
 
-func (c *ServiceClientMemory) GetSubTree(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32) ([]NodeResponse, error) {
+func (c *ServiceClientMemory) GetSubTree(_ context.Context, bktInfo *data.BucketInfo, treeID string, rootID []uint64, depth uint32, sort bool) ([]NodeResponse, error) {
 	cnr, ok := c.containers[bktInfo.CID.EncodeToString()]
 	if !ok {
 		return nil, nil
@@ -254,6 +254,10 @@ func (c *ServiceClientMemory) GetSubTree(_ context.Context, bktInfo *data.Bucket
 		return nil, ErrNodeNotFound
 	}
 
+	if sort {
+		sortNode(tr.treeData)
+	}
+
 	// we depth-1 in case of uint32 and 0 as mark to get all subtree leads to overflow and depth is getting quite big to walk all tree levels
 	return node.listNodes(nil, depth-1), nil
 }