Release v0.29.1

Signed-off-by: Alex Vanin <a.vanin@yadro.com>

CHANGELOG.md

@@ -4,6 +4,14 @@ This document outlines major changes between releases.
 
 ## [Unreleased]
 
+## [0.29.1] - 2024-06-20
+
+### Fixed
+- OPTIONS request processing for object operations (#399)
+
+### Added
+- Retries of put-bucket-setting operation during container creation (#398)
+
 ## [0.29.0] - Zemu - 2024-05-27
 
 ### Fixed
@@ -175,4 +183,5 @@ To see CHANGELOG for older versions, refer to https://github.com/nspcc-dev/neofs
 [0.28.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.28.0...v0.28.1
 [0.28.2]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.28.1...v0.28.2
 [0.29.0]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.28.2...v0.29.0
-[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.0...master
+[0.29.1]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.0...v0.29.1
+[Unreleased]: https://git.frostfs.info/TrueCloudLab/frostfs-s3-gw/compare/v0.29.1...master

VERSION

@@ -1 +1 @@
-v0.29.0
+v0.29.1

api/handler/api.go

@@ -47,6 +47,9 @@ type (
 		BypassContentEncodingInChunks() bool
 		MD5Enabled() bool
 		ACLEnabled() bool
+		RetryMaxAttempts() int
+		RetryMaxBackoff() time.Duration
+		RetryStrategy() RetryStrategy
 	}
 
 	FrostFSID interface {
@@ -63,6 +66,13 @@ type (
 	}
 )
 
+type RetryStrategy string
+
+const (
+	RetryStrategyExponential = "exponential"
+	RetryStrategyConstant    = "constant"
+)
+
 var _ api.Handler = (*handler)(nil)
 
 // New creates new api.Handler using given logger and client.

api/handler/cors.go

@@ -187,8 +187,8 @@ func (h *handler) Preflight(w http.ResponseWriter, r *http.Request) {
 						if !checkSubslice(rule.AllowedHeaders, headers) {
 							continue
 						}
-						w.Header().Set(api.AccessControlAllowOrigin, o)
+						w.Header().Set(api.AccessControlAllowOrigin, origin)
-						w.Header().Set(api.AccessControlAllowMethods, strings.Join(rule.AllowedMethods, ", "))
+						w.Header().Set(api.AccessControlAllowMethods, method)
 						if headers != nil {
 							w.Header().Set(api.AccessControlAllowHeaders, requestHeaders)
 						}

api/handler/cors_test.go

@@ -7,6 +7,7 @@ import (
 
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
+	"github.com/stretchr/testify/require"
 )
 
 func TestCORSOriginWildcard(t *testing.T) {
@@ -39,3 +40,181 @@ func TestCORSOriginWildcard(t *testing.T) {
 	hc.Handler().GetBucketCorsHandler(w, r)
 	assertStatus(t, w, http.StatusOK)
 }
+
+func TestPreflight(t *testing.T) {
+	body := `
+<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+	<CORSRule>
+		<AllowedMethod>GET</AllowedMethod>
+		<AllowedOrigin>http://www.example.com</AllowedOrigin>
+		<AllowedHeader>Authorization</AllowedHeader>
+		<ExposeHeader>x-amz-*</ExposeHeader>
+		<ExposeHeader>X-Amz-*</ExposeHeader>
+		<MaxAgeSeconds>600</MaxAgeSeconds>
+	</CORSRule>
+</CORSConfiguration>
+`
+	hc := prepareHandlerContext(t)
+
+	bktName := "bucket-preflight-test"
+	box, _ := createAccessBox(t)
+	w, r := prepareTestRequest(hc, bktName, "", nil)
+	ctx := middleware.SetBoxData(r.Context(), box)
+	r = r.WithContext(ctx)
+	hc.Handler().CreateBucketHandler(w, r)
+	assertStatus(t, w, http.StatusOK)
+
+	w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
+	ctx = middleware.SetBoxData(r.Context(), box)
+	r = r.WithContext(ctx)
+	hc.Handler().PutBucketCorsHandler(w, r)
+	assertStatus(t, w, http.StatusOK)
+
+	for _, tc := range []struct {
+		name           string
+		origin         string
+		method         string
+		headers        string
+		expectedStatus int
+	}{
+		{
+			name:           "Valid",
+			origin:         "http://www.example.com",
+			method:         "GET",
+			headers:        "Authorization",
+			expectedStatus: http.StatusOK,
+		},
+		{
+			name:           "Empty origin",
+			method:         "GET",
+			headers:        "Authorization",
+			expectedStatus: http.StatusBadRequest,
+		},
+		{
+			name:           "Empty request method",
+			origin:         "http://www.example.com",
+			headers:        "Authorization",
+			expectedStatus: http.StatusBadRequest,
+		},
+		{
+			name:           "Not allowed method",
+			origin:         "http://www.example.com",
+			method:         "PUT",
+			headers:        "Authorization",
+			expectedStatus: http.StatusForbidden,
+		},
+		{
+			name:           "Not allowed headers",
+			origin:         "http://www.example.com",
+			method:         "GET",
+			headers:        "Authorization, Last-Modified",
+			expectedStatus: http.StatusForbidden,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
+			r.Header.Set(api.Origin, tc.origin)
+			r.Header.Set(api.AccessControlRequestMethod, tc.method)
+			r.Header.Set(api.AccessControlRequestHeaders, tc.headers)
+			hc.Handler().Preflight(w, r)
+			assertStatus(t, w, tc.expectedStatus)
+
+			if tc.expectedStatus == http.StatusOK {
+				require.Equal(t, tc.origin, w.Header().Get(api.AccessControlAllowOrigin))
+				require.Equal(t, tc.method, w.Header().Get(api.AccessControlAllowMethods))
+				require.Equal(t, tc.headers, w.Header().Get(api.AccessControlAllowHeaders))
+				require.Equal(t, "x-amz-*, X-Amz-*", w.Header().Get(api.AccessControlExposeHeaders))
+				require.Equal(t, "true", w.Header().Get(api.AccessControlAllowCredentials))
+				require.Equal(t, "600", w.Header().Get(api.AccessControlMaxAge))
+			}
+		})
+	}
+}
+
+func TestPreflightWildcardOrigin(t *testing.T) {
+	body := `
+<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+	<CORSRule>
+		<AllowedMethod>GET</AllowedMethod>
+		<AllowedMethod>PUT</AllowedMethod>
+		<AllowedOrigin>*</AllowedOrigin>
+		<AllowedHeader>*</AllowedHeader>
+	</CORSRule>
+</CORSConfiguration>
+`
+	hc := prepareHandlerContext(t)
+
+	bktName := "bucket-preflight-wildcard-test"
+	box, _ := createAccessBox(t)
+	w, r := prepareTestRequest(hc, bktName, "", nil)
+	ctx := middleware.SetBoxData(r.Context(), box)
+	r = r.WithContext(ctx)
+	hc.Handler().CreateBucketHandler(w, r)
+	assertStatus(t, w, http.StatusOK)
+
+	w, r = prepareTestPayloadRequest(hc, bktName, "", strings.NewReader(body))
+	ctx = middleware.SetBoxData(r.Context(), box)
+	r = r.WithContext(ctx)
+	hc.Handler().PutBucketCorsHandler(w, r)
+	assertStatus(t, w, http.StatusOK)
+
+	for _, tc := range []struct {
+		name           string
+		origin         string
+		method         string
+		headers        string
+		expectedStatus int
+	}{
+		{
+			name:           "Valid get",
+			origin:         "http://www.example.com",
+			method:         "GET",
+			headers:        "Authorization, Last-Modified",
+			expectedStatus: http.StatusOK,
+		},
+		{
+			name:           "Valid put",
+			origin:         "http://example.com",
+			method:         "PUT",
+			headers:        "Authorization, Content-Type",
+			expectedStatus: http.StatusOK,
+		},
+		{
+			name:           "Empty origin",
+			method:         "GET",
+			headers:        "Authorization, Last-Modified",
+			expectedStatus: http.StatusBadRequest,
+		},
+		{
+			name:           "Empty request method",
+			origin:         "http://www.example.com",
+			headers:        "Authorization, Last-Modified",
+			expectedStatus: http.StatusBadRequest,
+		},
+		{
+			name:           "Not allowed method",
+			origin:         "http://www.example.com",
+			method:         "DELETE",
+			headers:        "Authorization, Last-Modified",
+			expectedStatus: http.StatusForbidden,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			w, r = prepareTestPayloadRequest(hc, bktName, "", nil)
+			r.Header.Set(api.Origin, tc.origin)
+			r.Header.Set(api.AccessControlRequestMethod, tc.method)
+			r.Header.Set(api.AccessControlRequestHeaders, tc.headers)
+			hc.Handler().Preflight(w, r)
+			assertStatus(t, w, tc.expectedStatus)
+
+			if tc.expectedStatus == http.StatusOK {
+				require.Equal(t, tc.origin, w.Header().Get(api.AccessControlAllowOrigin))
+				require.Equal(t, tc.method, w.Header().Get(api.AccessControlAllowMethods))
+				require.Equal(t, tc.headers, w.Header().Get(api.AccessControlAllowHeaders))
+				require.Empty(t, w.Header().Get(api.AccessControlExposeHeaders))
+				require.Empty(t, w.Header().Get(api.AccessControlAllowCredentials))
+				require.Equal(t, "0", w.Header().Get(api.AccessControlMaxAge))
+			}
+		})
+	}
+}

api/handler/handlers_test.go

@@ -130,6 +130,18 @@ func (c *configMock) ResolveNamespaceAlias(ns string) string {
 	return ns
 }
 
+func (c *configMock) RetryMaxAttempts() int {
+	return 1
+}
+
+func (c *configMock) RetryMaxBackoff() time.Duration {
+	return 0
+}
+
+func (c *configMock) RetryStrategy() RetryStrategy {
+	return RetryStrategyConstant
+}
+
 func prepareHandlerContext(t *testing.T) *handlerContext {
 	return prepareHandlerContextBase(t, layer.DefaultCachesConfigs(zap.NewExample()))
 }

api/handler/put.go

@@ -26,12 +26,16 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/retryer"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/eacl"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
 	"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
 	"git.frostfs.info/TrueCloudLab/policy-engine/schema/native"
 	"git.frostfs.info/TrueCloudLab/policy-engine/schema/s3"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
 	"go.uber.org/zap"
 )
@@ -916,7 +920,10 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
 		sp.Settings.Versioning = data.VersioningEnabled
 	}
 
-	if err = h.obj.PutBucketSettings(ctx, sp); err != nil {
+	err = retryer.MakeWithRetry(ctx, func() error {
+		return h.obj.PutBucketSettings(ctx, sp)
+	}, h.putBucketSettingsRetryer())
+	if err != nil {
 		h.logAndSendError(w, "couldn't save bucket settings", reqInfo, err,
 			zap.String("container_id", bktInfo.CID.EncodeToString()))
 		return
@@ -928,6 +935,27 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
 	}
 }
 
+func (h *handler) putBucketSettingsRetryer() aws.RetryerV2 {
+	return retry.NewStandard(func(options *retry.StandardOptions) {
+		options.MaxAttempts = h.cfg.RetryMaxAttempts()
+		options.MaxBackoff = h.cfg.RetryMaxBackoff()
+		if h.cfg.RetryStrategy() == RetryStrategyConstant {
+			options.Backoff = retry.NewExponentialJitterBackoff(options.MaxBackoff)
+		} else {
+			options.Backoff = retry.BackoffDelayerFunc(func(int, error) (time.Duration, error) {
+				return options.MaxBackoff, nil
+			})
+		}
+
+		options.Retryables = []retry.IsErrorRetryable{retry.IsErrorRetryableFunc(func(err error) aws.Ternary {
+			if stderrors.Is(err, tree.ErrNodeAccessDenied) {
+				return aws.TrueTernary
+			}
+			return aws.FalseTernary
+		})}
+	})
+}
+
 func (h *handler) createBucketHandlerACL(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	reqInfo := middleware.GetReqInfo(ctx)

api/middleware/constants.go

@@ -5,7 +5,7 @@ const (
 
 	// bucket operations.
 
-	OptionsOperation                   = "Options"
+	OptionsBucketOperation             = "OptionsBucket"
 	HeadBucketOperation                = "HeadBucket"
 	ListMultipartUploadsOperation      = "ListMultipartUploads"
 	GetBucketLocationOperation         = "GetBucketLocation"
@@ -50,6 +50,7 @@ const (
 
 	// object operations.
 
+	OptionsObjectOperation           = "OptionsObject"
 	HeadObjectOperation              = "HeadObject"
 	ListPartsOperation               = "ListParts"
 	GetObjectACLOperation            = "GetObjectACL"

api/middleware/metrics.go

@@ -103,7 +103,7 @@ func stats(f http.HandlerFunc, resolveCID cidResolveFunc, appMetrics *metrics.Ap
 
 func requestTypeFromAPI(api string) metrics.RequestType {
 	switch api {
-	case OptionsOperation, HeadObjectOperation, HeadBucketOperation:
+	case OptionsBucketOperation, OptionsObjectOperation, HeadObjectOperation, HeadBucketOperation:
 		return metrics.HEADRequest
 	case CreateMultipartUploadOperation, UploadPartCopyOperation, UploadPartOperation, CompleteMultipartUploadOperation,
 		PutObjectACLOperation, PutObjectTaggingOperation, CopyObjectOperation, PutObjectRetentionOperation, PutObjectLegalHoldOperation,

api/middleware/policy.go

@@ -215,7 +215,7 @@ func determineBucketOperation(r *http.Request) string {
 	query := r.URL.Query()
 	switch r.Method {
 	case http.MethodOptions:
-		return OptionsOperation
+		return OptionsBucketOperation
 	case http.MethodHead:
 		return HeadBucketOperation
 	case http.MethodGet:
@@ -318,6 +318,8 @@ func determineBucketOperation(r *http.Request) string {
 func determineObjectOperation(r *http.Request) string {
 	query := r.URL.Query()
 	switch r.Method {
+	case http.MethodOptions:
+		return OptionsObjectOperation
 	case http.MethodHead:
 		return HeadObjectOperation
 	case http.MethodGet:

api/router.go

@@ -217,7 +217,7 @@ func bucketRouter(h Handler, log *zap.Logger) chi.Router {
 
 	bktRouter.Mount("/", objectRouter(h, log))
 
-	bktRouter.Options("/", h.Preflight)
+	bktRouter.Options("/", named(s3middleware.OptionsBucketOperation, h.Preflight))
 
 	bktRouter.Head("/", named(s3middleware.HeadBucketOperation, h.HeadBucketHandler))
 
@@ -363,6 +363,8 @@ func objectRouter(h Handler, l *zap.Logger) chi.Router {
 	objRouter := chi.NewRouter()
 	objRouter.Use(s3middleware.AddObjectName(l))
 
+	objRouter.Options("/*", named(s3middleware.OptionsObjectOperation, h.Preflight))
+
 	objRouter.Head("/*", named(s3middleware.HeadObjectOperation, h.HeadObjectHandler))
 
 	// GET method handlers

cmd/s3-gw/app.go

@@ -106,6 +106,9 @@ type (
 		defaultNamespaces             []string
 		authorizedControlAPIKeys      [][]byte
 		policyDenyByDefault           bool
+		retryMaxAttempts              int
+		retryMaxBackoff               time.Duration
+		retryStrategy                 handler.RetryStrategy
 	}
 
 	maxClientsConfig struct {
@@ -231,6 +234,9 @@ func (s *appSettings) update(v *viper.Viper, log *zap.Logger, key *keys.PrivateK
 	s.setMD5Enabled(v.GetBool(cfgMD5Enabled))
 	s.setAuthorizedControlAPIKeys(append(fetchAuthorizedKeys(log, v), key.PublicKey()))
 	s.setPolicyDenyByDefault(v.GetBool(cfgPolicyDenyByDefault))
+	s.setRetryMaxAttempts(fetchRetryMaxAttempts(v))
+	s.setRetryMaxBackoff(fetchRetryMaxBackoff(v))
+	s.setRetryStrategy(fetchRetryStrategy(v))
 }
 
 func (s *appSettings) updateNamespacesSettings(v *viper.Viper, log *zap.Logger) {
@@ -425,6 +431,42 @@ func (s *appSettings) setPolicyDenyByDefault(policyDenyByDefault bool) {
 	s.mu.Unlock()
 }
 
+func (s *appSettings) setRetryMaxAttempts(maxAttempts int) {
+	s.mu.Lock()
+	s.retryMaxAttempts = maxAttempts
+	s.mu.Unlock()
+}
+
+func (s *appSettings) RetryMaxAttempts() int {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.retryMaxAttempts
+}
+
+func (s *appSettings) setRetryMaxBackoff(maxBackoff time.Duration) {
+	s.mu.Lock()
+	s.retryMaxBackoff = maxBackoff
+	s.mu.Unlock()
+}
+
+func (s *appSettings) RetryMaxBackoff() time.Duration {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.retryMaxBackoff
+}
+
+func (s *appSettings) setRetryStrategy(strategy handler.RetryStrategy) {
+	s.mu.Lock()
+	s.retryStrategy = strategy
+	s.mu.Unlock()
+}
+
+func (s *appSettings) RetryStrategy() handler.RetryStrategy {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return s.retryStrategy
+}
+
 func (a *App) initAPI(ctx context.Context) {
 	a.initLayer(ctx)
 	a.initHandler()

cmd/s3-gw/app_settings.go

@@ -59,6 +59,10 @@ const (
 	defaultConstraintName = "default"
 
 	defaultNamespace = ""
+
+	defaultRetryMaxAttempts = 4
+	defaultRetryMaxBackoff  = 30 * time.Second
+	defaultRetryStrategy    = handler.RetryStrategyExponential
 )
 
 var (
@@ -176,6 +180,11 @@ const ( // Settings.
 	cfgWebWriteTimeout      = "web.write_timeout"
 	cfgWebIdleTimeout       = "web.idle_timeout"
 
+	// Retry.
+	cfgRetryMaxAttempts = "retry.max_attempts"
+	cfgRetryMaxBackoff  = "retry.max_backoff"
+	cfgRetryStrategy    = "retry.strategy"
+
 	// Namespaces.
 	cfgNamespacesConfig = "namespaces.config"
 
@@ -309,6 +318,33 @@ func fetchSoftMemoryLimit(cfg *viper.Viper) int64 {
 	return int64(softMemoryLimit)
 }
 
+func fetchRetryMaxAttempts(cfg *viper.Viper) int {
+	val := cfg.GetInt(cfgRetryMaxAttempts)
+	if val <= 0 {
+		val = defaultRetryMaxAttempts
+	}
+
+	return val
+}
+
+func fetchRetryMaxBackoff(cfg *viper.Viper) time.Duration {
+	val := cfg.GetDuration(cfgRetryMaxBackoff)
+	if val <= 0 {
+		val = defaultRetryMaxBackoff
+	}
+
+	return val
+}
+
+func fetchRetryStrategy(cfg *viper.Viper) handler.RetryStrategy {
+	val := handler.RetryStrategy(cfg.GetString(cfgRetryStrategy))
+	if val != handler.RetryStrategyExponential && val != handler.RetryStrategyConstant {
+		val = defaultRetryStrategy
+	}
+
+	return val
+}
+
 func fetchDefaultPolicy(l *zap.Logger, cfg *viper.Viper) netmap.PlacementPolicy {
 	var policy netmap.PlacementPolicy
 
@@ -737,6 +773,10 @@ func newSettings() *viper.Viper {
 	// resolve
 	v.SetDefault(cfgResolveNamespaceHeader, defaultNamespaceHeader)
 
+	// retry
+	v.SetDefault(cfgRetryMaxAttempts, defaultRetryMaxAttempts)
+	v.SetDefault(cfgRetryMaxBackoff, defaultRetryMaxBackoff)
+
 	// Bind flags
 	if err := bindFlags(v, flags); err != nil {
 		panic(fmt.Errorf("bind flags: %w", err))

config/config.env

@@ -217,3 +217,12 @@ S3_GW_PROXY_CONTRACT=proxy.frostfs
 
 # Namespaces configuration
 S3_GW_NAMESPACES_CONFIG=namespaces.json
+
+# Retry strategy configuration.
+# Max amount of request attempts. Currently only for updating bucket settings request.
+S3_GW_RETRY_MAX_ATTEMPTS=4
+# Max delay before next attempt.
+S3_GW_RETRY_MAX_BACKOFF=30s
+# Backoff strategy. `exponential` and `constant` are allowed.
+S3_GW_RETRY_STRATEGY=exponential
+

config/config.yaml

@@ -257,3 +257,12 @@ proxy:
 
 namespaces:
   config: namespaces.json
+
+# Retry strategy configuration.
+retry:
+  # Max amount of request attempts. Currently only for updating bucket settings request.
+  max_attempts: 4
+  # Max delay before next attempt.
+  max_backoff: 30s
+  # Backoff strategy. `exponential` and `constant` are allowed.
+  strategy: exponential

docs/configuration.md

@@ -193,6 +193,7 @@ There are some custom types used for brevity:
 | `policy`           | [Policy contract configuration](#policy-section)               |
 | `proxy`            | [Proxy contract configuration](#proxy-section)                 |
 | `namespaces`       | [Namespaces configuration](#namespaces-section)                |
+| `retry`            | [Retry configuration](#retry-section)                          |
 
 ### General section
 
@@ -735,3 +736,21 @@ To override config values for default namespaces use namespace names that are pr
   }
 }
 ```
+
+# `retry` section
+
+Retry strategy configuration.
+
+```yaml
+retry:
+  max_attempts: 4
+  max_backoff: 30s
+  strategy: exponential
+```
+
+| Parameter     | Type       | SIGHUP reload | Default value | Description                                                                          |
+|---------------|------------|---------------|---------------|--------------------------------------------------------------------------------------|
+| `max_attemps` | `int`      | yes           | `4`           | Max amount of request attempts. Currently only for updating bucket settings request. |
+| `max_backoff` | `duration` | yes           | `30s`         | Max delay before next attempt.                                                       |
+| `strategy`    | `string`   | yes           | `exponential` | Backoff strategy. `exponential` and `constant` are allowed.                          |
+

go.mod

@@ -6,10 +6,11 @@ require (
 	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240306101814-c1c7b344b9c0
 	git.frostfs.info/TrueCloudLab/frostfs-contract v0.19.3-0.20240409115729-6eb492025bdd
 	git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6
-	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240402141532-e5040d35e99d
+	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240620142203-8412b075a5be
 	git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a
 	git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02
 	github.com/aws/aws-sdk-go v1.44.6
+	github.com/aws/aws-sdk-go-v2 v1.18.1
 	github.com/bluele/gcache v0.0.2
 	github.com/go-chi/chi/v5 v5.0.8
 	github.com/google/uuid v1.3.1
@@ -40,6 +41,7 @@ require (
 	git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
 	git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
+	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect

go.sum

@@ -44,8 +44,8 @@ git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 h1:FxqFDhQYYgpe41qsIHVOcdzSV
 git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0/go.mod h1:RUIKZATQLJ+TaYQa60X2fTDwfuhMfm8Ar60bQ5fr+vU=
 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6 h1:aGQ6QaAnTerQ5Dq5b2/f9DUQtSqPkZZ/bkMx/HKuLCo=
 git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6/go.mod h1:W8Nn08/l6aQ7UlIbpF7FsQou7TVpcRD1ZT1KG4TrFhE=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240402141532-e5040d35e99d h1:MYlPcYV/8j3PP0Yv0t6kJwPwOnhHNTybHi05g+ofyw0=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240620142203-8412b075a5be h1:c8v5pw5VtW55Ue3gkvYLyGDXxjA4FTwkrQ8/IPCw0GU=
-git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240402141532-e5040d35e99d/go.mod h1:s2ISRBm7AjfTdfZ3aF6gQt9VXz+n8cwSZcRiaVUMVI0=
+git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20240620142203-8412b075a5be/go.mod h1:s2ISRBm7AjfTdfZ3aF6gQt9VXz+n8cwSZcRiaVUMVI0=
 git.frostfs.info/TrueCloudLab/hrw v1.2.1 h1:ccBRK21rFvY5R1WotI6LNoPlizk7qSvdfD8lNIRudVc=
 git.frostfs.info/TrueCloudLab/hrw v1.2.1/go.mod h1:C1Ygde2n843yTZEQ0FP69jYiuaYV0kriLvP4zm8JuvM=
 git.frostfs.info/TrueCloudLab/policy-engine v0.0.0-20240412130734-0e69e485115a h1:wbndKvHbwDQiSMQWL75RxiTZCeUyCi7NUj1lsfdAGkc=
@@ -64,6 +64,10 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8
 github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/aws/aws-sdk-go v1.44.6 h1:Y+uHxmZfhRTLX2X3khkdxCoTZAyGEX21aOUHe1U6geg=
 github.com/aws/aws-sdk-go v1.44.6/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo=
+github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
+github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
+github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c=
@@ -168,6 +172,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=

pkg/retryer/retryer.go

@@ -0,0 +1,52 @@
+package retryer
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+)
+
+func MakeWithRetry(ctx context.Context, fn func() error, retryer aws.RetryerV2) (err error) {
+	var attemptNum int
+
+	maxAttempts := retryer.MaxAttempts()
+
+	for {
+		attemptNum++
+
+		err = fn()
+
+		if !retryer.IsErrorRetryable(err) {
+			return err
+		}
+
+		if attemptNum >= maxAttempts {
+			return fmt.Errorf("max retry attempts exausted, max %d: %w", maxAttempts, err)
+		}
+
+		retryDelay, reqErr := retryer.RetryDelay(attemptNum, err)
+		if reqErr != nil {
+			return reqErr
+		}
+
+		if reqErr = sleepWithContext(ctx, retryDelay); reqErr != nil {
+			return reqErr
+		}
+	}
+}
+
+func sleepWithContext(ctx context.Context, dur time.Duration) error {
+	t := time.NewTimer(dur)
+	defer t.Stop()
+
+	select {
+	case <-t.C:
+		break
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+
+	return nil
+}

pkg/retryer/retryer_test.go

@@ -0,0 +1,59 @@
+package retryer
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/service/tree"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRetryer(t *testing.T) {
+	ctx := context.Background()
+
+	retryer := retry.NewStandard(func(options *retry.StandardOptions) {
+		options.MaxAttempts = 3
+		options.Backoff = retry.BackoffDelayerFunc(func(int, error) (time.Duration, error) { return 0, nil })
+		options.Retryables = []retry.IsErrorRetryable{retry.IsErrorRetryableFunc(func(err error) aws.Ternary {
+			if errors.Is(err, tree.ErrNodeAccessDenied) {
+				return aws.TrueTernary
+			}
+			return aws.FalseTernary
+		})}
+	})
+
+	t.Run("no retryable", func(t *testing.T) {
+		count := 0
+		err := MakeWithRetry(ctx, func() error {
+			count++
+			if count == 1 {
+				return tree.ErrNodeNotFound
+			}
+			return tree.ErrNodeAccessDenied
+		}, retryer)
+		require.ErrorIs(t, err, tree.ErrNodeNotFound)
+	})
+
+	t.Run("retry", func(t *testing.T) {
+		count := 0
+		err := MakeWithRetry(ctx, func() error {
+			count++
+			if count < 3 {
+				return tree.ErrNodeAccessDenied
+			}
+			return nil
+		}, retryer)
+		require.NoError(t, err)
+	})
+
+	t.Run("retry exhausted", func(t *testing.T) {
+		err := MakeWithRetry(ctx, func() error {
+			return tree.ErrNodeAccessDenied
+		}, retryer)
+		require.ErrorIs(t, err, tree.ErrNodeAccessDenied)
+	})
+}