[#553] authmate: Add retryer to create access box

After using AddChain to provide access to container we have to wait:
* tx with APE chain be accepted by blockchain
* cache in storage node be updated

it takes a while. So we add retry
 (the same as when we add bucket settings during bucket creation)

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>

authmate/authmate.go

@@ -14,9 +14,12 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
 	sessionv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/retryer"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
 	frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
@@ -25,6 +28,8 @@ import (
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	"github.com/google/uuid"
 	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
 	"go.uber.org/zap"
@@ -85,11 +90,56 @@ type FrostFS interface {
 type Agent struct {
 	frostFS FrostFS
 	log     *zap.Logger
+	cfg     *config
+}
+
+type config struct {
+	RetryMaxAttempts int
+	RetryMaxBackoff  time.Duration
+	RetryStrategy    handler.RetryStrategy
+}
+
+func defaultConfig() *config {
+	return &config{
+		RetryMaxAttempts: 4,
+		RetryMaxBackoff:  30 * time.Second,
+		RetryStrategy:    handler.RetryStrategyExponential,
+	}
+}
+
+type Option func(cfg *config)
+
+func WithRetryMaxAttempts(attempts int) func(*config) {
+	return func(cfg *config) {
+		cfg.RetryMaxAttempts = attempts
+	}
+}
+
+func WithRetryMaxBackoff(backoff time.Duration) func(*config) {
+	return func(cfg *config) {
+		cfg.RetryMaxBackoff = backoff
+	}
+}
+
+func WithRetryStrategy(strategy handler.RetryStrategy) func(*config) {
+	return func(cfg *config) {
+		cfg.RetryStrategy = strategy
+	}
 }
 
 // New creates an object of type Agent that consists of Client and logger.
-func New(log *zap.Logger, frostFS FrostFS) *Agent {
-	return &Agent{log: log, frostFS: frostFS}
+func New(log *zap.Logger, frostFS FrostFS, options ...Option) *Agent {
+	cfg := defaultConfig()
+
+	for _, opt := range options {
+		opt(cfg)
+	}
+
+	return &Agent{
+		log:     log,
+		frostFS: frostFS,
+		cfg:     cfg,
+	}
 }
 
 type (
@@ -275,7 +325,13 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
 		CustomAttributes: options.CustomAttributes,
 	}
 
-	addr, err := creds.Put(ctx, prm)
+	var addr oid.Address
+	err = retryer.MakeWithRetry(ctx, func() error {
+		var inErr error
+		addr, inErr = creds.Put(ctx, prm)
+		return inErr
+	}, a.credsPutRetryer())
+
 	if err != nil {
 		return fmt.Errorf("failed to put creds: %w", err)
 	}
@@ -431,6 +487,27 @@ func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSe
 	return enc.Encode(or)
 }
 
+func (a *Agent) credsPutRetryer() aws.RetryerV2 {
+	return retry.NewStandard(func(options *retry.StandardOptions) {
+		options.MaxAttempts = a.cfg.RetryMaxAttempts
+		options.MaxBackoff = a.cfg.RetryMaxBackoff
+		if a.cfg.RetryStrategy == handler.RetryStrategyExponential {
+			options.Backoff = retry.NewExponentialJitterBackoff(options.MaxBackoff)
+		} else {
+			options.Backoff = retry.BackoffDelayerFunc(func(int, error) (time.Duration, error) {
+				return options.MaxBackoff, nil
+			})
+		}
+
+		options.Retryables = []retry.IsErrorRetryable{retry.IsErrorRetryableFunc(func(err error) aws.Ternary {
+			if errors.Is(err, frostfs.ErrAccessDenied) {
+				return aws.TrueTernary
+			}
+			return aws.FalseTernary
+		})}
+	})
+}
+
 func buildBearerToken(key *keys.PrivateKey, impersonate bool, lifetime lifetimeOptions, gateKey *keys.PublicKey) (*bearer.Token, error) {
 	var ownerID user.ID
 	user.IDFromKey(&ownerID, (ecdsa.PublicKey)(*gateKey))

cmd/s3-authmate/modules/issue-secret.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
@@ -47,6 +48,9 @@ const (
 	containerPolicyFlag          = "container-policy"
 	awsCLICredentialFlag         = "aws-cli-credentials"
 	attributesFlag               = "attributes"
+	retryMaxAttemptsFlag         = "retry-max-attempts"
+	retryMaxBackoffFlag          = "retry-max-backoff"
+	retryStrategyFlag            = "retry-strategy"
 )
 
 const walletPassphraseCfg = "wallet.passphrase"
@@ -58,6 +62,10 @@ const (
 	defaultPoolHealthcheckTimeout = 5 * time.Second
 	defaultPoolRebalanceInterval  = 30 * time.Second
 	defaultPoolStreamTimeout      = 10 * time.Second
+
+	defaultRetryMaxAttempts = 4
+	defaultRetryMaxBackoff  = 30 * time.Second
+	defaultRetryStrategy    = handler.RetryStrategyExponential
 )
 
 const (
@@ -90,6 +98,9 @@ func initIssueSecretCmd() {
 	issueSecretCmd.Flags().String(accessKeyIDFlag, "", "Access key id of s3 credential that must be created")
 	issueSecretCmd.Flags().String(secretAccessKeyFlag, "", "Secret access key of s3 credential that must be used")
 	issueSecretCmd.Flags().String(rpcEndpointFlag, "", "NEO node RPC address (must be provided if container-id is nns name)")
+	issueSecretCmd.Flags().Int(retryMaxAttemptsFlag, defaultRetryMaxAttempts, "Max amount of request attempts")
+	issueSecretCmd.Flags().Duration(retryMaxBackoffFlag, defaultRetryMaxBackoff, "Max delay before next attempt")
+	issueSecretCmd.Flags().String(retryStrategyFlag, defaultRetryStrategy, "Backoff strategy. `exponential` and `constant` are allowed")
 
 	_ = issueSecretCmd.MarkFlagRequired(walletFlag)
 	_ = issueSecretCmd.MarkFlagRequired(peerFlag)
@@ -180,7 +191,13 @@ func runIssueSecretCmd(cmd *cobra.Command, _ []string) error {
 		CustomAttributes:      customAttrs,
 	}
 
-	if err = authmate.New(log, frostFS).IssueSecret(ctx, os.Stdout, issueSecretOptions); err != nil {
+	options := []authmate.Option{
+		authmate.WithRetryMaxAttempts(viper.GetInt(retryMaxAttemptsFlag)),
+		authmate.WithRetryMaxBackoff(viper.GetDuration(retryMaxBackoffFlag)),
+		authmate.WithRetryStrategy(handler.RetryStrategy(viper.GetString(retryStrategyFlag))),
+	}
+
+	if err = authmate.New(log, frostFS, options...).IssueSecret(ctx, os.Stdout, issueSecretOptions); err != nil {
 		return wrapBusinessLogicError(fmt.Errorf("failed to issue secret: %s", err))
 	}
 	return nil