diff --git a/authmate/authmate.go b/authmate/authmate.go index 616ec986..ff7e5d94 100644 --- a/authmate/authmate.go +++ b/authmate/authmate.go @@ -14,9 +14,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" sessionv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/retryer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" @@ -25,6 +28,8 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.uber.org/zap" @@ -85,11 +90,56 @@ type FrostFS interface { type Agent struct { frostFS FrostFS log *zap.Logger + cfg *config +} + +type config struct { + RetryMaxAttempts int + RetryMaxBackoff time.Duration + RetryStrategy handler.RetryStrategy +} + +func defaultConfig() *config { + return &config{ + RetryMaxAttempts: 4, + RetryMaxBackoff: 30 * time.Second, + RetryStrategy: handler.RetryStrategyExponential, + } +} + +type Option func(cfg *config) + +func WithRetryMaxAttempts(attempts int) func(*config) { + return func(cfg *config) { + cfg.RetryMaxAttempts = attempts + } +} + +func WithRetryMaxBackoff(backoff time.Duration) func(*config) { + return func(cfg *config) { + cfg.RetryMaxBackoff = backoff + } +} + +func WithRetryStrategy(strategy handler.RetryStrategy) func(*config) { + return func(cfg *config) { + cfg.RetryStrategy = strategy + } } // New creates an object of type Agent that consists of Client and logger. -func New(log *zap.Logger, frostFS FrostFS) *Agent { - return &Agent{log: log, frostFS: frostFS} +func New(log *zap.Logger, frostFS FrostFS, options ...Option) *Agent { + cfg := defaultConfig() + + for _, opt := range options { + opt(cfg) + } + + return &Agent{ + log: log, + frostFS: frostFS, + cfg: cfg, + } } type ( @@ -275,7 +325,13 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr CustomAttributes: options.CustomAttributes, } - addr, err := creds.Put(ctx, prm) + var addr oid.Address + err = retryer.MakeWithRetry(ctx, func() error { + var inErr error + addr, inErr = creds.Put(ctx, prm) + return inErr + }, a.credsPutRetryer()) + if err != nil { return fmt.Errorf("failed to put creds: %w", err) } @@ -431,6 +487,27 @@ func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSe return enc.Encode(or) } +func (a *Agent) credsPutRetryer() aws.RetryerV2 { + return retry.NewStandard(func(options *retry.StandardOptions) { + options.MaxAttempts = a.cfg.RetryMaxAttempts + options.MaxBackoff = a.cfg.RetryMaxBackoff + if a.cfg.RetryStrategy == handler.RetryStrategyExponential { + options.Backoff = retry.NewExponentialJitterBackoff(options.MaxBackoff) + } else { + options.Backoff = retry.BackoffDelayerFunc(func(int, error) (time.Duration, error) { + return options.MaxBackoff, nil + }) + } + + options.Retryables = []retry.IsErrorRetryable{retry.IsErrorRetryableFunc(func(err error) aws.Ternary { + if errors.Is(err, frostfs.ErrAccessDenied) { + return aws.TrueTernary + } + return aws.FalseTernary + })} + }) +} + func buildBearerToken(key *keys.PrivateKey, impersonate bool, lifetime lifetimeOptions, gateKey *keys.PublicKey) (*bearer.Token, error) { var ownerID user.ID user.IDFromKey(&ownerID, (ecdsa.PublicKey)(*gateKey)) diff --git a/cmd/s3-authmate/modules/issue-secret.go b/cmd/s3-authmate/modules/issue-secret.go index 7970baf0..b8fca1de 100644 --- a/cmd/s3-authmate/modules/issue-secret.go +++ b/cmd/s3-authmate/modules/issue-secret.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util" @@ -47,6 +48,9 @@ const ( containerPolicyFlag = "container-policy" awsCLICredentialFlag = "aws-cli-credentials" attributesFlag = "attributes" + retryMaxAttemptsFlag = "retry-max-attempts" + retryMaxBackoffFlag = "retry-max-backoff" + retryStrategyFlag = "retry-strategy" ) const walletPassphraseCfg = "wallet.passphrase" @@ -58,6 +62,10 @@ const ( defaultPoolHealthcheckTimeout = 5 * time.Second defaultPoolRebalanceInterval = 30 * time.Second defaultPoolStreamTimeout = 10 * time.Second + + defaultRetryMaxAttempts = 4 + defaultRetryMaxBackoff = 30 * time.Second + defaultRetryStrategy = handler.RetryStrategyExponential ) const ( @@ -90,6 +98,9 @@ func initIssueSecretCmd() { issueSecretCmd.Flags().String(accessKeyIDFlag, "", "Access key id of s3 credential that must be created") issueSecretCmd.Flags().String(secretAccessKeyFlag, "", "Secret access key of s3 credential that must be used") issueSecretCmd.Flags().String(rpcEndpointFlag, "", "NEO node RPC address (must be provided if container-id is nns name)") + issueSecretCmd.Flags().Int(retryMaxAttemptsFlag, defaultRetryMaxAttempts, "Max amount of request attempts") + issueSecretCmd.Flags().Duration(retryMaxBackoffFlag, defaultRetryMaxBackoff, "Max delay before next attempt") + issueSecretCmd.Flags().String(retryStrategyFlag, defaultRetryStrategy, "Backoff strategy. `exponential` and `constant` are allowed") _ = issueSecretCmd.MarkFlagRequired(walletFlag) _ = issueSecretCmd.MarkFlagRequired(peerFlag) @@ -180,7 +191,13 @@ func runIssueSecretCmd(cmd *cobra.Command, _ []string) error { CustomAttributes: customAttrs, } - if err = authmate.New(log, frostFS).IssueSecret(ctx, os.Stdout, issueSecretOptions); err != nil { + options := []authmate.Option{ + authmate.WithRetryMaxAttempts(viper.GetInt(retryMaxAttemptsFlag)), + authmate.WithRetryMaxBackoff(viper.GetDuration(retryMaxBackoffFlag)), + authmate.WithRetryStrategy(handler.RetryStrategy(viper.GetString(retryStrategyFlag))), + } + + if err = authmate.New(log, frostFS, options...).IssueSecret(ctx, os.Stdout, issueSecretOptions); err != nil { return wrapBusinessLogicError(fmt.Errorf("failed to issue secret: %s", err)) } return nil