From b1775f9478d8eb9be2ed21395489b9f838fb37ff Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Tue, 19 Nov 2024 11:22:39 +0300 Subject: [PATCH] [#553] authmate: Add retryer to create access box After using AddChain to provide access to container we have to wait: * tx with APE chain be accepted by blockchain * cache in storage node be updated it takes a while. So we add retry (the same as when we add bucket settings during bucket creation) Signed-off-by: Denis Kirillov --- authmate/authmate.go | 83 ++++++++++++++++++++++++- cmd/s3-authmate/modules/issue-secret.go | 19 +++++- 2 files changed, 98 insertions(+), 4 deletions(-) diff --git a/authmate/authmate.go b/authmate/authmate.go index 616ec98..ff7e5d9 100644 --- a/authmate/authmate.go +++ b/authmate/authmate.go @@ -14,9 +14,12 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl" sessionv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/retryer" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" @@ -25,6 +28,8 @@ import ( oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/retry" "github.com/google/uuid" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.uber.org/zap" @@ -85,11 +90,56 @@ type FrostFS interface { type Agent struct { frostFS FrostFS log *zap.Logger + cfg *config +} + +type config struct { + RetryMaxAttempts int + RetryMaxBackoff time.Duration + RetryStrategy handler.RetryStrategy +} + +func defaultConfig() *config { + return &config{ + RetryMaxAttempts: 4, + RetryMaxBackoff: 30 * time.Second, + RetryStrategy: handler.RetryStrategyExponential, + } +} + +type Option func(cfg *config) + +func WithRetryMaxAttempts(attempts int) func(*config) { + return func(cfg *config) { + cfg.RetryMaxAttempts = attempts + } +} + +func WithRetryMaxBackoff(backoff time.Duration) func(*config) { + return func(cfg *config) { + cfg.RetryMaxBackoff = backoff + } +} + +func WithRetryStrategy(strategy handler.RetryStrategy) func(*config) { + return func(cfg *config) { + cfg.RetryStrategy = strategy + } } // New creates an object of type Agent that consists of Client and logger. -func New(log *zap.Logger, frostFS FrostFS) *Agent { - return &Agent{log: log, frostFS: frostFS} +func New(log *zap.Logger, frostFS FrostFS, options ...Option) *Agent { + cfg := defaultConfig() + + for _, opt := range options { + opt(cfg) + } + + return &Agent{ + log: log, + frostFS: frostFS, + cfg: cfg, + } } type ( @@ -275,7 +325,13 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr CustomAttributes: options.CustomAttributes, } - addr, err := creds.Put(ctx, prm) + var addr oid.Address + err = retryer.MakeWithRetry(ctx, func() error { + var inErr error + addr, inErr = creds.Put(ctx, prm) + return inErr + }, a.credsPutRetryer()) + if err != nil { return fmt.Errorf("failed to put creds: %w", err) } @@ -431,6 +487,27 @@ func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSe return enc.Encode(or) } +func (a *Agent) credsPutRetryer() aws.RetryerV2 { + return retry.NewStandard(func(options *retry.StandardOptions) { + options.MaxAttempts = a.cfg.RetryMaxAttempts + options.MaxBackoff = a.cfg.RetryMaxBackoff + if a.cfg.RetryStrategy == handler.RetryStrategyExponential { + options.Backoff = retry.NewExponentialJitterBackoff(options.MaxBackoff) + } else { + options.Backoff = retry.BackoffDelayerFunc(func(int, error) (time.Duration, error) { + return options.MaxBackoff, nil + }) + } + + options.Retryables = []retry.IsErrorRetryable{retry.IsErrorRetryableFunc(func(err error) aws.Ternary { + if errors.Is(err, frostfs.ErrAccessDenied) { + return aws.TrueTernary + } + return aws.FalseTernary + })} + }) +} + func buildBearerToken(key *keys.PrivateKey, impersonate bool, lifetime lifetimeOptions, gateKey *keys.PublicKey) (*bearer.Token, error) { var ownerID user.ID user.IDFromKey(&ownerID, (ecdsa.PublicKey)(*gateKey)) diff --git a/cmd/s3-authmate/modules/issue-secret.go b/cmd/s3-authmate/modules/issue-secret.go index 7970baf..b8fca1d 100644 --- a/cmd/s3-authmate/modules/issue-secret.go +++ b/cmd/s3-authmate/modules/issue-secret.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util" @@ -47,6 +48,9 @@ const ( containerPolicyFlag = "container-policy" awsCLICredentialFlag = "aws-cli-credentials" attributesFlag = "attributes" + retryMaxAttemptsFlag = "retry-max-attempts" + retryMaxBackoffFlag = "retry-max-backoff" + retryStrategyFlag = "retry-strategy" ) const walletPassphraseCfg = "wallet.passphrase" @@ -58,6 +62,10 @@ const ( defaultPoolHealthcheckTimeout = 5 * time.Second defaultPoolRebalanceInterval = 30 * time.Second defaultPoolStreamTimeout = 10 * time.Second + + defaultRetryMaxAttempts = 4 + defaultRetryMaxBackoff = 30 * time.Second + defaultRetryStrategy = handler.RetryStrategyExponential ) const ( @@ -90,6 +98,9 @@ func initIssueSecretCmd() { issueSecretCmd.Flags().String(accessKeyIDFlag, "", "Access key id of s3 credential that must be created") issueSecretCmd.Flags().String(secretAccessKeyFlag, "", "Secret access key of s3 credential that must be used") issueSecretCmd.Flags().String(rpcEndpointFlag, "", "NEO node RPC address (must be provided if container-id is nns name)") + issueSecretCmd.Flags().Int(retryMaxAttemptsFlag, defaultRetryMaxAttempts, "Max amount of request attempts") + issueSecretCmd.Flags().Duration(retryMaxBackoffFlag, defaultRetryMaxBackoff, "Max delay before next attempt") + issueSecretCmd.Flags().String(retryStrategyFlag, defaultRetryStrategy, "Backoff strategy. `exponential` and `constant` are allowed") _ = issueSecretCmd.MarkFlagRequired(walletFlag) _ = issueSecretCmd.MarkFlagRequired(peerFlag) @@ -180,7 +191,13 @@ func runIssueSecretCmd(cmd *cobra.Command, _ []string) error { CustomAttributes: customAttrs, } - if err = authmate.New(log, frostFS).IssueSecret(ctx, os.Stdout, issueSecretOptions); err != nil { + options := []authmate.Option{ + authmate.WithRetryMaxAttempts(viper.GetInt(retryMaxAttemptsFlag)), + authmate.WithRetryMaxBackoff(viper.GetDuration(retryMaxBackoffFlag)), + authmate.WithRetryStrategy(handler.RetryStrategy(viper.GetString(retryStrategyFlag))), + } + + if err = authmate.New(log, frostFS, options...).IssueSecret(ctx, os.Stdout, issueSecretOptions); err != nil { return wrapBusinessLogicError(fmt.Errorf("failed to issue secret: %s", err)) } return nil