[#553] authmate: Add retryer to create access box

After using AddChain to provide access to container we have to wait:
* tx with APE chain be accepted by blockchain
* cache in storage node be updated

it takes a while. So we add retry
 (the same as when we add bucket settings during bucket creation)

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>

api/layer/container.go

@@ -141,7 +141,6 @@ func (n *Layer) createContainer(ctx context.Context, p *CreateBucketParams) (*da
 		SessionToken:         p.SessionContainerCreation,
 		CreationTime:         bktInfo.Created,
 		AdditionalAttributes: attributes,
-		BasicACL:             0, // means APE
 	})
 	if err != nil {
 		return nil, fmt.Errorf("create container: %w", err)

api/layer/frostfs/frostfs.go

@@ -9,13 +9,13 @@ import (
 
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
+	"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
 )
 
 // PrmContainerCreate groups parameters of FrostFS.CreateContainer operation.
@@ -38,13 +38,16 @@ type PrmContainerCreate struct {
 	// Token of the container's creation session. Nil means session absence.
 	SessionToken *session.Container
 
-	// Basic ACL of the container.
-	BasicACL acl.Basic
-
 	// Attributes for optional parameters.
 	AdditionalAttributes [][2]string
 }
 
+// PrmAddContainerPolicyChain groups parameter of FrostFS.AddContainerPolicyChain operation.
+type PrmAddContainerPolicyChain struct {
+	ContainerID cid.ID
+	Chain       chain.Chain
+}
+
 // PrmContainer groups parameters of FrostFS.Container operation.
 type PrmContainer struct {
 	// Container identifier.
@@ -239,6 +242,10 @@ type FrostFS interface {
 	// prevented the container from being created.
 	CreateContainer(context.Context, PrmContainerCreate) (*ContainerCreateResult, error)
 
+	// AddContainerPolicyChain create new policy chain for container.
+	// Can be invoked only by container owner.
+	AddContainerPolicyChain(context.Context, PrmAddContainerPolicyChain) error
+
 	// Container reads a container from FrostFS by ID.
 	//
 	// It returns exactly one non-nil value. It returns any error encountered which

api/layer/frostfs_mock.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/rand"
 	"crypto/sha256"
+	"errors"
 	"fmt"
 	"io"
 	"strings"
@@ -25,6 +26,7 @@ import (
 	oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
+	"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
 	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
 )
 
@@ -61,13 +63,14 @@ func (k *FeatureSettingsMock) FormContainerZone(ns string) string {
 	return ns + ".ns"
 }
 
-type TestFrostFS struct {
-	frostfs.FrostFS
+var _ frostfs.FrostFS = (*TestFrostFS)(nil)
 
+type TestFrostFS struct {
 	objects         map[string]*object.Object
 	objectErrors    map[string]error
 	objectPutErrors map[string]error
 	containers      map[string]*container.Container
+	chains          map[string][]chain.Chain
 	currentEpoch    uint64
 	key             *keys.PrivateKey
 }
@@ -78,6 +81,7 @@ func NewTestFrostFS(key *keys.PrivateKey) *TestFrostFS {
 		objectErrors:    make(map[string]error),
 		objectPutErrors: make(map[string]error),
 		containers:      make(map[string]*container.Container),
+		chains:          make(map[string][]chain.Chain),
 		key:             key,
 	}
 }
@@ -145,7 +149,6 @@ func (t *TestFrostFS) CreateContainer(_ context.Context, prm frostfs.PrmContaine
 	cnr.Init()
 	cnr.SetOwner(prm.Creator)
 	cnr.SetPlacementPolicy(prm.Policy)
-	cnr.SetBasicACL(prm.BasicACL)
 
 	creationTime := prm.CreationTime
 	if creationTime.IsZero() {
@@ -174,6 +177,7 @@ func (t *TestFrostFS) CreateContainer(_ context.Context, prm frostfs.PrmContaine
 	var id cid.ID
 	id.SetSHA256(sha256.Sum256(b))
 	t.containers[id.EncodeToString()] = &cnr
+	t.chains[id.EncodeToString()] = []chain.Chain{}
 
 	return &frostfs.ContainerCreateResult{ContainerID: id}, nil
 }
@@ -455,6 +459,17 @@ func (t *TestFrostFS) PatchObject(ctx context.Context, prm frostfs.PrmObjectPatc
 	return newID, nil
 }
 
+func (t *TestFrostFS) AddContainerPolicyChain(_ context.Context, prm frostfs.PrmAddContainerPolicyChain) error {
+	list, ok := t.chains[prm.ContainerID.EncodeToString()]
+	if !ok {
+		return errors.New("container not found")
+	}
+
+	t.chains[prm.ContainerID.EncodeToString()] = append(list, prm.Chain)
+
+	return nil
+}
+
 func (t *TestFrostFS) checkAccess(cnrID cid.ID, owner user.ID) bool {
 	cnr, ok := t.containers[cnrID.EncodeToString()]
 	if !ok {

authmate/authmate.go

@@ -14,9 +14,12 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/acl"
 	sessionv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/session"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/pkg/retryer"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/bearer"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
 	frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa"
@@ -25,6 +28,8 @@ import (
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/session"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	"github.com/google/uuid"
 	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
 	"go.uber.org/zap"
@@ -33,7 +38,7 @@ import (
 // PrmContainerCreate groups parameters of containers created by authmate.
 type PrmContainerCreate struct {
 	// FrostFS identifier of the container creator.
-	Owner user.ID
+	Owner *keys.PublicKey
 
 	// Container placement policy.
 	Policy netmap.PlacementPolicy
@@ -85,11 +90,56 @@ type FrostFS interface {
 type Agent struct {
 	frostFS FrostFS
 	log     *zap.Logger
+	cfg     *config
+}
+
+type config struct {
+	RetryMaxAttempts int
+	RetryMaxBackoff  time.Duration
+	RetryStrategy    handler.RetryStrategy
+}
+
+func defaultConfig() *config {
+	return &config{
+		RetryMaxAttempts: 4,
+		RetryMaxBackoff:  30 * time.Second,
+		RetryStrategy:    handler.RetryStrategyExponential,
+	}
+}
+
+type Option func(cfg *config)
+
+func WithRetryMaxAttempts(attempts int) func(*config) {
+	return func(cfg *config) {
+		cfg.RetryMaxAttempts = attempts
+	}
+}
+
+func WithRetryMaxBackoff(backoff time.Duration) func(*config) {
+	return func(cfg *config) {
+		cfg.RetryMaxBackoff = backoff
+	}
+}
+
+func WithRetryStrategy(strategy handler.RetryStrategy) func(*config) {
+	return func(cfg *config) {
+		cfg.RetryStrategy = strategy
+	}
 }
 
 // New creates an object of type Agent that consists of Client and logger.
-func New(log *zap.Logger, frostFS FrostFS) *Agent {
-	return &Agent{log: log, frostFS: frostFS}
+func New(log *zap.Logger, frostFS FrostFS, options ...Option) *Agent {
+	cfg := defaultConfig()
+
+	for _, opt := range options {
+		opt(cfg)
+	}
+
+	return &Agent{
+		log:     log,
+		frostFS: frostFS,
+		cfg:     cfg,
+	}
 }
 
 type (
@@ -275,7 +325,13 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr
 		CustomAttributes: options.CustomAttributes,
 	}
 
-	addr, err := creds.Put(ctx, prm)
+	var addr oid.Address
+	err = retryer.MakeWithRetry(ctx, func() error {
+		var inErr error
+		addr, inErr = creds.Put(ctx, prm)
+		return inErr
+	}, a.credsPutRetryer())
+
 	if err != nil {
 		return fmt.Errorf("failed to put creds: %w", err)
 	}
@@ -431,6 +487,27 @@ func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSe
 	return enc.Encode(or)
 }
 
+func (a *Agent) credsPutRetryer() aws.RetryerV2 {
+	return retry.NewStandard(func(options *retry.StandardOptions) {
+		options.MaxAttempts = a.cfg.RetryMaxAttempts
+		options.MaxBackoff = a.cfg.RetryMaxBackoff
+		if a.cfg.RetryStrategy == handler.RetryStrategyExponential {
+			options.Backoff = retry.NewExponentialJitterBackoff(options.MaxBackoff)
+		} else {
+			options.Backoff = retry.BackoffDelayerFunc(func(int, error) (time.Duration, error) {
+				return options.MaxBackoff, nil
+			})
+		}
+
+		options.Retryables = []retry.IsErrorRetryable{retry.IsErrorRetryableFunc(func(err error) aws.Ternary {
+			if errors.Is(err, frostfs.ErrAccessDenied) {
+				return aws.TrueTernary
+			}
+			return aws.FalseTernary
+		})}
+	})
+}
+
 func buildBearerToken(key *keys.PrivateKey, impersonate bool, lifetime lifetimeOptions, gateKey *keys.PublicKey) (*bearer.Token, error) {
 	var ownerID user.ID
 	user.IDFromKey(&ownerID, (ecdsa.PublicKey)(*gateKey))

cmd/s3-authmate/modules/issue-secret.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/handler"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
@@ -14,7 +15,6 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
 	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -48,6 +48,9 @@ const (
 	containerPolicyFlag          = "container-policy"
 	awsCLICredentialFlag         = "aws-cli-credentials"
 	attributesFlag               = "attributes"
+	retryMaxAttemptsFlag         = "retry-max-attempts"
+	retryMaxBackoffFlag          = "retry-max-backoff"
+	retryStrategyFlag            = "retry-strategy"
 )
 
 const walletPassphraseCfg = "wallet.passphrase"
@@ -59,6 +62,10 @@ const (
 	defaultPoolHealthcheckTimeout = 5 * time.Second
 	defaultPoolRebalanceInterval  = 30 * time.Second
 	defaultPoolStreamTimeout      = 10 * time.Second
+
+	defaultRetryMaxAttempts = 4
+	defaultRetryMaxBackoff  = 30 * time.Second
+	defaultRetryStrategy    = handler.RetryStrategyExponential
 )
 
 const (
@@ -91,6 +98,9 @@ func initIssueSecretCmd() {
 	issueSecretCmd.Flags().String(accessKeyIDFlag, "", "Access key id of s3 credential that must be created")
 	issueSecretCmd.Flags().String(secretAccessKeyFlag, "", "Secret access key of s3 credential that must be used")
 	issueSecretCmd.Flags().String(rpcEndpointFlag, "", "NEO node RPC address (must be provided if container-id is nns name)")
+	issueSecretCmd.Flags().Int(retryMaxAttemptsFlag, defaultRetryMaxAttempts, "Max amount of request attempts")
+	issueSecretCmd.Flags().Duration(retryMaxBackoffFlag, defaultRetryMaxBackoff, "Max delay before next attempt")
+	issueSecretCmd.Flags().String(retryStrategyFlag, defaultRetryStrategy, "Backoff strategy. `exponential` and `constant` are allowed")
 
 	_ = issueSecretCmd.MarkFlagRequired(walletFlag)
 	_ = issueSecretCmd.MarkFlagRequired(peerFlag)
@@ -181,7 +191,13 @@ func runIssueSecretCmd(cmd *cobra.Command, _ []string) error {
 		CustomAttributes:      customAttrs,
 	}
 
-	if err = authmate.New(log, frostFS).IssueSecret(ctx, os.Stdout, issueSecretOptions); err != nil {
+	options := []authmate.Option{
+		authmate.WithRetryMaxAttempts(viper.GetInt(retryMaxAttemptsFlag)),
+		authmate.WithRetryMaxBackoff(viper.GetDuration(retryMaxBackoffFlag)),
+		authmate.WithRetryStrategy(handler.RetryStrategy(viper.GetString(retryStrategyFlag))),
+	}
+
+	if err = authmate.New(log, frostFS, options...).IssueSecret(ctx, os.Stdout, issueSecretOptions); err != nil {
 		return wrapBusinessLogicError(fmt.Errorf("failed to issue secret: %s", err))
 	}
 	return nil
@@ -227,10 +243,9 @@ func createAccessBox(ctx context.Context, frostFS *frostfs.AuthmateFrostFS, key
 
 	prm := authmate.PrmContainerCreate{
 		FriendlyName: friendlyName,
+		Owner:        key.PublicKey(),
 	}
 
-	user.IDFromKey(&prm.Owner, key.PrivateKey.PublicKey)
-
 	if err := prm.Policy.DecodeString(placementPolicy); err != nil {
 		return cid.ID{}, fmt.Errorf("failed to build placement policy: %w", err)
 	}

internal/frostfs/authmate.go

@@ -3,6 +3,7 @@ package frostfs
 import (
 	"bytes"
 	"context"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"strconv"
@@ -16,10 +17,12 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/crdt"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/acl"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
+	"git.frostfs.info/TrueCloudLab/policy-engine/pkg/chain"
+	"git.frostfs.info/TrueCloudLab/policy-engine/schema/native"
 	"go.uber.org/zap"
 )
 
@@ -55,21 +58,61 @@ func (x *AuthmateFrostFS) TimeToEpoch(ctx context.Context, futureTime time.Time)
 
 // CreateContainer implements authmate.FrostFS interface method.
 func (x *AuthmateFrostFS) CreateContainer(ctx context.Context, prm authmate.PrmContainerCreate) (cid.ID, error) {
-	basicACL := acl.Private
-	// allow reading objects to OTHERS in order to provide read access to S3 gateways
-	basicACL.AllowOp(acl.OpObjectGet, acl.RoleOthers)
-	basicACL.AllowOp(acl.OpObjectHead, acl.RoleOthers)
-	basicACL.AllowOp(acl.OpObjectSearch, acl.RoleOthers)
+	var owner user.ID
+	owner.SetScriptHash(prm.Owner.GetScriptHash())
 
 	res, err := x.frostFS.CreateContainer(ctx, frostfs.PrmContainerCreate{
-		Creator:  prm.Owner,
-		Policy:   prm.Policy,
-		Name:     prm.FriendlyName,
-		BasicACL: basicACL,
+		Creator: owner,
+		Policy:  prm.Policy,
+		Name:    prm.FriendlyName,
 	})
 	if err != nil {
 		return cid.ID{}, err
 	}
+
+	ch := chain.Chain{
+		ID: chain.ID("authmate/" + owner.String()),
+		Rules: []chain.Rule{
+			{
+				Status:  chain.Allow,
+				Actions: chain.Actions{Names: []string{"*"}},
+				Resources: chain.Resources{Names: []string{
+					fmt.Sprintf(native.ResourceFormatRootContainer, res.ContainerID),
+					fmt.Sprintf(native.ResourceFormatRootContainerObjects, res.ContainerID),
+				}},
+				Condition: []chain.Condition{{
+					Op:    chain.CondStringEquals,
+					Kind:  chain.KindRequest,
+					Key:   native.PropertyKeyActorPublicKey,
+					Value: hex.EncodeToString(prm.Owner.Bytes()),
+				}},
+			},
+			{
+				Status: chain.Allow,
+				Actions: chain.Actions{Names: []string{
+					native.MethodGetContainer,
+					native.MethodGetObject,
+					native.MethodHeadObject,
+					native.MethodSearchObject,
+					native.MethodRangeObject,
+					native.MethodHashObject,
+				}},
+				Resources: chain.Resources{Names: []string{
+					fmt.Sprintf(native.ResourceFormatRootContainer, res.ContainerID),
+					fmt.Sprintf(native.ResourceFormatRootContainerObjects, res.ContainerID),
+				}},
+			},
+		},
+	}
+
+	err = x.frostFS.AddContainerPolicyChain(ctx, frostfs.PrmAddContainerPolicyChain{
+		ContainerID: res.ContainerID,
+		Chain:       ch,
+	})
+	if err != nil {
+		return cid.ID{}, err
+	}
+
 	return res.ContainerID, nil
 }
 

internal/frostfs/authmate_test.go

@@ -47,7 +47,7 @@ func TestCredsObject(t *testing.T) {
 
 	cnrID, err := frostfs.CreateContainer(ctx, authmate.PrmContainerCreate{
 		FriendlyName: bktName,
-		Owner:        userID,
+		Owner:        key.PublicKey(),
 	})
 	require.NoError(t, err)
 

internal/frostfs/frostfs.go

@@ -12,6 +12,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/layer/frostfs"
 	frosterr "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/errors"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ape"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
@@ -93,7 +94,6 @@ func (x *FrostFS) CreateContainer(ctx context.Context, prm frostfs.PrmContainerC
 	cnr.Init()
 	cnr.SetPlacementPolicy(prm.Policy)
 	cnr.SetOwner(prm.Creator)
-	cnr.SetBasicACL(prm.BasicACL)
 
 	creationTime := prm.CreationTime
 	if creationTime.IsZero() {
@@ -135,6 +135,25 @@ func (x *FrostFS) CreateContainer(ctx context.Context, prm frostfs.PrmContainerC
 	}, handleObjectError("save container via connection pool", err)
 }
 
+// AddContainerPolicyChain implements frostfs.FrostFS interface method.
+func (x *FrostFS) AddContainerPolicyChain(ctx context.Context, prm frostfs.PrmAddContainerPolicyChain) error {
+	data, err := prm.Chain.MarshalBinary()
+	if err != nil {
+		return err
+	}
+
+	prmAddAPEChain := pool.PrmAddAPEChain{
+		Target: ape.ChainTarget{
+			TargetType: ape.TargetTypeContainer,
+			Name:       prm.ContainerID.EncodeToString(),
+		},
+		Chain: ape.Chain{Raw: data},
+	}
+
+	err = x.pool.AddAPEChain(ctx, prmAddAPEChain)
+	return handleObjectError("add ape chain to container", err)
+}
+
 // UserContainers implements layer.FrostFS interface method.
 func (x *FrostFS) UserContainers(ctx context.Context, layerPrm frostfs.PrmUserContainers) ([]cid.ID, error) {
 	prm := pool.PrmContainerList{