diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cfd407..3926633 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This document outlines major changes between releases. - Support new param `frostfs.graceful_close_on_switch_timeout` (#475) - Support patch object method (#479) - Add `sign` command to `frostfs-s3-authmate` (#467) +- Support custom aws credentials (#509) ### Changed - Update go version to go1.19 (#470) diff --git a/api/auth/center.go b/api/auth/center.go index 5b27e64..e654c25 100644 --- a/api/auth/center.go +++ b/api/auth/center.go @@ -18,6 +18,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/middleware" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/tokens" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/aws/aws-sdk-go/aws/credentials" ) @@ -34,6 +35,11 @@ type ( postReg *RegexpSubmatcher cli tokens.Credentials allowedAccessKeyIDPrefixes []string // empty slice means all access key ids are allowed + settings CenterSettings + } + + CenterSettings interface { + AccessBoxContainer() (cid.ID, bool) } //nolint:revive @@ -50,7 +56,6 @@ type ( ) const ( - accessKeyPartsNum = 2 authHeaderPartsNum = 6 maxFormSizeMemory = 50 * 1048576 // 50 MB @@ -82,12 +87,13 @@ var ContentSHA256HeaderStandardValue = map[string]struct{}{ } // New creates an instance of AuthCenter. -func New(creds tokens.Credentials, prefixes []string) *Center { +func New(creds tokens.Credentials, prefixes []string, settings CenterSettings) *Center { return &Center{ cli: creds, reg: NewRegexpMatcher(AuthorizationFieldRegexp), postReg: NewRegexpMatcher(postPolicyCredentialRegexp), allowedAccessKeyIDPrefixes: prefixes, + settings: settings, } } @@ -97,11 +103,6 @@ func (c *Center) parseAuthHeader(header string) (*AuthHeader, error) { return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrAuthorizationHeaderMalformed), header) } - accessKey := strings.Split(submatches["access_key_id"], "0") - if len(accessKey) != accessKeyPartsNum { - return nil, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrInvalidAccessKeyID), accessKey) - } - signedFields := strings.Split(submatches["signed_header_fields"], ";") return &AuthHeader{ @@ -114,15 +115,6 @@ func (c *Center) parseAuthHeader(header string) (*AuthHeader, error) { }, nil } -func getAddress(accessKeyID string) (oid.Address, error) { - var addr oid.Address - if err := addr.DecodeString(strings.ReplaceAll(accessKeyID, "0", "/")); err != nil { - return addr, fmt.Errorf("%w: %s", apierr.GetAPIError(apierr.ErrInvalidAccessKeyID), accessKeyID) - } - - return addr, nil -} - func IsStandardContentSHA256(key string) bool { _, ok := ContentSHA256HeaderStandardValue[key] return ok @@ -181,14 +173,14 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) { return nil, err } - addr, err := getAddress(authHdr.AccessKeyID) + cnrID, err := c.getAccessBoxContainer(authHdr.AccessKeyID) if err != nil { return nil, err } - box, attrs, err := c.cli.GetBox(r.Context(), addr) + box, attrs, err := c.cli.GetBox(r.Context(), cnrID, authHdr.AccessKeyID) if err != nil { - return nil, fmt.Errorf("get box '%s': %w", addr, err) + return nil, fmt.Errorf("get box by access key '%s': %w", authHdr.AccessKeyID, err) } if err = checkFormatHashContentSHA256(r.Header.Get(AmzContentSHA256)); err != nil { @@ -216,6 +208,20 @@ func (c *Center) Authenticate(r *http.Request) (*middleware.Box, error) { return result, nil } +func (c *Center) getAccessBoxContainer(accessKeyID string) (cid.ID, error) { + var addr oid.Address + if err := addr.DecodeString(strings.ReplaceAll(accessKeyID, "0", "/")); err == nil { + return addr.Container(), nil + } + + cnrID, ok := c.settings.AccessBoxContainer() + if ok { + return cnrID, nil + } + + return cid.ID{}, fmt.Errorf("%w: unknown container for creds '%s'", apierr.GetAPIError(apierr.ErrInvalidAccessKeyID), accessKeyID) +} + func checkFormatHashContentSHA256(hash string) error { if !IsStandardContentSHA256(hash) { hashBinary, err := hex.DecodeString(hash) @@ -272,14 +278,14 @@ func (c *Center) checkFormData(r *http.Request) (*middleware.Box, error) { accessKeyID := submatches["access_key_id"] - addr, err := getAddress(accessKeyID) + cnrID, err := c.getAccessBoxContainer(accessKeyID) if err != nil { return nil, err } - box, attrs, err := c.cli.GetBox(r.Context(), addr) + box, attrs, err := c.cli.GetBox(r.Context(), cnrID, accessKeyID) if err != nil { - return nil, fmt.Errorf("get box '%s': %w", addr, err) + return nil, fmt.Errorf("get box by accessKeyID '%s': %w", accessKeyID, err) } secret := box.Gate.SecretKey diff --git a/api/cache/accessbox.go b/api/cache/accessbox.go index 9401804..87eaee6 100644 --- a/api/cache/accessbox.go +++ b/api/cache/accessbox.go @@ -7,7 +7,6 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/creds/accessbox" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/bluele/gcache" "go.uber.org/zap" ) @@ -57,8 +56,8 @@ func NewAccessBoxCache(config *Config) *AccessBoxCache { } // Get returns a cached accessbox. -func (o *AccessBoxCache) Get(address oid.Address) *AccessBoxCacheValue { - entry, err := o.cache.Get(address) +func (o *AccessBoxCache) Get(accessKeyID string) *AccessBoxCacheValue { + entry, err := o.cache.Get(accessKeyID) if err != nil { return nil } @@ -74,16 +73,16 @@ func (o *AccessBoxCache) Get(address oid.Address) *AccessBoxCacheValue { } // Put stores an accessbox to cache. -func (o *AccessBoxCache) Put(address oid.Address, box *accessbox.Box, attrs []object.Attribute) error { +func (o *AccessBoxCache) Put(accessKeyID string, box *accessbox.Box, attrs []object.Attribute) error { val := &AccessBoxCacheValue{ Box: box, Attributes: attrs, PutTime: time.Now(), } - return o.cache.Set(address, val) + return o.cache.Set(accessKeyID, val) } // Delete removes an accessbox from cache. -func (o *AccessBoxCache) Delete(address oid.Address) { - o.cache.Remove(address) +func (o *AccessBoxCache) Delete(accessKeyID string) { + o.cache.Remove(accessKeyID) } diff --git a/authmate/authmate.go b/authmate/authmate.go index 401b26f..ef48d8c 100644 --- a/authmate/authmate.go +++ b/authmate/authmate.go @@ -98,7 +98,9 @@ type ( // IssueSecretOptions contains options for passing to Agent.IssueSecret method. IssueSecretOptions struct { - Container ContainerOptions + Container cid.ID + AccessKeyID string + SecretAccessKey string FrostFSKey *keys.PrivateKey GatesPublicKeys []*keys.PublicKey Impersonate bool @@ -114,7 +116,9 @@ type ( UpdateSecretOptions struct { FrostFSKey *keys.PrivateKey GatesPublicKeys []*keys.PublicKey - Address oid.Address + IsCustom bool + AccessKeyID string + ContainerID cid.ID GatePrivateKey *keys.PrivateKey CustomAttributes []object.Attribute } @@ -141,7 +145,8 @@ type ( // ObtainSecretOptions contains options for passing to Agent.ObtainSecret method. ObtainSecretOptions struct { - SecretAddress string + Container cid.ID + AccessKeyID string GatePrivateKey *keys.PrivateKey } ) @@ -168,32 +173,9 @@ type ( } ) -func (a *Agent) checkContainer(ctx context.Context, opts ContainerOptions, idOwner user.ID) (cid.ID, error) { - if !opts.ID.Equals(cid.ID{}) { - a.log.Info(logs.CheckContainer, zap.Stringer("cid", opts.ID)) - return opts.ID, a.frostFS.ContainerExists(ctx, opts.ID) - } - - a.log.Info(logs.CreateContainer, - zap.String("friendly_name", opts.FriendlyName), - zap.String("placement_policy", opts.PlacementPolicy)) - - var prm PrmContainerCreate - - err := prm.Policy.DecodeString(opts.PlacementPolicy) - if err != nil { - return cid.ID{}, fmt.Errorf("failed to build placement policy: %w", err) - } - - prm.Owner = idOwner - prm.FriendlyName = opts.FriendlyName - - cnrID, err := a.frostFS.CreateContainer(ctx, prm) - if err != nil { - return cid.ID{}, fmt.Errorf("create container in FrostFS: %w", err) - } - - return cnrID, nil +func (a *Agent) checkContainer(ctx context.Context, cnrID cid.ID) error { + a.log.Info(logs.CheckContainer, zap.Stringer("cid", cnrID)) + return a.frostFS.ContainerExists(ctx, cnrID) } func checkPolicy(policyString string) (*netmap.PlacementPolicy, error) { @@ -255,20 +237,24 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr return fmt.Errorf("create tokens: %w", err) } - box, secrets, err := accessbox.PackTokens(gatesData, nil) + var secret []byte + isCustom := options.AccessKeyID != "" + if isCustom { + secret = []byte(options.SecretAccessKey) + } + box, secrets, err := accessbox.PackTokens(gatesData, secret, isCustom) if err != nil { return fmt.Errorf("pack tokens: %w", err) } box.ContainerPolicy = policies - var idOwner user.ID - user.IDFromKey(&idOwner, options.FrostFSKey.PrivateKey.PublicKey) - id, err := a.checkContainer(ctx, options.Container, idOwner) - if err != nil { + if err = a.checkContainer(ctx, options.Container); err != nil { return fmt.Errorf("check container: %w", err) } + var idOwner user.ID + user.IDFromKey(&idOwner, options.FrostFSKey.PrivateKey.PublicKey) a.log.Info(logs.StoreBearerTokenIntoFrostFS, zap.Stringer("owner_tkn", idOwner)) @@ -281,26 +267,31 @@ func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecr creds := tokens.New(cfg) prm := tokens.CredentialsParam{ - OwnerID: idOwner, + Container: options.Container, + AccessKeyID: options.AccessKeyID, AccessBox: box, Expiration: lifetime.Exp, Keys: options.GatesPublicKeys, CustomAttributes: options.CustomAttributes, } - addr, err := creds.Put(ctx, id, prm) + addr, err := creds.Put(ctx, prm) if err != nil { return fmt.Errorf("failed to put creds: %w", err) } - accessKeyID := accessKeyIDFromAddr(addr) + accessKeyID := options.AccessKeyID + if accessKeyID == "" { + accessKeyID = accessKeyIDFromAddr(addr) + } + ir := &issuingResult{ InitialAccessKeyID: accessKeyID, AccessKeyID: accessKeyID, SecretAccessKey: secrets.SecretKey, OwnerPrivateKey: hex.EncodeToString(secrets.EphemeralKey.Bytes()), WalletPublicKey: hex.EncodeToString(options.FrostFSKey.PublicKey().Bytes()), - ContainerID: id.EncodeToString(), + ContainerID: options.Container.EncodeToString(), } enc := json.NewEncoder(w) @@ -337,13 +328,15 @@ func (a *Agent) UpdateSecret(ctx context.Context, w io.Writer, options *UpdateSe creds := tokens.New(cfg) - box, _, err := creds.GetBox(ctx, options.Address) + box, _, err := creds.GetBox(ctx, options.ContainerID, options.AccessKeyID) if err != nil { return fmt.Errorf("get accessbox: %w", err) } - secret, err := hex.DecodeString(box.Gate.SecretKey) - if err != nil { + var secret []byte + if options.IsCustom { + secret = []byte(box.Gate.SecretKey) + } else if secret, err = hex.DecodeString(box.Gate.SecretKey); err != nil { return fmt.Errorf("failed to decode secret key access box: %w", err) } @@ -360,7 +353,7 @@ func (a *Agent) UpdateSecret(ctx context.Context, w io.Writer, options *UpdateSe return fmt.Errorf("create tokens: %w", err) } - updatedBox, secrets, err := accessbox.PackTokens(gatesData, secret) + updatedBox, secrets, err := accessbox.PackTokens(gatesData, secret, options.IsCustom) if err != nil { return fmt.Errorf("pack tokens: %w", err) } @@ -371,22 +364,26 @@ func (a *Agent) UpdateSecret(ctx context.Context, w io.Writer, options *UpdateSe zap.Stringer("owner_tkn", idOwner)) prm := tokens.CredentialsParam{ - OwnerID: idOwner, + Container: options.ContainerID, AccessBox: updatedBox, Expiration: lifetime.Exp, Keys: options.GatesPublicKeys, CustomAttributes: options.CustomAttributes, } - oldAddr := options.Address - addr, err := creds.Update(ctx, oldAddr, prm) + addr, err := creds.Update(ctx, prm) if err != nil { return fmt.Errorf("failed to update creds: %w", err) } + accessKeyID := options.AccessKeyID + if !options.IsCustom { + accessKeyID = accessKeyIDFromAddr(addr) + } + ir := &issuingResult{ - AccessKeyID: accessKeyIDFromAddr(addr), - InitialAccessKeyID: accessKeyIDFromAddr(oldAddr), + AccessKeyID: accessKeyID, + InitialAccessKeyID: options.AccessKeyID, SecretAccessKey: secrets.SecretKey, OwnerPrivateKey: hex.EncodeToString(secrets.EphemeralKey.Bytes()), WalletPublicKey: hex.EncodeToString(options.FrostFSKey.PublicKey().Bytes()), @@ -419,12 +416,7 @@ func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSe bearerCreds := tokens.New(cfg) - var addr oid.Address - if err := addr.DecodeString(options.SecretAddress); err != nil { - return fmt.Errorf("failed to parse secret address: %w", err) - } - - box, _, err := bearerCreds.GetBox(ctx, addr) + box, _, err := bearerCreds.GetBox(ctx, options.Container, options.AccessKeyID) if err != nil { return fmt.Errorf("failed to get tokens: %w", err) } diff --git a/cmd/s3-authmate/modules/issue-secret.go b/cmd/s3-authmate/modules/issue-secret.go index 6895a19..b9df1eb 100644 --- a/cmd/s3-authmate/modules/issue-secret.go +++ b/cmd/s3-authmate/modules/issue-secret.go @@ -4,22 +4,34 @@ import ( "context" "fmt" "os" + "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet" cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/cobra" "github.com/spf13/viper" + "go.uber.org/zap" ) var issueSecretCmd = &cobra.Command{ Use: "issue-secret", Short: "Issue a secret in FrostFS network", Long: "Creates new s3 credentials to use with frostfs-s3-gw", - Example: `frostfs-s3-authmate issue-secret --wallet wallet.json --peer s01.frostfs.devenv:8080 --gate-public-key 031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a -frostfs-s3-authmate issue-secret --wallet wallet.json --peer s01.frostfs.devenv:8080 --gate-public-key 031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a --attributes LOGIN=NUUb82KR2JrVByHs2YSKgtK29gKnF5q6Vt`, + Example: `To create new s3 credentials use: +frostfs-s3-authmate issue-secret --wallet wallet.json --peer s01.frostfs.devenv:8080 --gate-public-key 031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a +frostfs-s3-authmate issue-secret --wallet wallet.json --peer s01.frostfs.devenv:8080 --gate-public-key 031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a --attributes LOGIN=NUUb82KR2JrVByHs2YSKgtK29gKnF5q6Vt + +To create new s3 credentials using specific access key id and secret access key use: +frostfs-s3-authmate issue-secret --wallet wallet.json --peer s01.frostfs.devenv:8080 --gate-public-key 031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a --access-key-id my-access-key-id --secret-access-key my-secret-key --container-id BpExV76416Vo7GrkJsGwXGoLM35xsBwup8voedDZR3c6 +`, RunE: runIssueSecretCmd, } @@ -54,6 +66,9 @@ const ( poolHealthcheckTimeoutFlag = "pool-healthcheck-timeout" poolRebalanceIntervalFlag = "pool-rebalance-interval" poolStreamTimeoutFlag = "pool-stream-timeout" + + accessKeyIDFlag = "access-key-id" + secretAccessKeyFlag = "secret-access-key" ) func initIssueSecretCmd() { @@ -73,6 +88,9 @@ func initIssueSecretCmd() { issueSecretCmd.Flags().Duration(poolRebalanceIntervalFlag, defaultPoolRebalanceInterval, "Interval for updating nodes health status") issueSecretCmd.Flags().Duration(poolStreamTimeoutFlag, defaultPoolStreamTimeout, "Timeout for individual operation in streaming RPC") issueSecretCmd.Flags().String(attributesFlag, "", "User attributes in form of Key1=Value1,Key2=Value2 (note: you cannot override system attributes)") + issueSecretCmd.Flags().String(accessKeyIDFlag, "", "Access key id of s3 credential that must be created") + issueSecretCmd.Flags().String(secretAccessKeyFlag, "", "Secret access key of s3 credential that must be used") + issueSecretCmd.Flags().String(rpcEndpointFlag, "", "NEO node RPC address (must be provided if container-id is nns name)") _ = issueSecretCmd.MarkFlagRequired(walletFlag) _ = issueSecretCmd.MarkFlagRequired(peerFlag) @@ -91,14 +109,6 @@ func runIssueSecretCmd(cmd *cobra.Command, _ []string) error { return wrapPreparationError(fmt.Errorf("failed to load frostfs private key: %s", err)) } - var cnrID cid.ID - containerID := viper.GetString(containerIDFlag) - if len(containerID) > 0 { - if err = cnrID.DecodeString(containerID); err != nil { - return wrapPreparationError(fmt.Errorf("failed to parse auth container id: %s", err)) - } - } - var gatesPublicKeys []*keys.PublicKey for _, keyStr := range viper.GetStringSlice(gatePublicKeyFlag) { gpk, err := keys.NewPublicKeyFromString(keyStr) @@ -137,17 +147,29 @@ func runIssueSecretCmd(cmd *cobra.Command, _ []string) error { return wrapFrostFSInitError(fmt.Errorf("failed to create FrostFS component: %s", err)) } + var accessBox cid.ID + if viper.IsSet(containerIDFlag) { + if accessBox, err = util.ResolveContainerID(viper.GetString(containerIDFlag), viper.GetString(rpcEndpointFlag)); err != nil { + return wrapPreparationError(fmt.Errorf("resolve accessbox container id (make sure you provided %s): %w", rpcEndpointFlag, err)) + } + } else if accessBox, err = createAccessBox(ctx, frostFS, key, log); err != nil { + return wrapPreparationError(err) + } + + accessKeyID, secretAccessKey, err := parseAccessKeys() + if err != nil { + return wrapPreparationError(err) + } + customAttrs, err := parseObjectAttrs(viper.GetString(attributesFlag)) if err != nil { return wrapPreparationError(fmt.Errorf("failed to parse attributes: %s", err)) } issueSecretOptions := &authmate.IssueSecretOptions{ - Container: authmate.ContainerOptions{ - ID: cnrID, - FriendlyName: viper.GetString(containerFriendlyNameFlag), - PlacementPolicy: viper.GetString(containerPlacementPolicyFlag), - }, + Container: accessBox, + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, FrostFSKey: key, GatesPublicKeys: gatesPublicKeys, Impersonate: true, @@ -164,3 +186,59 @@ func runIssueSecretCmd(cmd *cobra.Command, _ []string) error { } return nil } + +func parseAccessKeys() (accessKeyID, secretAccessKey string, err error) { + accessKeyID = viper.GetString(accessKeyIDFlag) + secretAccessKey = viper.GetString(secretAccessKeyFlag) + + if accessKeyID == "" && secretAccessKey != "" || accessKeyID != "" && secretAccessKey == "" { + return "", "", fmt.Errorf("flags %s and %s must be both provided or not", accessKeyIDFlag, secretAccessKeyFlag) + } + + if accessKeyID != "" { + if !isCustomCreds(accessKeyID) { + return "", "", fmt.Errorf("invalid custom AccessKeyID format: %s", accessKeyID) + } + if !checkAccessKeyLength(accessKeyID) { + return "", "", fmt.Errorf("invalid custom AccessKeyID length: %s", accessKeyID) + } + if !checkAccessKeyLength(secretAccessKey) { + return "", "", fmt.Errorf("invalid custom SecretAccessKey length: %s", secretAccessKey) + } + } + + return accessKeyID, secretAccessKey, nil +} + +func isCustomCreds(accessKeyID string) bool { + var addr oid.Address + return addr.DecodeString(strings.ReplaceAll(accessKeyID, "0", "/")) != nil +} + +func checkAccessKeyLength(key string) bool { + return 4 <= len(key) && len(key) <= 128 +} + +func createAccessBox(ctx context.Context, frostFS *frostfs.AuthmateFrostFS, key *keys.PrivateKey, log *zap.Logger) (cid.ID, error) { + friendlyName := viper.GetString(containerFriendlyNameFlag) + placementPolicy := viper.GetString(containerPlacementPolicyFlag) + + log.Info(logs.CreateContainer, zap.String("friendly_name", friendlyName), zap.String("placement_policy", placementPolicy)) + + prm := authmate.PrmContainerCreate{ + FriendlyName: friendlyName, + } + + user.IDFromKey(&prm.Owner, key.PrivateKey.PublicKey) + + if err := prm.Policy.DecodeString(placementPolicy); err != nil { + return cid.ID{}, fmt.Errorf("failed to build placement policy: %w", err) + } + + accessBox, err := frostFS.CreateContainer(ctx, prm) + if err != nil { + return cid.ID{}, fmt.Errorf("create container in FrostFS: %w", err) + } + + return accessBox, nil +} diff --git a/cmd/s3-authmate/modules/obtain-secret.go b/cmd/s3-authmate/modules/obtain-secret.go index b7d5130..ab0b9ce 100644 --- a/cmd/s3-authmate/modules/obtain-secret.go +++ b/cmd/s3-authmate/modules/obtain-secret.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "strings" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet" @@ -24,7 +23,6 @@ var obtainSecretCmd = &cobra.Command{ const ( gateWalletFlag = "gate-wallet" gateAddressFlag = "gate-address" - accessKeyIDFlag = "access-key-id" ) const ( @@ -38,10 +36,12 @@ func initObtainSecretCmd() { obtainSecretCmd.Flags().String(gateWalletFlag, "", "Path to the s3 gateway wallet to decrypt accessbox") obtainSecretCmd.Flags().String(gateAddressFlag, "", "Address of the s3 gateway wallet account") obtainSecretCmd.Flags().String(accessKeyIDFlag, "", "Access key id of s3 credential for which secret must be obtained") + obtainSecretCmd.Flags().String(containerIDFlag, "", "CID or NNS name of auth container that contains provided credential (must be provided if custom access key id is used)") obtainSecretCmd.Flags().Duration(poolDialTimeoutFlag, defaultPoolDialTimeout, "Timeout for connection to the node in pool to be established") obtainSecretCmd.Flags().Duration(poolHealthcheckTimeoutFlag, defaultPoolHealthcheckTimeout, "Timeout for request to node to decide if it is alive") obtainSecretCmd.Flags().Duration(poolRebalanceIntervalFlag, defaultPoolRebalanceInterval, "Interval for updating nodes health status") obtainSecretCmd.Flags().Duration(poolStreamTimeoutFlag, defaultPoolStreamTimeout, "Timeout for individual operation in streaming RPC") + obtainSecretCmd.Flags().String(rpcEndpointFlag, "", "NEO node RPC address (must be provided if container-id is nns name)") _ = obtainSecretCmd.MarkFlagRequired(walletFlag) _ = obtainSecretCmd.MarkFlagRequired(peerFlag) @@ -81,8 +81,14 @@ func runObtainSecretCmd(cmd *cobra.Command, _ []string) error { return wrapFrostFSInitError(cli.Exit(fmt.Sprintf("failed to create FrostFS component: %s", err), 2)) } + accessBox, accessKeyID, _, err := getAccessBoxID() + if err != nil { + return wrapPreparationError(err) + } + obtainSecretOptions := &authmate.ObtainSecretOptions{ - SecretAddress: strings.Replace(viper.GetString(accessKeyIDFlag), "0", "/", 1), + Container: accessBox, + AccessKeyID: accessKeyID, GatePrivateKey: gateKey, } diff --git a/cmd/s3-authmate/modules/update-secret.go b/cmd/s3-authmate/modules/update-secret.go index 7250691..8bc6fd0 100644 --- a/cmd/s3-authmate/modules/update-secret.go +++ b/cmd/s3-authmate/modules/update-secret.go @@ -4,11 +4,9 @@ import ( "context" "fmt" "os" - "strings" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/wallet" - oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -33,13 +31,15 @@ func initUpdateSecretCmd() { updateSecretCmd.Flags().String(peerFlag, "", "Address of a frostfs peer to connect to") updateSecretCmd.Flags().String(gateWalletFlag, "", "Path to the s3 gateway wallet to decrypt accessbox") updateSecretCmd.Flags().String(gateAddressFlag, "", "Address of the s3 gateway wallet account") - updateSecretCmd.Flags().String(accessKeyIDFlag, "", "Access key id of s3 credential for which secret must be obtained") + updateSecretCmd.Flags().String(accessKeyIDFlag, "", "Access key id of s3 credential for which secret must be updatedd") + updateSecretCmd.Flags().String(containerIDFlag, "", "CID or NNS name of auth container that contains provided credential (must be provided if custom access key id is used)") updateSecretCmd.Flags().StringSlice(gatePublicKeyFlag, nil, "Public 256r1 key of a gate (use flags repeatedly for multiple gates or separate them by comma)") updateSecretCmd.Flags().Duration(poolDialTimeoutFlag, defaultPoolDialTimeout, "Timeout for connection to the node in pool to be established") updateSecretCmd.Flags().Duration(poolHealthcheckTimeoutFlag, defaultPoolHealthcheckTimeout, "Timeout for request to node to decide if it is alive") updateSecretCmd.Flags().Duration(poolRebalanceIntervalFlag, defaultPoolRebalanceInterval, "Interval for updating nodes health status") updateSecretCmd.Flags().Duration(poolStreamTimeoutFlag, defaultPoolStreamTimeout, "Timeout for individual operation in streaming RPC") updateSecretCmd.Flags().String(attributesFlag, "", "User attributes in form of Key1=Value1,Key2=Value2 (note: you cannot override system attributes)") + updateSecretCmd.Flags().String(rpcEndpointFlag, "", "NEO node RPC address (must be provided if container-id is nns name)") _ = updateSecretCmd.MarkFlagRequired(walletFlag) _ = updateSecretCmd.MarkFlagRequired(peerFlag) @@ -66,10 +66,9 @@ func runUpdateSecretCmd(cmd *cobra.Command, _ []string) error { return wrapPreparationError(fmt.Errorf("failed to load s3 gate private key: %s", err)) } - var accessBoxAddress oid.Address - credAddr := strings.Replace(viper.GetString(accessKeyIDFlag), "0", "/", 1) - if err = accessBoxAddress.DecodeString(credAddr); err != nil { - return wrapPreparationError(fmt.Errorf("failed to parse creds address: %w", err)) + accessBox, accessKeyID, isCustom, err := getAccessBoxID() + if err != nil { + return wrapPreparationError(err) } var gatesPublicKeys []*keys.PublicKey @@ -101,7 +100,9 @@ func runUpdateSecretCmd(cmd *cobra.Command, _ []string) error { } updateSecretOptions := &authmate.UpdateSecretOptions{ - Address: accessBoxAddress, + ContainerID: accessBox, + AccessKeyID: accessKeyID, + IsCustom: isCustom, FrostFSKey: key, GatesPublicKeys: gatesPublicKeys, GatePrivateKey: gateKey, diff --git a/cmd/s3-authmate/modules/utils.go b/cmd/s3-authmate/modules/utils.go index b7924e8..d88db35 100644 --- a/cmd/s3-authmate/modules/utils.go +++ b/cmd/s3-authmate/modules/utils.go @@ -3,6 +3,7 @@ package modules import ( "context" "encoding/json" + "errors" "fmt" "os" "strings" @@ -11,8 +12,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/authmate" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs" + "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/frostfs/util" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "github.com/spf13/viper" @@ -163,3 +167,23 @@ func parseObjectAttrs(attributes string) ([]object.Attribute, error) { return attrs, nil } + +func getAccessBoxID() (cid.ID, string, bool, error) { + accessKeyID := viper.GetString(accessKeyIDFlag) + + var accessBoxAddress oid.Address + if err := accessBoxAddress.DecodeString(strings.Replace(accessKeyID, "0", "/", 1)); err == nil { + return accessBoxAddress.Container(), accessKeyID, false, nil + } + + if !viper.IsSet(containerIDFlag) { + return cid.ID{}, "", false, errors.New("accessbox parameter must be set when custom access key id is used") + } + + accessBox, err := util.ResolveContainerID(viper.GetString(containerIDFlag), viper.GetString(rpcEndpointFlag)) + if err != nil { + return cid.ID{}, "", false, fmt.Errorf("resolve accessbox container id (make sure you provided %s): %w", rpcEndpointFlag, err) + } + + return accessBox, accessKeyID, true, nil +} diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go index d3fcca2..ee8ea33 100644 --- a/cmd/s3-gw/app.go +++ b/cmd/s3-gw/app.go @@ -96,6 +96,7 @@ type ( resolveZoneList []string isResolveListAllow bool // True if ResolveZoneList contains allowed zones frostfsidValidation bool + accessbox *cid.ID mu sync.RWMutex namespaces Namespaces @@ -132,18 +133,7 @@ type ( func newApp(ctx context.Context, log *Logger, v *viper.Viper) *App { objPool, treePool, key := getPools(ctx, log.logger, v) - cfg := tokens.Config{ - FrostFS: frostfs.NewAuthmateFrostFS(frostfs.NewFrostFS(objPool, key), log.logger), - Key: key, - CacheConfig: getAccessBoxCacheConfig(v, log.logger), - RemovingCheckAfterDurations: fetchRemovingCheckInterval(v, log.logger), - } - - // prepare auth center - ctr := auth.New(tokens.New(cfg), v.GetStringSlice(cfgAllowedAccessKeyIDPrefixes)) - app := &App{ - ctr: ctr, log: log.logger, cfg: v, pool: objPool, @@ -162,6 +152,7 @@ func newApp(ctx context.Context, log *Logger, v *viper.Viper) *App { } func (a *App) init(ctx context.Context) { + a.initAuthCenter(ctx) a.setRuntimeParameters() a.initFrostfsID(ctx) a.initPolicyStorage(ctx) @@ -171,6 +162,25 @@ func (a *App) init(ctx context.Context) { a.initTracing(ctx) } +func (a *App) initAuthCenter(ctx context.Context) { + if a.cfg.IsSet(cfgContainersAccessBox) { + cnrID, err := a.resolveContainerID(ctx, cfgContainersAccessBox) + if err != nil { + a.log.Fatal(logs.CouldNotFetchAccessBoxContainerInfo, zap.Error(err)) + } + a.settings.accessbox = &cnrID + } + + cfg := tokens.Config{ + FrostFS: frostfs.NewAuthmateFrostFS(frostfs.NewFrostFS(a.pool, a.key), a.log), + Key: a.key, + CacheConfig: getAccessBoxCacheConfig(a.cfg, a.log), + RemovingCheckAfterDurations: fetchRemovingCheckInterval(a.cfg, a.log), + } + + a.ctr = auth.New(tokens.New(cfg), a.cfg.GetStringSlice(cfgAllowedAccessKeyIDPrefixes), a.settings) +} + func (a *App) initLayer(ctx context.Context) { a.initResolver() @@ -484,6 +494,14 @@ func (s *appSettings) RetryStrategy() handler.RetryStrategy { return s.retryStrategy } +func (s *appSettings) AccessBoxContainer() (cid.ID, bool) { + if s.accessbox != nil { + return *s.accessbox, true + } + + return cid.ID{}, false +} + func (a *App) initAPI(ctx context.Context) { a.initLayer(ctx) a.initHandler() @@ -1104,21 +1122,30 @@ func (a *App) tryReconnect(ctx context.Context, sr *http.Server) bool { } func (a *App) fetchContainerInfo(ctx context.Context, cfgKey string) (info *data.BucketInfo, err error) { + cnrID, err := a.resolveContainerID(ctx, cfgKey) + if err != nil { + return nil, err + } + + return getContainerInfo(ctx, cnrID, a.pool) +} + +func (a *App) resolveContainerID(ctx context.Context, cfgKey string) (cid.ID, error) { containerString := a.cfg.GetString(cfgKey) var id cid.ID - if err = id.DecodeString(containerString); err != nil { + if err := id.DecodeString(containerString); err != nil { i := strings.Index(containerString, ".") if i < 0 { - return nil, fmt.Errorf("invalid container address: %s", containerString) + return cid.ID{}, fmt.Errorf("invalid container address: %s", containerString) } if id, err = a.bucketResolver.Resolve(ctx, containerString[i+1:], containerString[:i]); err != nil { - return nil, fmt.Errorf("resolve container address %s: %w", containerString, err) + return cid.ID{}, fmt.Errorf("resolve container address %s: %w", containerString, err) } } - return getContainerInfo(ctx, id, a.pool) + return id, nil } func getContainerInfo(ctx context.Context, id cid.ID, frostFSPool *pool.Pool) (*data.BucketInfo, error) { diff --git a/cmd/s3-gw/app_settings.go b/cmd/s3-gw/app_settings.go index 94db37d..7071623 100644 --- a/cmd/s3-gw/app_settings.go +++ b/cmd/s3-gw/app_settings.go @@ -208,6 +208,7 @@ const ( // Settings. // Containers. cfgContainersCORS = "containers.cors" cfgContainersLifecycle = "containers.lifecycle" + cfgContainersAccessBox = "containers.accessbox" // Command line args. cmdHelp = "help" diff --git a/creds/accessbox/accessbox.go b/creds/accessbox/accessbox.go index 690c171..64cbdcd 100644 --- a/creds/accessbox/accessbox.go +++ b/creds/accessbox/accessbox.go @@ -99,7 +99,7 @@ func (x *AccessBox) Unmarshal(data []byte) error { // PackTokens adds bearer and session tokens to BearerTokens and SessionToken lists respectively. // Session token can be nil. // Secret can be nil. In such case secret will be generated. -func PackTokens(gatesData []*GateData, secret []byte) (*AccessBox, *Secrets, error) { +func PackTokens(gatesData []*GateData, secret []byte, isCustomSecret bool) (*AccessBox, *Secrets, error) { box := &AccessBox{} ephemeralKey, err := keys.NewPrivateKey() if err != nil { @@ -118,11 +118,16 @@ func PackTokens(gatesData []*GateData, secret []byte) (*AccessBox, *Secrets, err return nil, nil, fmt.Errorf("failed to add tokens to accessbox: %w", err) } - return box, &Secrets{hex.EncodeToString(secret), ephemeralKey}, err + secretKey := string(secret) + if !isCustomSecret { + secretKey = hex.EncodeToString(secret) + } + + return box, &Secrets{SecretKey: secretKey, EphemeralKey: ephemeralKey}, err } // GetTokens returns gate tokens from AccessBox. -func (x *AccessBox) GetTokens(owner *keys.PrivateKey) (*GateData, error) { +func (x *AccessBox) GetTokens(owner *keys.PrivateKey, isCustomSecret bool) (*GateData, error) { seedKey, err := keys.NewPublicKeyFromBytes(x.SeedKey, elliptic.P256()) if err != nil { return nil, fmt.Errorf("couldn't unmarshal SeedKey: %w", err) @@ -133,7 +138,7 @@ func (x *AccessBox) GetTokens(owner *keys.PrivateKey) (*GateData, error) { continue } - gateData, err := decodeGate(gate, owner, seedKey) + gateData, err := decodeGate(gate, owner, seedKey, isCustomSecret) if err != nil { return nil, fmt.Errorf("failed to decode gate: %w", err) } @@ -161,8 +166,8 @@ func (x *AccessBox) GetPlacementPolicy() ([]*ContainerPolicy, error) { } // GetBox parses AccessBox to Box. -func (x *AccessBox) GetBox(owner *keys.PrivateKey) (*Box, error) { - tokens, err := x.GetTokens(owner) +func (x *AccessBox) GetBox(owner *keys.PrivateKey, isCustomSecret bool) (*Box, error) { + tokens, err := x.GetTokens(owner, isCustomSecret) if err != nil { return nil, fmt.Errorf("get tokens: %w", err) } @@ -217,7 +222,7 @@ func encodeGate(ephemeralKey *keys.PrivateKey, seedKey *keys.PublicKey, tokens * return gate, nil } -func decodeGate(gate *AccessBox_Gate, owner *keys.PrivateKey, seedKey *keys.PublicKey) (*GateData, error) { +func decodeGate(gate *AccessBox_Gate, owner *keys.PrivateKey, seedKey *keys.PublicKey, isCustomSecret bool) (*GateData, error) { data, err := decrypt(owner, seedKey, gate.Tokens) if err != nil { return nil, fmt.Errorf("decrypt tokens: %w", err) @@ -243,7 +248,11 @@ func decodeGate(gate *AccessBox_Gate, owner *keys.PrivateKey, seedKey *keys.Publ gateData := NewGateData(owner.PublicKey(), &bearerTkn) gateData.SessionTokens = sessionTkns - gateData.SecretKey = hex.EncodeToString(tokens.SecretKey) + if isCustomSecret { + gateData.SecretKey = string(tokens.SecretKey) + } else { + gateData.SecretKey = hex.EncodeToString(tokens.SecretKey) + } return gateData, nil } diff --git a/creds/tokens/credentials.go b/creds/tokens/credentials.go index 573f362..985ba3b 100644 --- a/creds/tokens/credentials.go +++ b/creds/tokens/credentials.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strconv" + "strings" "time" "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/api/cache" @@ -14,7 +15,6 @@ import ( cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" - "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "github.com/nspcc-dev/neo-go/pkg/crypto/keys" "go.uber.org/zap" ) @@ -22,13 +22,14 @@ import ( type ( // Credentials is a bearer token get/put interface. Credentials interface { - GetBox(context.Context, oid.Address) (*accessbox.Box, []object.Attribute, error) - Put(context.Context, cid.ID, CredentialsParam) (oid.Address, error) - Update(context.Context, oid.Address, CredentialsParam) (oid.Address, error) + GetBox(context.Context, cid.ID, string) (*accessbox.Box, []object.Attribute, error) + Put(context.Context, CredentialsParam) (oid.Address, error) + Update(context.Context, CredentialsParam) (oid.Address, error) } CredentialsParam struct { - OwnerID user.ID + Container cid.ID + AccessKeyID string AccessBox *accessbox.AccessBox Expiration uint64 Keys keys.PublicKeys @@ -53,9 +54,6 @@ type ( // PrmObjectCreate groups parameters of objects created by credential tool. type PrmObjectCreate struct { - // FrostFS identifier of the object creator. - Creator user.ID - // FrostFS container to store the object. Container cid.ID @@ -64,7 +62,12 @@ type PrmObjectCreate struct { // Optional. // If provided cred object will be created using crdt approach. - NewVersionFor *oid.ID + NewVersionForAccessKeyID string + + // Optional. + // If provided cred object will contain specific crdt name attribute for first accessbox object version. + // If NewVersionForAccessKeyID is provided this field isn't used. + CustomAccessKey string // Last FrostFS epoch of the object lifetime. ExpirationEpoch uint64 @@ -76,6 +79,17 @@ type PrmObjectCreate struct { CustomAttributes []object.Attribute } +// PrmGetCredsObject groups parameters of getting credential object. +type PrmGetCredsObject struct { + // FrostFS container to get the object. + Container cid.ID + + // S3 access key id. + AccessKeyID string +} + +var ErrCustomAccessKeyIDNotFound = errors.New("custom AccessKeyId not found") + // FrostFS represents virtual connection to FrostFS network. type FrostFS interface { // CreateObject creates and saves a parameterized object in the specified @@ -92,8 +106,9 @@ type FrostFS interface { // // It returns exactly one non-nil value. It returns any error encountered which // prevented the object payload from being read. + // Returns ErrCustomAccessKeyIDNotFound if provided AccessKey is custom, and it was not found. // Object must contain full payload. - GetCredsObject(context.Context, oid.Address) (*object.Object, error) + GetCredsObject(context.Context, PrmGetCredsObject) (*object.Object, error) } var ( @@ -116,61 +131,66 @@ func New(cfg Config) Credentials { } } -func (c *cred) GetBox(ctx context.Context, addr oid.Address) (*accessbox.Box, []object.Attribute, error) { - cachedBoxValue := c.cache.Get(addr) +func (c *cred) GetBox(ctx context.Context, cnrID cid.ID, accessKeyID string) (*accessbox.Box, []object.Attribute, error) { + isCustomSecret := isCustom(accessKeyID) + cachedBoxValue := c.cache.Get(accessKeyID) if cachedBoxValue != nil { - return c.checkIfCredentialsAreRemoved(ctx, addr, cachedBoxValue) + return c.checkIfCredentialsAreRemoved(ctx, cnrID, accessKeyID, cachedBoxValue, isCustomSecret) } - box, attrs, err := c.getAccessBox(ctx, addr) + box, attrs, err := c.getAccessBox(ctx, cnrID, accessKeyID) if err != nil { return nil, nil, fmt.Errorf("get access box: %w", err) } - cachedBox, err := box.GetBox(c.key) + cachedBox, err := box.GetBox(c.key, isCustomSecret) if err != nil { return nil, nil, fmt.Errorf("get gate box: %w", err) } - c.putBoxToCache(addr, cachedBox, attrs) + c.putBoxToCache(accessKeyID, cachedBox, attrs) return cachedBox, attrs, nil } -func (c *cred) checkIfCredentialsAreRemoved(ctx context.Context, addr oid.Address, cachedBoxValue *cache.AccessBoxCacheValue) (*accessbox.Box, []object.Attribute, error) { +func (c *cred) checkIfCredentialsAreRemoved(ctx context.Context, cnrID cid.ID, accessKeyID string, cachedBoxValue *cache.AccessBoxCacheValue, isCustomSecret bool) (*accessbox.Box, []object.Attribute, error) { if time.Since(cachedBoxValue.PutTime) < c.removingCheckDuration { return cachedBoxValue.Box, cachedBoxValue.Attributes, nil } - box, attrs, err := c.getAccessBox(ctx, addr) + box, attrs, err := c.getAccessBox(ctx, cnrID, accessKeyID) if err != nil { if client.IsErrObjectAlreadyRemoved(err) { - c.cache.Delete(addr) + c.cache.Delete(accessKeyID) return nil, nil, fmt.Errorf("get access box: %w", err) } return cachedBoxValue.Box, cachedBoxValue.Attributes, nil } - cachedBox, err := box.GetBox(c.key) + cachedBox, err := box.GetBox(c.key, isCustomSecret) if err != nil { - c.cache.Delete(addr) + c.cache.Delete(accessKeyID) return nil, nil, fmt.Errorf("get gate box: %w", err) } // we need this to reset PutTime // to don't check for removing each time after removingCheckDuration interval - c.putBoxToCache(addr, cachedBox, attrs) + c.putBoxToCache(accessKeyID, cachedBox, attrs) return cachedBoxValue.Box, attrs, nil } -func (c *cred) putBoxToCache(addr oid.Address, box *accessbox.Box, attrs []object.Attribute) { - if err := c.cache.Put(addr, box, attrs); err != nil { - c.log.Warn(logs.CouldntPutAccessBoxIntoCache, zap.String("address", addr.EncodeToString())) +func (c *cred) putBoxToCache(accessKeyID string, box *accessbox.Box, attrs []object.Attribute) { + if err := c.cache.Put(accessKeyID, box, attrs); err != nil { + c.log.Warn(logs.CouldntPutAccessBoxIntoCache, zap.String("accessKeyID", accessKeyID)) } } -func (c *cred) getAccessBox(ctx context.Context, addr oid.Address) (*accessbox.AccessBox, []object.Attribute, error) { - obj, err := c.frostFS.GetCredsObject(ctx, addr) +func (c *cred) getAccessBox(ctx context.Context, cnrID cid.ID, accessKeyID string) (*accessbox.AccessBox, []object.Attribute, error) { + prm := PrmGetCredsObject{ + Container: cnrID, + AccessKeyID: accessKeyID, + } + obj, err := c.frostFS.GetCredsObject(ctx, prm) if err != nil { return nil, nil, fmt.Errorf("read payload and attributes: %w", err) } @@ -184,16 +204,29 @@ func (c *cred) getAccessBox(ctx context.Context, addr oid.Address) (*accessbox.A return &box, obj.Attributes(), nil } -func (c *cred) Put(ctx context.Context, idCnr cid.ID, prm CredentialsParam) (oid.Address, error) { - return c.createObject(ctx, idCnr, nil, prm) +func (c *cred) Put(ctx context.Context, prm CredentialsParam) (oid.Address, error) { + if prm.AccessKeyID != "" { + c.log.Info(logs.CheckCustomAccessKeyIDUniqueness, zap.String("access_key_id", prm.AccessKeyID)) + credsPrm := PrmGetCredsObject{ + Container: prm.Container, + AccessKeyID: prm.AccessKeyID, + } + + if _, err := c.frostFS.GetCredsObject(ctx, credsPrm); err == nil { + return oid.Address{}, fmt.Errorf("access key id '%s' already exists", prm.AccessKeyID) + } else if !errors.Is(err, ErrCustomAccessKeyIDNotFound) { + return oid.Address{}, fmt.Errorf("check AccessKeyID uniqueness: %w", err) + } + } + + return c.createObject(ctx, prm, false) } -func (c *cred) Update(ctx context.Context, addr oid.Address, prm CredentialsParam) (oid.Address, error) { - objID := addr.Object() - return c.createObject(ctx, addr.Container(), &objID, prm) +func (c *cred) Update(ctx context.Context, prm CredentialsParam) (oid.Address, error) { + return c.createObject(ctx, prm, true) } -func (c *cred) createObject(ctx context.Context, cnrID cid.ID, newVersionFor *oid.ID, prm CredentialsParam) (oid.Address, error) { +func (c *cred) createObject(ctx context.Context, prm CredentialsParam, update bool) (oid.Address, error) { if len(prm.Keys) == 0 { return oid.Address{}, ErrEmptyPublicKeys } else if prm.AccessBox == nil { @@ -204,14 +237,19 @@ func (c *cred) createObject(ctx context.Context, cnrID cid.ID, newVersionFor *oi return oid.Address{}, fmt.Errorf("marshall box: %w", err) } + var newVersionFor string + if update { + newVersionFor = prm.AccessKeyID + } + idObj, err := c.frostFS.CreateObject(ctx, PrmObjectCreate{ - Creator: prm.OwnerID, - Container: cnrID, - Filepath: strconv.FormatInt(time.Now().Unix(), 10) + "_access.box", - ExpirationEpoch: prm.Expiration, - NewVersionFor: newVersionFor, - Payload: data, - CustomAttributes: prm.CustomAttributes, + Container: prm.Container, + Filepath: strconv.FormatInt(time.Now().Unix(), 10) + "_access.box", + ExpirationEpoch: prm.Expiration, + CustomAccessKey: prm.AccessKeyID, + NewVersionForAccessKeyID: newVersionFor, + Payload: data, + CustomAttributes: prm.CustomAttributes, }) if err != nil { return oid.Address{}, fmt.Errorf("create object: %w", err) @@ -219,7 +257,11 @@ func (c *cred) createObject(ctx context.Context, cnrID cid.ID, newVersionFor *oi var addr oid.Address addr.SetObject(idObj) - addr.SetContainer(cnrID) + addr.SetContainer(prm.Container) return addr, nil } + +func isCustom(accessKeyID string) bool { + return (&oid.Address{}).DecodeString(strings.ReplaceAll(accessKeyID, "0", "/")) != nil +} diff --git a/docs/configuration.md b/docs/configuration.md index 205a901..6ff92e7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -761,12 +761,14 @@ Section for well-known containers to store s3-related data and settings. containers: cors: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj lifecycle: AZjLTXfK4vs4ovxMic2xEJKSymMNLqdwq9JT64ASFCRj + accessbox: ExnA1gSY3kzgomi2wJxNyWo1ytWv9VAKXRE55fNXEPL2 ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------| -| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. | -| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-------------|----------|---------------|---------------|-------------------------------------------------------------------------------------------------------------------------| +| `cors` | `string` | no | | Container name for CORS configurations. If not set, container of the bucket is used. | +| `lifecycle` | `string` | no | | Container name for lifecycle configurations. If not set, container of the bucket is used. | +| `accessbox` | `string` | no | | Container name to lookup accessbox if custom aws credentials is used. If not set, custom credentials are not supported. | # `vhs` section diff --git a/internal/frostfs/authmate.go b/internal/frostfs/authmate.go index 2936d3d..09eed96 100644 --- a/internal/frostfs/authmate.go +++ b/internal/frostfs/authmate.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "strconv" + "strings" "time" objectv2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" @@ -73,19 +74,26 @@ func (x *AuthmateFrostFS) CreateContainer(ctx context.Context, prm authmate.PrmC } // GetCredsObject implements authmate.FrostFS interface method. -func (x *AuthmateFrostFS) GetCredsObject(ctx context.Context, addr oid.Address) (*object.Object, error) { - versions, err := x.getCredVersions(ctx, addr) +func (x *AuthmateFrostFS) GetCredsObject(ctx context.Context, prm tokens.PrmGetCredsObject) (*object.Object, error) { + versions, err := x.getCredVersions(ctx, prm.Container, prm.AccessKeyID) if err != nil { return nil, err } - credObjID := addr.Object() + var addr oid.Address + isCustom := addr.DecodeString(strings.ReplaceAll(prm.AccessKeyID, "0", "/")) != nil + + var credObjID oid.ID if last := versions.GetLast(); last != nil { credObjID = last.ObjID + } else if !isCustom { + credObjID = addr.Object() + } else { + return nil, fmt.Errorf("%w: '%s'", tokens.ErrCustomAccessKeyIDNotFound, prm.AccessKeyID) } res, err := x.frostFS.GetObject(ctx, frostfs.PrmObjectGet{ - Container: addr.Container(), + Container: prm.Container, Object: credObjID, }) if err != nil { @@ -111,17 +119,20 @@ func (x *AuthmateFrostFS) GetCredsObject(ctx context.Context, addr oid.Address) func (x *AuthmateFrostFS) CreateObject(ctx context.Context, prm tokens.PrmObjectCreate) (oid.ID, error) { attributes := [][2]string{{objectv2.SysAttributeExpEpoch, strconv.FormatUint(prm.ExpirationEpoch, 10)}} - if prm.NewVersionFor != nil { - var addr oid.Address - addr.SetContainer(prm.Container) - addr.SetObject(*prm.NewVersionFor) - - versions, err := x.getCredVersions(ctx, addr) + if prm.NewVersionForAccessKeyID != "" { + versions, err := x.getCredVersions(ctx, prm.Container, prm.NewVersionForAccessKeyID) if err != nil { return oid.ID{}, err } if versions.GetLast() == nil { + var addr oid.Address + isCustom := addr.DecodeString(strings.ReplaceAll(prm.NewVersionForAccessKeyID, "0", "/")) != nil + + if isCustom { + return oid.ID{}, fmt.Errorf("creds object for accessKeyId '%s' not found", prm.NewVersionForAccessKeyID) + } + versions.AppendVersion(&crdt.ObjectVersion{ObjID: addr.Object()}) } @@ -130,6 +141,8 @@ func (x *AuthmateFrostFS) CreateObject(ctx context.Context, prm tokens.PrmObject } attributes = append(attributes, [2]string{accessBoxCRDTNameAttr, versions.Name()}) + } else if prm.CustomAccessKey != "" { + attributes = append(attributes, [2]string{accessBoxCRDTNameAttr, prm.CustomAccessKey}) } for _, attr := range prm.CustomAttributes { @@ -150,21 +163,20 @@ func (x *AuthmateFrostFS) CreateObject(ctx context.Context, prm tokens.PrmObject return res.ObjectID, nil } -func (x *AuthmateFrostFS) getCredVersions(ctx context.Context, addr oid.Address) (*crdt.ObjectVersions, error) { - objCredSystemName := credVersionSysName(addr.Container(), addr.Object()) +func (x *AuthmateFrostFS) getCredVersions(ctx context.Context, cnrID cid.ID, accessKeyID string) (*crdt.ObjectVersions, error) { credVersions, err := x.frostFS.SearchObjects(ctx, frostfs.PrmObjectSearch{ - Container: addr.Container(), - ExactAttribute: [2]string{accessBoxCRDTNameAttr, objCredSystemName}, + Container: cnrID, + ExactAttribute: [2]string{accessBoxCRDTNameAttr, accessKeyID}, }) if err != nil { return nil, fmt.Errorf("search s3 access boxes: %w", err) } - versions := crdt.NewObjectVersions(objCredSystemName) + versions := crdt.NewObjectVersions(accessKeyID) for _, id := range credVersions { objVersion, err := x.frostFS.HeadObject(ctx, frostfs.PrmObjectHead{ - Container: addr.Container(), + Container: cnrID, Object: id, }) if err != nil { @@ -184,7 +196,3 @@ func (x *AuthmateFrostFS) reqLogger(ctx context.Context) *zap.Logger { } return x.log } - -func credVersionSysName(cnrID cid.ID, objID oid.ID) string { - return cnrID.EncodeToString() + "0" + objID.EncodeToString() -} diff --git a/internal/frostfs/util/util.go b/internal/frostfs/util/util.go index 00a9905..aa1be48 100644 --- a/internal/frostfs/util/util.go +++ b/internal/frostfs/util/util.go @@ -7,6 +7,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/ns" "github.com/nspcc-dev/neo-go/pkg/util" @@ -36,6 +37,31 @@ func ResolveContractHash(contractHash, rpcAddress string) (util.Uint160, error) return nns.ResolveContractHash(domain) } +// ResolveContainerID determine container id by resolving NNS name. +func ResolveContainerID(containerID, rpcAddress string) (cid.ID, error) { + var cnrID cid.ID + if err := cnrID.DecodeString(containerID); err == nil { + return cnrID, nil + } + + splitName := strings.Split(containerID, ".") + if len(splitName) != 2 { + return cid.ID{}, fmt.Errorf("invalid container name: '%s'", containerID) + } + + var domain container.Domain + domain.SetName(splitName[0]) + domain.SetZone(splitName[1]) + + var nns ns.NNS + if err := nns.Dial(rpcAddress); err != nil { + return cid.ID{}, fmt.Errorf("dial nns '%s': %w", rpcAddress, err) + } + defer nns.Close() + + return nns.ResolveContainerDomain(domain) +} + func TimeToEpoch(ni *netmap.NetworkInfo, now, t time.Time) (uint64, error) { duration := t.Sub(now) durationAbs := duration.Abs() diff --git a/internal/logs/logs.go b/internal/logs/logs.go index dd7a659..81f0136 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -159,6 +159,7 @@ const ( FoundSeveralSystemNodes = "found several system nodes" FailedToParsePartInfo = "failed to parse part info" CouldNotFetchCORSContainerInfo = "couldn't fetch CORS container info" + CouldNotFetchAccessBoxContainerInfo = "couldn't fetch AccessBox container info" CloseCredsObjectPayload = "close creds object payload" CouldntDeleteLifecycleObject = "couldn't delete lifecycle configuration object" CouldntCacheLifecycleConfiguration = "couldn't cache lifecycle configuration" @@ -171,4 +172,5 @@ const ( FailedToRemoveOldPartNode = "failed to remove old part node" CouldntCacheNetworkInfo = "couldn't cache network info" NotSupported = "not supported" + CheckCustomAccessKeyIDUniqueness = "check custom access key id uniqueness" )