From 50e4eb3b4374e3e9a67be3fa0092a0cf0801f4c0 Mon Sep 17 00:00:00 2001 From: Angira Kekteeva Date: Tue, 18 May 2021 21:49:09 +0300 Subject: [PATCH] [#46] authmate: Copy files from cdn-authmate repository to neofs-s3-gw Copy authmate main file and s3 agent Signed-off-by: Angira Kekteeva --- authmate/s3.go | 229 +++++++++++++++++++++++++ cmd/authmate/main.go | 392 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 621 insertions(+) create mode 100644 authmate/s3.go create mode 100644 cmd/authmate/main.go diff --git a/authmate/s3.go b/authmate/s3.go new file mode 100644 index 00000000..c36b4c4e --- /dev/null +++ b/authmate/s3.go @@ -0,0 +1,229 @@ +package s3 + +import ( + "context" + "crypto/ecdsa" + "encoding/json" + "io" + "math" + "strconv" + "time" + + sdk "github.com/nspcc-dev/cdn-sdk" + "github.com/nspcc-dev/cdn-sdk/creds/bearer" + "github.com/nspcc-dev/cdn-sdk/creds/hcs" + "github.com/nspcc-dev/cdn-sdk/creds/neofs" + "github.com/nspcc-dev/cdn-sdk/creds/s3" + "github.com/nspcc-dev/neofs-api-go/pkg/acl/eacl" + "github.com/nspcc-dev/neofs-api-go/pkg/container" + "github.com/nspcc-dev/neofs-api-go/pkg/netmap" + "github.com/nspcc-dev/neofs-api-go/pkg/object" + "github.com/nspcc-dev/neofs-api-go/pkg/owner" + "github.com/nspcc-dev/neofs-api-go/pkg/token" + "github.com/nspcc-dev/neofs-node/pkg/policy" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +const defaultAuthContainerBasicACL uint32 = 0b00111100100011001000110011001100 + +type Agent struct { + cli sdk.Client + log *zap.Logger +} + +func New(log *zap.Logger, client sdk.Client) *Agent { + return &Agent{log: log, cli: client} +} + +type ( + IssueSecretOptions struct { + ContainerID *container.ID + ContainerFriendlyName string + NEOFSCreds neofs.Credentials + OwnerPrivateKey hcs.PrivateKey + GatesPublicKeys []hcs.PublicKey + EACLRules []byte + } + + ObtainSecretOptions struct { + SecretAddress string + GatePrivateKey hcs.PrivateKey + } +) + +type ( + issuingResult struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + OwnerPrivateKey string `json:"owner_private_key"` + } + + obtainingResult struct { + BearerToken *token.BearerToken `json:"-"` + SecretAccessKey string `json:"secret_access_key"` + } +) + +func (a *Agent) checkContainer(ctx context.Context, cid *container.ID, friendlyName string) (*container.ID, error) { + if cid != nil { + // check that container exists + _, err := a.cli.Container().Get(ctx, cid) + return cid, err + } + + pp, err := buildPlacementPolicy("") + if err != nil { + return nil, errors.Wrap(err, "failed to build placement policy") + } + + cnr := container.New( + container.WithPolicy(pp), + container.WithCustomBasicACL(defaultAuthContainerBasicACL), + container.WithAttribute(container.AttributeName, friendlyName), + container.WithAttribute(container.AttributeTimestamp, strconv.FormatInt(time.Now().Unix(), 10))) + + return a.cli.Container().Put(ctx, cnr, + sdk.ContainerPutAndWait(), + sdk.ContainerPutWithTimeout(120*time.Second)) +} + +func (a *Agent) IssueSecret(ctx context.Context, w io.Writer, options *IssueSecretOptions) error { + var ( + err error + cid *container.ID + ) + + a.log.Info("check container", zap.Stringer("cid", options.ContainerID)) + if cid, err = a.checkContainer(ctx, options.ContainerID, options.ContainerFriendlyName); err != nil { + return err + } + + a.log.Info("prepare eACL table") + + table, err := buildEACLTable(cid, options.EACLRules) + if err != nil { + return errors.Wrap(err, "failed to build eacl table") + } + + tkn, err := buildBearerToken(options.NEOFSCreds.PrivateKey(), options.NEOFSCreds.Owner(), table) + if err != nil { + return errors.Wrap(err, "failed to build bearer token") + } + + a.log.Info("store bearer token into NeoFS", + zap.Stringer("owner_key", options.NEOFSCreds.Owner()), + zap.Stringer("owner_tkn", tkn.Issuer())) + + address, err := bearer. + New(a.cli.Object(), options.OwnerPrivateKey). + Put(ctx, cid, tkn, options.GatesPublicKeys...) + if err != nil { + return errors.Wrap(err, "failed to put bearer token") + } + + secret, err := s3.SecretAccessKey(tkn) + if err != nil { + return errors.Wrap(err, "failed to get bearer token secret key") + } + + ir := &issuingResult{ + AccessKeyID: address.String(), + SecretAccessKey: secret, + OwnerPrivateKey: options.OwnerPrivateKey.String(), + } + + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(ir) +} + +func (a *Agent) ObtainSecret(ctx context.Context, w io.Writer, options *ObtainSecretOptions) error { + bearerCreds := bearer.New(a.cli.Object(), options.GatePrivateKey) + address := object.NewAddress() + if err := address.Parse(options.SecretAddress); err != nil { + return errors.Wrap(err, "failed to parse secret address") + } + + tkn, err := bearerCreds.Get(ctx, address) + if err != nil { + return errors.Wrap(err, "failed to get bearer token") + } + + secret, err := s3.SecretAccessKey(tkn) + if err != nil { + return errors.Wrap(err, "failed to get bearer token secret key") + } + + or := &obtainingResult{ + BearerToken: tkn, + SecretAccessKey: secret, + } + + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(or) +} + +func buildPlacementPolicy(placementRules string) (*netmap.PlacementPolicy, error) { + if len(placementRules) != 0 { + return policy.Parse(placementRules) + } + + /* + REP 1 IN X // place one copy of object + CBF 1 + SELECT 2 From * AS X // in container of two nodes + */ + pp := new(netmap.PlacementPolicy) + pp.SetContainerBackupFactor(1) + pp.SetReplicas([]*netmap.Replica{newReplica("X", 1)}...) + pp.SetSelectors([]*netmap.Selector{newSimpleSelector("X", 2)}...) + + return pp, nil +} + +// selects nodes in container without any additional attributes +func newSimpleSelector(name string, count uint32) (s *netmap.Selector) { + s = new(netmap.Selector) + s.SetCount(count) + s.SetFilter("*") + s.SetName(name) + return +} + +func newReplica(name string, count uint32) (r *netmap.Replica) { + r = new(netmap.Replica) + r.SetCount(count) + r.SetSelector(name) + return +} + +func buildEACLTable(cid *container.ID, eaclTable []byte) (*eacl.Table, error) { + table := eacl.NewTable() + if len(eaclTable) != 0 { + return table, table.UnmarshalJSON(eaclTable) + } + + record := eacl.NewRecord() + record.SetOperation(eacl.OperationGet) + record.SetAction(eacl.ActionAllow) + // TODO: Change this later. + // from := eacl.HeaderFromObject + // matcher := eacl.MatchStringEqual + // record.AddFilter(from eacl.FilterHeaderType, matcher eacl.Match, name string, value string) + eacl.AddFormedTarget(record, eacl.RoleOthers) + table.SetCID(cid) + table.AddRecord(record) + + return table, nil +} + +func buildBearerToken(key *ecdsa.PrivateKey, oid *owner.ID, table *eacl.Table) (*token.BearerToken, error) { + bearerToken := token.NewBearerToken() + bearerToken.SetEACLTable(table) + bearerToken.SetOwner(oid) + bearerToken.SetLifetime(math.MaxUint64, 0, 0) + + return bearerToken, bearerToken.SignToken(key) +} diff --git a/cmd/authmate/main.go b/cmd/authmate/main.go new file mode 100644 index 00000000..b87e9668 --- /dev/null +++ b/cmd/authmate/main.go @@ -0,0 +1,392 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/json" + "fmt" + "os" + "time" + + "github.com/nspcc-dev/cdn-authmate/agents/s3" + sdk "github.com/nspcc-dev/cdn-sdk" + "github.com/nspcc-dev/cdn-sdk/creds/hcs" + "github.com/nspcc-dev/cdn-sdk/creds/neofs" + "github.com/nspcc-dev/cdn-sdk/grace" + "github.com/nspcc-dev/cdn-sdk/pool" + "github.com/nspcc-dev/neofs-api-go/pkg/container" + "github.com/pkg/errors" + "github.com/urfave/cli/v2" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +type gateKey struct { + PrivateKey string `json:"private_key"` + PublicKey string `json:"public_key"` +} + +const ( + poolConnectTimeout = 5 * time.Second + poolRequestTimeout = 5 * time.Second +) + +var ( + // Build = "now" + Version = "dev" +) + +var ( + neoFSKeyPathFlag string + peerAddressFlag string + eaclRulesFlag string + gatePrivateKeyFlag string + secretAddressFlag string + ownerPrivateKeyFlag string + containerIDFlag string + containerFriendlyName string + gatesPublicKeysFlag cli.StringSlice + gatesKeysCountFlag int + logEnabledFlag bool + logDebugEnabledFlag bool +) + +var zapConfig = zap.Config{ + Development: true, + Encoding: "console", + Level: zap.NewAtomicLevelAt(zapcore.FatalLevel), + OutputPaths: []string{"stdout"}, + EncoderConfig: zapcore.EncoderConfig{ + MessageKey: "message", + LevelKey: "level", + EncodeLevel: zapcore.CapitalLevelEncoder, + TimeKey: "time", + EncodeTime: zapcore.ISO8601TimeEncoder, + CallerKey: "caller", + EncodeCaller: zapcore.ShortCallerEncoder, + }, +} + +func prepare() (context.Context, *zap.Logger) { + var ( + err error + log = zap.NewNop() + ) + + if !logEnabledFlag { + return grace.Context(log), log + } else if logDebugEnabledFlag { + zapConfig.Level = zap.NewAtomicLevelAt(zapcore.DebugLevel) + } + + if log, err = zapConfig.Build(); err != nil { + panic(err) + } + + return grace.Context(log), log +} + +func main() { + app := &cli.App{ + Name: "NeoFS gate authentication manager", + Usage: "Helps manage delegated access via gates to data stored in NeoFS network", + Version: Version, + Flags: appFlags(), + Commands: appCommands(), + } + + if err := app.Run(os.Args); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "error: %s\n", err) + os.Exit(100) + } +} + +func appFlags() []cli.Flag { + return []cli.Flag{ + &cli.BoolFlag{ + Name: "with-log", + Usage: "Enable logger", + Destination: &logEnabledFlag, + }, + &cli.BoolFlag{ + Name: "debug", + Usage: "Enable debug logger level", + Destination: &logDebugEnabledFlag, + }, + } +} + +func appCommands() []*cli.Command { + return []*cli.Command{ + issueSecret(), + obtainSecret(), + generateKeys(), + } +} + +func generateGatesKeys(count int) ([]hcs.Credentials, error) { + var ( + err error + res = make([]hcs.Credentials, count, count) + ) + + for i := 0; i < count; i++ { + if res[i], err = hcs.Generate(rand.Reader); err != nil { + return nil, err + } + } + + return res, nil +} + +func generateKeys() *cli.Command { + return &cli.Command{ + Name: "generate-keys", + Usage: "Generate key pairs for gates", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "count", + Usage: "number of x25519 key pairs to generate", + Value: 1, + Destination: &gatesKeysCountFlag, + }, + }, + Action: func(c *cli.Context) error { + _, log := prepare() + + log.Info("start generating x25519 keys") + + csl, err := generateGatesKeys(gatesKeysCountFlag) + if err != nil { + return cli.Exit(fmt.Sprintf("failed to create key pairs of gates: %s", err), 1) + } + + log.Info("generated x25519 keys") + + gatesKeys := make([]gateKey, len(csl), len(csl)) + for i, cs := range csl { + privateKey, publicKey := cs.PrivateKey().String(), cs.PublicKey().String() + gatesKeys[i] = gateKey{PrivateKey: privateKey, PublicKey: publicKey} + } + + keys, err := json.MarshalIndent(gatesKeys, "", " ") + if err != nil { + return cli.Exit(fmt.Sprintf("failed to marshal key pairs of gates: %s", err), 2) + } + + fmt.Println(string(keys)) + return nil + }, + } +} + +func issueSecret() *cli.Command { + return &cli.Command{ + Name: "issue-secret", + Usage: "Issue a secret in NeoFS network", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "neofs-key", + Value: "", + Usage: "path to owner's neofs private ecdsa key", + Required: true, + Destination: &neoFSKeyPathFlag, + }, + &cli.StringFlag{ + Name: "peer", + Value: "", + Usage: "address of a neofs peer to connect to", + Required: true, + Destination: &peerAddressFlag, + }, + &cli.StringFlag{ + Name: "rules", + Usage: "eacl rules as plain json string", + Required: false, + Destination: &eaclRulesFlag, + }, + &cli.StringSliceFlag{ + Name: "gate-public-key", + Usage: "public x25519 key of a gate (use flags repeatedly for multiple gates)", + Required: true, + Destination: &gatesPublicKeysFlag, + }, + &cli.StringFlag{ + Name: "owner-private-key", + Usage: "owner's private x25519 key", + Required: false, + Destination: &ownerPrivateKeyFlag, + }, + &cli.StringFlag{ + Name: "container-id", + Usage: "auth container id to put the secret into", + Required: false, + Destination: &containerIDFlag, + }, + &cli.StringFlag{ + Name: "container-friendly-name", + Usage: "friendly name of auth container to put the secret into", + Required: false, + Destination: &containerFriendlyName, + Value: "auth-container", + }, + }, + Action: func(c *cli.Context) error { + ctx, log := prepare() + + neofsCreds, err := neofs.New(neoFSKeyPathFlag) + if err != nil { + return cli.Exit(fmt.Sprintf("failed to load neofs private key: %s", err), 1) + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + client, err := createSDKClient(ctx, log, neofsCreds, peerAddressFlag) + if err != nil { + return cli.Exit(fmt.Sprintf("failed to create sdk client: %s", err), 2) + } + + agent := s3.New(log, client) + var cid *container.ID + if len(containerIDFlag) > 0 { + cid = container.NewID() + if err := cid.Parse(containerIDFlag); err != nil { + return cli.Exit(fmt.Sprintf("failed to parse auth container id: %s", err), 3) + } + } + + var owner hcs.Credentials + if owner, err = fetchHCSCredentials(ownerPrivateKeyFlag); err != nil { + return cli.Exit(fmt.Sprintf("failed to create owner's private key: %s", err), 4) + } + + var gatesPublicKeys []hcs.PublicKey + for _, key := range gatesPublicKeysFlag.Value() { + gpk, err := hcs.LoadPublicKey(key) + if err != nil { + return cli.Exit(fmt.Sprintf("failed to load gate's public key: %s", err), 5) + } + gatesPublicKeys = append(gatesPublicKeys, gpk) + } + + issueSecretOptions := &s3.IssueSecretOptions{ + ContainerID: cid, + ContainerFriendlyName: containerFriendlyName, + NEOFSCreds: neofsCreds, + OwnerPrivateKey: owner.PrivateKey(), + GatesPublicKeys: gatesPublicKeys, + EACLRules: []byte(eaclRulesFlag), + } + + if err = agent.IssueSecret(ctx, os.Stdout, issueSecretOptions); err != nil { + return cli.Exit(fmt.Sprintf("failed to issue secret: %s", err), 6) + } + + return nil + }, + } +} + +func obtainSecret() *cli.Command { + command := &cli.Command{ + Name: "obtain-secret", + Usage: "Obtain a secret from NeoFS network", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "neofs-key", + Value: "", + Usage: "path to owner's neofs private ecdsa key", + Required: true, + Destination: &neoFSKeyPathFlag, + }, + &cli.StringFlag{ + Name: "peer", + Value: "", + Usage: "address of neofs peer to connect to", + Required: true, + Destination: &peerAddressFlag, + }, + &cli.StringFlag{ + Name: "gate-private-key", + Usage: "gate's private x25519 key", + Required: true, + Destination: &gatePrivateKeyFlag, + }, + &cli.StringFlag{ + Name: "secret-address", + Usage: "address of a secret (i.e. access key id for s3)", + Required: true, + Destination: &secretAddressFlag, + }, + }, + Action: func(c *cli.Context) error { + ctx, log := prepare() + + neofsCreds, err := neofs.New(neoFSKeyPathFlag) + if err != nil { + return cli.Exit(fmt.Sprintf("failed to load neofs private key: %s", err), 1) + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + client, err := createSDKClient(ctx, log, neofsCreds, peerAddressFlag) + if err != nil { + return cli.Exit(fmt.Sprintf("failed to create sdk client: %s", err), 2) + } + + agent := s3.New(log, client) + + var _ = agent + + gateCreds, err := hcs.NewCredentials(gatePrivateKeyFlag) + if err != nil { + return cli.Exit(fmt.Sprintf("failed to create owner's private key: %s", err), 4) + } + + obtainSecretOptions := &s3.ObtainSecretOptions{ + SecretAddress: secretAddressFlag, + GatePrivateKey: gateCreds.PrivateKey(), + } + + if err = agent.ObtainSecret(ctx, os.Stdout, obtainSecretOptions); err != nil { + return cli.Exit(fmt.Sprintf("failed to obtain secret: %s", err), 5) + } + + return nil + }, + } + return command +} + +func fetchHCSCredentials(val string) (hcs.Credentials, error) { + if val == "" { + return hcs.Generate(rand.Reader) + } + + return hcs.NewCredentials(val) +} + +func createSDKClient(ctx context.Context, log *zap.Logger, neofsCreds neofs.Credentials, peerAddress string) (sdk.Client, error) { + log.Debug("prepare connection pool") + + p, err := pool.New(ctx, + pool.WithLogger(log), + pool.WithAddress(peerAddress), + pool.WithCredentials(neofsCreds), + pool.WithAPIPreparer(sdk.APIPreparer), + pool.WithConnectTimeout(poolConnectTimeout), + pool.WithRequestTimeout(poolRequestTimeout)) + if err != nil { + return nil, errors.Wrap(err, "failed to create connection pool") + } + + log.Debug("prepare sdk client") + + return sdk.New(ctx, + sdk.WithLogger(log), + sdk.WithCredentials(neofsCreds), + sdk.WithConnectionPool(p), + sdk.WithAPIPreparer(sdk.APIPreparer)) +}