From 020281a0e9349f6723754b9594d741b3060cd748 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 7 Mar 2024 12:03:58 +0300 Subject: [PATCH] [#9999] node: Add billing service Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/billing.go | 61 ++++++++++++ cmd/frostfs-node/config.go | 5 + cmd/frostfs-node/config/billing/config.go | 58 ++++++++++++ .../config/billing/config_test.go | 37 ++++++++ cmd/frostfs-node/main.go | 1 + config/example/node.env | 4 + config/example/node.json | 9 ++ config/example/node.yaml | 7 ++ dev/.vscode-example/launch.json | 2 + docs/storage-node-configuration.md | 15 +++ internal/logs/logs.go | 2 + .../billing/server/list_containers.go | 12 +++ pkg/services/billing/server/server.go | 34 +++++++ pkg/services/billing/server/sign.go | 92 +++++++++++++++++++ 14 files changed, 339 insertions(+) create mode 100644 cmd/frostfs-node/billing.go create mode 100644 cmd/frostfs-node/config/billing/config.go create mode 100644 cmd/frostfs-node/config/billing/config_test.go create mode 100644 pkg/services/billing/server/list_containers.go create mode 100644 pkg/services/billing/server/server.go create mode 100644 pkg/services/billing/server/sign.go diff --git a/cmd/frostfs-node/billing.go b/cmd/frostfs-node/billing.go new file mode 100644 index 000000000..e09999257 --- /dev/null +++ b/cmd/frostfs-node/billing.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "net" + + billingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/billing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/billing" + billingSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/billing/server" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +const serviceNameBilling = "billing" + +func initBillingService(c *cfg) { + endpoint := billingconfig.GRPC(c.appCfg).Endpoint() + if endpoint == billingconfig.GRPCEndpointDefault { + c.log.Info(logs.FrostFSNodeBillingServiceDisabled) + return + } + + pubs := billingconfig.AuthorizedKeys(c.appCfg) + rawPubs := make([][]byte, 0, len(pubs)+1) + rawPubs = append(rawPubs, c.key.PublicKey().Bytes()) + + for i := range pubs { + rawPubs = append(rawPubs, pubs[i].Bytes()) + } + + billingSvc := billingSvc.New( + &c.key.PrivateKey, + rawPubs, + c.cfgObject.cnrSource, + c.cfgObject.cfgLocalStorage.localStorage, + ) + + lis, err := net.Listen("tcp", endpoint) + if err != nil { + c.log.Error(logs.FrostFSNodeCantListenGRPCEndpointBilling, zap.Error(err)) + return + } + + c.cfgBillingService.server = grpc.NewServer() + + c.onShutdown(func() { + stopGRPC("FrostFS Billing API", c.cfgBillingService.server, c.log) + }) + + billing.RegisterBillingServiceServer(c.cfgBillingService.server, billingSvc) + + c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { + runAndLog(ctx, c, serviceNameBilling, false, func(context.Context, *cfg) { + c.log.Info(logs.FrostFSNodeStartListeningEndpoint, + zap.String("service", serviceNameBilling), + zap.String("endpoint", endpoint)) + fatalOnErr(c.cfgBillingService.server.Serve(lis)) + }) + })) +} diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 2b185cfc8..f8dc0d4ab 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -476,6 +476,7 @@ type cfg struct { cfgNetmap cfgNetmap cfgControlService cfgControlService cfgObject cfgObject + cfgBillingService cfgBillingService } // ReadCurrentNetMap reads network map which has been cached at the @@ -656,6 +657,10 @@ type cfgControlService struct { server *grpc.Server } +type cfgBillingService struct { + server *grpc.Server +} + var persistateSideChainLastBlockKey = []byte("side_chain_last_processed_block") func initCfg(appCfg *config.Config) *cfg { diff --git a/cmd/frostfs-node/config/billing/config.go b/cmd/frostfs-node/config/billing/config.go new file mode 100644 index 000000000..15d4149d0 --- /dev/null +++ b/cmd/frostfs-node/config/billing/config.go @@ -0,0 +1,58 @@ +package billing + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +type GRPCConfig struct { + cfg *config.Config +} + +const ( + subsection = "billing" + grpcSubsection = "grpc" + + // GRPCEndpointDefault is a default endpoint of gRPC Billing service. + GRPCEndpointDefault = "" +) + +// AuthorizedKeys parses and returns an array of "authorized_keys" config +// parameter from "control" section. +// +// Returns an empty list if not set. +func AuthorizedKeys(c *config.Config) keys.PublicKeys { + strKeys := config.StringSliceSafe(c.Sub(subsection), "authorized_keys") + pubs := make(keys.PublicKeys, 0, len(strKeys)) + + for i := range strKeys { + pub, err := keys.NewPublicKeyFromString(strKeys[i]) + if err != nil { + panic(fmt.Errorf("invalid permitted key for Billing service %s: %w", strKeys[i], err)) + } + + pubs = append(pubs, pub) + } + + return pubs +} + +func GRPC(c *config.Config) GRPCConfig { + return GRPCConfig{ + c.Sub(subsection).Sub(grpcSubsection), + } +} + +// Endpoint returns the value of "endpoint" config parameter. +// +// Returns GRPCEndpointDefault if the value is not a non-empty string. +func (g GRPCConfig) Endpoint() string { + v := config.String(g.cfg, "endpoint") + if v != "" { + return v + } + + return GRPCEndpointDefault +} diff --git a/cmd/frostfs-node/config/billing/config_test.go b/cmd/frostfs-node/config/billing/config_test.go new file mode 100644 index 000000000..85251e9e5 --- /dev/null +++ b/cmd/frostfs-node/config/billing/config_test.go @@ -0,0 +1,37 @@ +package billing_test + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + billingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/billing" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestBillingSection(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + empty := configtest.EmptyConfig() + + require.Empty(t, billingconfig.AuthorizedKeys(empty)) + require.Equal(t, billingconfig.GRPCEndpointDefault, billingconfig.GRPC(empty).Endpoint()) + }) + + const path = "../../../../config/example/node" + + pubs := make(keys.PublicKeys, 2) + pubs[0], _ = keys.NewPublicKeyFromString("035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11") + pubs[1], _ = keys.NewPublicKeyFromString("028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6") + + fileConfigTest := func(c *config.Config) { + require.Equal(t, pubs, billingconfig.AuthorizedKeys(c)) + require.Equal(t, "localhost:8092", billingconfig.GRPC(c).Endpoint()) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) +} diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index e4f0a434c..3eb7fcd95 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -115,6 +115,7 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(c, "tree", initTreeService) initAndLog(c, "apemanager", initAPEManagerService) initAndLog(c, "control", initControlService) + initAndLog(c, "billing", initBillingService) initAndLog(c, "morph notifications", func(c *cfg) { listenMorphNotifications(ctx, c) }) } diff --git a/config/example/node.env b/config/example/node.env index 72f56e96c..60cdfcf8f 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -49,6 +49,10 @@ FROSTFS_GRPC_1_TLS_ENABLED=false FROSTFS_CONTROL_AUTHORIZED_KEYS="035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6" FROSTFS_CONTROL_GRPC_ENDPOINT=localhost:8090 +# Billing service section +FROSTFS_BILLING_AUTHORIZED_KEYS="035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6" +FROSTFS_BILLING_GRPC_ENDPOINT=localhost:8092 + # Contracts section FROSTFS_CONTRACTS_BALANCE=5263abba1abedbf79bb57f3e40b50b4425d2d6cd FROSTFS_CONTRACTS_CONTAINER=5d084790d7aa36cea7b53fe897380dab11d2cd3c diff --git a/config/example/node.json b/config/example/node.json index b9dc6014c..1ef5979b4 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -82,6 +82,15 @@ "endpoint": "localhost:8090" } }, + "billing": { + "authorized_keys": [ + "035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11", + "028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6" + ], + "grpc": { + "endpoint": "localhost:8092" + } + }, "contracts": { "balance": "5263abba1abedbf79bb57f3e40b50b4425d2d6cd", "container": "5d084790d7aa36cea7b53fe897380dab11d2cd3c", diff --git a/config/example/node.yaml b/config/example/node.yaml index bad67816a..df5fb1c7e 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -69,6 +69,13 @@ control: grpc: endpoint: localhost:8090 # endpoint that is listened by the Control Service +billing: + authorized_keys: # list of hex-encoded public keys that have rights to use the Billing Service + - 035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 + - 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6 + grpc: + endpoint: localhost:8092 # endpoint that is listened by the Billing Service + contracts: # side chain NEOFS contract script hashes; optional, override values retrieved from NNS contract balance: 5263abba1abedbf79bb57f3e40b50b4425d2d6cd container: 5d084790d7aa36cea7b53fe897380dab11d2cd3c diff --git a/dev/.vscode-example/launch.json b/dev/.vscode-example/launch.json index 990fd42a8..e09a952ba 100644 --- a/dev/.vscode-example/launch.json +++ b/dev/.vscode-example/launch.json @@ -49,6 +49,8 @@ "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8080", "FROSTFS_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8081", "FROSTFS_CONTROL_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", + "FROSTFS_BILLING_GRPC_ENDPOINT":"127.0.0.1:8082", + "FROSTFS_BILLING_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-node-s1-state", diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 5389bfbb5..0db423b33 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -18,6 +18,7 @@ There are some custom types used for brevity: | `pprof` | [PProf configuration](#pprof-section) | | `prometheus` | [Prometheus metrics configuration](#prometheus-section) | | `control` | [Control service configuration](#control-section) | +| `billing` | [Billing service configuration](#billing-section) | | `contracts` | [Override FrostFS contracts hashes](#contracts-section) | | `morph` | [N3 blockchain client configuration](#morph-section) | | `apiclient` | [FrostFS API client configuration](#apiclient-section) | @@ -42,6 +43,20 @@ control: | `authorized_keys` | `[]public key` | empty | List of public keys which are used to authorize requests to the control service. | | `grpc.endpoint` | `string` | empty | Address that control service listener binds to. | +# `billing` section +```yaml +billing: + authorized_keys: + - 035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 + - 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6 + grpc: + endpoint: 127.0.0.1:8092 +``` +| Parameter | Type | Default value | Description | +|-------------------|----------------|---------------|----------------------------------------------------------------------------------| +| `authorized_keys` | `[]public key` | empty | List of public keys which are used to authorize requests to the billing service. | +| `grpc.endpoint` | `string` | empty | Address that billing service listener binds to. | + # `grpc` section ```yaml grpc: diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 78f00c4ee..e53432e8d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -443,6 +443,8 @@ const ( FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" + FrostFSNodeCantListenGRPCEndpointBilling = "can't listen gRPC endpoint (billing)" + FrostFSNodeBillingServiceDisabled = "billing service is disabled" FrostFSNodePolicerIsDisabled = "policer is disabled" CommonApplicationStarted = "application started" ShardGCCollectingExpiredObjectsStarted = "collecting expired objects started" diff --git a/pkg/services/billing/server/list_containers.go b/pkg/services/billing/server/list_containers.go new file mode 100644 index 000000000..e121e37f7 --- /dev/null +++ b/pkg/services/billing/server/list_containers.go @@ -0,0 +1,12 @@ +package server + +import ( + "context" + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/billing" +) + +func (s *Server) ListContainers(context.Context, *billing.ListContainersRequest) (*billing.ListContainersResponse, error) { + return nil, errors.New("not implemented") +} diff --git a/pkg/services/billing/server/server.go b/pkg/services/billing/server/server.go new file mode 100644 index 000000000..0e01cb72d --- /dev/null +++ b/pkg/services/billing/server/server.go @@ -0,0 +1,34 @@ +package server + +import ( + "crypto/ecdsa" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" +) + +type cfg struct { + key *ecdsa.PrivateKey + allowedKeys [][]byte + cnrSrc container.Source + se *engine.StorageEngine +} + +type Server struct { + *cfg +} + +func New(key *ecdsa.PrivateKey, + allowedKeys [][]byte, + cnrSrc container.Source, + se *engine.StorageEngine, +) *Server { + return &Server{ + cfg: &cfg{ + key: key, + allowedKeys: allowedKeys, + cnrSrc: cnrSrc, + se: se, + }, + } +} diff --git a/pkg/services/billing/server/sign.go b/pkg/services/billing/server/sign.go new file mode 100644 index 000000000..579d43baf --- /dev/null +++ b/pkg/services/billing/server/sign.go @@ -0,0 +1,92 @@ +package server + +import ( + "bytes" + "crypto/ecdsa" + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/billing" + frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" + frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" +) + +// SignedMessage is an interface of Control service message. +type SignedMessage interface { + ReadSignedData([]byte) ([]byte, error) + GetSignature() *billing.Signature + SetSignature(*billing.Signature) +} + +var ( + errDisallowedKey = errors.New("key is not in the allowed list") + errMissingSignature = errors.New("missing signature") + errInvalidSignature = errors.New("invalid signature") +) + +func (s *Server) isValidRequest(req SignedMessage) error { + sign := req.GetSignature() + if sign == nil { + return errMissingSignature + } + + var ( + key = sign.GetKey() + allowed = false + ) + + for i := range s.allowedKeys { + if allowed = bytes.Equal(s.allowedKeys[i], key); allowed { + break + } + } + if !allowed { + return errDisallowedKey + } + + binBody, err := req.ReadSignedData(nil) + if err != nil { + return fmt.Errorf("marshal request body: %w", err) + } + + var sigV2 refs.Signature + sigV2.SetKey(sign.GetKey()) + sigV2.SetSign(sign.GetSignature()) + sigV2.SetScheme(refs.ECDSA_SHA512) + + var sig frostfscrypto.Signature + if err := sig.ReadFromV2(sigV2); err != nil { + return fmt.Errorf("can't read signature: %w", err) + } + + if !sig.Verify(binBody) { + return errInvalidSignature + } + + return nil +} + +func SignMessage(key *ecdsa.PrivateKey, msg SignedMessage) error { + binBody, err := msg.ReadSignedData(nil) + if err != nil { + return fmt.Errorf("marshal request body: %w", err) + } + + var sig frostfscrypto.Signature + + err = sig.Calculate(frostfsecdsa.Signer(*key), binBody) + if err != nil { + return fmt.Errorf("calculate signature: %w", err) + } + + var sigV2 refs.Signature + sig.WriteToV2(&sigV2) + + var sigBilling billing.Signature + sigBilling.Key = sigV2.GetKey() + sigBilling.Signature = sigV2.GetSign() + msg.SetSignature(&sigBilling) + + return nil +}