[#607] cmd/node: Serve gRPC on multiple interfaces

Generalize single gRPC interface of the storage node to a group of
interfaces. Each interface calls the same RPC handler.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-06-22 20:25:18 +03:00 committed by Leonard Lyubich
parent d1eb9c3b0f
commit 8060735732
13 changed files with 220 additions and 189 deletions

View file

@ -16,8 +16,7 @@ func initAccountingService(c *cfg) {
balanceMorphWrapper, err := wrapper.NewFromMorph(c.cfgMorph.client, c.cfgAccounting.scriptHash, 0) balanceMorphWrapper, err := wrapper.NewFromMorph(c.cfgMorph.client, c.cfgAccounting.scriptHash, 0)
fatalOnErr(err) fatalOnErr(err)
accountingGRPC.RegisterAccountingServiceServer(c.cfgGRPC.server, server := accountingTransportGRPC.New(
accountingTransportGRPC.New(
accountingService.NewSignService( accountingService.NewSignService(
&c.key.PrivateKey, &c.key.PrivateKey,
accountingService.NewResponseService( accountingService.NewResponseService(
@ -27,6 +26,9 @@ func initAccountingService(c *cfg) {
c.respSvc, c.respSvc,
), ),
), ),
),
) )
for _, srv := range c.cfgGRPC.servers {
accountingGRPC.RegisterAccountingServiceServer(srv, server)
}
} }

View file

@ -17,7 +17,6 @@ import (
contractsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/contracts" contractsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/contracts"
engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine" engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine"
shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard" shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
grpcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/grpc"
loggerconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/logger" loggerconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/logger"
metricsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/metrics" metricsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/metrics"
nodeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/node" nodeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/node"
@ -112,17 +111,13 @@ type cfg struct {
} }
type cfgGRPC struct { type cfgGRPC struct {
listener net.Listener listeners []net.Listener
server *grpc.Server servers []*grpc.Server
maxChunkSize uint64 maxChunkSize uint64
maxAddrAmount uint64 maxAddrAmount uint64
tlsEnabled bool
tlsCertFile string
tlsKeyFile string
} }
type cfgMorph struct { type cfgMorph struct {
@ -225,24 +220,6 @@ func initCfg(path string) *cfg {
maxChunkSize := uint64(maxMsgSize) * 3 / 4 // 25% to meta, 75% to payload maxChunkSize := uint64(maxMsgSize) * 3 / 4 // 25% to meta, 75% to payload
maxAddrAmount := uint64(maxChunkSize) / addressSize // each address is about 72 bytes maxAddrAmount := uint64(maxChunkSize) / addressSize // each address is about 72 bytes
var (
tlsEnabled bool
tlsCertFile string
tlsKeyFile string
tlsConfig = grpcconfig.TLS(appCfg)
)
if tlsConfig.Enabled() {
tlsEnabled = true
tlsCertFile = tlsConfig.CertificateFile()
tlsKeyFile = tlsConfig.KeyFile()
}
if tlsEnabled {
netAddr.AddTLS()
}
state := newNetworkState() state := newNetworkState()
containerWorkerPool, err := ants.NewPool(notificationHandlerPoolSize) containerWorkerPool, err := ants.NewPool(notificationHandlerPoolSize)
@ -281,9 +258,6 @@ func initCfg(path string) *cfg {
cfgGRPC: cfgGRPC{ cfgGRPC: cfgGRPC{
maxChunkSize: maxChunkSize, maxChunkSize: maxChunkSize,
maxAddrAmount: maxAddrAmount, maxAddrAmount: maxAddrAmount,
tlsEnabled: tlsEnabled,
tlsCertFile: tlsCertFile,
tlsKeyFile: tlsKeyFile,
}, },
localAddr: network.GroupFromAddress(netAddr), localAddr: network.GroupFromAddress(netAddr),
respSvc: response.NewService( respSvc: response.NewService(

View file

@ -2,33 +2,28 @@ package grpcconfig
import ( import (
"errors" "errors"
"strconv"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
) )
const (
subsection = "grpc"
tlsSubsection = "tls"
)
var ( var (
errEndpointNotSet = errors.New("empty/not set endpoint, see `grpc.endpoint` section") errEndpointNotSet = errors.New("empty/not set endpoint, see `grpc.endpoint` section")
errTLSKeyNotSet = errors.New("empty/not set TLS key file path, see `grpc.tls.key` section") errTLSKeyNotSet = errors.New("empty/not set TLS key file path, see `grpc.tls.key` section")
errTLSCertNotSet = errors.New("empty/not set TLS certificate file path, see `grpc.tls.certificate` section") errTLSCertNotSet = errors.New("empty/not set TLS certificate file path, see `grpc.tls.certificate` section")
) )
// TLSConfig is a wrapper over "tls" config section which provides access // Config is a wrapper over the config section
// to TLS configuration of gRPC connection. // which provides access to gRPC server configurations.
type TLSConfig struct { type Config config.Config
cfg *config.Config
}
// Endpoint returns value of "endpoint" config parameter // Endpoint returns value of "endpoint" config parameter.
// from "grpc" section.
// //
// Panics if value is not a non-empty string. // Panics if value is not a non-empty string.
func Endpoint(c *config.Config) string { func (x *Config) Endpoint() string {
v := config.StringSafe(c.Sub(subsection), "endpoint") v := config.StringSafe(
(*config.Config)(x),
"endpoint")
if v == "" { if v == "" {
panic(errEndpointNotSet) panic(errEndpointNotSet)
} }
@ -36,19 +31,27 @@ func Endpoint(c *config.Config) string {
return v return v
} }
// TLS returns structure that provides access to "tls" subsection of // TLS returns "tls" subsection as a TLSConfig.
// "grpc" section. //
func TLS(c *config.Config) TLSConfig { // Returns nil if "enabled" value of "tls" subsection is false.
return TLSConfig{ func (x *Config) TLS() *TLSConfig {
cfg: c.Sub(subsection).Sub(tlsSubsection), sub := (*config.Config)(x).
Sub("tls")
if !config.BoolSafe(sub, "enabled") {
return nil
}
return &TLSConfig{
cfg: sub,
} }
} }
// Enabled returns value of "enabled" config parameter. // TLSConfig is a wrapper over the config section
// // which provides access to TLS configurations
// Returns false if value is not set. // of the gRPC server.
func (tls TLSConfig) Enabled() bool { type TLSConfig struct {
return config.BoolSafe(tls.cfg, "enabled") cfg *config.Config
} }
// KeyFile returns value of "key" config parameter. // KeyFile returns value of "key" config parameter.
@ -74,3 +77,24 @@ func (tls TLSConfig) CertificateFile() string {
return v return v
} }
// IterateEndpoints iterates over subsections ["0":"N") (N - "num" value)
// of "grpc" section of c, wrap them into Config and passes to f.
//
// Panics if N is not a positive number.
func IterateEndpoints(c *config.Config, f func(*Config)) {
c = c.Sub("grpc")
num := config.Uint(c, "num")
if num == 0 {
panic("no gRPC server configured")
}
for i := uint64(0); i < num; i++ {
si := strconv.FormatUint(i, 10)
sc := (*Config)(c.Sub(si))
f(sc)
}
}

View file

@ -10,49 +10,35 @@ import (
func TestGRPCSection(t *testing.T) { func TestGRPCSection(t *testing.T) {
t.Run("defaults", func(t *testing.T) { t.Run("defaults", func(t *testing.T) {
empty := configtest.EmptyConfig() require.Panics(t, func() {
IterateEndpoints(configtest.EmptyConfig(), nil)
tlsEnabled := TLS(empty).Enabled() })
require.Equal(t, false, tlsEnabled)
require.PanicsWithError(
t,
errEndpointNotSet.Error(),
func() {
Endpoint(empty)
},
)
require.PanicsWithError(
t,
errTLSKeyNotSet.Error(),
func() {
TLS(empty).KeyFile()
},
)
require.PanicsWithError(
t,
errTLSCertNotSet.Error(),
func() {
TLS(empty).CertificateFile()
},
)
}) })
const path = "../../../../config/example/node" const path = "../../../../config/example/node"
var fileConfigTest = func(c *config.Config) { var fileConfigTest = func(c *config.Config) {
addr := Endpoint(c) num := 0
tlsEnabled := TLS(c).Enabled()
tlsCert := TLS(c).CertificateFile()
tlsKey := TLS(c).KeyFile()
require.Equal(t, "s01.neofs.devenv:8080", addr) IterateEndpoints(c, func(sc *Config) {
require.Equal(t, true, tlsEnabled) defer func() {
require.Equal(t, "/path/to/cert", tlsCert) num++
require.Equal(t, "/path/to/key", tlsKey) }()
tls := sc.TLS()
switch num {
case 0:
require.Equal(t, "s01.neofs.devenv:8080", sc.Endpoint())
require.Equal(t, "/path/to/cert", tls.CertificateFile())
require.Equal(t, "/path/to/key", tls.KeyFile())
case 1:
require.Equal(t, "s02.neofs.devenv:8080", sc.Endpoint())
require.Nil(t, tls)
}
})
} }
configtest.ForEachFileType(path, fileConfigTest) configtest.ForEachFileType(path, fileConfigTest)

View file

@ -114,8 +114,7 @@ func initContainerService(c *cfg) {
}) })
}) })
containerGRPC.RegisterContainerServiceServer(c.cfgGRPC.server, server := containerTransportGRPC.New(
containerTransportGRPC.New(
containerService.NewSignService( containerService.NewSignService(
&c.key.PrivateKey, &c.key.PrivateKey,
containerService.NewResponseService( containerService.NewResponseService(
@ -129,8 +128,11 @@ func initContainerService(c *cfg) {
c.respSvc, c.respSvc,
), ),
), ),
),
) )
for _, srv := range c.cfgGRPC.servers {
containerGRPC.RegisterContainerServiceServer(srv, server)
}
} }
// addContainerNotificationHandler adds handler that will be executed synchronously // addContainerNotificationHandler adds handler that will be executed synchronously

View file

@ -12,41 +12,52 @@ import (
) )
func initGRPC(c *cfg) { func initGRPC(c *cfg) {
var err error grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) {
lis, err := net.Listen("tcp", sc.Endpoint())
c.cfgGRPC.listener, err = net.Listen("tcp", grpcconfig.Endpoint(c.appCfg))
fatalOnErr(err) fatalOnErr(err)
c.cfgGRPC.listeners = append(c.cfgGRPC.listeners, lis)
serverOpts := []grpc.ServerOption{ serverOpts := []grpc.ServerOption{
grpc.MaxSendMsgSize(maxMsgSize), grpc.MaxSendMsgSize(maxMsgSize),
} }
if c.cfgGRPC.tlsEnabled { tlsCfg := sc.TLS()
creds, err := credentials.NewServerTLSFromFile(c.cfgGRPC.tlsCertFile, c.cfgGRPC.tlsKeyFile)
if tlsCfg != nil {
creds, err := credentials.NewServerTLSFromFile(tlsCfg.CertificateFile(), tlsCfg.KeyFile())
fatalOnErrDetails("could not read credentials from file", err) fatalOnErrDetails("could not read credentials from file", err)
serverOpts = append(serverOpts, grpc.Creds(creds)) serverOpts = append(serverOpts, grpc.Creds(creds))
} }
c.cfgGRPC.server = grpc.NewServer(serverOpts...) srv := grpc.NewServer(serverOpts...)
c.onShutdown(func() { c.onShutdown(func() {
stopGRPC("NeoFS Public API", c.cfgGRPC.server, c.log) stopGRPC("NeoFS Public API", srv, c.log)
})
c.cfgGRPC.servers = append(c.cfgGRPC.servers, srv)
}) })
} }
func serveGRPC(c *cfg) { func serveGRPC(c *cfg) {
for i := range c.cfgGRPC.servers {
c.wg.Add(1) c.wg.Add(1)
srv := c.cfgGRPC.servers[i]
lis := c.cfgGRPC.listeners[i]
go func() { go func() {
defer func() { defer func() {
c.wg.Done() c.wg.Done()
}() }()
if err := c.cfgGRPC.server.Serve(c.cfgGRPC.listener); err != nil { if err := srv.Serve(lis); err != nil {
fmt.Println("gRPC server error", err) fmt.Println("gRPC server error", err)
} }
}() }()
}
} }
func stopGRPC(name string, s *grpc.Server, l *logger.Logger) { func stopGRPC(name string, s *grpc.Server, l *logger.Logger) {

View file

@ -86,8 +86,7 @@ func initNetmapService(c *cfg) {
initMorphComponents(c) initMorphComponents(c)
} }
netmapGRPC.RegisterNetmapServiceServer(c.cfgGRPC.server, server := netmapTransportGRPC.New(
netmapTransportGRPC.New(
netmapService.NewSignService( netmapService.NewSignService(
&c.key.PrivateKey, &c.key.PrivateKey,
netmapService.NewResponseService( netmapService.NewResponseService(
@ -102,9 +101,12 @@ func initNetmapService(c *cfg) {
c.respSvc, c.respSvc,
), ),
), ),
),
) )
for _, srv := range c.cfgGRPC.servers {
netmapGRPC.RegisterNetmapServiceServer(srv, server)
}
addNewEpochNotificationHandler(c, func(ev event.Event) { addNewEpochNotificationHandler(c, func(ev event.Event) {
c.cfgNetmap.state.setCurrentEpoch(ev.(netmapEvent.NewEpoch).EpochNumber()) c.cfgNetmap.state.setCurrentEpoch(ev.(netmapEvent.NewEpoch).EpochNumber())
}) })

View file

@ -371,9 +371,11 @@ func initObjectService(c *cfg) {
firstSvc = objectService.NewMetricCollector(aclSvc, c.metricsCollector) firstSvc = objectService.NewMetricCollector(aclSvc, c.metricsCollector)
} }
objectGRPC.RegisterObjectServiceServer(c.cfgGRPC.server, server := objectTransportGRPC.New(firstSvc)
objectTransportGRPC.New(firstSvc),
) for _, srv := range c.cfgGRPC.servers {
objectGRPC.RegisterObjectServiceServer(srv, server)
}
} }
type morphEACLStorage struct { type morphEACLStorage struct {

View file

@ -186,8 +186,7 @@ func initReputationService(c *cfg) {
}, },
) )
v2reputationgrpc.RegisterReputationServiceServer(c.cfgGRPC.server, server := grpcreputation.New(
grpcreputation.New(
reputationrpc.NewSignService( reputationrpc.NewSignService(
&c.key.PrivateKey, &c.key.PrivateKey,
reputationrpc.NewResponseService( reputationrpc.NewResponseService(
@ -201,9 +200,12 @@ func initReputationService(c *cfg) {
c.respSvc, c.respSvc,
), ),
), ),
),
) )
for _, srv := range c.cfgGRPC.servers {
v2reputationgrpc.RegisterReputationServiceServer(srv, server)
}
// initialize eigen trust block timer // initialize eigen trust block timer
durationMeter := NewEigenTrustDuration(c.cfgNetmap.wrapper) durationMeter := NewEigenTrustDuration(c.cfgNetmap.wrapper)

View file

@ -10,8 +10,7 @@ import (
func initSessionService(c *cfg) { func initSessionService(c *cfg) {
c.privateTokenStore = storage.New() c.privateTokenStore = storage.New()
sessionGRPC.RegisterSessionServiceServer(c.cfgGRPC.server, server := sessionTransportGRPC.New(
sessionTransportGRPC.New(
sessionSvc.NewSignService( sessionSvc.NewSignService(
&c.key.PrivateKey, &c.key.PrivateKey,
sessionSvc.NewResponseService( sessionSvc.NewResponseService(
@ -21,6 +20,9 @@ func initSessionService(c *cfg) {
c.respSvc, c.respSvc,
), ),
), ),
),
) )
for _, srv := range c.cfgGRPC.servers {
sessionGRPC.RegisterSessionServiceServer(srv, server)
}
} }

View file

@ -17,10 +17,18 @@ NEOFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MSK
NEOFS_NODE_RELAY=true NEOFS_NODE_RELAY=true
# gRPC section # gRPC section
NEOFS_GRPC_ENDPOINT=s01.neofs.devenv:8080 NEOFS_GRPC_NUM=2
NEOFS_GRPC_TLS_ENABLED=true ## 0 server
NEOFS_GRPC_TLS_CERTIFICATE=/path/to/cert NEOFS_GRPC_0_ENDPOINT=s01.neofs.devenv:8080
NEOFS_GRPC_TLS_KEY=/path/to/key ### TLS config
NEOFS_GRPC_0_TLS_ENABLED=true
NEOFS_GRPC_0_TLS_CERTIFICATE=/path/to/cert
NEOFS_GRPC_0_TLS_KEY=/path/to/key
## 1 server
NEOFS_GRPC_1_ENDPOINT=s02.neofs.devenv:8080
### TLS config
NEOFS_GRPC_1_TLS_ENABLED=false
# Control service section # Control service section
NEOFS_CONTROL_AUTHORIZED_KEYS=035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6 NEOFS_CONTROL_AUTHORIZED_KEYS=035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6

View file

@ -23,6 +23,8 @@
"relay": true "relay": true
}, },
"grpc": { "grpc": {
"num": 2,
"0": {
"endpoint": "s01.neofs.devenv:8080", "endpoint": "s01.neofs.devenv:8080",
"tls": { "tls": {
"enabled": true, "enabled": true,
@ -30,6 +32,13 @@
"key": "/path/to/key" "key": "/path/to/key"
} }
}, },
"1": {
"endpoint": "s02.neofs.devenv:8080",
"tls": {
"enabled": false
}
}
},
"control": { "control": {
"authorized_keys": [ "authorized_keys": [
"035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11", "035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11",

View file

@ -21,12 +21,19 @@ node:
relay: true relay: true
grpc: grpc:
num: 2
0:
endpoint: s01.neofs.devenv:8080 endpoint: s01.neofs.devenv:8080
tls: tls:
enabled: true enabled: true
certificate: /path/to/cert certificate: /path/to/cert
key: /path/to/key key: /path/to/key
1:
endpoint: s02.neofs.devenv:8080
tls:
enabled: false
control: control:
authorized_keys: authorized_keys:
- 035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 - 035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11