[#607] cmd/node: Serve gRPC on multiple interfaces

Generalize single gRPC interface of the storage node to a group of
interfaces. Each interface calls the same RPC handler.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-06-22 20:25:18 +03:00 committed by Leonard Lyubich
parent d1eb9c3b0f
commit 8060735732
13 changed files with 220 additions and 189 deletions

View file

@ -16,17 +16,19 @@ func initAccountingService(c *cfg) {
balanceMorphWrapper, err := wrapper.NewFromMorph(c.cfgMorph.client, c.cfgAccounting.scriptHash, 0) balanceMorphWrapper, err := wrapper.NewFromMorph(c.cfgMorph.client, c.cfgAccounting.scriptHash, 0)
fatalOnErr(err) fatalOnErr(err)
accountingGRPC.RegisterAccountingServiceServer(c.cfgGRPC.server, server := accountingTransportGRPC.New(
accountingTransportGRPC.New( accountingService.NewSignService(
accountingService.NewSignService( &c.key.PrivateKey,
&c.key.PrivateKey, accountingService.NewResponseService(
accountingService.NewResponseService( accountingService.NewExecutionService(
accountingService.NewExecutionService( accounting.NewExecutor(balanceMorphWrapper),
accounting.NewExecutor(balanceMorphWrapper),
),
c.respSvc,
), ),
c.respSvc,
), ),
), ),
) )
for _, srv := range c.cfgGRPC.servers {
accountingGRPC.RegisterAccountingServiceServer(srv, server)
}
} }

View file

@ -17,7 +17,6 @@ import (
contractsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/contracts" contractsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/contracts"
engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine" engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine"
shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard" shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
grpcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/grpc"
loggerconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/logger" loggerconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/logger"
metricsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/metrics" metricsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/metrics"
nodeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/node" nodeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/node"
@ -112,17 +111,13 @@ type cfg struct {
} }
type cfgGRPC struct { type cfgGRPC struct {
listener net.Listener listeners []net.Listener
server *grpc.Server servers []*grpc.Server
maxChunkSize uint64 maxChunkSize uint64
maxAddrAmount uint64 maxAddrAmount uint64
tlsEnabled bool
tlsCertFile string
tlsKeyFile string
} }
type cfgMorph struct { type cfgMorph struct {
@ -225,24 +220,6 @@ func initCfg(path string) *cfg {
maxChunkSize := uint64(maxMsgSize) * 3 / 4 // 25% to meta, 75% to payload maxChunkSize := uint64(maxMsgSize) * 3 / 4 // 25% to meta, 75% to payload
maxAddrAmount := uint64(maxChunkSize) / addressSize // each address is about 72 bytes maxAddrAmount := uint64(maxChunkSize) / addressSize // each address is about 72 bytes
var (
tlsEnabled bool
tlsCertFile string
tlsKeyFile string
tlsConfig = grpcconfig.TLS(appCfg)
)
if tlsConfig.Enabled() {
tlsEnabled = true
tlsCertFile = tlsConfig.CertificateFile()
tlsKeyFile = tlsConfig.KeyFile()
}
if tlsEnabled {
netAddr.AddTLS()
}
state := newNetworkState() state := newNetworkState()
containerWorkerPool, err := ants.NewPool(notificationHandlerPoolSize) containerWorkerPool, err := ants.NewPool(notificationHandlerPoolSize)
@ -281,9 +258,6 @@ func initCfg(path string) *cfg {
cfgGRPC: cfgGRPC{ cfgGRPC: cfgGRPC{
maxChunkSize: maxChunkSize, maxChunkSize: maxChunkSize,
maxAddrAmount: maxAddrAmount, maxAddrAmount: maxAddrAmount,
tlsEnabled: tlsEnabled,
tlsCertFile: tlsCertFile,
tlsKeyFile: tlsKeyFile,
}, },
localAddr: network.GroupFromAddress(netAddr), localAddr: network.GroupFromAddress(netAddr),
respSvc: response.NewService( respSvc: response.NewService(

View file

@ -2,33 +2,28 @@ package grpcconfig
import ( import (
"errors" "errors"
"strconv"
"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config" "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
) )
const (
subsection = "grpc"
tlsSubsection = "tls"
)
var ( var (
errEndpointNotSet = errors.New("empty/not set endpoint, see `grpc.endpoint` section") errEndpointNotSet = errors.New("empty/not set endpoint, see `grpc.endpoint` section")
errTLSKeyNotSet = errors.New("empty/not set TLS key file path, see `grpc.tls.key` section") errTLSKeyNotSet = errors.New("empty/not set TLS key file path, see `grpc.tls.key` section")
errTLSCertNotSet = errors.New("empty/not set TLS certificate file path, see `grpc.tls.certificate` section") errTLSCertNotSet = errors.New("empty/not set TLS certificate file path, see `grpc.tls.certificate` section")
) )
// TLSConfig is a wrapper over "tls" config section which provides access // Config is a wrapper over the config section
// to TLS configuration of gRPC connection. // which provides access to gRPC server configurations.
type TLSConfig struct { type Config config.Config
cfg *config.Config
}
// Endpoint returns value of "endpoint" config parameter // Endpoint returns value of "endpoint" config parameter.
// from "grpc" section.
// //
// Panics if value is not a non-empty string. // Panics if value is not a non-empty string.
func Endpoint(c *config.Config) string { func (x *Config) Endpoint() string {
v := config.StringSafe(c.Sub(subsection), "endpoint") v := config.StringSafe(
(*config.Config)(x),
"endpoint")
if v == "" { if v == "" {
panic(errEndpointNotSet) panic(errEndpointNotSet)
} }
@ -36,19 +31,27 @@ func Endpoint(c *config.Config) string {
return v return v
} }
// TLS returns structure that provides access to "tls" subsection of // TLS returns "tls" subsection as a TLSConfig.
// "grpc" section. //
func TLS(c *config.Config) TLSConfig { // Returns nil if "enabled" value of "tls" subsection is false.
return TLSConfig{ func (x *Config) TLS() *TLSConfig {
cfg: c.Sub(subsection).Sub(tlsSubsection), sub := (*config.Config)(x).
Sub("tls")
if !config.BoolSafe(sub, "enabled") {
return nil
}
return &TLSConfig{
cfg: sub,
} }
} }
// Enabled returns value of "enabled" config parameter. // TLSConfig is a wrapper over the config section
// // which provides access to TLS configurations
// Returns false if value is not set. // of the gRPC server.
func (tls TLSConfig) Enabled() bool { type TLSConfig struct {
return config.BoolSafe(tls.cfg, "enabled") cfg *config.Config
} }
// KeyFile returns value of "key" config parameter. // KeyFile returns value of "key" config parameter.
@ -74,3 +77,24 @@ func (tls TLSConfig) CertificateFile() string {
return v return v
} }
// IterateEndpoints iterates over subsections ["0":"N") (N - "num" value)
// of "grpc" section of c, wrap them into Config and passes to f.
//
// Panics if N is not a positive number.
func IterateEndpoints(c *config.Config, f func(*Config)) {
c = c.Sub("grpc")
num := config.Uint(c, "num")
if num == 0 {
panic("no gRPC server configured")
}
for i := uint64(0); i < num; i++ {
si := strconv.FormatUint(i, 10)
sc := (*Config)(c.Sub(si))
f(sc)
}
}

View file

@ -10,49 +10,35 @@ import (
func TestGRPCSection(t *testing.T) { func TestGRPCSection(t *testing.T) {
t.Run("defaults", func(t *testing.T) { t.Run("defaults", func(t *testing.T) {
empty := configtest.EmptyConfig() require.Panics(t, func() {
IterateEndpoints(configtest.EmptyConfig(), nil)
tlsEnabled := TLS(empty).Enabled() })
require.Equal(t, false, tlsEnabled)
require.PanicsWithError(
t,
errEndpointNotSet.Error(),
func() {
Endpoint(empty)
},
)
require.PanicsWithError(
t,
errTLSKeyNotSet.Error(),
func() {
TLS(empty).KeyFile()
},
)
require.PanicsWithError(
t,
errTLSCertNotSet.Error(),
func() {
TLS(empty).CertificateFile()
},
)
}) })
const path = "../../../../config/example/node" const path = "../../../../config/example/node"
var fileConfigTest = func(c *config.Config) { var fileConfigTest = func(c *config.Config) {
addr := Endpoint(c) num := 0
tlsEnabled := TLS(c).Enabled()
tlsCert := TLS(c).CertificateFile()
tlsKey := TLS(c).KeyFile()
require.Equal(t, "s01.neofs.devenv:8080", addr) IterateEndpoints(c, func(sc *Config) {
require.Equal(t, true, tlsEnabled) defer func() {
require.Equal(t, "/path/to/cert", tlsCert) num++
require.Equal(t, "/path/to/key", tlsKey) }()
tls := sc.TLS()
switch num {
case 0:
require.Equal(t, "s01.neofs.devenv:8080", sc.Endpoint())
require.Equal(t, "/path/to/cert", tls.CertificateFile())
require.Equal(t, "/path/to/key", tls.KeyFile())
case 1:
require.Equal(t, "s02.neofs.devenv:8080", sc.Endpoint())
require.Nil(t, tls)
}
})
} }
configtest.ForEachFileType(path, fileConfigTest) configtest.ForEachFileType(path, fileConfigTest)

View file

@ -114,23 +114,25 @@ func initContainerService(c *cfg) {
}) })
}) })
containerGRPC.RegisterContainerServiceServer(c.cfgGRPC.server, server := containerTransportGRPC.New(
containerTransportGRPC.New( containerService.NewSignService(
containerService.NewSignService( &c.key.PrivateKey,
&c.key.PrivateKey, containerService.NewResponseService(
containerService.NewResponseService( &usedSpaceService{
&usedSpaceService{ Server: containerService.NewExecutionService(containerMorph.NewExecutor(wrap)),
Server: containerService.NewExecutionService(containerMorph.NewExecutor(wrap)), loadWriterProvider: loadRouter,
loadWriterProvider: loadRouter, loadPlacementBuilder: loadPlacementBuilder,
loadPlacementBuilder: loadPlacementBuilder, routeBuilder: routeBuilder,
routeBuilder: routeBuilder, cfg: c,
cfg: c, },
}, c.respSvc,
c.respSvc,
),
), ),
), ),
) )
for _, srv := range c.cfgGRPC.servers {
containerGRPC.RegisterContainerServiceServer(srv, server)
}
} }
// addContainerNotificationHandler adds handler that will be executed synchronously // addContainerNotificationHandler adds handler that will be executed synchronously

View file

@ -12,41 +12,52 @@ import (
) )
func initGRPC(c *cfg) { func initGRPC(c *cfg) {
var err error grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) {
lis, err := net.Listen("tcp", sc.Endpoint())
fatalOnErr(err)
c.cfgGRPC.listener, err = net.Listen("tcp", grpcconfig.Endpoint(c.appCfg)) c.cfgGRPC.listeners = append(c.cfgGRPC.listeners, lis)
fatalOnErr(err)
serverOpts := []grpc.ServerOption{ serverOpts := []grpc.ServerOption{
grpc.MaxSendMsgSize(maxMsgSize), grpc.MaxSendMsgSize(maxMsgSize),
} }
if c.cfgGRPC.tlsEnabled { tlsCfg := sc.TLS()
creds, err := credentials.NewServerTLSFromFile(c.cfgGRPC.tlsCertFile, c.cfgGRPC.tlsKeyFile)
fatalOnErrDetails("could not read credentials from file", err)
serverOpts = append(serverOpts, grpc.Creds(creds)) if tlsCfg != nil {
} creds, err := credentials.NewServerTLSFromFile(tlsCfg.CertificateFile(), tlsCfg.KeyFile())
fatalOnErrDetails("could not read credentials from file", err)
c.cfgGRPC.server = grpc.NewServer(serverOpts...) serverOpts = append(serverOpts, grpc.Creds(creds))
}
c.onShutdown(func() { srv := grpc.NewServer(serverOpts...)
stopGRPC("NeoFS Public API", c.cfgGRPC.server, c.log)
c.onShutdown(func() {
stopGRPC("NeoFS Public API", srv, c.log)
})
c.cfgGRPC.servers = append(c.cfgGRPC.servers, srv)
}) })
} }
func serveGRPC(c *cfg) { func serveGRPC(c *cfg) {
c.wg.Add(1) for i := range c.cfgGRPC.servers {
c.wg.Add(1)
go func() { srv := c.cfgGRPC.servers[i]
defer func() { lis := c.cfgGRPC.listeners[i]
c.wg.Done()
go func() {
defer func() {
c.wg.Done()
}()
if err := srv.Serve(lis); err != nil {
fmt.Println("gRPC server error", err)
}
}() }()
}
if err := c.cfgGRPC.server.Serve(c.cfgGRPC.listener); err != nil {
fmt.Println("gRPC server error", err)
}
}()
} }
func stopGRPC(name string, s *grpc.Server, l *logger.Logger) { func stopGRPC(name string, s *grpc.Server, l *logger.Logger) {

View file

@ -86,25 +86,27 @@ func initNetmapService(c *cfg) {
initMorphComponents(c) initMorphComponents(c)
} }
netmapGRPC.RegisterNetmapServiceServer(c.cfgGRPC.server, server := netmapTransportGRPC.New(
netmapTransportGRPC.New( netmapService.NewSignService(
netmapService.NewSignService( &c.key.PrivateKey,
&c.key.PrivateKey, netmapService.NewResponseService(
netmapService.NewResponseService( netmapService.NewExecutionService(
netmapService.NewExecutionService( c,
c, c.apiVersion,
c.apiVersion, &netInfo{
&netInfo{ netState: c.cfgNetmap.state,
netState: c.cfgNetmap.state, magic: c.cfgMorph.client,
magic: c.cfgMorph.client, },
},
),
c.respSvc,
), ),
c.respSvc,
), ),
), ),
) )
for _, srv := range c.cfgGRPC.servers {
netmapGRPC.RegisterNetmapServiceServer(srv, server)
}
addNewEpochNotificationHandler(c, func(ev event.Event) { addNewEpochNotificationHandler(c, func(ev event.Event) {
c.cfgNetmap.state.setCurrentEpoch(ev.(netmapEvent.NewEpoch).EpochNumber()) c.cfgNetmap.state.setCurrentEpoch(ev.(netmapEvent.NewEpoch).EpochNumber())
}) })

View file

@ -371,9 +371,11 @@ func initObjectService(c *cfg) {
firstSvc = objectService.NewMetricCollector(aclSvc, c.metricsCollector) firstSvc = objectService.NewMetricCollector(aclSvc, c.metricsCollector)
} }
objectGRPC.RegisterObjectServiceServer(c.cfgGRPC.server, server := objectTransportGRPC.New(firstSvc)
objectTransportGRPC.New(firstSvc),
) for _, srv := range c.cfgGRPC.servers {
objectGRPC.RegisterObjectServiceServer(srv, server)
}
} }
type morphEACLStorage struct { type morphEACLStorage struct {

View file

@ -186,24 +186,26 @@ func initReputationService(c *cfg) {
}, },
) )
v2reputationgrpc.RegisterReputationServiceServer(c.cfgGRPC.server, server := grpcreputation.New(
grpcreputation.New( reputationrpc.NewSignService(
reputationrpc.NewSignService( &c.key.PrivateKey,
&c.key.PrivateKey, reputationrpc.NewResponseService(
reputationrpc.NewResponseService( &reputationServer{
&reputationServer{ cfg: c,
cfg: c, log: c.log,
log: c.log, localRouter: localTrustRouter,
localRouter: localTrustRouter, intermediateRouter: intermediateTrustRouter,
intermediateRouter: intermediateTrustRouter, routeBuilder: localRouteBuilder,
routeBuilder: localRouteBuilder, },
}, c.respSvc,
c.respSvc,
),
), ),
), ),
) )
for _, srv := range c.cfgGRPC.servers {
v2reputationgrpc.RegisterReputationServiceServer(srv, server)
}
// initialize eigen trust block timer // initialize eigen trust block timer
durationMeter := NewEigenTrustDuration(c.cfgNetmap.wrapper) durationMeter := NewEigenTrustDuration(c.cfgNetmap.wrapper)

View file

@ -10,17 +10,19 @@ import (
func initSessionService(c *cfg) { func initSessionService(c *cfg) {
c.privateTokenStore = storage.New() c.privateTokenStore = storage.New()
sessionGRPC.RegisterSessionServiceServer(c.cfgGRPC.server, server := sessionTransportGRPC.New(
sessionTransportGRPC.New( sessionSvc.NewSignService(
sessionSvc.NewSignService( &c.key.PrivateKey,
&c.key.PrivateKey, sessionSvc.NewResponseService(
sessionSvc.NewResponseService( sessionSvc.NewExecutionService(
sessionSvc.NewExecutionService( c.privateTokenStore,
c.privateTokenStore,
),
c.respSvc,
), ),
c.respSvc,
), ),
), ),
) )
for _, srv := range c.cfgGRPC.servers {
sessionGRPC.RegisterSessionServiceServer(srv, server)
}
} }

View file

@ -17,10 +17,18 @@ NEOFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MSK
NEOFS_NODE_RELAY=true NEOFS_NODE_RELAY=true
# gRPC section # gRPC section
NEOFS_GRPC_ENDPOINT=s01.neofs.devenv:8080 NEOFS_GRPC_NUM=2
NEOFS_GRPC_TLS_ENABLED=true ## 0 server
NEOFS_GRPC_TLS_CERTIFICATE=/path/to/cert NEOFS_GRPC_0_ENDPOINT=s01.neofs.devenv:8080
NEOFS_GRPC_TLS_KEY=/path/to/key ### TLS config
NEOFS_GRPC_0_TLS_ENABLED=true
NEOFS_GRPC_0_TLS_CERTIFICATE=/path/to/cert
NEOFS_GRPC_0_TLS_KEY=/path/to/key
## 1 server
NEOFS_GRPC_1_ENDPOINT=s02.neofs.devenv:8080
### TLS config
NEOFS_GRPC_1_TLS_ENABLED=false
# Control service section # Control service section
NEOFS_CONTROL_AUTHORIZED_KEYS=035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6 NEOFS_CONTROL_AUTHORIZED_KEYS=035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6

View file

@ -23,11 +23,20 @@
"relay": true "relay": true
}, },
"grpc": { "grpc": {
"endpoint": "s01.neofs.devenv:8080", "num": 2,
"tls": { "0": {
"enabled": true, "endpoint": "s01.neofs.devenv:8080",
"certificate": "/path/to/cert", "tls": {
"key": "/path/to/key" "enabled": true,
"certificate": "/path/to/cert",
"key": "/path/to/key"
}
},
"1": {
"endpoint": "s02.neofs.devenv:8080",
"tls": {
"enabled": false
}
} }
}, },
"control": { "control": {

View file

@ -21,11 +21,18 @@ node:
relay: true relay: true
grpc: grpc:
endpoint: s01.neofs.devenv:8080 num: 2
tls: 0:
enabled: true endpoint: s01.neofs.devenv:8080
certificate: /path/to/cert tls:
key: /path/to/key enabled: true
certificate: /path/to/cert
key: /path/to/key
1:
endpoint: s02.neofs.devenv:8080
tls:
enabled: false
control: control:
authorized_keys: authorized_keys: