[#607] cmd/node: Serve gRPC on multiple interfaces

Generalize single gRPC interface of the storage node to a group of
interfaces. Each interface calls the same RPC handler.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>

cmd/neofs-node/accounting.go

@@ -16,17 +16,19 @@ func initAccountingService(c *cfg) {
 	balanceMorphWrapper, err := wrapper.NewFromMorph(c.cfgMorph.client, c.cfgAccounting.scriptHash, 0)
 	fatalOnErr(err)
 
-	accountingGRPC.RegisterAccountingServiceServer(c.cfgGRPC.server,
+	server := accountingTransportGRPC.New(
-		accountingTransportGRPC.New(
+		accountingService.NewSignService(
-			accountingService.NewSignService(
+			&c.key.PrivateKey,
-				&c.key.PrivateKey,
+			accountingService.NewResponseService(
-				accountingService.NewResponseService(
+				accountingService.NewExecutionService(
-					accountingService.NewExecutionService(
+					accounting.NewExecutor(balanceMorphWrapper),
-						accounting.NewExecutor(balanceMorphWrapper),
-					),
-					c.respSvc,
 				),
+				c.respSvc,
 			),
 		),
 	)
+
+	for _, srv := range c.cfgGRPC.servers {
+		accountingGRPC.RegisterAccountingServiceServer(srv, server)
+	}
 }

cmd/neofs-node/config.go

@@ -17,7 +17,6 @@ import (
 	contractsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/contracts"
 	engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine"
 	shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
-	grpcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/grpc"
 	loggerconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/logger"
 	metricsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/metrics"
 	nodeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/node"
@@ -112,17 +111,13 @@ type cfg struct {
 }
 
 type cfgGRPC struct {
-	listener net.Listener
+	listeners []net.Listener
 
-	server *grpc.Server
+	servers []*grpc.Server
 
 	maxChunkSize uint64
 
 	maxAddrAmount uint64
-
-	tlsEnabled  bool
-	tlsCertFile string
-	tlsKeyFile  string
 }
 
 type cfgMorph struct {
@@ -225,24 +220,6 @@ func initCfg(path string) *cfg {
 	maxChunkSize := uint64(maxMsgSize) * 3 / 4          // 25% to meta, 75% to payload
 	maxAddrAmount := uint64(maxChunkSize) / addressSize // each address is about 72 bytes
 
-	var (
-		tlsEnabled  bool
-		tlsCertFile string
-		tlsKeyFile  string
-
-		tlsConfig = grpcconfig.TLS(appCfg)
-	)
-
-	if tlsConfig.Enabled() {
-		tlsEnabled = true
-		tlsCertFile = tlsConfig.CertificateFile()
-		tlsKeyFile = tlsConfig.KeyFile()
-	}
-
-	if tlsEnabled {
-		netAddr.AddTLS()
-	}
-
 	state := newNetworkState()
 
 	containerWorkerPool, err := ants.NewPool(notificationHandlerPoolSize)
@@ -281,9 +258,6 @@ func initCfg(path string) *cfg {
 		cfgGRPC: cfgGRPC{
 			maxChunkSize:  maxChunkSize,
 			maxAddrAmount: maxAddrAmount,
-			tlsEnabled:    tlsEnabled,
-			tlsCertFile:   tlsCertFile,
-			tlsKeyFile:    tlsKeyFile,
 		},
 		localAddr: network.GroupFromAddress(netAddr),
 		respSvc: response.NewService(

cmd/neofs-node/config/grpc/config.go

@@ -2,33 +2,28 @@ package grpcconfig
 
 import (
 	"errors"
+	"strconv"
 
 	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
 )
 
-const (
-	subsection    = "grpc"
-	tlsSubsection = "tls"
-)
-
 var (
 	errEndpointNotSet = errors.New("empty/not set endpoint, see `grpc.endpoint` section")
 	errTLSKeyNotSet   = errors.New("empty/not set TLS key file path, see `grpc.tls.key` section")
 	errTLSCertNotSet  = errors.New("empty/not set TLS certificate file path, see `grpc.tls.certificate` section")
 )
 
-// TLSConfig is a wrapper over "tls" config section which provides access
+// Config is a wrapper over the config section
-// to TLS configuration of gRPC connection.
+// which provides access to gRPC server configurations.
-type TLSConfig struct {
+type Config config.Config
-	cfg *config.Config
-}
 
-// Endpoint returns value of "endpoint" config parameter
+// Endpoint returns value of "endpoint" config parameter.
-// from "grpc" section.
 //
 // Panics if value is not a non-empty string.
-func Endpoint(c *config.Config) string {
+func (x *Config) Endpoint() string {
-	v := config.StringSafe(c.Sub(subsection), "endpoint")
+	v := config.StringSafe(
+		(*config.Config)(x),
+		"endpoint")
 	if v == "" {
 		panic(errEndpointNotSet)
 	}
@@ -36,19 +31,27 @@ func Endpoint(c *config.Config) string {
 	return v
 }
 
-// TLS returns structure that provides access to "tls" subsection of
+// TLS returns "tls" subsection as a TLSConfig.
-// "grpc" section.
+//
-func TLS(c *config.Config) TLSConfig {
+// Returns nil if "enabled" value of "tls" subsection is false.
-	return TLSConfig{
+func (x *Config) TLS() *TLSConfig {
-		cfg: c.Sub(subsection).Sub(tlsSubsection),
+	sub := (*config.Config)(x).
+		Sub("tls")
+
+	if !config.BoolSafe(sub, "enabled") {
+		return nil
+	}
+
+	return &TLSConfig{
+		cfg: sub,
 	}
 }
 
-// Enabled returns value of "enabled" config parameter.
+// TLSConfig is a wrapper over the config section
-//
+// which provides access to TLS configurations
-// Returns false if value is not set.
+// of the gRPC server.
-func (tls TLSConfig) Enabled() bool {
+type TLSConfig struct {
-	return config.BoolSafe(tls.cfg, "enabled")
+	cfg *config.Config
 }
 
 // KeyFile returns value of "key" config parameter.
@@ -74,3 +77,24 @@ func (tls TLSConfig) CertificateFile() string {
 
 	return v
 }
+
+// IterateEndpoints iterates over subsections ["0":"N") (N - "num" value)
+// of "grpc" section of c, wrap them into Config and passes to f.
+//
+// Panics if N is not a positive number.
+func IterateEndpoints(c *config.Config, f func(*Config)) {
+	c = c.Sub("grpc")
+
+	num := config.Uint(c, "num")
+	if num == 0 {
+		panic("no gRPC server configured")
+	}
+
+	for i := uint64(0); i < num; i++ {
+		si := strconv.FormatUint(i, 10)
+
+		sc := (*Config)(c.Sub(si))
+
+		f(sc)
+	}
+}

cmd/neofs-node/config/grpc/config_test.go

@@ -10,49 +10,35 @@ import (
 
 func TestGRPCSection(t *testing.T) {
 	t.Run("defaults", func(t *testing.T) {
-		empty := configtest.EmptyConfig()
+		require.Panics(t, func() {
-
+			IterateEndpoints(configtest.EmptyConfig(), nil)
-		tlsEnabled := TLS(empty).Enabled()
+		})
-
-		require.Equal(t, false, tlsEnabled)
-
-		require.PanicsWithError(
-			t,
-			errEndpointNotSet.Error(),
-			func() {
-				Endpoint(empty)
-			},
-		)
-
-		require.PanicsWithError(
-			t,
-			errTLSKeyNotSet.Error(),
-			func() {
-				TLS(empty).KeyFile()
-			},
-		)
-
-		require.PanicsWithError(
-			t,
-			errTLSCertNotSet.Error(),
-			func() {
-				TLS(empty).CertificateFile()
-			},
-		)
 	})
 
 	const path = "../../../../config/example/node"
 
 	var fileConfigTest = func(c *config.Config) {
-		addr := Endpoint(c)
+		num := 0
-		tlsEnabled := TLS(c).Enabled()
-		tlsCert := TLS(c).CertificateFile()
-		tlsKey := TLS(c).KeyFile()
 
-		require.Equal(t, "s01.neofs.devenv:8080", addr)
+		IterateEndpoints(c, func(sc *Config) {
-		require.Equal(t, true, tlsEnabled)
+			defer func() {
-		require.Equal(t, "/path/to/cert", tlsCert)
+				num++
-		require.Equal(t, "/path/to/key", tlsKey)
+			}()
+
+			tls := sc.TLS()
+
+			switch num {
+			case 0:
+				require.Equal(t, "s01.neofs.devenv:8080", sc.Endpoint())
+
+				require.Equal(t, "/path/to/cert", tls.CertificateFile())
+				require.Equal(t, "/path/to/key", tls.KeyFile())
+			case 1:
+				require.Equal(t, "s02.neofs.devenv:8080", sc.Endpoint())
+
+				require.Nil(t, tls)
+			}
+		})
 	}
 
 	configtest.ForEachFileType(path, fileConfigTest)

cmd/neofs-node/container.go

@@ -114,23 +114,25 @@ func initContainerService(c *cfg) {
 		})
 	})
 
-	containerGRPC.RegisterContainerServiceServer(c.cfgGRPC.server,
+	server := containerTransportGRPC.New(
-		containerTransportGRPC.New(
+		containerService.NewSignService(
-			containerService.NewSignService(
+			&c.key.PrivateKey,
-				&c.key.PrivateKey,
+			containerService.NewResponseService(
-				containerService.NewResponseService(
+				&usedSpaceService{
-					&usedSpaceService{
+					Server:               containerService.NewExecutionService(containerMorph.NewExecutor(wrap)),
-						Server:               containerService.NewExecutionService(containerMorph.NewExecutor(wrap)),
+					loadWriterProvider:   loadRouter,
-						loadWriterProvider:   loadRouter,
+					loadPlacementBuilder: loadPlacementBuilder,
-						loadPlacementBuilder: loadPlacementBuilder,
+					routeBuilder:         routeBuilder,
-						routeBuilder:         routeBuilder,
+					cfg:                  c,
-						cfg:                  c,
+				},
-					},
+				c.respSvc,
-					c.respSvc,
-				),
 			),
 		),
 	)
+
+	for _, srv := range c.cfgGRPC.servers {
+		containerGRPC.RegisterContainerServiceServer(srv, server)
+	}
 }
 
 // addContainerNotificationHandler adds handler that will be executed synchronously

cmd/neofs-node/grpc.go

@@ -12,41 +12,52 @@ import (
 )
 
 func initGRPC(c *cfg) {
-	var err error
+	grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) {
+		lis, err := net.Listen("tcp", sc.Endpoint())
+		fatalOnErr(err)
 
-	c.cfgGRPC.listener, err = net.Listen("tcp", grpcconfig.Endpoint(c.appCfg))
+		c.cfgGRPC.listeners = append(c.cfgGRPC.listeners, lis)
-	fatalOnErr(err)
 
-	serverOpts := []grpc.ServerOption{
+		serverOpts := []grpc.ServerOption{
-		grpc.MaxSendMsgSize(maxMsgSize),
+			grpc.MaxSendMsgSize(maxMsgSize),
-	}
+		}
 
-	if c.cfgGRPC.tlsEnabled {
+		tlsCfg := sc.TLS()
-		creds, err := credentials.NewServerTLSFromFile(c.cfgGRPC.tlsCertFile, c.cfgGRPC.tlsKeyFile)
-		fatalOnErrDetails("could not read credentials from file", err)
 
-		serverOpts = append(serverOpts, grpc.Creds(creds))
+		if tlsCfg != nil {
-	}
+			creds, err := credentials.NewServerTLSFromFile(tlsCfg.CertificateFile(), tlsCfg.KeyFile())
+			fatalOnErrDetails("could not read credentials from file", err)
 
-	c.cfgGRPC.server = grpc.NewServer(serverOpts...)
+			serverOpts = append(serverOpts, grpc.Creds(creds))
+		}
 
-	c.onShutdown(func() {
+		srv := grpc.NewServer(serverOpts...)
-		stopGRPC("NeoFS Public API", c.cfgGRPC.server, c.log)
+
+		c.onShutdown(func() {
+			stopGRPC("NeoFS Public API", srv, c.log)
+		})
+
+		c.cfgGRPC.servers = append(c.cfgGRPC.servers, srv)
 	})
 }
 
 func serveGRPC(c *cfg) {
-	c.wg.Add(1)
+	for i := range c.cfgGRPC.servers {
+		c.wg.Add(1)
 
-	go func() {
+		srv := c.cfgGRPC.servers[i]
-		defer func() {
+		lis := c.cfgGRPC.listeners[i]
-			c.wg.Done()
+
+		go func() {
+			defer func() {
+				c.wg.Done()
+			}()
+
+			if err := srv.Serve(lis); err != nil {
+				fmt.Println("gRPC server error", err)
+			}
 		}()
-
+	}
-		if err := c.cfgGRPC.server.Serve(c.cfgGRPC.listener); err != nil {
-			fmt.Println("gRPC server error", err)
-		}
-	}()
 }
 
 func stopGRPC(name string, s *grpc.Server, l *logger.Logger) {

cmd/neofs-node/netmap.go

@@ -86,25 +86,27 @@ func initNetmapService(c *cfg) {
 		initMorphComponents(c)
 	}
 
-	netmapGRPC.RegisterNetmapServiceServer(c.cfgGRPC.server,
+	server := netmapTransportGRPC.New(
-		netmapTransportGRPC.New(
+		netmapService.NewSignService(
-			netmapService.NewSignService(
+			&c.key.PrivateKey,
-				&c.key.PrivateKey,
+			netmapService.NewResponseService(
-				netmapService.NewResponseService(
+				netmapService.NewExecutionService(
-					netmapService.NewExecutionService(
+					c,
-						c,
+					c.apiVersion,
-						c.apiVersion,
+					&netInfo{
-						&netInfo{
+						netState: c.cfgNetmap.state,
-							netState: c.cfgNetmap.state,
+						magic:    c.cfgMorph.client,
-							magic:    c.cfgMorph.client,
+					},
-						},
-					),
-					c.respSvc,
 				),
+				c.respSvc,
 			),
 		),
 	)
 
+	for _, srv := range c.cfgGRPC.servers {
+		netmapGRPC.RegisterNetmapServiceServer(srv, server)
+	}
+
 	addNewEpochNotificationHandler(c, func(ev event.Event) {
 		c.cfgNetmap.state.setCurrentEpoch(ev.(netmapEvent.NewEpoch).EpochNumber())
 	})

cmd/neofs-node/object.go

@@ -371,9 +371,11 @@ func initObjectService(c *cfg) {
 		firstSvc = objectService.NewMetricCollector(aclSvc, c.metricsCollector)
 	}
 
-	objectGRPC.RegisterObjectServiceServer(c.cfgGRPC.server,
+	server := objectTransportGRPC.New(firstSvc)
-		objectTransportGRPC.New(firstSvc),
+
-	)
+	for _, srv := range c.cfgGRPC.servers {
+		objectGRPC.RegisterObjectServiceServer(srv, server)
+	}
 }
 
 type morphEACLStorage struct {

cmd/neofs-node/reputation.go

@@ -186,24 +186,26 @@ func initReputationService(c *cfg) {
 		},
 	)
 
-	v2reputationgrpc.RegisterReputationServiceServer(c.cfgGRPC.server,
+	server := grpcreputation.New(
-		grpcreputation.New(
+		reputationrpc.NewSignService(
-			reputationrpc.NewSignService(
+			&c.key.PrivateKey,
-				&c.key.PrivateKey,
+			reputationrpc.NewResponseService(
-				reputationrpc.NewResponseService(
+				&reputationServer{
-					&reputationServer{
+					cfg:                c,
-						cfg:                c,
+					log:                c.log,
-						log:                c.log,
+					localRouter:        localTrustRouter,
-						localRouter:        localTrustRouter,
+					intermediateRouter: intermediateTrustRouter,
-						intermediateRouter: intermediateTrustRouter,
+					routeBuilder:       localRouteBuilder,
-						routeBuilder:       localRouteBuilder,
+				},
-					},
+				c.respSvc,
-					c.respSvc,
-				),
 			),
 		),
 	)
 
+	for _, srv := range c.cfgGRPC.servers {
+		v2reputationgrpc.RegisterReputationServiceServer(srv, server)
+	}
+
 	// initialize eigen trust block timer
 	durationMeter := NewEigenTrustDuration(c.cfgNetmap.wrapper)
 

cmd/neofs-node/session.go

@@ -10,17 +10,19 @@ import (
 func initSessionService(c *cfg) {
 	c.privateTokenStore = storage.New()
 
-	sessionGRPC.RegisterSessionServiceServer(c.cfgGRPC.server,
+	server := sessionTransportGRPC.New(
-		sessionTransportGRPC.New(
+		sessionSvc.NewSignService(
-			sessionSvc.NewSignService(
+			&c.key.PrivateKey,
-				&c.key.PrivateKey,
+			sessionSvc.NewResponseService(
-				sessionSvc.NewResponseService(
+				sessionSvc.NewExecutionService(
-					sessionSvc.NewExecutionService(
+					c.privateTokenStore,
-						c.privateTokenStore,
-					),
-					c.respSvc,
 				),
+				c.respSvc,
 			),
 		),
 	)
+
+	for _, srv := range c.cfgGRPC.servers {
+		sessionGRPC.RegisterSessionServiceServer(srv, server)
+	}
 }

config/example/node.env

@@ -17,10 +17,18 @@ NEOFS_NODE_ATTRIBUTE_1=UN-LOCODE:RU MSK
 NEOFS_NODE_RELAY=true
 
 # gRPC section
-NEOFS_GRPC_ENDPOINT=s01.neofs.devenv:8080
+NEOFS_GRPC_NUM=2
-NEOFS_GRPC_TLS_ENABLED=true
+## 0 server
-NEOFS_GRPC_TLS_CERTIFICATE=/path/to/cert
+NEOFS_GRPC_0_ENDPOINT=s01.neofs.devenv:8080
-NEOFS_GRPC_TLS_KEY=/path/to/key
+### TLS config
+NEOFS_GRPC_0_TLS_ENABLED=true
+NEOFS_GRPC_0_TLS_CERTIFICATE=/path/to/cert
+NEOFS_GRPC_0_TLS_KEY=/path/to/key
+
+## 1 server
+NEOFS_GRPC_1_ENDPOINT=s02.neofs.devenv:8080
+### TLS config
+NEOFS_GRPC_1_TLS_ENABLED=false
 
 # Control service section
 NEOFS_CONTROL_AUTHORIZED_KEYS=035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6

config/example/node.json

@@ -23,11 +23,20 @@
     "relay": true
   },
   "grpc": {
-    "endpoint": "s01.neofs.devenv:8080",
+    "num": 2,
-    "tls": {
+    "0": {
-      "enabled": true,
+      "endpoint": "s01.neofs.devenv:8080",
-      "certificate": "/path/to/cert",
+      "tls": {
-      "key": "/path/to/key"
+        "enabled": true,
+        "certificate": "/path/to/cert",
+        "key": "/path/to/key"
+      }
+    },
+    "1": {
+      "endpoint": "s02.neofs.devenv:8080",
+      "tls": {
+        "enabled": false
+      }
     }
   },
   "control": {

config/example/node.yaml

@@ -21,11 +21,18 @@ node:
   relay: true
 
 grpc:
-  endpoint: s01.neofs.devenv:8080
+  num: 2
-  tls:
+  0:
-    enabled: true
+    endpoint: s01.neofs.devenv:8080
-    certificate: /path/to/cert
+    tls:
-    key: /path/to/key
+      enabled: true
+      certificate: /path/to/cert
+      key: /path/to/key
+
+  1:
+    endpoint: s02.neofs.devenv:8080
+    tls:
+      enabled: false
 
 control:
   authorized_keys: