frostfs-node/cmd/frostfs-node/grpc.go

package main

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"time"

	grpcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/grpc"
	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
	metrics "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics/grpc"
	tracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc"
	"go.uber.org/zap"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

const maxRecvMsgSize = 256 << 20

func initGRPC(c *cfg) {
	var successCount int
	grpcconfig.IterateEndpoints(c.appCfg, func(sc *grpcconfig.Config) {
		serverOpts := []grpc.ServerOption{
			grpc.MaxRecvMsgSize(maxRecvMsgSize),
			grpc.ChainUnaryInterceptor(
				metrics.NewUnaryServerInterceptor(),
				tracing.NewUnaryServerInterceptor(),
			),
			grpc.ChainStreamInterceptor(
				metrics.NewStreamServerInterceptor(),
				tracing.NewStreamServerInterceptor(),
			),
		}

		tlsCfg := sc.TLS()

		if tlsCfg != nil {
			cert, err := tls.LoadX509KeyPair(tlsCfg.CertificateFile(), tlsCfg.KeyFile())
			if err != nil {
				c.log.Error(logs.FrostFSNodeCouldNotReadCertificateFromFile, zap.Error(err))
				return
			}

			var cipherSuites []uint16
			if !tlsCfg.UseInsecureCrypto() {
				// This more or less follows the list in https://wiki.mozilla.org/Security/Server_Side_TLS
				// excluding:
				// 1. TLS 1.3 suites need not be specified here.
				// 2. Suites that use DH key exchange are not implemented by stdlib.
				cipherSuites = []uint16{
					tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
					tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
					tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
					tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
					tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
					tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
				}
			}
			creds := credentials.NewTLS(&tls.Config{
				MinVersion:   tls.VersionTLS12,
				CipherSuites: cipherSuites,
				Certificates: []tls.Certificate{cert},
			})

			serverOpts = append(serverOpts, grpc.Creds(creds))
		}

		lis, err := net.Listen("tcp", sc.Endpoint())
		if err != nil {
			c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(sc.Endpoint())
			c.log.Error(logs.FrostFSNodeCantListenGRPCEndpoint, zap.Error(err))
			return
		}
		c.metricsCollector.GrpcServerMetrics().MarkHealthy(sc.Endpoint())

		c.cfgGRPC.listeners = append(c.cfgGRPC.listeners, lis)
		c.cfgGRPC.endpoints = append(c.cfgGRPC.endpoints, sc.Endpoint())

		srv := grpc.NewServer(serverOpts...)

		c.onShutdown(func() {
			stopGRPC("FrostFS Public API", srv, c.log)
		})

		c.cfgGRPC.servers = append(c.cfgGRPC.servers, srv)
		successCount++
	})

	if successCount == 0 {
		fatalOnErr(errors.New("could not listen to any gRPC endpoints"))
	}
}

func serveGRPC(c *cfg) {
	for i := range c.cfgGRPC.servers {
		c.wg.Add(1)

		srv := c.cfgGRPC.servers[i]
		lis := c.cfgGRPC.listeners[i]
		endpoint := c.cfgGRPC.endpoints[i]

		go func() {
			defer func() {
				c.log.Info(logs.FrostFSNodeStopListeningGRPCEndpoint,
					zap.Stringer("endpoint", lis.Addr()),
				)

				c.wg.Done()
			}()

			c.log.Info(logs.FrostFSNodeStartListeningGRPCEndpoint,
				zap.Stringer("endpoint", lis.Addr()),
			)

			if err := srv.Serve(lis); err != nil {
				c.metricsCollector.GrpcServerMetrics().MarkUnhealthy(endpoint)
				fmt.Println("gRPC server error", err)
			}
		}()
	}
}

func stopGRPC(name string, s *grpc.Server, l *logger.Logger) {
	l = &logger.Logger{Logger: l.With(zap.String("name", name))}

	l.Info(logs.FrostFSNodeStoppingGRPCServer)

	// GracefulStop() may freeze forever, see #1270
	done := make(chan struct{})
	go func() {
		s.GracefulStop()
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(1 * time.Minute):
		l.Info(logs.FrostFSNodeGRPCCannotShutdownGracefullyForcingStop)
		s.Stop()
	}

	l.Info(logs.FrostFSNodeGRPCServerStoppedSuccessfully)
}