From 3de3c102fc4f346f45e30a21ad18cdaf0bd2e1ce Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Fri, 25 Mar 2022 13:28:07 +0300 Subject: [PATCH] [#1270] neofs-node: Add timeout for grpc GracefulStop() GracefulStop() may be blocked until all server-side streams are finished. There is no control over such streams yet, so application may be frozen in shutdown stage. Naive solution is to add timeout for GracefulStop(). At this point healthy connection will be finished and unhealthy connections will be terminated by Stop(). Signed-off-by: Alex Vanin (cherry picked from commit 683439970a2ae01f805fc74cc77b2b5e6e9a2639) --- cmd/neofs-node/grpc.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cmd/neofs-node/grpc.go b/cmd/neofs-node/grpc.go index ee7f462f8..1521c5602 100644 --- a/cmd/neofs-node/grpc.go +++ b/cmd/neofs-node/grpc.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "fmt" "net" + "time" grpcconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/grpc" "github.com/nspcc-dev/neofs-node/pkg/util/logger" @@ -95,7 +96,19 @@ func stopGRPC(name string, s *grpc.Server, l *logger.Logger) { l.Info("stopping gRPC server...") - s.GracefulStop() + // GracefulStop() may freeze forever, see #1270 + done := make(chan struct{}) + go func() { + s.GracefulStop() + close(done) + }() + + select { + case <-done: + case <-time.After(1 * time.Minute): + l.Info("gRPC cannot shutdown gracefully, forcing stop") + s.Stop() + } l.Info("gRPC server stopped successfully") }