[#835] grpc: Try to reconnect if endpoint listen failed

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

cmd/frostfs-node/config.go

@@ -460,18 +460,80 @@ func (c *cfg) ReadCurrentNetMap(msg *netmapV2.NetMap) error {
 	return nil
 }
 
+type grpcServer struct {
+	Listener net.Listener
+	Server   *grpc.Server
+	Endpoint string
+}
+
 type cfgGRPC struct {
-	listeners []net.Listener
-
-	servers []*grpc.Server
-
-	endpoints []string
-
-	maxChunkSize uint64
+	// guard protects connections and handlers
+	guard sync.RWMutex
+	// servers must be protected with guard
+	servers []grpcServer
+	// handlers must be protected with guard
+	handlers []func(e string, l net.Listener, s *grpc.Server)
 
+	maxChunkSize  uint64
 	maxAddrAmount uint64
 }
 
+func (c *cfgGRPC) append(e string, l net.Listener, s *grpc.Server) {
+	c.guard.Lock()
+	defer c.guard.Unlock()
+
+	c.servers = append(c.servers, grpcServer{
+		Listener: l,
+		Server:   s,
+		Endpoint: e,
+	})
+}
+
+func (c *cfgGRPC) appendAndHandle(e string, l net.Listener, s *grpc.Server) {
+	c.guard.Lock()
+	defer c.guard.Unlock()
+
+	c.servers = append(c.servers, grpcServer{
+		Listener: l,
+		Server:   s,
+		Endpoint: e,
+	})
+
+	for _, h := range c.handlers {
+		h(e, l, s)
+	}
+}
+
+func (c *cfgGRPC) performAndSave(handler func(e string, l net.Listener, s *grpc.Server)) {
+	c.guard.Lock()
+	defer c.guard.Unlock()
+
+	for _, conn := range c.servers {
+		handler(conn.Endpoint, conn.Listener, conn.Server)
+	}
+
+	c.handlers = append(c.handlers, handler)
+}
+
+func (c *cfgGRPC) dropConnection(endpoint string) {
+	c.guard.Lock()
+	defer c.guard.Unlock()
+
+	pos := -1
+	for idx, srv := range c.servers {
+		if srv.Endpoint == endpoint {
+			pos = idx
+			break
+		}
+	}
+	if pos < 0 {
+		return
+	}
+
+	c.servers[pos].Server.Stop() // closes listener
+	c.servers = append(c.servers[0:pos], c.servers[pos+1:]...)
+}
+
 type cfgMorph struct {
 	client *client.Client
 
@@ -1232,7 +1294,7 @@ func (c *cfg) shutdown() {
 	}
 
 	c.ctxCancel()
-	c.done <- struct{}{}
+	close(c.done)
 	for i := range c.closers {
 		c.closers[len(c.closers)-1-i].fn()
 	}