From 2bfaa6545564c96197f2469d2ff75dc7397b1f99 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 18 Dec 2023 17:55:24 +0300 Subject: [PATCH] [#879] node: Drain internal error's channel This fixes shutdown panic: 1. Some morph connection gets error and passes it to internalErr channel. 2. Storage node starts to shutdow and closes internalErr channel. 3. Other morph connection gets error and tries to pass it to internalErr channel. Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 +-- cmd/frostfs-node/main.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d74eff476..88d55ae23 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -1162,9 +1162,8 @@ func (c *cfg) shutdown() { c.setHealthStatus(control.HealthStatus_SHUTTING_DOWN) c.ctxCancel() - c.done <- struct{}{} + close(c.done) for i := range c.closers { c.closers[len(c.closers)-1-i].fn() } - close(c.internalErr) } diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 88032ebdb..014f94bed 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "os" + "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" @@ -148,9 +149,22 @@ func wait(c *cfg) { <-c.done // graceful shutdown + drain := &sync.WaitGroup{} + drain.Add(1) + go func() { + defer drain.Done() + for err := range c.internalErr { + c.log.Warn(logs.FrostFSNodeInternalApplicationError, + zap.String("message", err.Error())) + } + }() + c.log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop) c.wg.Wait() + + close(c.internalErr) + drain.Wait() } func (c *cfg) onShutdown(f func()) { -- 2.45.2