[#879] node: Drain internal error's channel
Some checks failed
Vulncheck / Vulncheck (pull_request) Failing after 1m20s
DCO action / DCO (pull_request) Successful in 3m6s
Build / Build Components (1.20) (pull_request) Successful in 3m31s
Build / Build Components (1.21) (pull_request) Successful in 3m48s
Tests and linters / Staticcheck (pull_request) Successful in 8m7s
Tests and linters / Lint (pull_request) Successful in 8m34s
Tests and linters / Tests with -race (pull_request) Failing after 14m37s
Tests and linters / Tests (1.20) (pull_request) Failing after 15m3s
Tests and linters / Tests (1.21) (pull_request) Successful in 16m37s

This fixes shutdown panic:
1. Some morph connection gets error and passes it to internalErr channel.
2. Storage node starts to shutdow and closes internalErr channel.
3. Other morph connection gets error and tries to pass it to internalErr channel.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-12-18 17:55:24 +03:00
parent e39db63827
commit 2bfaa65455
2 changed files with 15 additions and 2 deletions

View file

@ -1162,9 +1162,8 @@ func (c *cfg) shutdown() {
c.setHealthStatus(control.HealthStatus_SHUTTING_DOWN) c.setHealthStatus(control.HealthStatus_SHUTTING_DOWN)
c.ctxCancel() c.ctxCancel()
c.done <- struct{}{} close(c.done)
for i := range c.closers { for i := range c.closers {
c.closers[len(c.closers)-1-i].fn() c.closers[len(c.closers)-1-i].fn()
} }
close(c.internalErr)
} }

View file

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"log" "log"
"os" "os"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
@ -148,9 +149,22 @@ func wait(c *cfg) {
<-c.done // graceful shutdown <-c.done // graceful shutdown
drain := &sync.WaitGroup{}
drain.Add(1)
go func() {
defer drain.Done()
for err := range c.internalErr {
c.log.Warn(logs.FrostFSNodeInternalApplicationError,
zap.String("message", err.Error()))
}
}()
c.log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop) c.log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop)
c.wg.Wait() c.wg.Wait()
close(c.internalErr)
drain.Wait()
} }
func (c *cfg) onShutdown(f func()) { func (c *cfg) onShutdown(f func()) {