[#49] node: React on SIGHUP only when node in READY state

Add more info in logs when node is going to shut down,
but initialization process still in progress.

Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
This commit is contained in:
Anton Nikiforov 2023-10-20 09:52:25 +03:00 committed by Evgenii Stratonikov
parent 58b6224dd8
commit b0cf100427
6 changed files with 35 additions and 7 deletions

View file

@ -1040,7 +1040,6 @@ func (c *cfg) signalWatcher(ctx context.Context) {
c.reloadConfig(ctx) c.reloadConfig(ctx)
case syscall.SIGTERM, syscall.SIGINT: case syscall.SIGTERM, syscall.SIGINT:
c.log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) c.log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)
// TODO (@acid-ant): #49 need to cover case when stuck at the middle(node health UNDEFINED or STARTING)
c.shutdown() c.shutdown()
@ -1062,6 +1061,12 @@ func (c *cfg) signalWatcher(ctx context.Context) {
func (c *cfg) reloadConfig(ctx context.Context) { func (c *cfg) reloadConfig(ctx context.Context) {
c.log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) c.log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration)
if !c.compareAndSwapHealthStatus(control.HealthStatus_READY, control.HealthStatus_RECONFIGURING) {
c.log.Info(logs.FrostFSNodeSIGHUPSkip)
return
}
defer c.compareAndSwapHealthStatus(control.HealthStatus_RECONFIGURING, control.HealthStatus_READY)
err := c.readConfig(c.appCfg) err := c.readConfig(c.appCfg)
if err != nil { if err != nil {
c.log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) c.log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err))
@ -1142,7 +1147,14 @@ func (c *cfg) createTombstoneSource() *tombstone.ExpirationChecker {
} }
func (c *cfg) shutdown() { func (c *cfg) shutdown() {
c.setHealthStatus(control.HealthStatus_SHUTTING_DOWN) old := c.swapHealthStatus(control.HealthStatus_SHUTTING_DOWN)
if old == control.HealthStatus_SHUTTING_DOWN {
c.log.Info(logs.FrostFSNodeShutdownSkip)
return
}
if old == control.HealthStatus_STARTING {
c.log.Warn(logs.FrostFSNodeShutdownWhenNotReady)
}
c.ctxCancel() c.ctxCancel()
c.done <- struct{}{} c.done <- struct{}{}

View file

@ -83,10 +83,20 @@ func (c *cfg) NetmapStatus() control.NetmapStatus {
func (c *cfg) setHealthStatus(st control.HealthStatus) { func (c *cfg) setHealthStatus(st control.HealthStatus) {
c.healthStatus.Store(int32(st)) c.healthStatus.Store(int32(st))
if c.metricsCollector != nil {
c.metricsCollector.State().SetHealth(int32(st)) c.metricsCollector.State().SetHealth(int32(st))
}
func (c *cfg) compareAndSwapHealthStatus(oldSt, newSt control.HealthStatus) (swapped bool) {
if swapped = c.healthStatus.CompareAndSwap(int32(oldSt), int32(newSt)); swapped {
c.metricsCollector.State().SetHealth(int32(newSt))
} }
return
}
func (c *cfg) swapHealthStatus(st control.HealthStatus) (old control.HealthStatus) {
old = control.HealthStatus(c.healthStatus.Swap(int32(st)))
c.metricsCollector.State().SetHealth(int32(st))
return
} }
func (c *cfg) HealthStatus() control.HealthStatus { func (c *cfg) HealthStatus() control.HealthStatus {

View file

@ -60,13 +60,13 @@ func main() {
var ctx context.Context var ctx context.Context
ctx, c.ctxCancel = context.WithCancel(context.Background()) ctx, c.ctxCancel = context.WithCancel(context.Background())
initApp(ctx, c)
c.setHealthStatus(control.HealthStatus_STARTING) c.setHealthStatus(control.HealthStatus_STARTING)
initApp(ctx, c)
bootUp(ctx, c) bootUp(ctx, c)
c.setHealthStatus(control.HealthStatus_READY) c.compareAndSwapHealthStatus(control.HealthStatus_STARTING, control.HealthStatus_READY)
wait(c) wait(c)
} }

View file

@ -436,6 +436,9 @@ const (
FrostFSNodeInternalApplicationError = "internal application error" FrostFSNodeInternalApplicationError = "internal application error"
FrostFSNodeInternalErrorProcessingIsComplete = "internal error processing is complete" FrostFSNodeInternalErrorProcessingIsComplete = "internal error processing is complete"
FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration = "SIGHUP has been received, rereading configuration..." FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration = "SIGHUP has been received, rereading configuration..."
FrostFSNodeSIGHUPSkip = "node not ready for reconfiguration, skipped SIGHUP"
FrostFSNodeShutdownSkip = "node already is going to shutting down, skipped shutdown"
FrostFSNodeShutdownWhenNotReady = "node is going to shutting down when subsystems still initializing"
FrostFSNodeConfigurationReading = "configuration reading" FrostFSNodeConfigurationReading = "configuration reading"
FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation" FrostFSNodeLoggerConfigurationPreparation = "logger configuration preparation"
FrostFSNodeTracingConfigationUpdated = "tracing configation updated" FrostFSNodeTracingConfigationUpdated = "tracing configation updated"

Binary file not shown.

View file

@ -115,6 +115,9 @@ enum HealthStatus {
// Storage node application is shutting down. // Storage node application is shutting down.
SHUTTING_DOWN = 3; SHUTTING_DOWN = 3;
// Storage node application is reconfiguring.
RECONFIGURING = 4;
} }
// Shard description. // Shard description.