From 14f83b8aa9211ca8968e3a48a279693a7d5261dd Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 25 Apr 2023 13:10:24 +0300 Subject: [PATCH] [#125] ir: Change log level on SIGHUP Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 2 + cmd/frostfs-ir/config.go | 73 +++++++++++++++++++++++++++++ cmd/frostfs-ir/defaults.go | 19 -------- cmd/frostfs-ir/main.go | 94 +++++++++++++++++++++----------------- 4 files changed, 127 insertions(+), 61 deletions(-) create mode 100644 cmd/frostfs-ir/config.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e6792c5b3..e4c86fa37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Changelog for FrostFS Node ### Added - Support impersonate bearer token (#229) +- Change log level on SIGHUP for ir (#125) + ### Changed ### Fixed - Take network settings into account during netmap contract update (#100) diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go new file mode 100644 index 000000000..c4787f965 --- /dev/null +++ b/cmd/frostfs-ir/config.go @@ -0,0 +1,73 @@ +package main + +import ( + "os" + "os/signal" + "syscall" + + configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "github.com/spf13/viper" + "go.uber.org/zap" +) + +func newConfig() (*viper.Viper, error) { + var err error + var dv = viper.New() + + defaultConfiguration(dv) + + _, err = configViper.CreateViper(configViper.WithConfigFile(*configFile), + configViper.WithConfigDir(*configDir), configViper.WithEnvPrefix(EnvPrefix), + configViper.WithViper(dv)) + if err != nil { + return nil, err + } + + return dv, err +} + +func reloadConfig() error { + err := configViper.ReloadViper(configViper.WithConfigFile(*configFile), + configViper.WithConfigDir(*configDir), configViper.WithEnvPrefix(EnvPrefix), + configViper.WithViper(cfg)) + if err != nil { + return err + } + err = logPrm.SetLevelString(cfg.GetString("logger.level")) + if err != nil { + return err + } + return logPrm.Reload() +} + +func watchForSignal(cancel func()) { + ch := make(chan os.Signal, 1) + signal.Notify(ch, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) + + for { + select { + case err := <-intErr: + log.Info(logs.FrostFSIRInternalError, zap.String("msg", err.Error())) + cancel() + shutdown() + return + case sig := <-ch: + switch sig { + case syscall.SIGHUP: + log.Info(logs.FrostFSNodeSIGHUPHasBeenReceivedRereadingConfiguration) + err := reloadConfig() + if err != nil { + log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) + } + log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) + case syscall.SIGTERM, syscall.SIGINT: + log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) + cancel() + shutdown() + log.Info(logs.FrostFSNodeTerminationSignalProcessingIsComplete) + return + } + } + } +} diff --git a/cmd/frostfs-ir/defaults.go b/cmd/frostfs-ir/defaults.go index a3bd867d6..eaad05f2e 100644 --- a/cmd/frostfs-ir/defaults.go +++ b/cmd/frostfs-ir/defaults.go @@ -3,28 +3,9 @@ package main import ( "time" - configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config" "github.com/spf13/viper" ) -func newConfig(path, directory string) (*viper.Viper, error) { - const envPrefix = "FROSTFS_IR" - - var err error - var dv = viper.New() - - defaultConfiguration(dv) - - _, _, err = configViper.CreateViper(configViper.WithConfigFile(path), - configViper.WithConfigDir(directory), configViper.WithEnvPrefix(envPrefix), - configViper.WithViper(dv)) - if err != nil { - return nil, err - } - - return dv, err -} - func defaultConfiguration(cfg *viper.Viper) { cfg.SetDefault("logger.level", "info") diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 5db1db6b6..932b90404 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -6,8 +6,7 @@ import ( "fmt" "net/http" "os" - "os/signal" - "syscall" + "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" @@ -25,6 +24,20 @@ const ( // SuccessReturnCode returns when application closed without panic. SuccessReturnCode = 0 + + EnvPrefix = "FROSTFS_IR" +) + +var ( + wg = new(sync.WaitGroup) + intErr = make(chan error) // internal inner ring errors + logPrm = new(logger.Prm) + innerRing *innerring.Server + httpServers []*httputil.Server + log *logger.Logger + cfg *viper.Viper + configFile *string + configDir *string ) func exitErr(err error) { @@ -35,8 +48,8 @@ func exitErr(err error) { } func main() { - configFile := flag.String("config", "", "path to config") - configDir := flag.String("config-dir", "", "path to config directory") + configFile = flag.String("config", "", "path to config") + configDir = flag.String("config-dir", "", "path to config directory") versionFlag := flag.Bool("version", false, "frostfs-ir node version") flag.Parse() @@ -46,35 +59,32 @@ func main() { os.Exit(SuccessReturnCode) } - cfg, err := newConfig(*configFile, *configDir) + var err error + cfg, err = newConfig() exitErr(err) - var logPrm logger.Prm - err = logPrm.SetLevelString( cfg.GetString("logger.level"), ) exitErr(err) - log, err := logger.NewLogger(&logPrm) + log, err = logger.NewLogger(logPrm) exitErr(err) - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) - defer cancel() + ctx, cancel := context.WithCancel(context.Background()) - intErr := make(chan error) // internal inner ring errors + initHTTPServers(cfg) - httpServers := initHTTPServers(cfg, log) - - innerRing, err := innerring.New(ctx, log, cfg, intErr) + innerRing, err = innerring.New(ctx, log, cfg, intErr) exitErr(err) // start HTTP servers - for i := range httpServers { - srv := httpServers[i] - go func() { + for _, srv := range httpServers { + wg.Add(1) + go func(srv *httputil.Server) { exitErr(srv.Serve()) - }() + wg.Done() + }(srv) } // start inner ring @@ -84,32 +94,16 @@ func main() { log.Info(logs.CommonApplicationStarted, zap.String("version", misc.Version)) - select { - case <-ctx.Done(): - case err := <-intErr: - log.Info(logs.FrostFSIRInternalError, zap.String("msg", err.Error())) - } + watchForSignal(cancel) - innerRing.Stop() - - // shut down HTTP servers - for i := range httpServers { - srv := httpServers[i] - - go func() { - err := srv.Shutdown() - if err != nil { - log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, - zap.String("error", err.Error()), - ) - } - }() - } + <-ctx.Done() // graceful shutdown + log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop) + wg.Wait() log.Info(logs.FrostFSIRApplicationStopped) } -func initHTTPServers(cfg *viper.Viper, log *logger.Logger) []*httputil.Server { +func initHTTPServers(cfg *viper.Viper) { items := []struct { cfgPrefix string handler func() http.Handler @@ -118,7 +112,7 @@ func initHTTPServers(cfg *viper.Viper, log *logger.Logger) []*httputil.Server { {"prometheus", promhttp.Handler}, } - httpServers := make([]*httputil.Server, 0, len(items)) + httpServers = make([]*httputil.Server, 0, len(items)) for _, item := range items { if !cfg.GetBool(item.cfgPrefix + ".enabled") { @@ -141,6 +135,22 @@ func initHTTPServers(cfg *viper.Viper, log *logger.Logger) []*httputil.Server { ), ) } - - return httpServers +} + +func shutdown() { + innerRing.Stop() + + // shut down HTTP servers + for _, srv := range httpServers { + wg.Add(1) + go func(srv *httputil.Server) { + err := srv.Shutdown() + if err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), + ) + } + wg.Done() + }(srv) + } }