From 800eb5e983fba1b9dc4e5048df1f71229486c5bb Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Wed, 26 Apr 2023 15:45:57 +0300 Subject: [PATCH] [#125] ir: Reconfigure pprof and metrics on SIGHUP Signed-off-by: Anton Nikiforov --- CHANGELOG.md | 1 + cmd/frostfs-ir/config.go | 2 + cmd/frostfs-ir/httpcomponent.go | 80 ++++++++++++++++++++++++++++ cmd/frostfs-ir/main.go | 94 +++++++++------------------------ cmd/frostfs-ir/metrics.go | 21 ++++++++ cmd/frostfs-ir/pprof.go | 21 ++++++++ pkg/innerring/initialization.go | 9 ---- pkg/innerring/innerring.go | 2 +- pkg/metrics/innerring.go | 4 +- 9 files changed, 154 insertions(+), 80 deletions(-) create mode 100644 cmd/frostfs-ir/httpcomponent.go create mode 100644 cmd/frostfs-ir/metrics.go create mode 100644 cmd/frostfs-ir/pprof.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ed434da5..58fe9c3ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Changelog for FrostFS Node ### Added - Support impersonate bearer token (#229) - Change log level on SIGHUP for ir (#125) +- Reload pprof and metrics on SIGHUP for ir (#125) ### Changed ### Fixed diff --git a/cmd/frostfs-ir/config.go b/cmd/frostfs-ir/config.go index c4787f965..2e2aa1613 100644 --- a/cmd/frostfs-ir/config.go +++ b/cmd/frostfs-ir/config.go @@ -60,6 +60,8 @@ func watchForSignal(cancel func()) { if err != nil { log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) } + pprofCmp.reload() + metricsCmp.reload() log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) case syscall.SIGTERM, syscall.SIGINT: log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go new file mode 100644 index 000000000..d3cda8930 --- /dev/null +++ b/cmd/frostfs-ir/httpcomponent.go @@ -0,0 +1,80 @@ +package main + +import ( + "fmt" + "net/http" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" + "go.uber.org/zap" +) + +type httpComponent struct { + srv *httputil.Server + address string + addressKey string + name string + handler http.Handler + shutdownDur time.Duration + shutdownTimeoutKey string + enabled bool + enabledKey string +} + +func (c *httpComponent) init() { + log.Info(fmt.Sprintf("init %s", c.name)) + c.enabled = cfg.GetBool(c.enabledKey) + c.address = cfg.GetString(c.addressKey) + c.shutdownDur = cfg.GetDuration(c.shutdownTimeoutKey) + + if c.enabled { + c.srv = httputil.New( + httputil.HTTPSrvPrm{ + Address: c.address, + Handler: c.handler, + }, + httputil.WithShutdownTimeout(c.shutdownDur), + ) + } else { + log.Info(fmt.Sprintf("%s is disabled, skip", c.name)) + c.srv = nil + } +} + +func (c *httpComponent) start() { + if c.srv != nil { + log.Info(fmt.Sprintf("start %s", c.name)) + wg.Add(1) + go func() { + exitErr(c.srv.Serve()) + wg.Done() + }() + } +} + +func (c *httpComponent) shutdown() error { + if c.srv != nil { + log.Info(fmt.Sprintf("shutdown %s", c.name)) + return c.srv.Shutdown() + } + return nil +} + +func (c *httpComponent) reload() { + log.Info(fmt.Sprintf("reload %s", c.name)) + enabled := cfg.GetBool(c.enabledKey) + address := cfg.GetString(c.addressKey) + dur := cfg.GetDuration(c.shutdownTimeoutKey) + if enabled != c.enabled || address != c.address || dur != c.shutdownDur { + log.Info(fmt.Sprintf("%s config updated", c.name)) + if err := c.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), + ) + } else { + c.init() + c.start() + } + } +} diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 932b90404..84dff9101 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -4,16 +4,13 @@ import ( "context" "flag" "fmt" - "net/http" "os" "sync" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" - httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/viper" "go.uber.org/zap" ) @@ -29,15 +26,16 @@ const ( ) var ( - wg = new(sync.WaitGroup) - intErr = make(chan error) // internal inner ring errors - logPrm = new(logger.Prm) - innerRing *innerring.Server - httpServers []*httputil.Server - log *logger.Logger - cfg *viper.Viper - configFile *string - configDir *string + wg = new(sync.WaitGroup) + intErr = make(chan error) // internal inner ring errors + logPrm = new(logger.Prm) + innerRing *innerring.Server + pprofCmp *httpComponent + metricsCmp *httpComponent + log *logger.Logger + cfg *viper.Viper + configFile *string + configDir *string ) func exitErr(err error) { @@ -73,19 +71,17 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) - initHTTPServers(cfg) + pprofCmp = newPprofComponent() + pprofCmp.init() + + metricsCmp = newMetricsComponent() + metricsCmp.init() innerRing, err = innerring.New(ctx, log, cfg, intErr) exitErr(err) - // start HTTP servers - for _, srv := range httpServers { - wg.Add(1) - go func(srv *httputil.Server) { - exitErr(srv.Serve()) - wg.Done() - }(srv) - } + pprofCmp.start() + metricsCmp.start() // start inner ring err = innerRing.Start(ctx, intErr) @@ -103,54 +99,16 @@ func main() { log.Info(logs.FrostFSIRApplicationStopped) } -func initHTTPServers(cfg *viper.Viper) { - items := []struct { - cfgPrefix string - handler func() http.Handler - }{ - {"pprof", httputil.Handler}, - {"prometheus", promhttp.Handler}, +func shutdown() { + innerRing.Stop() + if err := metricsCmp.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), + ) } - - httpServers = make([]*httputil.Server, 0, len(items)) - - for _, item := range items { - if !cfg.GetBool(item.cfgPrefix + ".enabled") { - log.Info(item.cfgPrefix + " is disabled, skip") - continue - } - - addr := cfg.GetString(item.cfgPrefix + ".address") - - var prm httputil.HTTPSrvPrm - - prm.Address = addr - prm.Handler = item.handler() - - httpServers = append(httpServers, - httputil.New(prm, - httputil.WithShutdownTimeout( - cfg.GetDuration(item.cfgPrefix+".shutdown_timeout"), - ), - ), + if err := pprofCmp.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error()), ) } } - -func shutdown() { - innerRing.Stop() - - // shut down HTTP servers - for _, srv := range httpServers { - wg.Add(1) - go func(srv *httputil.Server) { - err := srv.Shutdown() - if err != nil { - log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, - zap.String("error", err.Error()), - ) - } - wg.Done() - }(srv) - } -} diff --git a/cmd/frostfs-ir/metrics.go b/cmd/frostfs-ir/metrics.go new file mode 100644 index 000000000..debeb7d60 --- /dev/null +++ b/cmd/frostfs-ir/metrics.go @@ -0,0 +1,21 @@ +package main + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +const ( + prometheusEnabledKey = "prometheus.enabled" + prometheusAddressKey = "prometheus.address" + prometheusShutdownTimeoutKey = "prometheus.shutdown_timeout" +) + +func newMetricsComponent() *httpComponent { + return &httpComponent{ + name: "prometheus", + enabledKey: prometheusEnabledKey, + addressKey: prometheusAddressKey, + shutdownTimeoutKey: prometheusShutdownTimeoutKey, + handler: metrics.Handler(), + } +} diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go new file mode 100644 index 000000000..0d48c11de --- /dev/null +++ b/cmd/frostfs-ir/pprof.go @@ -0,0 +1,21 @@ +package main + +import ( + httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" +) + +const ( + pprofEnabledKey = "pprof.enabled" + pprofAddressKey = "pprof.address" + pprofShutdownTimeoutKey = "pprof.shutdown_timeout" +) + +func newPprofComponent() *httpComponent { + return &httpComponent{ + name: "pprof", + enabledKey: pprofEnabledKey, + addressKey: pprofAddressKey, + shutdownTimeoutKey: pprofShutdownTimeoutKey, + handler: httputil.Handler(), + } +} diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 84d08c4c6..1dc1d40ea 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -16,7 +16,6 @@ import ( nodevalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation" addrvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/maddress" statevalidation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" @@ -535,11 +534,3 @@ func (s *Server) initKey(cfg *viper.Viper) error { s.key = acc.PrivateKey() return nil } - -func (s *Server) initMetrics(cfg *viper.Viper) { - if cfg.GetString("prometheus.address") == "" { - return - } - m := metrics.NewInnerRingMetrics() - s.metrics = &m -} diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index deb546f08..9119ff201 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -396,7 +396,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan return nil, err } - server.initMetrics(cfg) + server.metrics = metrics.NewInnerRingMetrics() return server, nil } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 05b76f9c9..79db424b8 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -11,7 +11,7 @@ type InnerRingServiceMetrics struct { } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. -func NewInnerRingMetrics() InnerRingServiceMetrics { +func NewInnerRingMetrics() *InnerRingServiceMetrics { var ( epoch = newGauge(prometheus.GaugeOpts{ Namespace: namespace, @@ -30,7 +30,7 @@ func NewInnerRingMetrics() InnerRingServiceMetrics { mustRegister(epoch) mustRegister(health) - return InnerRingServiceMetrics{ + return &InnerRingServiceMetrics{ epoch: epoch, health: health, }