[#125] ir: Reconfigure pprof and metrics on SIGHUP

Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>

CHANGELOG.md

@@ -6,6 +6,7 @@ Changelog for FrostFS Node
 ### Added
 - Support impersonate bearer token (#229)
 - Change log level on SIGHUP for ir (#125)
+- Reload pprof and metrics on SIGHUP for ir (#125)
 
 ### Changed
 ### Fixed

cmd/frostfs-ir/config.go

@@ -60,6 +60,8 @@ func watchForSignal(cancel func()) {
 				if err != nil {
 					log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err))
 				}
+				pprofCmp.reload()
+				metricsCmp.reload()
 				log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully)
 			case syscall.SIGTERM, syscall.SIGINT:
 				log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)

cmd/frostfs-ir/httpcomponent.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
+	httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
+	"go.uber.org/zap"
+)
+
+type httpComponent struct {
+	srv                *httputil.Server
+	address            string
+	addressKey         string
+	name               string
+	handler            http.Handler
+	shutdownDur        time.Duration
+	shutdownTimeoutKey string
+	enabled            bool
+	enabledKey         string
+}
+
+func (c *httpComponent) init() {
+	log.Info(fmt.Sprintf("init %s", c.name))
+	c.enabled = cfg.GetBool(c.enabledKey)
+	c.address = cfg.GetString(c.addressKey)
+	c.shutdownDur = cfg.GetDuration(c.shutdownTimeoutKey)
+
+	if c.enabled {
+		c.srv = httputil.New(
+			httputil.HTTPSrvPrm{
+				Address: c.address,
+				Handler: c.handler,
+			},
+			httputil.WithShutdownTimeout(c.shutdownDur),
+		)
+	} else {
+		log.Info(fmt.Sprintf("%s is disabled, skip", c.name))
+		c.srv = nil
+	}
+}
+
+func (c *httpComponent) start() {
+	if c.srv != nil {
+		log.Info(fmt.Sprintf("start %s", c.name))
+		wg.Add(1)
+		go func() {
+			exitErr(c.srv.Serve())
+			wg.Done()
+		}()
+	}
+}
+
+func (c *httpComponent) shutdown() error {
+	if c.srv != nil {
+		log.Info(fmt.Sprintf("shutdown %s", c.name))
+		return c.srv.Shutdown()
+	}
+	return nil
+}
+
+func (c *httpComponent) reload() {
+	log.Info(fmt.Sprintf("reload %s", c.name))
+	enabled := cfg.GetBool(c.enabledKey)
+	address := cfg.GetString(c.addressKey)
+	dur := cfg.GetDuration(c.shutdownTimeoutKey)
+	if enabled != c.enabled || address != c.address || dur != c.shutdownDur {
+		log.Info(fmt.Sprintf("%s config updated", c.name))
+		if err := c.shutdown(); err != nil {
+			log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
+				zap.String("error", err.Error()),
+			)
+		} else {
+			c.init()
+			c.start()
+		}
+	}
+}

cmd/frostfs-ir/main.go

@@ -4,16 +4,13 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"net/http"
 	"os"
 	"sync"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/misc"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring"
-	httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/viper"
 	"go.uber.org/zap"
 )
@@ -29,15 +26,16 @@ const (
 )
 
 var (
-	wg          = new(sync.WaitGroup)
+	wg         = new(sync.WaitGroup)
-	intErr      = make(chan error) // internal inner ring errors
+	intErr     = make(chan error) // internal inner ring errors
-	logPrm      = new(logger.Prm)
+	logPrm     = new(logger.Prm)
-	innerRing   *innerring.Server
+	innerRing  *innerring.Server
-	httpServers []*httputil.Server
+	pprofCmp   *httpComponent
-	log         *logger.Logger
+	metricsCmp *httpComponent
-	cfg         *viper.Viper
+	log        *logger.Logger
-	configFile  *string
+	cfg        *viper.Viper
-	configDir   *string
+	configFile *string
+	configDir  *string
 )
 
 func exitErr(err error) {
@@ -73,19 +71,17 @@ func main() {
 
 	ctx, cancel := context.WithCancel(context.Background())
 
-	initHTTPServers(cfg)
+	pprofCmp = newPprofComponent()
+	pprofCmp.init()
+
+	metricsCmp = newMetricsComponent()
+	metricsCmp.init()
 
 	innerRing, err = innerring.New(ctx, log, cfg, intErr)
 	exitErr(err)
 
-	// start HTTP servers
+	pprofCmp.start()
-	for _, srv := range httpServers {
+	metricsCmp.start()
-		wg.Add(1)
-		go func(srv *httputil.Server) {
-			exitErr(srv.Serve())
-			wg.Done()
-		}(srv)
-	}
 
 	// start inner ring
 	err = innerRing.Start(ctx, intErr)
@@ -103,54 +99,16 @@ func main() {
 	log.Info(logs.FrostFSIRApplicationStopped)
 }
 
-func initHTTPServers(cfg *viper.Viper) {
+func shutdown() {
-	items := []struct {
+	innerRing.Stop()
-		cfgPrefix string
+	if err := metricsCmp.shutdown(); err != nil {
-		handler   func() http.Handler
+		log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
-	}{
+			zap.String("error", err.Error()),
-		{"pprof", httputil.Handler},
+		)
-		{"prometheus", promhttp.Handler},
 	}
-
+	if err := pprofCmp.shutdown(); err != nil {
-	httpServers = make([]*httputil.Server, 0, len(items))
+		log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
-
+			zap.String("error", err.Error()),
-	for _, item := range items {
-		if !cfg.GetBool(item.cfgPrefix + ".enabled") {
-			log.Info(item.cfgPrefix + " is disabled, skip")
-			continue
-		}
-
-		addr := cfg.GetString(item.cfgPrefix + ".address")
-
-		var prm httputil.HTTPSrvPrm
-
-		prm.Address = addr
-		prm.Handler = item.handler()
-
-		httpServers = append(httpServers,
-			httputil.New(prm,
-				httputil.WithShutdownTimeout(
-					cfg.GetDuration(item.cfgPrefix+".shutdown_timeout"),
-				),
-			),
 		)
 	}
 }
-
-func shutdown() {
-	innerRing.Stop()
-
-	// shut down HTTP servers
-	for _, srv := range httpServers {
-		wg.Add(1)
-		go func(srv *httputil.Server) {
-			err := srv.Shutdown()
-			if err != nil {
-				log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
-					zap.String("error", err.Error()),
-				)
-			}
-			wg.Done()
-		}(srv)
-	}
-}

cmd/frostfs-ir/metrics.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
+)
+
+const (
+	prometheusEnabledKey         = "prometheus.enabled"
+	prometheusAddressKey         = "prometheus.address"
+	prometheusShutdownTimeoutKey = "prometheus.shutdown_timeout"
+)
+
+func newMetricsComponent() *httpComponent {
+	return &httpComponent{
+		name:               "prometheus",
+		enabledKey:         prometheusEnabledKey,
+		addressKey:         prometheusAddressKey,
+		shutdownTimeoutKey: prometheusShutdownTimeoutKey,
+		handler:            metrics.Handler(),
+	}
+}

cmd/frostfs-ir/pprof.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
+)
+
+const (
+	pprofEnabledKey         = "pprof.enabled"
+	pprofAddressKey         = "pprof.address"
+	pprofShutdownTimeoutKey = "pprof.shutdown_timeout"
+)
+
+func newPprofComponent() *httpComponent {
+	return &httpComponent{
+		name:               "pprof",
+		enabledKey:         pprofEnabledKey,
+		addressKey:         pprofAddressKey,
+		shutdownTimeoutKey: pprofShutdownTimeoutKey,
+		handler:            httputil.Handler(),
+	}
+}

pkg/innerring/initialization.go

@@ -16,7 +16,6 @@ import (
 	nodevalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation"
 	addrvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/maddress"
 	statevalidation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state"
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
 	balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
@@ -535,11 +534,3 @@ func (s *Server) initKey(cfg *viper.Viper) error {
 	s.key = acc.PrivateKey()
 	return nil
 }
-
-func (s *Server) initMetrics(cfg *viper.Viper) {
-	if cfg.GetString("prometheus.address") == "" {
-		return
-	}
-	m := metrics.NewInnerRingMetrics()
-	s.metrics = &m
-}

pkg/innerring/innerring.go

@@ -396,7 +396,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan
 		return nil, err
 	}
 
-	server.initMetrics(cfg)
+	server.metrics = metrics.NewInnerRingMetrics()
 
 	return server, nil
 }

pkg/metrics/innerring.go

@@ -11,7 +11,7 @@ type InnerRingServiceMetrics struct {
 }
 
 // NewInnerRingMetrics returns new instance of metrics collectors for inner ring.
-func NewInnerRingMetrics() InnerRingServiceMetrics {
+func NewInnerRingMetrics() *InnerRingServiceMetrics {
 	var (
 		epoch = newGauge(prometheus.GaugeOpts{
 			Namespace: namespace,
@@ -30,7 +30,7 @@ func NewInnerRingMetrics() InnerRingServiceMetrics {
 	mustRegister(epoch)
 	mustRegister(health)
 
-	return InnerRingServiceMetrics{
+	return &InnerRingServiceMetrics{
 		epoch:  epoch,
 		health: health,
 	}