[#125] ir: Reconfigure pprof and metrics on SIGHUP

Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
This commit is contained in:
Anton Nikiforov 2023-04-26 15:45:57 +03:00
parent a181c9e434
commit 800eb5e983
9 changed files with 154 additions and 80 deletions

View file

@ -6,6 +6,7 @@ Changelog for FrostFS Node
### Added ### Added
- Support impersonate bearer token (#229) - Support impersonate bearer token (#229)
- Change log level on SIGHUP for ir (#125) - Change log level on SIGHUP for ir (#125)
- Reload pprof and metrics on SIGHUP for ir (#125)
### Changed ### Changed
### Fixed ### Fixed

View file

@ -60,6 +60,8 @@ func watchForSignal(cancel func()) {
if err != nil { if err != nil {
log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err)) log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err))
} }
pprofCmp.reload()
metricsCmp.reload()
log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully) log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully)
case syscall.SIGTERM, syscall.SIGINT: case syscall.SIGTERM, syscall.SIGINT:
log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping) log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)

View file

@ -0,0 +1,80 @@
package main
import (
"fmt"
"net/http"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
"go.uber.org/zap"
)
type httpComponent struct {
srv *httputil.Server
address string
addressKey string
name string
handler http.Handler
shutdownDur time.Duration
shutdownTimeoutKey string
enabled bool
enabledKey string
}
func (c *httpComponent) init() {
log.Info(fmt.Sprintf("init %s", c.name))
c.enabled = cfg.GetBool(c.enabledKey)
c.address = cfg.GetString(c.addressKey)
c.shutdownDur = cfg.GetDuration(c.shutdownTimeoutKey)
if c.enabled {
c.srv = httputil.New(
httputil.HTTPSrvPrm{
Address: c.address,
Handler: c.handler,
},
httputil.WithShutdownTimeout(c.shutdownDur),
)
} else {
log.Info(fmt.Sprintf("%s is disabled, skip", c.name))
c.srv = nil
}
}
func (c *httpComponent) start() {
if c.srv != nil {
log.Info(fmt.Sprintf("start %s", c.name))
wg.Add(1)
go func() {
exitErr(c.srv.Serve())
wg.Done()
}()
}
}
func (c *httpComponent) shutdown() error {
if c.srv != nil {
log.Info(fmt.Sprintf("shutdown %s", c.name))
return c.srv.Shutdown()
}
return nil
}
func (c *httpComponent) reload() {
log.Info(fmt.Sprintf("reload %s", c.name))
enabled := cfg.GetBool(c.enabledKey)
address := cfg.GetString(c.addressKey)
dur := cfg.GetDuration(c.shutdownTimeoutKey)
if enabled != c.enabled || address != c.address || dur != c.shutdownDur {
log.Info(fmt.Sprintf("%s config updated", c.name))
if err := c.shutdown(); err != nil {
log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.String("error", err.Error()),
)
} else {
c.init()
c.start()
}
}
}

View file

@ -4,16 +4,13 @@ import (
"context" "context"
"flag" "flag"
"fmt" "fmt"
"net/http"
"os" "os"
"sync" "sync"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/misc"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring"
httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/viper" "github.com/spf13/viper"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -29,15 +26,16 @@ const (
) )
var ( var (
wg = new(sync.WaitGroup) wg = new(sync.WaitGroup)
intErr = make(chan error) // internal inner ring errors intErr = make(chan error) // internal inner ring errors
logPrm = new(logger.Prm) logPrm = new(logger.Prm)
innerRing *innerring.Server innerRing *innerring.Server
httpServers []*httputil.Server pprofCmp *httpComponent
log *logger.Logger metricsCmp *httpComponent
cfg *viper.Viper log *logger.Logger
configFile *string cfg *viper.Viper
configDir *string configFile *string
configDir *string
) )
func exitErr(err error) { func exitErr(err error) {
@ -73,19 +71,17 @@ func main() {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
initHTTPServers(cfg) pprofCmp = newPprofComponent()
pprofCmp.init()
metricsCmp = newMetricsComponent()
metricsCmp.init()
innerRing, err = innerring.New(ctx, log, cfg, intErr) innerRing, err = innerring.New(ctx, log, cfg, intErr)
exitErr(err) exitErr(err)
// start HTTP servers pprofCmp.start()
for _, srv := range httpServers { metricsCmp.start()
wg.Add(1)
go func(srv *httputil.Server) {
exitErr(srv.Serve())
wg.Done()
}(srv)
}
// start inner ring // start inner ring
err = innerRing.Start(ctx, intErr) err = innerRing.Start(ctx, intErr)
@ -103,54 +99,16 @@ func main() {
log.Info(logs.FrostFSIRApplicationStopped) log.Info(logs.FrostFSIRApplicationStopped)
} }
func initHTTPServers(cfg *viper.Viper) { func shutdown() {
items := []struct { innerRing.Stop()
cfgPrefix string if err := metricsCmp.shutdown(); err != nil {
handler func() http.Handler log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
}{ zap.String("error", err.Error()),
{"pprof", httputil.Handler}, )
{"prometheus", promhttp.Handler},
} }
if err := pprofCmp.shutdown(); err != nil {
httpServers = make([]*httputil.Server, 0, len(items)) log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.String("error", err.Error()),
for _, item := range items {
if !cfg.GetBool(item.cfgPrefix + ".enabled") {
log.Info(item.cfgPrefix + " is disabled, skip")
continue
}
addr := cfg.GetString(item.cfgPrefix + ".address")
var prm httputil.HTTPSrvPrm
prm.Address = addr
prm.Handler = item.handler()
httpServers = append(httpServers,
httputil.New(prm,
httputil.WithShutdownTimeout(
cfg.GetDuration(item.cfgPrefix+".shutdown_timeout"),
),
),
) )
} }
} }
func shutdown() {
innerRing.Stop()
// shut down HTTP servers
for _, srv := range httpServers {
wg.Add(1)
go func(srv *httputil.Server) {
err := srv.Shutdown()
if err != nil {
log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.String("error", err.Error()),
)
}
wg.Done()
}(srv)
}
}

21
cmd/frostfs-ir/metrics.go Normal file
View file

@ -0,0 +1,21 @@
package main
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
)
const (
prometheusEnabledKey = "prometheus.enabled"
prometheusAddressKey = "prometheus.address"
prometheusShutdownTimeoutKey = "prometheus.shutdown_timeout"
)
func newMetricsComponent() *httpComponent {
return &httpComponent{
name: "prometheus",
enabledKey: prometheusEnabledKey,
addressKey: prometheusAddressKey,
shutdownTimeoutKey: prometheusShutdownTimeoutKey,
handler: metrics.Handler(),
}
}

21
cmd/frostfs-ir/pprof.go Normal file
View file

@ -0,0 +1,21 @@
package main
import (
httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
)
const (
pprofEnabledKey = "pprof.enabled"
pprofAddressKey = "pprof.address"
pprofShutdownTimeoutKey = "pprof.shutdown_timeout"
)
func newPprofComponent() *httpComponent {
return &httpComponent{
name: "pprof",
enabledKey: pprofEnabledKey,
addressKey: pprofAddressKey,
shutdownTimeoutKey: pprofShutdownTimeoutKey,
handler: httputil.Handler(),
}
}

View file

@ -16,7 +16,6 @@ import (
nodevalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation" nodevalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation"
addrvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/maddress" addrvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/maddress"
statevalidation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state" statevalidation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
@ -535,11 +534,3 @@ func (s *Server) initKey(cfg *viper.Viper) error {
s.key = acc.PrivateKey() s.key = acc.PrivateKey()
return nil return nil
} }
func (s *Server) initMetrics(cfg *viper.Viper) {
if cfg.GetString("prometheus.address") == "" {
return
}
m := metrics.NewInnerRingMetrics()
s.metrics = &m
}

View file

@ -396,7 +396,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan
return nil, err return nil, err
} }
server.initMetrics(cfg) server.metrics = metrics.NewInnerRingMetrics()
return server, nil return server, nil
} }

View file

@ -11,7 +11,7 @@ type InnerRingServiceMetrics struct {
} }
// NewInnerRingMetrics returns new instance of metrics collectors for inner ring. // NewInnerRingMetrics returns new instance of metrics collectors for inner ring.
func NewInnerRingMetrics() InnerRingServiceMetrics { func NewInnerRingMetrics() *InnerRingServiceMetrics {
var ( var (
epoch = newGauge(prometheus.GaugeOpts{ epoch = newGauge(prometheus.GaugeOpts{
Namespace: namespace, Namespace: namespace,
@ -30,7 +30,7 @@ func NewInnerRingMetrics() InnerRingServiceMetrics {
mustRegister(epoch) mustRegister(epoch)
mustRegister(health) mustRegister(health)
return InnerRingServiceMetrics{ return &InnerRingServiceMetrics{
epoch: epoch, epoch: epoch,
health: health, health: health,
} }