a-savchuk/frostfs-node
1
0
Fork 0
forked from TrueCloudLab/frostfs-node
Code Pull requests Activity

[#125] ir: Reconfigure pprof and metrics on SIGHUP

Browse source 
Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
This commit is contained in:
Anton Nikiforov 2023-04-26 15:45:57 +03:00
parent a181c9e434
commit 800eb5e983
9 changed files with 154 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -6,6 +6,7 @@ Changelog for FrostFS Node
### Added
- Support impersonate bearer token (#229)
- Change log level on SIGHUP for ir (#125)
- Reload pprof and metrics on SIGHUP for ir (#125)
### Changed
### Fixed

2
cmd/frostfs-ir/config.go
View file

@ -60,6 +60,8 @@ func watchForSignal(cancel func()) {
if err != nil {
log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err))
}
pprofCmp.reload()
metricsCmp.reload()
log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully)
case syscall.SIGTERM, syscall.SIGINT:
log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)

80
cmd/frostfs-ir/httpcomponent.go Normal file
View file

@ -0,0 +1,80 @@
package main
import (
"fmt"
"net/http"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
"go.uber.org/zap"
)
type httpComponent struct {
srv *httputil.Server
address string
addressKey string
name string
handler http.Handler
shutdownDur time.Duration
shutdownTimeoutKey string
enabled bool
enabledKey string
}
func (c *httpComponent) init() {
log.Info(fmt.Sprintf("init %s", c.name))
c.enabled = cfg.GetBool(c.enabledKey)
c.address = cfg.GetString(c.addressKey)
c.shutdownDur = cfg.GetDuration(c.shutdownTimeoutKey)
if c.enabled {
c.srv = httputil.New(
httputil.HTTPSrvPrm{
Address: c.address,
Handler: c.handler,
},
httputil.WithShutdownTimeout(c.shutdownDur),
)
} else {
log.Info(fmt.Sprintf("%s is disabled, skip", c.name))
c.srv = nil
}
}
func (c *httpComponent) start() {
if c.srv != nil {
log.Info(fmt.Sprintf("start %s", c.name))
wg.Add(1)
go func() {
exitErr(c.srv.Serve())
wg.Done()
}()
}
}
func (c *httpComponent) shutdown() error {
if c.srv != nil {
log.Info(fmt.Sprintf("shutdown %s", c.name))
return c.srv.Shutdown()
}
return nil
}
func (c *httpComponent) reload() {
log.Info(fmt.Sprintf("reload %s", c.name))
enabled := cfg.GetBool(c.enabledKey)
address := cfg.GetString(c.addressKey)
dur := cfg.GetDuration(c.shutdownTimeoutKey)
if enabled != c.enabled || address != c.address || dur != c.shutdownDur {
log.Info(fmt.Sprintf("%s config updated", c.name))
if err := c.shutdown(); err != nil {
log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.String("error", err.Error()),
)
} else {
c.init()
c.start()
}
}
}

94
cmd/frostfs-ir/main.go
View file

@ -4,16 +4,13 @@ import (
"context"
"flag"
"fmt"
"net/http"
"os"
"sync"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/misc"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring"
httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/viper"
"go.uber.org/zap"
)
@ -29,15 +26,16 @@ const (
)
var (
wg = new(sync.WaitGroup)
intErr = make(chan error) // internal inner ring errors
logPrm = new(logger.Prm)
innerRing *innerring.Server
httpServers []*httputil.Server
log *logger.Logger
cfg *viper.Viper
configFile *string
configDir *string
wg = new(sync.WaitGroup)
intErr = make(chan error) // internal inner ring errors
logPrm = new(logger.Prm)
innerRing *innerring.Server
pprofCmp *httpComponent
metricsCmp *httpComponent
log *logger.Logger
cfg *viper.Viper
configFile *string
configDir *string
)
func exitErr(err error) {
@ -73,19 +71,17 @@ func main() {
ctx, cancel := context.WithCancel(context.Background())
initHTTPServers(cfg)
pprofCmp = newPprofComponent()
pprofCmp.init()
metricsCmp = newMetricsComponent()
metricsCmp.init()
innerRing, err = innerring.New(ctx, log, cfg, intErr)
exitErr(err)
// start HTTP servers
for _, srv := range httpServers {
wg.Add(1)
go func(srv *httputil.Server) {
exitErr(srv.Serve())
wg.Done()
}(srv)
}
pprofCmp.start()
metricsCmp.start()
// start inner ring
err = innerRing.Start(ctx, intErr)
@ -103,54 +99,16 @@ func main() {
log.Info(logs.FrostFSIRApplicationStopped)
}
func initHTTPServers(cfg *viper.Viper) {
items := []struct {
cfgPrefix string
handler func() http.Handler
}{
{"pprof", httputil.Handler},
{"prometheus", promhttp.Handler},
func shutdown() {
innerRing.Stop()
if err := metricsCmp.shutdown(); err != nil {
log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.String("error", err.Error()),
)
}
httpServers = make([]*httputil.Server, 0, len(items))
for _, item := range items {
if !cfg.GetBool(item.cfgPrefix + ".enabled") {
log.Info(item.cfgPrefix + " is disabled, skip")
continue
}
addr := cfg.GetString(item.cfgPrefix + ".address")
var prm httputil.HTTPSrvPrm
prm.Address = addr
prm.Handler = item.handler()
httpServers = append(httpServers,
httputil.New(prm,
httputil.WithShutdownTimeout(
cfg.GetDuration(item.cfgPrefix+".shutdown_timeout"),
),
),
if err := pprofCmp.shutdown(); err != nil {
log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.String("error", err.Error()),
)
}
}
func shutdown() {
innerRing.Stop()
// shut down HTTP servers
for _, srv := range httpServers {
wg.Add(1)
go func(srv *httputil.Server) {
err := srv.Shutdown()
if err != nil {
log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer,
zap.String("error", err.Error()),
)
}
wg.Done()
}(srv)
}
}

21
cmd/frostfs-ir/metrics.go Normal file
View file

@ -0,0 +1,21 @@
package main
import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
)
const (
prometheusEnabledKey = "prometheus.enabled"
prometheusAddressKey = "prometheus.address"
prometheusShutdownTimeoutKey = "prometheus.shutdown_timeout"
)
func newMetricsComponent() *httpComponent {
return &httpComponent{
name: "prometheus",
enabledKey: prometheusEnabledKey,
addressKey: prometheusAddressKey,
shutdownTimeoutKey: prometheusShutdownTimeoutKey,
handler: metrics.Handler(),
}
}

21
cmd/frostfs-ir/pprof.go Normal file
View file

@ -0,0 +1,21 @@
package main
import (
httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
)
const (
pprofEnabledKey = "pprof.enabled"
pprofAddressKey = "pprof.address"
pprofShutdownTimeoutKey = "pprof.shutdown_timeout"
)
func newPprofComponent() *httpComponent {
return &httpComponent{
name: "pprof",
enabledKey: pprofEnabledKey,
addressKey: pprofAddressKey,
shutdownTimeoutKey: pprofShutdownTimeoutKey,
handler: httputil.Handler(),
}
}

9
pkg/innerring/initialization.go
View file

@ -16,7 +16,6 @@ import (
nodevalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation"
addrvalidator "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/maddress"
statevalidation "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring/processors/netmap/nodevalidation/state"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
@ -535,11 +534,3 @@ func (s *Server) initKey(cfg *viper.Viper) error {
s.key = acc.PrivateKey()
return nil
}
func (s *Server) initMetrics(cfg *viper.Viper) {
if cfg.GetString("prometheus.address") == "" {
return
}
m := metrics.NewInnerRingMetrics()
s.metrics = &m
}

2
pkg/innerring/innerring.go
View file

@ -396,7 +396,7 @@ func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan
return nil, err
}
server.initMetrics(cfg)
server.metrics = metrics.NewInnerRingMetrics()
return server, nil
}

4
pkg/metrics/innerring.go
View file

@ -11,7 +11,7 @@ type InnerRingServiceMetrics struct {
}
// NewInnerRingMetrics returns new instance of metrics collectors for inner ring.
func NewInnerRingMetrics() InnerRingServiceMetrics {
func NewInnerRingMetrics() *InnerRingServiceMetrics {
var (
epoch = newGauge(prometheus.GaugeOpts{
Namespace: namespace,
@ -30,7 +30,7 @@ func NewInnerRingMetrics() InnerRingServiceMetrics {
mustRegister(epoch)
mustRegister(health)
return InnerRingServiceMetrics{
return &InnerRingServiceMetrics{
epoch: epoch,
health: health,
}