From f604d6bbdce327a3560020864b9f37857d0ebbf0 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 19 Apr 2023 20:02:02 +0300 Subject: [PATCH] [#39] ir: Add optional profilers Includes `block` and `mutex` profiles configuration. Signed-off-by: Pavel Karpy Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-ir/httpcomponent.go | 10 ++++-- cmd/frostfs-ir/main.go | 2 +- cmd/frostfs-ir/pprof.go | 64 ++++++++++++++++++++++++++++++--- config/example/ir.env | 2 ++ config/example/ir.yaml | 2 ++ 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-ir/httpcomponent.go b/cmd/frostfs-ir/httpcomponent.go index 38e936471..8f5e3753b 100644 --- a/cmd/frostfs-ir/httpcomponent.go +++ b/cmd/frostfs-ir/httpcomponent.go @@ -64,12 +64,16 @@ func (c *httpComponent) shutdown() error { return nil } -func (c *httpComponent) reload() { - log.Info(fmt.Sprintf("reload %s", c.name)) +func (c *httpComponent) needReload() bool { enabled := cfg.GetBool(c.name + enabledKeyPostfix) address := cfg.GetString(c.name + addressKeyPostfix) dur := cfg.GetDuration(c.name + shutdownTimeoutKeyPostfix) - if enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) { + return enabled != c.enabled || enabled && (address != c.address || dur != c.shutdownDur) +} + +func (c *httpComponent) reload() { + log.Info(fmt.Sprintf("reload %s", c.name)) + if c.needReload() { log.Info(fmt.Sprintf("%s config updated", c.name)) if err := c.shutdown(); err != nil { log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 84dff9101..1718a46ab 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -30,7 +30,7 @@ var ( intErr = make(chan error) // internal inner ring errors logPrm = new(logger.Prm) innerRing *innerring.Server - pprofCmp *httpComponent + pprofCmp *pprofComponent metricsCmp *httpComponent log *logger.Logger cfg *viper.Viper diff --git a/cmd/frostfs-ir/pprof.go b/cmd/frostfs-ir/pprof.go index 8228a0f53..d67c463fc 100644 --- a/cmd/frostfs-ir/pprof.go +++ b/cmd/frostfs-ir/pprof.go @@ -1,12 +1,68 @@ package main import ( + "fmt" + "runtime" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" + "go.uber.org/zap" ) -func newPprofComponent() *httpComponent { - return &httpComponent{ - name: "pprof", - handler: httputil.Handler(), +type pprofComponent struct { + httpComponent + blockRate int + mutexRate int +} + +const ( + pprofBlockRateKey = "pprof.block_rate" + pprofMutexRateKey = "pprof.mutex_rate" +) + +func newPprofComponent() *pprofComponent { + return &pprofComponent{ + httpComponent: httpComponent{ + name: "pprof", + handler: httputil.Handler(), + }, + } +} + +func (c *pprofComponent) init() { + c.httpComponent.init() + + if c.enabled { + c.blockRate = cfg.GetInt(pprofBlockRateKey) + c.mutexRate = cfg.GetInt(pprofMutexRateKey) + runtime.SetBlockProfileRate(c.blockRate) + runtime.SetMutexProfileFraction(c.mutexRate) + } else { + c.blockRate = 0 + c.mutexRate = 0 + runtime.SetBlockProfileRate(0) + runtime.SetMutexProfileFraction(0) + } +} + +func (c *pprofComponent) needReload() bool { + blockRate := cfg.GetInt(pprofBlockRateKey) + mutexRate := cfg.GetInt(pprofMutexRateKey) + return c.httpComponent.needReload() || + c.enabled && (c.blockRate != blockRate || c.mutexRate != mutexRate) +} + +func (c *pprofComponent) reload() { + log.Info(fmt.Sprintf("reload %s", c.name)) + if c.needReload() { + log.Info(fmt.Sprintf("%s config updated", c.name)) + if err := c.shutdown(); err != nil { + log.Debug(logs.FrostFSIRCouldNotShutdownHTTPServer, + zap.String("error", err.Error())) + return + } + + c.init() + c.start() } } diff --git a/config/example/ir.env b/config/example/ir.env index e3de23ac1..7b8f8a89d 100644 --- a/config/example/ir.env +++ b/config/example/ir.env @@ -74,6 +74,8 @@ FROSTFS_IR_CONTRACTS_ALPHABET_ZHIVETE=f584699bc2ff457d339fb09f16217042c1a42101 FROSTFS_IR_PPROF_ENABLED=true FROSTFS_IR_PPROF_ADDRESS=localhost:6060 FROSTFS_IR_PPROF_SHUTDOWN_TIMEOUT=30s +FROSTFS_IR_PPROF_BLOCK_RATE=10000 +FROSTFS_IR_PPROF_MUTEX_RATE=10000 FROSTFS_IR_PROMETHEUS_ENABLED=true FROSTFS_IR_PROMETHEUS_ADDRESS=localhost:9090 diff --git a/config/example/ir.yaml b/config/example/ir.yaml index bd56ec74b..1130a840b 100644 --- a/config/example/ir.yaml +++ b/config/example/ir.yaml @@ -113,6 +113,8 @@ pprof: enabled: true address: localhost:6060 # Endpoint for application pprof profiling; disabled by default shutdown_timeout: 30s # Timeout for profiling HTTP server graceful shutdown + block_rate: 10000 # sampling rate: an average of one blocking event per rate nanoseconds spent blocked is reported; "1" reports every blocking event; "0" disables profiler + mutex_rate: 10000 # sampling rate: on average 1/rate events are reported; "0" disables profiler prometheus: enabled: true