[#39] node: Add optional profilers

Include settings for block and mutex profilers.
They are disabled by default, as in Go runtime itself.

Signed-off-by: Pavel Karpy <p.karpy@yadro.com>
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
Pavel Karpy 2023-02-01 10:21:17 +03:00 committed by Evgenii Stratonikov
parent a6ee7a3087
commit 14c35d776e
8 changed files with 78 additions and 8 deletions

View file

@ -51,3 +51,27 @@ func Address(c *config.Config) string {
return AddressDefault
}
// BlockRates returns the value of "block_rate" config parameter
// from "pprof" section.
func BlockRate(c *config.Config) int {
s := c.Sub(subsection)
v := int(config.IntSafe(s, "block_rate"))
if v <= 0 {
return 0
}
return v
}
// MutexRate returns the value of "mutex_rate" config parameter
// from "pprof" section.
func MutexRate(c *config.Config) int {
s := c.Sub(subsection)
v := int(config.IntSafe(s, "mutex_rate"))
if v <= 0 {
return 0
}
return v
}

View file

@ -18,6 +18,9 @@ func TestProfilerSection(t *testing.T) {
require.Equal(t, profilerconfig.ShutdownTimeoutDefault, to)
require.Equal(t, profilerconfig.AddressDefault, addr)
require.False(t, profilerconfig.Enabled(configtest.EmptyConfig()))
require.Zero(t, profilerconfig.BlockRate(configtest.EmptyConfig()))
require.Zero(t, profilerconfig.MutexRate(configtest.EmptyConfig()))
})
const path = "../../../../config/example/node"
@ -29,6 +32,9 @@ func TestProfilerSection(t *testing.T) {
require.Equal(t, 15*time.Second, to)
require.Equal(t, "localhost:6060", addr)
require.True(t, profilerconfig.Enabled(c))
require.Equal(t, 10_000, profilerconfig.BlockRate(c))
require.Equal(t, 10_000, profilerconfig.MutexRate(c))
}
configtest.ForEachFileType(path, fileConfigTest)

View file

@ -83,9 +83,8 @@ func initApp(ctx context.Context, c *cfg) {
c.wg.Done()
}()
pprof, _ := pprofComponent(c)
metrics, _ := metricsComponent(c)
initAndLog(c, pprof.name, pprof.init)
initAndLog(c, "profiler", initProfilerService)
initAndLog(c, metrics.name, metrics.init)
initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) })

View file

@ -1,10 +1,19 @@
package main
import (
"runtime"
profilerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/profiler"
httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
)
func initProfilerService(c *cfg) {
tuneProfilers(c)
pprof, _ := pprofComponent(c)
pprof.init(c)
}
func pprofComponent(c *cfg) (*httpComponent, bool) {
var updated bool
// check if it has been inited before
@ -13,6 +22,7 @@ func pprofComponent(c *cfg) (*httpComponent, bool) {
c.dynamicConfiguration.pprof.cfg = c
c.dynamicConfiguration.pprof.name = "pprof"
c.dynamicConfiguration.pprof.handler = httputil.Handler()
c.dynamicConfiguration.pprof.preReload = tuneProfilers
updated = true
}
@ -35,3 +45,18 @@ func pprofComponent(c *cfg) (*httpComponent, bool) {
return c.dynamicConfiguration.pprof, updated
}
func tuneProfilers(c *cfg) {
// Disabled by default, see documentation for
// runtime.SetBlockProfileRate() and runtime.SetMutexProfileFraction().
blockRate := 0
mutexRate := 0
if profilerconfig.Enabled(c.appCfg) {
blockRate = profilerconfig.BlockRate(c.appCfg)
mutexRate = profilerconfig.MutexRate(c.appCfg)
}
runtime.SetBlockProfileRate(blockRate)
runtime.SetMutexProfileFraction(mutexRate)
}

View file

@ -3,6 +3,8 @@ FROSTFS_LOGGER_LEVEL=debug
FROSTFS_PPROF_ENABLED=true
FROSTFS_PPROF_ADDRESS=localhost:6060
FROSTFS_PPROF_SHUTDOWN_TIMEOUT=15s
FROSTFS_PPROF_BLOCK_RATE=10000
FROSTFS_PPROF_MUTEX_RATE=10000
FROSTFS_PROMETHEUS_ENABLED=true
FROSTFS_PROMETHEUS_ADDRESS=localhost:9090

View file

@ -5,7 +5,9 @@
"pprof": {
"enabled": true,
"address": "localhost:6060",
"shutdown_timeout": "15s"
"shutdown_timeout": "15s",
"block_rate": 10000,
"mutex_rate": 10000
},
"prometheus": {
"enabled": true,

View file

@ -5,6 +5,8 @@ pprof:
enabled: true
address: localhost:6060 # endpoint for Node profiling
shutdown_timeout: 15s # timeout for profiling HTTP server graceful shutdown
block_rate: 10000 # sampling rate: an average of one blocking event per rate nanoseconds spent blocked is reported; "1" reports every blocking event; "0" disables profiler
mutex_rate: 10000 # sampling rate: on average 1/rate events are reported; "0" disables profiler
prometheus:
enabled: true

View file

@ -76,12 +76,22 @@ element.
Contains configuration for the `pprof` profiler.
| Parameter | Type | Default value | Description |
|--------------------|------------|---------------|-----------------------------------------|
|--------------------|-----------------------------------|---------------|-----------------------------------------|
| `enabled` | `bool` | `false` | Flag to enable the service. |
| `address` | `string` | | Address that service listener binds to. |
| `shutdown_timeout` | `duration` | `30s` | Time to wait for a graceful shutdown. |
| `debug` | [Debug config](#debug-subsection) | | Optional profiles configuration |
## `debug` subsection
Contains optional profiles configuration.
| Parameter | Type | Default value | Description |
|--------------|-------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `block_rate` | `int` | `0` | Controls the block profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetBlockProfileRate. |
| `mutex_rate` | `int` | `0` | Controls the mutex profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetMutexProfileFraction. |
# `prometheus` section
Contains configuration for the `prometheus` metrics service.