From 14c35d776edcbe5c069052ad3d7bb3f4ecd845fc Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 1 Feb 2023 10:21:17 +0300 Subject: [PATCH] [#39] node: Add optional profilers Include settings for block and mutex profilers. They are disabled by default, as in Go runtime itself. Signed-off-by: Pavel Karpy Signed-off-by: Evgenii Stratonikov --- cmd/frostfs-node/config/profiler/config.go | 24 ++++++++++++++++++ .../config/profiler/config_test.go | 6 +++++ cmd/frostfs-node/main.go | 3 +-- cmd/frostfs-node/pprof.go | 25 +++++++++++++++++++ config/example/node.env | 2 ++ config/example/node.json | 4 ++- config/example/node.yaml | 2 ++ docs/storage-node-configuration.md | 20 +++++++++++---- 8 files changed, 78 insertions(+), 8 deletions(-) diff --git a/cmd/frostfs-node/config/profiler/config.go b/cmd/frostfs-node/config/profiler/config.go index f891833a6..191694970 100644 --- a/cmd/frostfs-node/config/profiler/config.go +++ b/cmd/frostfs-node/config/profiler/config.go @@ -51,3 +51,27 @@ func Address(c *config.Config) string { return AddressDefault } + +// BlockRates returns the value of "block_rate" config parameter +// from "pprof" section. +func BlockRate(c *config.Config) int { + s := c.Sub(subsection) + + v := int(config.IntSafe(s, "block_rate")) + if v <= 0 { + return 0 + } + return v +} + +// MutexRate returns the value of "mutex_rate" config parameter +// from "pprof" section. +func MutexRate(c *config.Config) int { + s := c.Sub(subsection) + + v := int(config.IntSafe(s, "mutex_rate")) + if v <= 0 { + return 0 + } + return v +} diff --git a/cmd/frostfs-node/config/profiler/config_test.go b/cmd/frostfs-node/config/profiler/config_test.go index bb1b20eb8..355874387 100644 --- a/cmd/frostfs-node/config/profiler/config_test.go +++ b/cmd/frostfs-node/config/profiler/config_test.go @@ -18,6 +18,9 @@ func TestProfilerSection(t *testing.T) { require.Equal(t, profilerconfig.ShutdownTimeoutDefault, to) require.Equal(t, profilerconfig.AddressDefault, addr) require.False(t, profilerconfig.Enabled(configtest.EmptyConfig())) + + require.Zero(t, profilerconfig.BlockRate(configtest.EmptyConfig())) + require.Zero(t, profilerconfig.MutexRate(configtest.EmptyConfig())) }) const path = "../../../../config/example/node" @@ -29,6 +32,9 @@ func TestProfilerSection(t *testing.T) { require.Equal(t, 15*time.Second, to) require.Equal(t, "localhost:6060", addr) require.True(t, profilerconfig.Enabled(c)) + + require.Equal(t, 10_000, profilerconfig.BlockRate(c)) + require.Equal(t, 10_000, profilerconfig.MutexRate(c)) } configtest.ForEachFileType(path, fileConfigTest) diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index 4fcc38e08..a2024706b 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -83,9 +83,8 @@ func initApp(ctx context.Context, c *cfg) { c.wg.Done() }() - pprof, _ := pprofComponent(c) metrics, _ := metricsComponent(c) - initAndLog(c, pprof.name, pprof.init) + initAndLog(c, "profiler", initProfilerService) initAndLog(c, metrics.name, metrics.init) initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) }) diff --git a/cmd/frostfs-node/pprof.go b/cmd/frostfs-node/pprof.go index 9be2dd9df..dcd320146 100644 --- a/cmd/frostfs-node/pprof.go +++ b/cmd/frostfs-node/pprof.go @@ -1,10 +1,19 @@ package main import ( + "runtime" + profilerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/profiler" httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http" ) +func initProfilerService(c *cfg) { + tuneProfilers(c) + + pprof, _ := pprofComponent(c) + pprof.init(c) +} + func pprofComponent(c *cfg) (*httpComponent, bool) { var updated bool // check if it has been inited before @@ -13,6 +22,7 @@ func pprofComponent(c *cfg) (*httpComponent, bool) { c.dynamicConfiguration.pprof.cfg = c c.dynamicConfiguration.pprof.name = "pprof" c.dynamicConfiguration.pprof.handler = httputil.Handler() + c.dynamicConfiguration.pprof.preReload = tuneProfilers updated = true } @@ -35,3 +45,18 @@ func pprofComponent(c *cfg) (*httpComponent, bool) { return c.dynamicConfiguration.pprof, updated } + +func tuneProfilers(c *cfg) { + // Disabled by default, see documentation for + // runtime.SetBlockProfileRate() and runtime.SetMutexProfileFraction(). + blockRate := 0 + mutexRate := 0 + + if profilerconfig.Enabled(c.appCfg) { + blockRate = profilerconfig.BlockRate(c.appCfg) + mutexRate = profilerconfig.MutexRate(c.appCfg) + } + + runtime.SetBlockProfileRate(blockRate) + runtime.SetMutexProfileFraction(mutexRate) +} diff --git a/config/example/node.env b/config/example/node.env index 77992d995..143bf0388 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -3,6 +3,8 @@ FROSTFS_LOGGER_LEVEL=debug FROSTFS_PPROF_ENABLED=true FROSTFS_PPROF_ADDRESS=localhost:6060 FROSTFS_PPROF_SHUTDOWN_TIMEOUT=15s +FROSTFS_PPROF_BLOCK_RATE=10000 +FROSTFS_PPROF_MUTEX_RATE=10000 FROSTFS_PROMETHEUS_ENABLED=true FROSTFS_PROMETHEUS_ADDRESS=localhost:9090 diff --git a/config/example/node.json b/config/example/node.json index b52eb6d96..04aabdd42 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -5,7 +5,9 @@ "pprof": { "enabled": true, "address": "localhost:6060", - "shutdown_timeout": "15s" + "shutdown_timeout": "15s", + "block_rate": 10000, + "mutex_rate": 10000 }, "prometheus": { "enabled": true, diff --git a/config/example/node.yaml b/config/example/node.yaml index 1669e0e86..bc665a688 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -5,6 +5,8 @@ pprof: enabled: true address: localhost:6060 # endpoint for Node profiling shutdown_timeout: 15s # timeout for profiling HTTP server graceful shutdown + block_rate: 10000 # sampling rate: an average of one blocking event per rate nanoseconds spent blocked is reported; "1" reports every blocking event; "0" disables profiler + mutex_rate: 10000 # sampling rate: on average 1/rate events are reported; "0" disables profiler prometheus: enabled: true diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 366c263a0..2c78cf6b1 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -75,13 +75,23 @@ element. Contains configuration for the `pprof` profiler. -| Parameter | Type | Default value | Description | -|--------------------|------------|---------------|-----------------------------------------| -| `enabled` | `bool` | `false` | Flag to enable the service. | -| `address` | `string` | | Address that service listener binds to. | -| `shutdown_timeout` | `duration` | `30s` | Time to wait for a graceful shutdown. | +| Parameter | Type | Default value | Description | +|--------------------|-----------------------------------|---------------|-----------------------------------------| +| `enabled` | `bool` | `false` | Flag to enable the service. | +| `address` | `string` | | Address that service listener binds to. | +| `shutdown_timeout` | `duration` | `30s` | Time to wait for a graceful shutdown. | +| `debug` | [Debug config](#debug-subsection) | | Optional profiles configuration | +## `debug` subsection + +Contains optional profiles configuration. + +| Parameter | Type | Default value | Description | +|--------------|-------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `block_rate` | `int` | `0` | Controls the block profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetBlockProfileRate. | +| `mutex_rate` | `int` | `0` | Controls the mutex profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetMutexProfileFraction. | + # `prometheus` section Contains configuration for the `prometheus` metrics service.