[#39] node: Add optional profilers

Include settings for block and mutex profilers.
They are disabled by default, as in Go runtime itself.

Signed-off-by: Pavel Karpy <p.karpy@yadro.com>
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>

cmd/frostfs-node/config/profiler/config.go

@@ -51,3 +51,27 @@ func Address(c *config.Config) string {
 
 	return AddressDefault
 }
+
+// BlockRates returns the value of "block_rate" config parameter
+// from "pprof" section.
+func BlockRate(c *config.Config) int {
+	s := c.Sub(subsection)
+
+	v := int(config.IntSafe(s, "block_rate"))
+	if v <= 0 {
+		return 0
+	}
+	return v
+}
+
+// MutexRate returns the value of "mutex_rate" config parameter
+// from "pprof" section.
+func MutexRate(c *config.Config) int {
+	s := c.Sub(subsection)
+
+	v := int(config.IntSafe(s, "mutex_rate"))
+	if v <= 0 {
+		return 0
+	}
+	return v
+}

cmd/frostfs-node/config/profiler/config_test.go

@@ -18,6 +18,9 @@ func TestProfilerSection(t *testing.T) {
 		require.Equal(t, profilerconfig.ShutdownTimeoutDefault, to)
 		require.Equal(t, profilerconfig.AddressDefault, addr)
 		require.False(t, profilerconfig.Enabled(configtest.EmptyConfig()))
+
+		require.Zero(t, profilerconfig.BlockRate(configtest.EmptyConfig()))
+		require.Zero(t, profilerconfig.MutexRate(configtest.EmptyConfig()))
 	})
 
 	const path = "../../../../config/example/node"
@@ -29,6 +32,9 @@ func TestProfilerSection(t *testing.T) {
 		require.Equal(t, 15*time.Second, to)
 		require.Equal(t, "localhost:6060", addr)
 		require.True(t, profilerconfig.Enabled(c))
+
+		require.Equal(t, 10_000, profilerconfig.BlockRate(c))
+		require.Equal(t, 10_000, profilerconfig.MutexRate(c))
 	}
 
 	configtest.ForEachFileType(path, fileConfigTest)

cmd/frostfs-node/main.go

@@ -83,9 +83,8 @@ func initApp(ctx context.Context, c *cfg) {
 		c.wg.Done()
 	}()
 
-	pprof, _ := pprofComponent(c)
 	metrics, _ := metricsComponent(c)
-	initAndLog(c, pprof.name, pprof.init)
+	initAndLog(c, "profiler", initProfilerService)
 	initAndLog(c, metrics.name, metrics.init)
 
 	initAndLog(c, "tracing", func(c *cfg) { initTracing(ctx, c) })

cmd/frostfs-node/pprof.go

@@ -1,10 +1,19 @@
 package main
 
 import (
+	"runtime"
+
 	profilerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/profiler"
 	httputil "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/http"
 )
 
+func initProfilerService(c *cfg) {
+	tuneProfilers(c)
+
+	pprof, _ := pprofComponent(c)
+	pprof.init(c)
+}
+
 func pprofComponent(c *cfg) (*httpComponent, bool) {
 	var updated bool
 	// check if it has been inited before
@@ -13,6 +22,7 @@ func pprofComponent(c *cfg) (*httpComponent, bool) {
 		c.dynamicConfiguration.pprof.cfg = c
 		c.dynamicConfiguration.pprof.name = "pprof"
 		c.dynamicConfiguration.pprof.handler = httputil.Handler()
+		c.dynamicConfiguration.pprof.preReload = tuneProfilers
 		updated = true
 	}
 
@@ -35,3 +45,18 @@ func pprofComponent(c *cfg) (*httpComponent, bool) {
 
 	return c.dynamicConfiguration.pprof, updated
 }
+
+func tuneProfilers(c *cfg) {
+	// Disabled by default, see documentation for
+	// runtime.SetBlockProfileRate() and runtime.SetMutexProfileFraction().
+	blockRate := 0
+	mutexRate := 0
+
+	if profilerconfig.Enabled(c.appCfg) {
+		blockRate = profilerconfig.BlockRate(c.appCfg)
+		mutexRate = profilerconfig.MutexRate(c.appCfg)
+	}
+
+	runtime.SetBlockProfileRate(blockRate)
+	runtime.SetMutexProfileFraction(mutexRate)
+}

config/example/node.env

@@ -3,6 +3,8 @@ FROSTFS_LOGGER_LEVEL=debug
 FROSTFS_PPROF_ENABLED=true
 FROSTFS_PPROF_ADDRESS=localhost:6060
 FROSTFS_PPROF_SHUTDOWN_TIMEOUT=15s
+FROSTFS_PPROF_BLOCK_RATE=10000
+FROSTFS_PPROF_MUTEX_RATE=10000
 
 FROSTFS_PROMETHEUS_ENABLED=true
 FROSTFS_PROMETHEUS_ADDRESS=localhost:9090

config/example/node.json

@@ -5,7 +5,9 @@
   "pprof": {
     "enabled": true,
     "address": "localhost:6060",
-    "shutdown_timeout": "15s"
+    "shutdown_timeout": "15s",
+    "block_rate": 10000,
+    "mutex_rate": 10000
   },
   "prometheus": {
     "enabled": true,

config/example/node.yaml

@@ -5,6 +5,8 @@ pprof:
   enabled: true
   address: localhost:6060  # endpoint for Node profiling
   shutdown_timeout: 15s  # timeout for profiling HTTP server graceful shutdown
+  block_rate: 10000  # sampling rate: an average of one blocking event per rate nanoseconds spent blocked is reported; "1" reports every blocking event; "0" disables profiler
+  mutex_rate: 10000  # sampling rate: on average 1/rate events are reported; "0" disables profiler
 
 prometheus:
   enabled: true

docs/storage-node-configuration.md

@@ -75,13 +75,23 @@ element.
 
 Contains configuration for the `pprof` profiler.
 
-| Parameter          | Type       | Default value | Description                             |
-|--------------------|------------|---------------|-----------------------------------------|
-| `enabled`          | `bool`     | `false`       | Flag to enable the service.             |
-| `address`          | `string`   |               | Address that service listener binds to. |
-| `shutdown_timeout` | `duration` | `30s`         | Time to wait for a graceful shutdown.   |
+| Parameter          | Type                              | Default value | Description                             |
+|--------------------|-----------------------------------|---------------|-----------------------------------------|
+| `enabled`          | `bool`                            | `false`       | Flag to enable the service.             |
+| `address`          | `string`                          |               | Address that service listener binds to. |
+| `shutdown_timeout` | `duration`                        | `30s`         | Time to wait for a graceful shutdown.   |
+| `debug`            | [Debug config](#debug-subsection) |               | Optional profiles configuration         |
 
 
+## `debug` subsection
+
+Contains optional profiles configuration.
+
+| Parameter    | Type  | Default value | Description                                                                                                                                                   |
+|--------------|-------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `block_rate` | `int` | `0`           | Controls the block profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetBlockProfileRate.     |
+| `mutex_rate` | `int` | `0`           | Controls the mutex profiler. Non-positive values disable profiler reports. For more information: https://pkg.go.dev/runtime@go1.20.3#SetMutexProfileFraction. |
+
 # `prometheus` section
 
 Contains configuration for the `prometheus` metrics service.