[#537] node: Add runtime.memory_limit config parameter

This parameter allows to set soft memory limit for Go GC.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2023-07-25 10:07:38 +03:00
parent 5ff82ff04f
commit 32c77f3a23
10 changed files with 107 additions and 0 deletions

View file

@ -1055,6 +1055,10 @@ func (c *cfg) reloadConfig(ctx context.Context) {
} }
components = append(components, dCmp{"logger", logPrm.Reload}) components = append(components, dCmp{"logger", logPrm.Reload})
components = append(components, dCmp{"runtime", func() error {
setRuntimeParameters(c)
return nil
}})
components = append(components, dCmp{"tracing", func() error { components = append(components, dCmp{"tracing", func() error {
updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg)) updated, err := tracing.Setup(ctx, *tracingconfig.ToTracingConfig(c.appCfg))
if updated { if updated {

View file

@ -0,0 +1,23 @@
package runtime
import (
"math"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
)
const (
subsection = "runtime"
memoryLimitDefault = math.MaxInt64
)
// GCMemoryLimitBytes returns the value of "soft_memory_limit" config parameter from "runtime" section.
func GCMemoryLimitBytes(c *config.Config) int64 {
l := config.SizeInBytesSafe(c.Sub(subsection), "soft_memory_limit")
if l > 0 {
return int64(l)
}
return memoryLimitDefault
}

View file

@ -0,0 +1,30 @@
package runtime
import (
"math"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test"
"github.com/stretchr/testify/require"
)
func TestGCMemoryLimit(t *testing.T) {
t.Run("defaults", func(t *testing.T) {
empty := configtest.EmptyConfig()
require.Equal(t, int64(math.MaxInt64), GCMemoryLimitBytes(empty))
})
const path = "../../../../config/example/node"
fileConfigTest := func(c *config.Config) {
require.Equal(t, int64(1073741824), GCMemoryLimitBytes(c))
}
configtest.ForEachFileType(path, fileConfigTest)
t.Run("ENV", func(t *testing.T) {
configtest.ForEnvFileType(t, path, fileConfigTest)
})
}

View file

@ -84,6 +84,7 @@ func initApp(ctx context.Context, c *cfg) {
c.wg.Done() c.wg.Done()
}() }()
setRuntimeParameters(c)
metrics, _ := metricsComponent(c) metrics, _ := metricsComponent(c)
initAndLog(c, "profiler", initProfilerService) initAndLog(c, "profiler", initProfilerService)
initAndLog(c, metrics.name, metrics.init) initAndLog(c, metrics.name, metrics.init)

View file

@ -0,0 +1,26 @@
package main
import (
"os"
"runtime/debug"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/runtime"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"go.uber.org/zap"
)
func setRuntimeParameters(c *cfg) {
if len(os.Getenv("GOMEMLIMIT")) != 0 {
// default limit < yaml limit < app env limit < GOMEMLIMIT
c.log.Warn(logs.RuntimeSoftMemoryDefinedWithGOMEMLIMIT)
return
}
memLimitBytes := runtime.GCMemoryLimitBytes(c.appCfg)
previous := debug.SetMemoryLimit(memLimitBytes)
if memLimitBytes != previous {
c.log.Info(logs.RuntimeSoftMemoryLimitUpdated,
zap.Int64("new_value", memLimitBytes),
zap.Int64("old_value", previous))
}
}

View file

@ -188,3 +188,5 @@ FROSTFS_STORAGE_SHARD_1_GC_REMOVER_SLEEP_INTERVAL=5m
FROSTFS_TRACING_ENABLED=true FROSTFS_TRACING_ENABLED=true
FROSTFS_TRACING_ENDPOINT="localhost" FROSTFS_TRACING_ENDPOINT="localhost"
FROSTFS_TRACING_EXPORTER="otlp_grpc" FROSTFS_TRACING_EXPORTER="otlp_grpc"
FROSTFS_RUNTIME_SOFT_MEMORY_LIMIT=1073741824

View file

@ -245,5 +245,8 @@
"enabled": true, "enabled": true,
"endpoint": "localhost:9090", "endpoint": "localhost:9090",
"exporter": "otlp_grpc" "exporter": "otlp_grpc"
},
"runtime": {
"soft_memory_limit": 1073741824
} }
} }

View file

@ -217,3 +217,6 @@ tracing:
enabled: true enabled: true
exporter: "otlp_grpc" exporter: "otlp_grpc"
endpoint: "localhost" endpoint: "localhost"
runtime:
soft_memory_limit: 1gb

View file

@ -24,6 +24,7 @@ There are some custom types used for brevity:
| `policer` | [Policer service configuration](#policer-section) | | `policer` | [Policer service configuration](#policer-section) |
| `replicator` | [Replicator service configuration](#replicator-section) | | `replicator` | [Replicator service configuration](#replicator-section) |
| `storage` | [Storage engine configuration](#storage-section) | | `storage` | [Storage engine configuration](#storage-section) |
| `runtime` | [Runtime configuration](#runtime-section) |
# `control` section # `control` section
@ -426,3 +427,15 @@ object:
| `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. | | `delete.tombstone_lifetime` | `int` | `5` | Tombstone lifetime for removed objects in epochs. |
| `put.pool_size_remote` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. | | `put.pool_size_remote` | `int` | `10` | Max pool size for performing remote `PUT` operations. Used by Policer and Replicator services. |
| `put.pool_size_local` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. | | `put.pool_size_local` | `int` | `10` | Max pool size for performing local `PUT` operations. Used by Policer and Replicator services. |
# `runtime` section
Contains runtime parameters.
```yaml
runtime:
soft_memory_limit: 1GB
```
| Parameter | Type | Default value | Description |
|---------------------|--------|---------------|--------------------------------------------------------------------------|
| `soft_memory_limit` | `size` | 0 | Soft memory limit for the runtime. Zero or no value stands for no limit. If `GOMEMLIMIT` environment variable is set, the value from the configuration file will be ignored. |

View file

@ -493,4 +493,6 @@ const (
FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap" FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap = "the node is under maintenance, skip initial bootstrap"
EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled" EngineCouldNotChangeShardModeToDisabled = "could not change shard mode to disabled"
NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap = "the node is already in candidate list with online state, skip initial bootstrap" NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap = "the node is already in candidate list with online state, skip initial bootstrap"
RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated"
RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped"
) )