[#1450] node/config: Allow to configure engine's InhumePoolSize
All checks were successful
DCO action / DCO (pull_request) Successful in 1m4s
Vulncheck / Vulncheck (pull_request) Successful in 2m0s
Tests and linters / Run gofumpt (pull_request) Successful in 2m4s
Build / Build Components (pull_request) Successful in 2m42s
Pre-commit hooks / Pre-commit (pull_request) Successful in 2m49s
Tests and linters / Staticcheck (pull_request) Successful in 3m12s
Tests and linters / gopls check (pull_request) Successful in 3m35s
Tests and linters / Tests with -race (pull_request) Successful in 3m55s
Tests and linters / Lint (pull_request) Successful in 4m12s
Tests and linters / Tests (pull_request) Successful in 4m13s

Signed-off-by: Aleksey Savchuk <a.savchuk@yadro.com>
This commit is contained in:
Aleksey Savchuk 2024-11-19 12:58:14 +03:00
parent 9706a03b89
commit 3fe0a84364
Signed by: a-savchuk
GPG key ID: 70C0A7FF6F9C4639
4 changed files with 24 additions and 0 deletions

View file

@ -118,6 +118,7 @@ type applicationConfiguration struct {
shardPoolSize uint32 shardPoolSize uint32
shards []shardCfg shards []shardCfg
lowMem bool lowMem bool
inhumePoolSize uint32
} }
// if need to run node in compatibility with other versions mode // if need to run node in compatibility with other versions mode
@ -249,6 +250,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error {
a.EngineCfg.errorThreshold = engineconfig.ShardErrorThreshold(c) a.EngineCfg.errorThreshold = engineconfig.ShardErrorThreshold(c)
a.EngineCfg.shardPoolSize = engineconfig.ShardPoolSize(c) a.EngineCfg.shardPoolSize = engineconfig.ShardPoolSize(c)
a.EngineCfg.lowMem = engineconfig.EngineLowMemoryConsumption(c) a.EngineCfg.lowMem = engineconfig.EngineLowMemoryConsumption(c)
a.EngineCfg.inhumePoolSize = engineconfig.InhumePoolSize(c)
return engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error { return a.updateShardConfig(c, sc) }) return engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error { return a.updateShardConfig(c, sc) })
} }
@ -881,6 +883,7 @@ func (c *cfg) engineOpts() []engine.Option {
engine.WithErrorThreshold(c.EngineCfg.errorThreshold), engine.WithErrorThreshold(c.EngineCfg.errorThreshold),
engine.WithLogger(c.log), engine.WithLogger(c.log),
engine.WithLowMemoryConsumption(c.EngineCfg.lowMem), engine.WithLowMemoryConsumption(c.EngineCfg.lowMem),
engine.WithInhumePoolSize(c.EngineCfg.inhumePoolSize),
) )
if c.metricsCollector != nil { if c.metricsCollector != nil {

View file

@ -15,6 +15,11 @@ const (
// ShardPoolSizeDefault is a default value of routine pool size per-shard to // ShardPoolSizeDefault is a default value of routine pool size per-shard to
// process object PUT operations in a storage engine. // process object PUT operations in a storage engine.
ShardPoolSizeDefault = 20 ShardPoolSizeDefault = 20
// InhumePoolSizeDefault is the default size of the engine-level
// worker pool, which is used to process objects during the INHUME
// operation.
InhumePoolSizeDefault = 50
) )
// ErrNoShardConfigured is returned when at least 1 shard is required but none are found. // ErrNoShardConfigured is returned when at least 1 shard is required but none are found.
@ -88,3 +93,16 @@ func ShardErrorThreshold(c *config.Config) uint32 {
func EngineLowMemoryConsumption(c *config.Config) bool { func EngineLowMemoryConsumption(c *config.Config) bool {
return config.BoolSafe(c.Sub(subsection), "low_mem") return config.BoolSafe(c.Sub(subsection), "low_mem")
} }
// InhumePoolSize returns the value of "inhume_pool_size" config parameter from
// "storage" section.
//
// Returns InhumePoolSizeDefault if the value is not a positive number.
func InhumePoolSize(c *config.Config) uint32 {
v := config.Uint32Safe(c.Sub(subsection), "inhume_pool_size")
if v > 0 {
return v
}
return InhumePoolSizeDefault
}

View file

@ -39,6 +39,7 @@ func TestEngineSection(t *testing.T) {
require.EqualValues(t, 0, engineconfig.ShardErrorThreshold(empty)) require.EqualValues(t, 0, engineconfig.ShardErrorThreshold(empty))
require.EqualValues(t, engineconfig.ShardPoolSizeDefault, engineconfig.ShardPoolSize(empty)) require.EqualValues(t, engineconfig.ShardPoolSizeDefault, engineconfig.ShardPoolSize(empty))
require.EqualValues(t, mode.ReadWrite, shardconfig.From(empty).Mode()) require.EqualValues(t, mode.ReadWrite, shardconfig.From(empty).Mode())
require.EqualValues(t, engineconfig.InhumePoolSizeDefault, engineconfig.InhumePoolSize(empty))
}) })
const path = "../../../../config/example/node" const path = "../../../../config/example/node"
@ -48,6 +49,7 @@ func TestEngineSection(t *testing.T) {
require.EqualValues(t, 100, engineconfig.ShardErrorThreshold(c)) require.EqualValues(t, 100, engineconfig.ShardErrorThreshold(c))
require.EqualValues(t, 15, engineconfig.ShardPoolSize(c)) require.EqualValues(t, 15, engineconfig.ShardPoolSize(c))
require.EqualValues(t, 50, engineconfig.InhumePoolSize(c))
err := engineconfig.IterateShards(c, true, func(sc *shardconfig.Config) error { err := engineconfig.IterateShards(c, true, func(sc *shardconfig.Config) error {
defer func() { defer func() {

View file

@ -124,6 +124,7 @@ storage:
# note: shard configuration can be omitted for relay node (see `node.relay`) # note: shard configuration can be omitted for relay node (see `node.relay`)
shard_pool_size: 15 # size of per-shard worker pools used for PUT operations shard_pool_size: 15 # size of per-shard worker pools used for PUT operations
shard_ro_error_threshold: 100 # amount of errors to occur before shard is made read-only (default: 0, ignore errors) shard_ro_error_threshold: 100 # amount of errors to occur before shard is made read-only (default: 0, ignore errors)
inhume_pool_size: 50 # size of engine-level worker pool used during INHUME operation
shard: shard:
default: # section with the default shard parameters default: # section with the default shard parameters