From 368e28041305ccacdc76f29b96907a97eb8c2407 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Wed, 8 Sep 2021 12:46:12 +0300 Subject: [PATCH] [#776] cmd/node: Configure write-cache size limit Add `size_limit` config value to write-cache section. Add `SizeLimit` method of `writecache.Config` type. Pass its value to `WithMaxCacheSize` option on app construction. Reflect config update in examples. Cover new value in unit test. Signed-off-by: Leonard Lyubich --- cmd/neofs-node/config.go | 1 + cmd/neofs-node/config/engine/config_test.go | 2 ++ .../config/engine/shard/writecache/config.go | 19 +++++++++++++++++++ config/example/node.env | 2 ++ config/example/node.json | 6 ++++-- config/example/node.yaml | 2 ++ 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/cmd/neofs-node/config.go b/cmd/neofs-node/config.go index 990c1a27a2..7724db1db9 100644 --- a/cmd/neofs-node/config.go +++ b/cmd/neofs-node/config.go @@ -369,6 +369,7 @@ func initShardOptions(c *cfg) { writecache.WithMaxObjectSize(writeCacheCfg.MaxObjectSize()), writecache.WithSmallObjectSize(writeCacheCfg.SmallObjectSize()), writecache.WithFlushWorkersCount(writeCacheCfg.WorkersNumber()), + writecache.WithMaxCacheSize(writeCacheCfg.SizeLimit()), } } diff --git a/cmd/neofs-node/config/engine/config_test.go b/cmd/neofs-node/config/engine/config_test.go index c4d6a29499..e5454cb432 100644 --- a/cmd/neofs-node/config/engine/config_test.go +++ b/cmd/neofs-node/config/engine/config_test.go @@ -44,6 +44,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 16384, wc.SmallObjectSize()) require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkersNumber()) + require.EqualValues(t, 3221225472, wc.SizeLimit()) require.Equal(t, "tmp/0/meta", meta.Path()) require.Equal(t, fs.FileMode(0644), meta.Perm()) @@ -69,6 +70,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 16384, wc.SmallObjectSize()) require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkersNumber()) + require.EqualValues(t, 4294967296, wc.SizeLimit()) require.Equal(t, "tmp/1/meta", meta.Path()) require.Equal(t, fs.FileMode(0644), meta.Perm()) diff --git a/cmd/neofs-node/config/engine/shard/writecache/config.go b/cmd/neofs-node/config/engine/shard/writecache/config.go index 2803d6093f..00ff447c20 100644 --- a/cmd/neofs-node/config/engine/shard/writecache/config.go +++ b/cmd/neofs-node/config/engine/shard/writecache/config.go @@ -21,6 +21,9 @@ const ( // WorkersNumberDefault is a default number of workers. WorkersNumberDefault = 20 + + // SizeLimitDefault is a default write-cache size limit. + SizeLimitDefault = 1 << 30 ) // From wraps config section into Config. @@ -107,3 +110,19 @@ func (x *Config) WorkersNumber() int { return WorkersNumberDefault } + +// SizeLimit returns value of "size_limit" config parameter. +// +// Returns SizeLimitDefault if value is not a positive number. +func (x *Config) SizeLimit() uint64 { + c := config.UintSafe( + (*config.Config)(x), + "size_limit", + ) + + if c > 0 { + return c + } + + return SizeLimitDefault +} diff --git a/config/example/node.env b/config/example/node.env index 212a2d07de..7676c12ffe 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -76,6 +76,7 @@ NEOFS_STORAGE_SHARD_0_WRITECACHE_MEM_SIZE=2147483648 NEOFS_STORAGE_SHARD_0_WRITECACHE_SMALL_SIZE=16384 NEOFS_STORAGE_SHARD_0_WRITECACHE_MAX_SIZE=134217728 NEOFS_STORAGE_SHARD_0_WRITECACHE_WORKERS_NUMBER=30 +NEOFS_STORAGE_SHARD_0_WRITECACHE_SIZE_LIMIT=3221225472 ### Metabase config NEOFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta NEOFS_STORAGE_SHARD_0_METABASE_PERM=0644 @@ -104,6 +105,7 @@ NEOFS_STORAGE_SHARD_1_WRITECACHE_MEM_SIZE=2147483648 NEOFS_STORAGE_SHARD_1_WRITECACHE_SMALL_SIZE=16384 NEOFS_STORAGE_SHARD_1_WRITECACHE_MAX_SIZE=134217728 NEOFS_STORAGE_SHARD_1_WRITECACHE_WORKERS_NUMBER=30 +NEOFS_STORAGE_SHARD_1_WRITECACHE_SIZE_LIMIT=4294967296 ### Metabase config NEOFS_STORAGE_SHARD_1_METABASE_PATH=tmp/1/meta NEOFS_STORAGE_SHARD_1_METABASE_PERM=0644 diff --git a/config/example/node.json b/config/example/node.json index ec0c4099e6..f26ea05bc5 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -110,7 +110,8 @@ "mem_size": 2147483648, "small_size": 16384, "max_size": 134217728, - "workers_number": 30 + "workers_number": 30, + "size_limit": 3221225472 }, "metabase": { "path": "tmp/0/meta", @@ -141,7 +142,8 @@ "mem_size": 2147483648, "small_size": 16384, "max_size": 134217728, - "workers_number": 30 + "workers_number": 30, + "size_limit": 4294967296 }, "metabase": { "path": "tmp/1/meta", diff --git a/config/example/node.yaml b/config/example/node.yaml index 6cdd59f805..2a22ef5857 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -98,6 +98,7 @@ storage: small_size: 16384 max_size: 134217728 workers_number: 30 + size_limit: 3221225472 metabase: path: tmp/0/meta @@ -129,6 +130,7 @@ storage: small_size: 16384 max_size: 134217728 workers_number: 30 + size_limit: 4294967296 metabase: path: tmp/1/meta