[#776] cmd/node: Configure write-cache size limit

Add `size_limit` config value to write-cache section. Add `SizeLimit` method
of `writecache.Config` type. Pass its value to `WithMaxCacheSize` option on
app construction.

Reflect config update in examples. Cover new value in unit test.

Signed-off-by: Leonard Lyubich <leonard@nspcc.ru>
This commit is contained in:
Leonard Lyubich 2021-09-08 12:46:12 +03:00 committed by Alex Vanin
parent a1696a81b6
commit 368e280413
6 changed files with 30 additions and 2 deletions

View file

@ -369,6 +369,7 @@ func initShardOptions(c *cfg) {
writecache.WithMaxObjectSize(writeCacheCfg.MaxObjectSize()), writecache.WithMaxObjectSize(writeCacheCfg.MaxObjectSize()),
writecache.WithSmallObjectSize(writeCacheCfg.SmallObjectSize()), writecache.WithSmallObjectSize(writeCacheCfg.SmallObjectSize()),
writecache.WithFlushWorkersCount(writeCacheCfg.WorkersNumber()), writecache.WithFlushWorkersCount(writeCacheCfg.WorkersNumber()),
writecache.WithMaxCacheSize(writeCacheCfg.SizeLimit()),
} }
} }

View file

@ -44,6 +44,7 @@ func TestEngineSection(t *testing.T) {
require.EqualValues(t, 16384, wc.SmallObjectSize()) require.EqualValues(t, 16384, wc.SmallObjectSize())
require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 134217728, wc.MaxObjectSize())
require.EqualValues(t, 30, wc.WorkersNumber()) require.EqualValues(t, 30, wc.WorkersNumber())
require.EqualValues(t, 3221225472, wc.SizeLimit())
require.Equal(t, "tmp/0/meta", meta.Path()) require.Equal(t, "tmp/0/meta", meta.Path())
require.Equal(t, fs.FileMode(0644), meta.Perm()) require.Equal(t, fs.FileMode(0644), meta.Perm())
@ -69,6 +70,7 @@ func TestEngineSection(t *testing.T) {
require.EqualValues(t, 16384, wc.SmallObjectSize()) require.EqualValues(t, 16384, wc.SmallObjectSize())
require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 134217728, wc.MaxObjectSize())
require.EqualValues(t, 30, wc.WorkersNumber()) require.EqualValues(t, 30, wc.WorkersNumber())
require.EqualValues(t, 4294967296, wc.SizeLimit())
require.Equal(t, "tmp/1/meta", meta.Path()) require.Equal(t, "tmp/1/meta", meta.Path())
require.Equal(t, fs.FileMode(0644), meta.Perm()) require.Equal(t, fs.FileMode(0644), meta.Perm())

View file

@ -21,6 +21,9 @@ const (
// WorkersNumberDefault is a default number of workers. // WorkersNumberDefault is a default number of workers.
WorkersNumberDefault = 20 WorkersNumberDefault = 20
// SizeLimitDefault is a default write-cache size limit.
SizeLimitDefault = 1 << 30
) )
// From wraps config section into Config. // From wraps config section into Config.
@ -107,3 +110,19 @@ func (x *Config) WorkersNumber() int {
return WorkersNumberDefault return WorkersNumberDefault
} }
// SizeLimit returns value of "size_limit" config parameter.
//
// Returns SizeLimitDefault if value is not a positive number.
func (x *Config) SizeLimit() uint64 {
c := config.UintSafe(
(*config.Config)(x),
"size_limit",
)
if c > 0 {
return c
}
return SizeLimitDefault
}

View file

@ -76,6 +76,7 @@ NEOFS_STORAGE_SHARD_0_WRITECACHE_MEM_SIZE=2147483648
NEOFS_STORAGE_SHARD_0_WRITECACHE_SMALL_SIZE=16384 NEOFS_STORAGE_SHARD_0_WRITECACHE_SMALL_SIZE=16384
NEOFS_STORAGE_SHARD_0_WRITECACHE_MAX_SIZE=134217728 NEOFS_STORAGE_SHARD_0_WRITECACHE_MAX_SIZE=134217728
NEOFS_STORAGE_SHARD_0_WRITECACHE_WORKERS_NUMBER=30 NEOFS_STORAGE_SHARD_0_WRITECACHE_WORKERS_NUMBER=30
NEOFS_STORAGE_SHARD_0_WRITECACHE_SIZE_LIMIT=3221225472
### Metabase config ### Metabase config
NEOFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta NEOFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta
NEOFS_STORAGE_SHARD_0_METABASE_PERM=0644 NEOFS_STORAGE_SHARD_0_METABASE_PERM=0644
@ -104,6 +105,7 @@ NEOFS_STORAGE_SHARD_1_WRITECACHE_MEM_SIZE=2147483648
NEOFS_STORAGE_SHARD_1_WRITECACHE_SMALL_SIZE=16384 NEOFS_STORAGE_SHARD_1_WRITECACHE_SMALL_SIZE=16384
NEOFS_STORAGE_SHARD_1_WRITECACHE_MAX_SIZE=134217728 NEOFS_STORAGE_SHARD_1_WRITECACHE_MAX_SIZE=134217728
NEOFS_STORAGE_SHARD_1_WRITECACHE_WORKERS_NUMBER=30 NEOFS_STORAGE_SHARD_1_WRITECACHE_WORKERS_NUMBER=30
NEOFS_STORAGE_SHARD_1_WRITECACHE_SIZE_LIMIT=4294967296
### Metabase config ### Metabase config
NEOFS_STORAGE_SHARD_1_METABASE_PATH=tmp/1/meta NEOFS_STORAGE_SHARD_1_METABASE_PATH=tmp/1/meta
NEOFS_STORAGE_SHARD_1_METABASE_PERM=0644 NEOFS_STORAGE_SHARD_1_METABASE_PERM=0644

View file

@ -110,7 +110,8 @@
"mem_size": 2147483648, "mem_size": 2147483648,
"small_size": 16384, "small_size": 16384,
"max_size": 134217728, "max_size": 134217728,
"workers_number": 30 "workers_number": 30,
"size_limit": 3221225472
}, },
"metabase": { "metabase": {
"path": "tmp/0/meta", "path": "tmp/0/meta",
@ -141,7 +142,8 @@
"mem_size": 2147483648, "mem_size": 2147483648,
"small_size": 16384, "small_size": 16384,
"max_size": 134217728, "max_size": 134217728,
"workers_number": 30 "workers_number": 30,
"size_limit": 4294967296
}, },
"metabase": { "metabase": {
"path": "tmp/1/meta", "path": "tmp/1/meta",

View file

@ -98,6 +98,7 @@ storage:
small_size: 16384 small_size: 16384
max_size: 134217728 max_size: 134217728
workers_number: 30 workers_number: 30
size_limit: 3221225472
metabase: metabase:
path: tmp/0/meta path: tmp/0/meta
@ -129,6 +130,7 @@ storage:
small_size: 16384 small_size: 16384
max_size: 134217728 max_size: 134217728
workers_number: 30 workers_number: 30
size_limit: 4294967296
metabase: metabase:
path: tmp/1/meta path: tmp/1/meta