forked from TrueCloudLab/frostfs-node
[#1745] neofs-node: Remove memcache_capacity
from the configuration
It is unused since ddaed283e9
.
Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
This commit is contained in:
parent
bda084f331
commit
177e8e01b1
11 changed files with 5 additions and 45 deletions
|
@ -415,7 +415,6 @@ func initShardOptions(c *cfg) {
|
||||||
writecache.WithLogger(c.log),
|
writecache.WithLogger(c.log),
|
||||||
writecache.WithMaxBatchSize(writeCacheCfg.BoltDB().MaxBatchSize()),
|
writecache.WithMaxBatchSize(writeCacheCfg.BoltDB().MaxBatchSize()),
|
||||||
writecache.WithMaxBatchDelay(writeCacheCfg.BoltDB().MaxBatchDelay()),
|
writecache.WithMaxBatchDelay(writeCacheCfg.BoltDB().MaxBatchDelay()),
|
||||||
writecache.WithMaxMemSize(writeCacheCfg.MemSize()),
|
|
||||||
writecache.WithMaxObjectSize(writeCacheCfg.MaxObjectSize()),
|
writecache.WithMaxObjectSize(writeCacheCfg.MaxObjectSize()),
|
||||||
writecache.WithSmallObjectSize(writeCacheCfg.SmallObjectSize()),
|
writecache.WithSmallObjectSize(writeCacheCfg.SmallObjectSize()),
|
||||||
writecache.WithFlushWorkersCount(writeCacheCfg.WorkersNumber()),
|
writecache.WithFlushWorkersCount(writeCacheCfg.WorkersNumber()),
|
||||||
|
|
|
@ -70,7 +70,6 @@ func TestEngineSection(t *testing.T) {
|
||||||
require.Equal(t, false, wc.Enabled())
|
require.Equal(t, false, wc.Enabled())
|
||||||
|
|
||||||
require.Equal(t, "tmp/0/cache", wc.Path())
|
require.Equal(t, "tmp/0/cache", wc.Path())
|
||||||
require.EqualValues(t, 2147483648, wc.MemSize())
|
|
||||||
require.EqualValues(t, 16384, wc.SmallObjectSize())
|
require.EqualValues(t, 16384, wc.SmallObjectSize())
|
||||||
require.EqualValues(t, 134217728, wc.MaxObjectSize())
|
require.EqualValues(t, 134217728, wc.MaxObjectSize())
|
||||||
require.EqualValues(t, 30, wc.WorkersNumber())
|
require.EqualValues(t, 30, wc.WorkersNumber())
|
||||||
|
@ -113,7 +112,6 @@ func TestEngineSection(t *testing.T) {
|
||||||
require.Equal(t, true, wc.Enabled())
|
require.Equal(t, true, wc.Enabled())
|
||||||
|
|
||||||
require.Equal(t, "tmp/1/cache", wc.Path())
|
require.Equal(t, "tmp/1/cache", wc.Path())
|
||||||
require.EqualValues(t, 2147483648, wc.MemSize())
|
|
||||||
require.EqualValues(t, 16384, wc.SmallObjectSize())
|
require.EqualValues(t, 16384, wc.SmallObjectSize())
|
||||||
require.EqualValues(t, 134217728, wc.MaxObjectSize())
|
require.EqualValues(t, 134217728, wc.MaxObjectSize())
|
||||||
require.EqualValues(t, 30, wc.WorkersNumber())
|
require.EqualValues(t, 30, wc.WorkersNumber())
|
||||||
|
|
|
@ -11,9 +11,6 @@ type Config config.Config
|
||||||
|
|
||||||
// config defaults
|
// config defaults
|
||||||
const (
|
const (
|
||||||
// MemSizeDefault is a default memory size.
|
|
||||||
MemSizeDefault = 1 << 30
|
|
||||||
|
|
||||||
// SmallSizeDefault is a default size of small objects.
|
// SmallSizeDefault is a default size of small objects.
|
||||||
SmallSizeDefault = 32 << 10
|
SmallSizeDefault = 32 << 10
|
||||||
|
|
||||||
|
@ -55,22 +52,6 @@ func (x *Config) Path() string {
|
||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
// MemSize returns the value of "memcache_capacity" config parameter.
|
|
||||||
//
|
|
||||||
// Returns MemSizeDefault if the value is not a positive number.
|
|
||||||
func (x *Config) MemSize() uint64 {
|
|
||||||
s := config.SizeInBytesSafe(
|
|
||||||
(*config.Config)(x),
|
|
||||||
"memcache_capacity",
|
|
||||||
)
|
|
||||||
|
|
||||||
if s > 0 {
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
return MemSizeDefault
|
|
||||||
}
|
|
||||||
|
|
||||||
// SmallObjectSize returns the value of "small_object_size" config parameter.
|
// SmallObjectSize returns the value of "small_object_size" config parameter.
|
||||||
//
|
//
|
||||||
// Returns SmallSizeDefault if the value is not a positive number.
|
// Returns SmallSizeDefault if the value is not a positive number.
|
||||||
|
|
|
@ -90,7 +90,6 @@ NEOFS_STORAGE_SHARD_0_MODE=read-only
|
||||||
### Write cache config
|
### Write cache config
|
||||||
NEOFS_STORAGE_SHARD_0_WRITECACHE_ENABLED=false
|
NEOFS_STORAGE_SHARD_0_WRITECACHE_ENABLED=false
|
||||||
NEOFS_STORAGE_SHARD_0_WRITECACHE_PATH=tmp/0/cache
|
NEOFS_STORAGE_SHARD_0_WRITECACHE_PATH=tmp/0/cache
|
||||||
NEOFS_STORAGE_SHARD_0_WRITECACHE_MEMCACHE_CAPACITY=2147483648
|
|
||||||
NEOFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384
|
NEOFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384
|
||||||
NEOFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728
|
NEOFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728
|
||||||
NEOFS_STORAGE_SHARD_0_WRITECACHE_WORKERS_NUMBER=30
|
NEOFS_STORAGE_SHARD_0_WRITECACHE_WORKERS_NUMBER=30
|
||||||
|
@ -135,7 +134,6 @@ NEOFS_STORAGE_SHARD_1_MODE=read-write
|
||||||
### Write cache config
|
### Write cache config
|
||||||
NEOFS_STORAGE_SHARD_1_WRITECACHE_ENABLED=true
|
NEOFS_STORAGE_SHARD_1_WRITECACHE_ENABLED=true
|
||||||
NEOFS_STORAGE_SHARD_1_WRITECACHE_PATH=tmp/1/cache
|
NEOFS_STORAGE_SHARD_1_WRITECACHE_PATH=tmp/1/cache
|
||||||
NEOFS_STORAGE_SHARD_1_WRITECACHE_MEMCACHE_CAPACITY=2147483648
|
|
||||||
NEOFS_STORAGE_SHARD_1_WRITECACHE_SMALL_OBJECT_SIZE=16384
|
NEOFS_STORAGE_SHARD_1_WRITECACHE_SMALL_OBJECT_SIZE=16384
|
||||||
NEOFS_STORAGE_SHARD_1_WRITECACHE_MAX_OBJECT_SIZE=134217728
|
NEOFS_STORAGE_SHARD_1_WRITECACHE_MAX_OBJECT_SIZE=134217728
|
||||||
NEOFS_STORAGE_SHARD_1_WRITECACHE_WORKERS_NUMBER=30
|
NEOFS_STORAGE_SHARD_1_WRITECACHE_WORKERS_NUMBER=30
|
||||||
|
|
|
@ -135,7 +135,6 @@
|
||||||
"writecache": {
|
"writecache": {
|
||||||
"enabled": false,
|
"enabled": false,
|
||||||
"path": "tmp/0/cache",
|
"path": "tmp/0/cache",
|
||||||
"memcache_capacity": 2147483648,
|
|
||||||
"small_object_size": 16384,
|
"small_object_size": 16384,
|
||||||
"max_object_size": 134217728,
|
"max_object_size": 134217728,
|
||||||
"workers_number": 30,
|
"workers_number": 30,
|
||||||
|
|
|
@ -113,7 +113,6 @@ storage:
|
||||||
|
|
||||||
writecache:
|
writecache:
|
||||||
enabled: true
|
enabled: true
|
||||||
memcache_capacity: 2147483648 # approximate RAM usage limit for "small" objects, bytes
|
|
||||||
small_object_size: 16k # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes
|
small_object_size: 16k # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes
|
||||||
max_object_size: 134217728 # size threshold for "big" objects which bypass write-cache and go to the storage directly, bytes
|
max_object_size: 134217728 # size threshold for "big" objects which bypass write-cache and go to the storage directly, bytes
|
||||||
workers_number: 30 # number of write-cache flusher threads
|
workers_number: 30 # number of write-cache flusher threads
|
||||||
|
|
|
@ -252,7 +252,6 @@ writecache:
|
||||||
enabled: true
|
enabled: true
|
||||||
path: /path/to/writecache
|
path: /path/to/writecache
|
||||||
capacity: 4294967296
|
capacity: 4294967296
|
||||||
memcache_capacity: 2147483648
|
|
||||||
small_object_size: 16384
|
small_object_size: 16384
|
||||||
max_object_size: 134217728
|
max_object_size: 134217728
|
||||||
workers_number: 30
|
workers_number: 30
|
||||||
|
@ -262,7 +261,6 @@ writecache:
|
||||||
|----------------------|------------|---------------|----------------------------------------------------------------------------------------------------------------------|
|
|----------------------|------------|---------------|----------------------------------------------------------------------------------------------------------------------|
|
||||||
| `path` | `string` | | Path to the metabase file. |
|
| `path` | `string` | | Path to the metabase file. |
|
||||||
| `capacity` | `size` | unrestricted | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. |
|
| `capacity` | `size` | unrestricted | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. |
|
||||||
| `memcache_capacity` | `size` | `1G` | Maximum size of the memory-cache. |
|
|
||||||
| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. |
|
| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. |
|
||||||
| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. |
|
| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. |
|
||||||
| `workers_number` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. |
|
| `workers_number` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. |
|
||||||
|
|
|
@ -65,7 +65,7 @@ func testShardGetRange(t *testing.T, hasWriteCache bool) {
|
||||||
}
|
}
|
||||||
|
|
||||||
sh := newCustomShard(t, t.TempDir(), hasWriteCache,
|
sh := newCustomShard(t, t.TempDir(), hasWriteCache,
|
||||||
[]writecache.Option{writecache.WithMaxMemSize(0), writecache.WithMaxObjectSize(writeCacheMaxSize)},
|
[]writecache.Option{writecache.WithMaxObjectSize(writeCacheMaxSize)},
|
||||||
[]blobstor.Option{blobstor.WithStorages([]blobstor.SubStorage{
|
[]blobstor.Option{blobstor.WithStorages([]blobstor.SubStorage{
|
||||||
{
|
{
|
||||||
Storage: blobovniczatree.NewBlobovniczaTree(
|
Storage: blobovniczatree.NewBlobovniczaTree(
|
||||||
|
|
|
@ -35,7 +35,7 @@ func (s epochState) CurrentEpoch() uint64 {
|
||||||
|
|
||||||
func newShard(t testing.TB, enableWriteCache bool) *shard.Shard {
|
func newShard(t testing.TB, enableWriteCache bool) *shard.Shard {
|
||||||
return newCustomShard(t, t.TempDir(), enableWriteCache,
|
return newCustomShard(t, t.TempDir(), enableWriteCache,
|
||||||
[]writecache.Option{writecache.WithMaxMemSize(0)},
|
nil,
|
||||||
nil)
|
nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -19,9 +19,6 @@ type options struct {
|
||||||
blobstor *blobstor.BlobStor
|
blobstor *blobstor.BlobStor
|
||||||
// metabase is the metabase instance.
|
// metabase is the metabase instance.
|
||||||
metabase *meta.DB
|
metabase *meta.DB
|
||||||
// maxMemSize is the maximum total size of all objects cached in memory.
|
|
||||||
// 1 GiB by default.
|
|
||||||
maxMemSize uint64
|
|
||||||
// maxObjectSize is the maximum size of the object stored in the write-cache.
|
// maxObjectSize is the maximum size of the object stored in the write-cache.
|
||||||
maxObjectSize uint64
|
maxObjectSize uint64
|
||||||
// smallObjectSize is the maximum size of the object stored in the database.
|
// smallObjectSize is the maximum size of the object stored in the database.
|
||||||
|
@ -67,13 +64,6 @@ func WithMetabase(db *meta.DB) Option {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithMaxMemSize sets maximum size for in-memory DB.
|
|
||||||
func WithMaxMemSize(sz uint64) Option {
|
|
||||||
return func(o *options) {
|
|
||||||
o.maxMemSize = sz
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// WithMaxObjectSize sets maximum object size to be stored in write-cache.
|
// WithMaxObjectSize sets maximum object size to be stored in write-cache.
|
||||||
func WithMaxObjectSize(sz uint64) Option {
|
func WithMaxObjectSize(sz uint64) Option {
|
||||||
return func(o *options) {
|
return func(o *options) {
|
||||||
|
|
|
@ -66,10 +66,9 @@ type objectInfo struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
maxInMemorySizeBytes = 1024 * 1024 * 1024 // 1 GiB
|
maxObjectSize = 64 * 1024 * 1024 // 64 MiB
|
||||||
maxObjectSize = 64 * 1024 * 1024 // 64 MiB
|
smallObjectSize = 32 * 1024 // 32 KiB
|
||||||
smallObjectSize = 32 * 1024 // 32 KiB
|
maxCacheSizeBytes = 1 << 30 // 1 GiB
|
||||||
maxCacheSizeBytes = 1 << 30 // 1 GiB
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
@ -85,7 +84,6 @@ func New(opts ...Option) Cache {
|
||||||
compressFlags: make(map[string]struct{}),
|
compressFlags: make(map[string]struct{}),
|
||||||
options: options{
|
options: options{
|
||||||
log: zap.NewNop(),
|
log: zap.NewNop(),
|
||||||
maxMemSize: maxInMemorySizeBytes,
|
|
||||||
maxObjectSize: maxObjectSize,
|
maxObjectSize: maxObjectSize,
|
||||||
smallObjectSize: smallObjectSize,
|
smallObjectSize: smallObjectSize,
|
||||||
workersCount: defaultFlushWorkersCount,
|
workersCount: defaultFlushWorkersCount,
|
||||||
|
|
Loading…
Reference in a new issue