From 08b1f18bca97c100831b8ded4d9e6633a5aae138 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 6 Aug 2024 15:09:12 +0300 Subject: [PATCH] [#1296] writecache: Add count limit Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 3 +++ cmd/frostfs-node/config/engine/config_test.go | 3 +++ .../config/engine/shard/writecache/config.go | 19 ++++++++++++++++++ config/example/node.env | 1 + config/example/node.json | 3 ++- config/example/node.yaml | 1 + docs/storage-node-configuration.md | 19 +++++++++--------- .../writecache/options.go | 10 ++++++++++ pkg/local_object_storage/writecache/put.go | 6 ++---- pkg/local_object_storage/writecache/state.go | 20 +++++++++++++------ 10 files changed, 65 insertions(+), 20 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 5b91e7819..a3dd80684 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -152,6 +152,7 @@ type shardCfg struct { maxObjSize uint64 flushWorkerCount int sizeLimit uint64 + countLimit uint64 noSync bool } @@ -275,6 +276,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg, wc.smallObjectSize = writeCacheCfg.SmallObjectSize() wc.flushWorkerCount = writeCacheCfg.WorkerCount() wc.sizeLimit = writeCacheCfg.SizeLimit() + wc.countLimit = writeCacheCfg.CountLimit() wc.noSync = writeCacheCfg.NoSync() } } @@ -867,6 +869,7 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option { writecache.WithSmallObjectSize(wcRead.smallObjectSize), writecache.WithFlushWorkersCount(wcRead.flushWorkerCount), writecache.WithMaxCacheSize(wcRead.sizeLimit), + writecache.WithMaxCacheCount(wcRead.countLimit), writecache.WithNoSync(wcRead.noSync), writecache.WithLogger(c.log), ) diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 7473afefb..da4eeabfd 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -12,6 +12,7 @@ import ( fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" gcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/gc" piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama" + writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "github.com/stretchr/testify/require" @@ -78,6 +79,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 3221225472, wc.SizeLimit()) + require.EqualValues(t, 49, wc.CountLimit()) require.Equal(t, "tmp/0/meta", meta.Path()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) @@ -133,6 +135,7 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 4294967296, wc.SizeLimit()) + require.EqualValues(t, writecacheconfig.CountLimitDefault, wc.CountLimit()) require.Equal(t, "tmp/1/meta", meta.Path()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) diff --git a/cmd/frostfs-node/config/engine/shard/writecache/config.go b/cmd/frostfs-node/config/engine/shard/writecache/config.go index 5e31e04ad..bfe8144df 100644 --- a/cmd/frostfs-node/config/engine/shard/writecache/config.go +++ b/cmd/frostfs-node/config/engine/shard/writecache/config.go @@ -21,6 +21,9 @@ const ( // SizeLimitDefault is a default write-cache size limit. SizeLimitDefault = 1 << 30 + + // CountLimitDefault is a default write-cache count limit. + CountLimitDefault = 0 ) // From wraps config section into Config. @@ -115,6 +118,22 @@ func (x *Config) SizeLimit() uint64 { return SizeLimitDefault } +// CountLimit returns the value of "max_object_count" config parameter. +// +// Returns CountLimitDefault if the value is not a positive number. +func (x *Config) CountLimit() uint64 { + c := config.SizeInBytesSafe( + (*config.Config)(x), + "max_object_count", + ) + + if c > 0 { + return c + } + + return CountLimitDefault +} + // NoSync returns the value of "no_sync" config parameter. // // Returns false if the value is not a boolean. diff --git a/config/example/node.env b/config/example/node.env index 00190eb39..fc42cc3bf 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -105,6 +105,7 @@ FROSTFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384 FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728 FROSTFS_STORAGE_SHARD_0_WRITECACHE_FLUSH_WORKER_COUNT=30 FROSTFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472 +FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_COUNT=49 ### Metabase config FROSTFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644 diff --git a/config/example/node.json b/config/example/node.json index 9051d2bb7..9aa0dac53 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -148,7 +148,8 @@ "small_object_size": 16384, "max_object_size": 134217728, "flush_worker_count": 30, - "capacity": 3221225472 + "capacity": 3221225472, + "max_object_count": 49 }, "metabase": { "path": "tmp/0/meta", diff --git a/config/example/node.yaml b/config/example/node.yaml index bcc8552b3..2e6affae9 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -171,6 +171,7 @@ storage: no_sync: true path: tmp/0/cache # write-cache root directory capacity: 3221225472 # approximate write-cache total size, bytes + max_object_count: 49 metabase: path: tmp/0/meta # metabase path diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 4a6e5ba6d..424172ff5 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -292,15 +292,16 @@ writecache: flush_worker_count: 30 ``` -| Parameter | Type | Default value | Description | -|----------------------|------------|---------------|----------------------------------------------------------------------------------------------------------------------| -| `path` | `string` | | Path to the metabase file. | -| `capacity` | `size` | unrestricted | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | -| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | -| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. | -| `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | -| `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | -| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | +| Parameter | Type | Default value | Description | +|----------------------|------------|---------------|-------------------------------------------------------------------------------------------------------------------------------| +| `path` | `string` | | Path to the metabase file. | +| `capacity` | `size` | `1G` | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | +| `max_object_count` | `int` | unrestricted | Approximate maximum objects count in the writecache. If the writecache is full, objects are written to the blobstor directly. | +| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | +| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. | +| `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | +| `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | +| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | # `node` section diff --git a/pkg/local_object_storage/writecache/options.go b/pkg/local_object_storage/writecache/options.go index c8eb1bc45..66cd79bea 100644 --- a/pkg/local_object_storage/writecache/options.go +++ b/pkg/local_object_storage/writecache/options.go @@ -29,6 +29,9 @@ type options struct { // maxCacheSize is the maximum total size of all objects saved in cache (DB + FS). // 1 GiB by default. maxCacheSize uint64 + // maxCacheCount is the maximum total count of all object saved in cache. + // 0 (no limit) by default. + maxCacheCount uint64 // objCounters contains atomic counters for the number of objects stored in cache. objCounters counters // maxBatchSize is the maximum batch size for the small object database. @@ -108,6 +111,13 @@ func WithMaxCacheSize(sz uint64) Option { } } +// WithMaxCacheCount sets maximum write-cache objects count. +func WithMaxCacheCount(v uint64) Option { + return func(o *options) { + o.maxCacheCount = v + } +} + // WithMaxBatchSize sets max batch size for the small object database. func WithMaxBatchSize(sz int) Option { return func(o *options) { diff --git a/pkg/local_object_storage/writecache/put.go b/pkg/local_object_storage/writecache/put.go index 0e419f95b..150399de8 100644 --- a/pkg/local_object_storage/writecache/put.go +++ b/pkg/local_object_storage/writecache/put.go @@ -76,8 +76,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro // putSmall persists small objects to the write-cache database and // pushes the to the flush workers queue. func (c *cache) putSmall(obj objectInfo) error { - cacheSize := c.estimateCacheSize() - if c.maxCacheSize < c.incSizeDB(cacheSize) { + if !c.hasEnoughSpaceDB() { return ErrOutOfSpace } @@ -107,8 +106,7 @@ func (c *cache) putSmall(obj objectInfo) error { // putBig writes object to FSTree and pushes it to the flush workers queue. func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) error { - cacheSz := c.estimateCacheSize() - if c.maxCacheSize < c.incSizeFS(cacheSz) { + if !c.hasEnoughSpaceFS() { return ErrOutOfSpace } diff --git a/pkg/local_object_storage/writecache/state.go b/pkg/local_object_storage/writecache/state.go index bc75aaf27..d03f4a63e 100644 --- a/pkg/local_object_storage/writecache/state.go +++ b/pkg/local_object_storage/writecache/state.go @@ -9,7 +9,7 @@ import ( "go.etcd.io/bbolt" ) -func (c *cache) estimateCacheSize() uint64 { +func (c *cache) estimateCacheSize() (uint64, uint64) { dbCount := c.objCounters.DB() fsCount := c.objCounters.FS() if fsCount > 0 { @@ -19,15 +19,23 @@ func (c *cache) estimateCacheSize() uint64 { fsSize := fsCount * c.maxObjectSize c.metrics.SetEstimateSize(dbSize, fsSize) c.metrics.SetActualCounters(dbCount, fsCount) - return dbSize + fsSize + return dbCount + fsCount, dbSize + fsSize } -func (c *cache) incSizeDB(sz uint64) uint64 { - return sz + c.smallObjectSize +func (c *cache) hasEnoughSpaceDB() bool { + return c.hasEnoughSpace(c.smallObjectSize) } -func (c *cache) incSizeFS(sz uint64) uint64 { - return sz + c.maxObjectSize +func (c *cache) hasEnoughSpaceFS() bool { + return c.hasEnoughSpace(c.maxObjectSize) +} + +func (c *cache) hasEnoughSpace(objectSize uint64) bool { + count, size := c.estimateCacheSize() + if c.maxCacheCount > 0 && count+1 > c.maxCacheCount { + return false + } + return c.maxCacheSize >= size+objectSize } var _ fstree.FileCounter = &counters{}