[#1296] writecache: Add count limit

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
Dmitrii Stepanov 2024-08-06 15:09:12 +03:00 committed by Evgenii Stratonikov
parent 36efccd862
commit 08b1f18bca
10 changed files with 65 additions and 20 deletions

View file

@ -152,6 +152,7 @@ type shardCfg struct {
maxObjSize uint64 maxObjSize uint64
flushWorkerCount int flushWorkerCount int
sizeLimit uint64 sizeLimit uint64
countLimit uint64
noSync bool noSync bool
} }
@ -275,6 +276,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg,
wc.smallObjectSize = writeCacheCfg.SmallObjectSize() wc.smallObjectSize = writeCacheCfg.SmallObjectSize()
wc.flushWorkerCount = writeCacheCfg.WorkerCount() wc.flushWorkerCount = writeCacheCfg.WorkerCount()
wc.sizeLimit = writeCacheCfg.SizeLimit() wc.sizeLimit = writeCacheCfg.SizeLimit()
wc.countLimit = writeCacheCfg.CountLimit()
wc.noSync = writeCacheCfg.NoSync() wc.noSync = writeCacheCfg.NoSync()
} }
} }
@ -867,6 +869,7 @@ func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option {
writecache.WithSmallObjectSize(wcRead.smallObjectSize), writecache.WithSmallObjectSize(wcRead.smallObjectSize),
writecache.WithFlushWorkersCount(wcRead.flushWorkerCount), writecache.WithFlushWorkersCount(wcRead.flushWorkerCount),
writecache.WithMaxCacheSize(wcRead.sizeLimit), writecache.WithMaxCacheSize(wcRead.sizeLimit),
writecache.WithMaxCacheCount(wcRead.countLimit),
writecache.WithNoSync(wcRead.noSync), writecache.WithNoSync(wcRead.noSync),
writecache.WithLogger(c.log), writecache.WithLogger(c.log),
) )

View file

@ -12,6 +12,7 @@ import (
fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree"
gcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/gc" gcconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/gc"
piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama" piloramaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/pilorama"
writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/writecache"
configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -78,6 +79,7 @@ func TestEngineSection(t *testing.T) {
require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 134217728, wc.MaxObjectSize())
require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 30, wc.WorkerCount())
require.EqualValues(t, 3221225472, wc.SizeLimit()) require.EqualValues(t, 3221225472, wc.SizeLimit())
require.EqualValues(t, 49, wc.CountLimit())
require.Equal(t, "tmp/0/meta", meta.Path()) require.Equal(t, "tmp/0/meta", meta.Path())
require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm())
@ -133,6 +135,7 @@ func TestEngineSection(t *testing.T) {
require.EqualValues(t, 134217728, wc.MaxObjectSize()) require.EqualValues(t, 134217728, wc.MaxObjectSize())
require.EqualValues(t, 30, wc.WorkerCount()) require.EqualValues(t, 30, wc.WorkerCount())
require.EqualValues(t, 4294967296, wc.SizeLimit()) require.EqualValues(t, 4294967296, wc.SizeLimit())
require.EqualValues(t, writecacheconfig.CountLimitDefault, wc.CountLimit())
require.Equal(t, "tmp/1/meta", meta.Path()) require.Equal(t, "tmp/1/meta", meta.Path())
require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm()) require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm())

View file

@ -21,6 +21,9 @@ const (
// SizeLimitDefault is a default write-cache size limit. // SizeLimitDefault is a default write-cache size limit.
SizeLimitDefault = 1 << 30 SizeLimitDefault = 1 << 30
// CountLimitDefault is a default write-cache count limit.
CountLimitDefault = 0
) )
// From wraps config section into Config. // From wraps config section into Config.
@ -115,6 +118,22 @@ func (x *Config) SizeLimit() uint64 {
return SizeLimitDefault return SizeLimitDefault
} }
// CountLimit returns the value of "max_object_count" config parameter.
//
// Returns CountLimitDefault if the value is not a positive number.
func (x *Config) CountLimit() uint64 {
c := config.SizeInBytesSafe(
(*config.Config)(x),
"max_object_count",
)
if c > 0 {
return c
}
return CountLimitDefault
}
// NoSync returns the value of "no_sync" config parameter. // NoSync returns the value of "no_sync" config parameter.
// //
// Returns false if the value is not a boolean. // Returns false if the value is not a boolean.

View file

@ -105,6 +105,7 @@ FROSTFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384
FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728 FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728
FROSTFS_STORAGE_SHARD_0_WRITECACHE_FLUSH_WORKER_COUNT=30 FROSTFS_STORAGE_SHARD_0_WRITECACHE_FLUSH_WORKER_COUNT=30
FROSTFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472 FROSTFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472
FROSTFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_COUNT=49
### Metabase config ### Metabase config
FROSTFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta FROSTFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta
FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644 FROSTFS_STORAGE_SHARD_0_METABASE_PERM=0644

View file

@ -148,7 +148,8 @@
"small_object_size": 16384, "small_object_size": 16384,
"max_object_size": 134217728, "max_object_size": 134217728,
"flush_worker_count": 30, "flush_worker_count": 30,
"capacity": 3221225472 "capacity": 3221225472,
"max_object_count": 49
}, },
"metabase": { "metabase": {
"path": "tmp/0/meta", "path": "tmp/0/meta",

View file

@ -171,6 +171,7 @@ storage:
no_sync: true no_sync: true
path: tmp/0/cache # write-cache root directory path: tmp/0/cache # write-cache root directory
capacity: 3221225472 # approximate write-cache total size, bytes capacity: 3221225472 # approximate write-cache total size, bytes
max_object_count: 49
metabase: metabase:
path: tmp/0/meta # metabase path path: tmp/0/meta # metabase path

View file

@ -292,15 +292,16 @@ writecache:
flush_worker_count: 30 flush_worker_count: 30
``` ```
| Parameter | Type | Default value | Description | | Parameter | Type | Default value | Description |
|----------------------|------------|---------------|----------------------------------------------------------------------------------------------------------------------| |----------------------|------------|---------------|-------------------------------------------------------------------------------------------------------------------------------|
| `path` | `string` | | Path to the metabase file. | | `path` | `string` | | Path to the metabase file. |
| `capacity` | `size` | unrestricted | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. | | `capacity` | `size` | `1G` | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. |
| `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. | | `max_object_count` | `int` | unrestricted | Approximate maximum objects count in the writecache. If the writecache is full, objects are written to the blobstor directly. |
| `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. | | `small_object_size` | `size` | `32K` | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system. |
| `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. | | `max_object_size` | `size` | `64M` | Maximum object size allowed to be stored in the writecache. |
| `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. | | `flush_worker_count` | `int` | `20` | Amount of background workers that move data from the writecache to the blobstor. |
| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. | | `max_batch_size` | `int` | `1000` | Maximum amount of small object `PUT` operations to perform in a single transaction. |
| `max_batch_delay` | `duration` | `10ms` | Maximum delay before a batch starts. |
# `node` section # `node` section

View file

@ -29,6 +29,9 @@ type options struct {
// maxCacheSize is the maximum total size of all objects saved in cache (DB + FS). // maxCacheSize is the maximum total size of all objects saved in cache (DB + FS).
// 1 GiB by default. // 1 GiB by default.
maxCacheSize uint64 maxCacheSize uint64
// maxCacheCount is the maximum total count of all object saved in cache.
// 0 (no limit) by default.
maxCacheCount uint64
// objCounters contains atomic counters for the number of objects stored in cache. // objCounters contains atomic counters for the number of objects stored in cache.
objCounters counters objCounters counters
// maxBatchSize is the maximum batch size for the small object database. // maxBatchSize is the maximum batch size for the small object database.
@ -108,6 +111,13 @@ func WithMaxCacheSize(sz uint64) Option {
} }
} }
// WithMaxCacheCount sets maximum write-cache objects count.
func WithMaxCacheCount(v uint64) Option {
return func(o *options) {
o.maxCacheCount = v
}
}
// WithMaxBatchSize sets max batch size for the small object database. // WithMaxBatchSize sets max batch size for the small object database.
func WithMaxBatchSize(sz int) Option { func WithMaxBatchSize(sz int) Option {
return func(o *options) { return func(o *options) {

View file

@ -76,8 +76,7 @@ func (c *cache) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, erro
// putSmall persists small objects to the write-cache database and // putSmall persists small objects to the write-cache database and
// pushes the to the flush workers queue. // pushes the to the flush workers queue.
func (c *cache) putSmall(obj objectInfo) error { func (c *cache) putSmall(obj objectInfo) error {
cacheSize := c.estimateCacheSize() if !c.hasEnoughSpaceDB() {
if c.maxCacheSize < c.incSizeDB(cacheSize) {
return ErrOutOfSpace return ErrOutOfSpace
} }
@ -107,8 +106,7 @@ func (c *cache) putSmall(obj objectInfo) error {
// putBig writes object to FSTree and pushes it to the flush workers queue. // putBig writes object to FSTree and pushes it to the flush workers queue.
func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) error { func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) error {
cacheSz := c.estimateCacheSize() if !c.hasEnoughSpaceFS() {
if c.maxCacheSize < c.incSizeFS(cacheSz) {
return ErrOutOfSpace return ErrOutOfSpace
} }

View file

@ -9,7 +9,7 @@ import (
"go.etcd.io/bbolt" "go.etcd.io/bbolt"
) )
func (c *cache) estimateCacheSize() uint64 { func (c *cache) estimateCacheSize() (uint64, uint64) {
dbCount := c.objCounters.DB() dbCount := c.objCounters.DB()
fsCount := c.objCounters.FS() fsCount := c.objCounters.FS()
if fsCount > 0 { if fsCount > 0 {
@ -19,15 +19,23 @@ func (c *cache) estimateCacheSize() uint64 {
fsSize := fsCount * c.maxObjectSize fsSize := fsCount * c.maxObjectSize
c.metrics.SetEstimateSize(dbSize, fsSize) c.metrics.SetEstimateSize(dbSize, fsSize)
c.metrics.SetActualCounters(dbCount, fsCount) c.metrics.SetActualCounters(dbCount, fsCount)
return dbSize + fsSize return dbCount + fsCount, dbSize + fsSize
} }
func (c *cache) incSizeDB(sz uint64) uint64 { func (c *cache) hasEnoughSpaceDB() bool {
return sz + c.smallObjectSize return c.hasEnoughSpace(c.smallObjectSize)
} }
func (c *cache) incSizeFS(sz uint64) uint64 { func (c *cache) hasEnoughSpaceFS() bool {
return sz + c.maxObjectSize return c.hasEnoughSpace(c.maxObjectSize)
}
func (c *cache) hasEnoughSpace(objectSize uint64) bool {
count, size := c.estimateCacheSize()
if c.maxCacheCount > 0 && count+1 > c.maxCacheCount {
return false
}
return c.maxCacheSize >= size+objectSize
} }
var _ fstree.FileCounter = &counters{} var _ fstree.FileCounter = &counters{}