TrueCloudLab/frostfs-node
20
1
Fork 28
Code Issues 109 Pull requests 4 Projects Releases 2 Packages 4 Activity Actions

[#421] Try using badger for the write-cache

Browse source 
Signed-off-by: Alejandro Lopez <a.lopez@yadro.com>
This commit is contained in:
Alejandro Lopez 2023-06-22 14:55:30 +03:00 • committed by Evgenii Stratonikov
parent 65c72f3e0b
commit 1a0cb0f34a
56 changed files with 2234 additions and 747 deletions
Download patch file Download diff file Expand all files Collapse all files

53
cmd/frostfs-node/config.go
View file

@ -40,7 +40,9 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
shardmode "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebadger"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/writecachebbolt"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
containerClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
@ -127,6 +129,7 @@ type shardCfg struct {
writecacheCfg struct {
enabled bool
typ writecacheconfig.Type
path string
maxBatchSize int
maxBatchDelay time.Duration
@ -135,6 +138,7 @@ type shardCfg struct {
flushWorkerCount int
sizeLimit uint64
noSync bool
gcInterval time.Duration
}
piloramaCfg struct {
@ -238,6 +242,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg,
wc := &newConfig.writecacheCfg
wc.enabled = true
wc.typ = writeCacheCfg.Type()
wc.path = writeCacheCfg.Path()
wc.maxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize()
wc.maxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay()
@ -246,6 +251,7 @@ func (a *applicationConfiguration) setShardWriteCacheConfig(newConfig *shardCfg,
wc.flushWorkerCount = writeCacheCfg.WorkersNumber()
wc.sizeLimit = writeCacheCfg.SizeLimit()
wc.noSync = writeCacheCfg.NoSync()
wc.gcInterval = writeCacheCfg.GCInterval()
}
}
@ -704,20 +710,37 @@ func (c *cfg) shardOpts() []shardOptsWithID {
return shards
}
func (c *cfg) getWriteCacheOpts(shCfg shardCfg) []writecache.Option {
var writeCacheOpts []writecache.Option
func (c *cfg) getWriteCacheOpts(shCfg shardCfg) writecacheconfig.Options {
var writeCacheOpts writecacheconfig.Options
if wcRead := shCfg.writecacheCfg; wcRead.enabled {
writeCacheOpts = append(writeCacheOpts,
writecache.WithPath(wcRead.path),
writecache.WithMaxBatchSize(wcRead.maxBatchSize),
writecache.WithMaxBatchDelay(wcRead.maxBatchDelay),
writecache.WithMaxObjectSize(wcRead.maxObjSize),
writecache.WithSmallObjectSize(wcRead.smallObjectSize),
writecache.WithFlushWorkersCount(wcRead.flushWorkerCount),
writecache.WithMaxCacheSize(wcRead.sizeLimit),
writecache.WithNoSync(wcRead.noSync),
writecache.WithLogger(c.log),
)
switch wcRead.typ {
case writecacheconfig.TypeBBolt:
writeCacheOpts.Type = writecacheconfig.TypeBBolt
writeCacheOpts.BBoltOptions = append(writeCacheOpts.BBoltOptions,
writecachebbolt.WithPath(wcRead.path),
writecachebbolt.WithMaxBatchSize(wcRead.maxBatchSize),
writecachebbolt.WithMaxBatchDelay(wcRead.maxBatchDelay),
writecachebbolt.WithMaxObjectSize(wcRead.maxObjSize),
writecachebbolt.WithSmallObjectSize(wcRead.smallObjectSize),
writecachebbolt.WithFlushWorkersCount(wcRead.flushWorkerCount),
writecachebbolt.WithMaxCacheSize(wcRead.sizeLimit),
writecachebbolt.WithNoSync(wcRead.noSync),
writecachebbolt.WithLogger(c.log),
)
case writecacheconfig.TypeBadger:
writeCacheOpts.Type = writecacheconfig.TypeBBolt
writeCacheOpts.BadgerOptions = append(writeCacheOpts.BadgerOptions,
writecachebadger.WithPath(wcRead.path),
writecachebadger.WithMaxObjectSize(wcRead.maxObjSize),
writecachebadger.WithFlushWorkersCount(wcRead.flushWorkerCount),
writecachebadger.WithMaxCacheSize(wcRead.sizeLimit),
writecachebadger.WithNoSync(wcRead.noSync),
writecachebadger.WithLogger(c.log),
writecachebadger.WithGCInterval(wcRead.gcInterval),
)
default:
panic(fmt.Sprintf("unknown writecache type: %q", wcRead.typ))
}
}
return writeCacheOpts
}
@ -836,7 +859,7 @@ func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID {
shard.WithMetaBaseOptions(mbOptions...),
shard.WithPiloramaOptions(piloramaOpts...),
shard.WithWriteCache(shCfg.writecacheCfg.enabled),
shard.WithWriteCacheOptions(writeCacheOpts...),
shard.WithWriteCacheOptions(writeCacheOpts),
shard.WithRemoverBatchSize(shCfg.gcCfg.removerBatchSize),
shard.WithGCRemoverSleepInterval(shCfg.gcCfg.removerSleepInterval),
shard.WithExpiredCollectorBatchSize(shCfg.gcCfg.expiredCollectorBatchSize),

36
cmd/frostfs-node/config/engine/shard/writecache/config.go
View file

@ -1,8 +1,12 @@
package writecacheconfig
import (
"fmt"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
boltdbconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/boltdb"
writecacheconfig "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache/config"
)
// Config is a wrapper over the config section
@ -21,6 +25,9 @@ const (
// SizeLimitDefault is a default write-cache size limit.
SizeLimitDefault = 1 << 30
// DefaultGCInterval is the default duration of the GC cycle interval.
DefaultGCInterval = 1 * time.Minute
)
// From wraps config section into Config.
@ -35,6 +42,22 @@ func (x *Config) Enabled() bool {
return config.Bool((*config.Config)(x), "enabled")
}
// Type returns the writecache implementation type to use.
//
// Panics if the type is not recognized.
func (x *Config) Type() writecacheconfig.Type {
t := config.String((*config.Config)(x), "type")
switch t {
case "bbolt", "":
return writecacheconfig.TypeBBolt
case "badger":
return writecacheconfig.TypeBadger
}
panic(fmt.Sprintf("invalid writecache type: %q", t))
}
// Path returns the value of "path" config parameter.
//
// Panics if the value is not a non-empty string.
@ -126,3 +149,16 @@ func (x *Config) NoSync() bool {
func (x *Config) BoltDB() *boltdbconfig.Config {
return (*boltdbconfig.Config)(x)
}
// GCInterval returns the value of "gc_interval" config parameter.
//
// Returns DefaultGCInterval if the value is not a positive duration.
func (x *Config) GCInterval() time.Duration {
d := config.DurationSafe((*config.Config)(x), "gc_interval")
if d > 0 {
return d
}
return DefaultGCInterval
}