From e9aed2245485a9927662113ee415a1aab6bfc228 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 25 Oct 2023 18:50:50 +0300 Subject: [PATCH 1/7] [#645] blobstor: Add Badger store Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 47 ++++++- .../shard/blobstor/badgerstore/config.go | 85 +++++++++++++ cmd/frostfs-node/validate.go | 3 +- .../blobstor/badgerstore/config.go | 117 ++++++++++++++++++ .../blobstor/badgerstore/control.go | 93 ++++++++++++++ .../blobstor/badgerstore/delete.go | 44 +++++++ .../blobstor/badgerstore/exists.go | 36 ++++++ .../blobstor/badgerstore/generic_test.go | 39 ++++++ .../blobstor/badgerstore/get.go | 107 ++++++++++++++++ .../blobstor/badgerstore/iterate.go | 113 +++++++++++++++++ .../blobstor/badgerstore/keys.go | 33 +++++ .../blobstor/badgerstore/put.go | 40 ++++++ .../blobstor/badgerstore/store.go | 71 +++++++++++ .../blobstor/perf_test.go | 9 ++ 14 files changed, 830 insertions(+), 7 deletions(-) create mode 100644 cmd/frostfs-node/config/engine/shard/blobstor/badgerstore/config.go create mode 100644 pkg/local_object_storage/blobstor/badgerstore/config.go create mode 100644 pkg/local_object_storage/blobstor/badgerstore/control.go create mode 100644 pkg/local_object_storage/blobstor/badgerstore/delete.go create mode 100644 pkg/local_object_storage/blobstor/badgerstore/exists.go create mode 100644 pkg/local_object_storage/blobstor/badgerstore/generic_test.go create mode 100644 pkg/local_object_storage/blobstor/badgerstore/get.go create mode 100644 pkg/local_object_storage/blobstor/badgerstore/iterate.go create mode 100644 pkg/local_object_storage/blobstor/badgerstore/keys.go create mode 100644 pkg/local_object_storage/blobstor/badgerstore/put.go create mode 100644 pkg/local_object_storage/blobstor/badgerstore/store.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d78a90cfc..7a3b6aa10 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -21,6 +21,7 @@ import ( contractsconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/contracts" engineconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine" shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard" + badgerstoreconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/badgerstore" blobovniczaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza" fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree" loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger" @@ -34,6 +35,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/badgerstore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" @@ -183,12 +185,20 @@ type subStorageCfg struct { noSync bool // blobovnicza-specific - size uint64 - width uint64 - leafWidth uint64 - openedCacheSize int - initWorkerCount int - initInAdvance bool + size uint64 + width uint64 + leafWidth uint64 + openedCacheSize int + initWorkerCount int + initInAdvance bool + + // badgerstore-specific + indexCacheSize int64 + memTablesCount int + compactorsCount int + gcInterval time.Duration + gcDiscardRatio float64 + valueLogFileSize int64 rebuildDropTimeout time.Duration } @@ -317,6 +327,14 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sub := fstreeconfig.From((*config.Config)(storagesCfg[i])) sCfg.depth = sub.Depth() sCfg.noSync = sub.NoSync() + case badgerstore.Type: + sub := badgerstoreconfig.From((*config.Config)(storagesCfg[i])) + sCfg.indexCacheSize = sub.IndexCacheSize() + sCfg.memTablesCount = sub.MemTablesCount() + sCfg.compactorsCount = sub.CompactorsCount() + sCfg.gcInterval = sub.GCInterval() + sCfg.gcDiscardRatio = sub.GCDiscardRatio() + sCfg.valueLogFileSize = sub.ValueLogFileSize() default: return fmt.Errorf("invalid storage type: %s", storagesCfg[i].Type()) } @@ -941,6 +959,23 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { return true }, }) + case badgerstore.Type: + badgerStoreOpts := []badgerstore.Option{ + badgerstore.WithPath(sRead.path), + badgerstore.WithPermissions(sRead.perm), + badgerstore.WithCompactorsCount(sRead.compactorsCount), + badgerstore.WithGCDiscardRatio(sRead.gcDiscardRatio), + badgerstore.WithGCInterval(sRead.gcInterval), + badgerstore.WithIndexCacheSize(sRead.indexCacheSize), + badgerstore.WithMemTablesCount(sRead.memTablesCount), + badgerstore.WithValueLogSize(sRead.valueLogFileSize), + } + ss = append(ss, blobstor.SubStorage{ + Storage: badgerstore.New(badgerStoreOpts...), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return uint64(len(data)) < shCfg.smallSizeObjectLimit + }, + }) default: // should never happen, that has already // been handled: when the config was read diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/badgerstore/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/badgerstore/config.go new file mode 100644 index 000000000..b80324d4e --- /dev/null +++ b/cmd/frostfs-node/config/engine/shard/blobstor/badgerstore/config.go @@ -0,0 +1,85 @@ +package badgerstore + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/badgerstore" +) + +type Config config.Config + +const ( + IndexCacheSizeDefault = 256 << 20 // 256MB + MemTablesCountDefault = 32 + CompactorsCountDefault = 64 + GCIntervalDefault = 10 * time.Minute + GCDiscardRatioDefault = 0.2 + ValueLogSizeDefault = 1 << 30 // 1GB +) + +// From wraps config section into Config. +func From(c *config.Config) *Config { + return (*Config)(c) +} + +// Type returns the storage type. +func (x *Config) Type() string { + return badgerstore.Type +} + +// IndexCacheSize returns `index_cache_size` value or IndexCacheSizeDefault. +func (x *Config) IndexCacheSize() int64 { + s := config.SizeInBytesSafe((*config.Config)(x), "index_cache_size") + if s > 0 { + return int64(s) + } + + return IndexCacheSizeDefault +} + +// MemTablesCount returns `mem_tables_count` value or MemTablesCountDefault. +func (x *Config) MemTablesCount() int { + v := config.IntSafe((*config.Config)(x), "mem_tables_count") + if v > 0 { + return int(v) + } + return MemTablesCountDefault +} + +// CompactorsCount returns `compactors_count` value or CompactorsCountDefault. +func (x *Config) CompactorsCount() int { + v := config.IntSafe((*config.Config)(x), "compactors_count") + if v > 0 { + return int(v) + } + return CompactorsCountDefault +} + +// GCInterval returns `gc_interval` value or GCIntervalDefault. +func (x *Config) GCInterval() time.Duration { + v := config.DurationSafe((*config.Config)(x), "gc_interval") + if v > 0 { + return v + } + return GCIntervalDefault +} + +// GCDiscardRatio returns `gc_discard_percent` value as ratio value (in range (0.0; 1.0)) or GCDiscardRatioDefault. +func (x *Config) GCDiscardRatio() float64 { + v := config.Uint32Safe((*config.Config)(x), "gc_discard_percent") + if v > 0 && v < 100 { + return float64(v) / (float64(100)) + } + return GCDiscardRatioDefault +} + +// ValueLogFileSize returns `value_log_file_size` value or ValueLogSizeDefault. +func (x *Config) ValueLogFileSize() int64 { + s := config.SizeInBytesSafe((*config.Config)(x), "value_log_file_size") + if s > 0 { + return int64(s) + } + + return ValueLogSizeDefault +} diff --git a/cmd/frostfs-node/validate.go b/cmd/frostfs-node/validate.go index ae52b9e4a..dde895692 100644 --- a/cmd/frostfs-node/validate.go +++ b/cmd/frostfs-node/validate.go @@ -9,6 +9,7 @@ import ( shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard" loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger" treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/badgerstore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" @@ -60,7 +61,7 @@ func validateConfig(c *config.Config) error { } for i := range blobstor { switch blobstor[i].Type() { - case fstree.Type, blobovniczatree.Type: + case fstree.Type, blobovniczatree.Type, badgerstore.Type: default: return fmt.Errorf("unexpected storage type: %s (shard %d)", blobstor[i].Type(), shardNum) } diff --git a/pkg/local_object_storage/blobstor/badgerstore/config.go b/pkg/local_object_storage/blobstor/badgerstore/config.go new file mode 100644 index 000000000..72a9f51bb --- /dev/null +++ b/pkg/local_object_storage/blobstor/badgerstore/config.go @@ -0,0 +1,117 @@ +package badgerstore + +import ( + "io/fs" + "math" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/badger/v4/options" +) + +type cfg struct { + permissions fs.FileMode + compression *compression.Config + db badger.Options + gcTimeout time.Duration + gcDiscardRatio float64 +} + +type Option func(*cfg) + +// defaultCfg creates default options to create Store. +// Default Badger options: +// BaseTableSize: 2MB +// BaseLevelSize: 10MB +// TableSizeMultiplier: 2 +// LevelSizeMultiplier: 10 +// MaxLevels: 7 +// NumLevelZeroTables: 5 +// ValueLogFileSize: 1GB +// +// Badger flushes MemTable directly to Level0. +// So for Level0 MemTableSize is used as TableSize https://github.com/dgraph-io/badger/blob/v4.1.0/levels.go#L403. +// There is no total size limit for Level0, only NumLevelZeroTables +// +// Badger uses Dynamic Level Sizes like RocksDB. +// See https://github.com/facebook/rocksdb/blob/v3.11/include/rocksdb/options.h#L366 for explanation. +func defaultCfg() *cfg { + opts := badger.DefaultOptions("/") + opts.BlockCacheSize = 0 // compression and encryption are disabled, so block cache should be disabled + opts.IndexCacheSize = 256 << 20 // 256MB, to not to keep all indicies in memory + opts.Compression = options.None // performed by cfg.compressor + opts.Logger = nil + opts.MetricsEnabled = false + opts.NumLevelZeroTablesStall = math.MaxInt // to not to stall because of Level0 slow compaction + opts.NumMemtables = 32 // default memtable size is 64MB, so max memory consumption will be 2GB before stall + opts.NumCompactors = 64 + opts.SyncWrites = true + opts.ValueLogMaxEntries = math.MaxUint32 // default vLog file size is 1GB, so size is more clear than entries count + opts.ValueThreshold = 0 // to store all values in vLog + opts.LmaxCompaction = true + + return &cfg{ + permissions: 0o700, + db: opts, + gcTimeout: 10 * time.Minute, + gcDiscardRatio: 0.2, // for 1GB vLog file GC will perform only if around 200MB could be free + } +} + +// WithPath sets BadgerStore directory. +func WithPath(dir string) Option { + return func(c *cfg) { + c.db.Dir = dir + c.db.ValueDir = dir + } +} + +// WithPermissions sets persmission flags. +func WithPermissions(p fs.FileMode) Option { + return func(c *cfg) { + c.permissions = p + } +} + +// WithIndexCacheSize sets BadgerStore index cache size. +func WithIndexCacheSize(sz int64) Option { + return func(c *cfg) { + c.db.IndexCacheSize = sz + } +} + +// WithMemTablesCount sets maximum count of memtables. +func WithMemTablesCount(count int) Option { + return func(c *cfg) { + c.db.NumMemtables = count + } +} + +// WithCompactorsCount sets count of concurrent compactors. +func WithCompactorsCount(count int) Option { + return func(c *cfg) { + c.db.NumCompactors = count + } +} + +// WithGCInterval sets GC interval value. +func WithGCInterval(d time.Duration) Option { + return func(c *cfg) { + c.gcTimeout = d + } +} + +// WithGCDiscardRatio sets GC discard ratio. +func WithGCDiscardRatio(r float64) Option { + return func(c *cfg) { + c.gcDiscardRatio = r + } +} + +// WithValueLogSize sets max value log size. +func WithValueLogSize(sz int64) Option { + return func(c *cfg) { + c.db.ValueLogFileSize = sz + } +} diff --git a/pkg/local_object_storage/blobstor/badgerstore/control.go b/pkg/local_object_storage/blobstor/badgerstore/control.go new file mode 100644 index 000000000..2b808d97b --- /dev/null +++ b/pkg/local_object_storage/blobstor/badgerstore/control.go @@ -0,0 +1,93 @@ +package badgerstore + +import ( + "context" + "errors" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" + "github.com/dgraph-io/badger/v4" +) + +var errStoreMustBeOpenedBeforeInit = errors.New("store must be opened before initialization") + +// Close implements common.Storage. +func (s *Store) Close() error { + s.modeMtx.Lock() + defer s.modeMtx.Unlock() + + if !s.opened { + return nil + } + + if s.gcCancel != nil { + s.gcCancel() + } + s.wg.Wait() + + if err := s.db.Close(); err != nil { + return err + } + s.opened = false + return nil +} + +// Init implements common.Storage. +func (s *Store) Init() error { + s.modeMtx.Lock() + defer s.modeMtx.Unlock() + + if !s.opened { + return errStoreMustBeOpenedBeforeInit + } + + s.startGC() + + return nil +} + +func (s *Store) startGC() { + ctx, cancel := context.WithCancel(context.Background()) + s.gcCancel = cancel + + t := time.NewTicker(s.cfg.gcTimeout) + s.wg.Add(1) + + go func() { + defer s.wg.Done() + + select { + case <-ctx.Done(): + return + case <-t.C: + if err := s.db.RunValueLogGC(s.cfg.gcDiscardRatio); err == nil { + _ = s.db.RunValueLogGC(s.cfg.gcDiscardRatio) // see https://dgraph.io/docs/badger/get-started/#garbage-collection + } + } + }() +} + +// Open implements common.Storage. +func (s *Store) Open(readOnly bool) error { + s.modeMtx.Lock() + defer s.modeMtx.Unlock() + + if s.opened { + return nil + } + + err := util.MkdirAllX(s.cfg.db.Dir, s.cfg.permissions) + if err != nil { + return err + } + s.cfg.db.ReadOnly = readOnly + if s.db, err = badger.Open(s.cfg.db); err != nil { + return err + } + s.opened = true + return nil +} + +func (s *Store) readOnly() bool { + return s.cfg.db.ReadOnly +} diff --git a/pkg/local_object_storage/blobstor/badgerstore/delete.go b/pkg/local_object_storage/blobstor/badgerstore/delete.go new file mode 100644 index 000000000..0bad88117 --- /dev/null +++ b/pkg/local_object_storage/blobstor/badgerstore/delete.go @@ -0,0 +1,44 @@ +package badgerstore + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + "github.com/dgraph-io/badger/v4" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// Delete implements common.Storage. +func (s *Store) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.Delete", + trace.WithAttributes( + attribute.String("path", s.cfg.db.Dir), + attribute.String("address", prm.Address.EncodeToString()), + )) + defer span.End() + + if s.readOnly() { + return common.DeleteRes{}, common.ErrReadOnly + } + + tx := s.db.NewTransaction(true) + defer tx.Discard() + + _, err := tx.Get(key(prm.Address)) + if err != nil { + if err == badger.ErrKeyNotFound { + return common.DeleteRes{}, logicerr.Wrap(new(apistatus.ObjectNotFound)) + } + return common.DeleteRes{}, err + } + + err = tx.Delete(key(prm.Address)) + if err != nil { + return common.DeleteRes{}, err + } + return common.DeleteRes{}, tx.Commit() +} diff --git a/pkg/local_object_storage/blobstor/badgerstore/exists.go b/pkg/local_object_storage/blobstor/badgerstore/exists.go new file mode 100644 index 000000000..5bf879f82 --- /dev/null +++ b/pkg/local_object_storage/blobstor/badgerstore/exists.go @@ -0,0 +1,36 @@ +package badgerstore + +import ( + "context" + "encoding/hex" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "github.com/dgraph-io/badger/v4" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// Exists implements common.Storage. +func (s *Store) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.Exists", + trace.WithAttributes( + attribute.String("path", s.cfg.db.Dir), + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + )) + defer span.End() + + tx := s.db.NewTransaction(false) + defer tx.Discard() + + _, err := tx.Get(key(prm.Address)) + if err != nil { + if err == badger.ErrKeyNotFound { + return common.ExistsRes{Exists: false}, nil + } + return common.ExistsRes{}, err + } + + return common.ExistsRes{Exists: true}, nil +} diff --git a/pkg/local_object_storage/blobstor/badgerstore/generic_test.go b/pkg/local_object_storage/blobstor/badgerstore/generic_test.go new file mode 100644 index 000000000..d34981471 --- /dev/null +++ b/pkg/local_object_storage/blobstor/badgerstore/generic_test.go @@ -0,0 +1,39 @@ +package badgerstore + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest" +) + +func TestGeneric(t *testing.T) { + const maxObjectSize = 1 << 16 + + helper := func(t *testing.T, dir string) common.Storage { + return New(WithPath(dir)) + } + + newStore := func(t *testing.T) common.Storage { + return helper(t, t.TempDir()) + } + + blobstortest.TestAll(t, newStore, 1024, maxObjectSize) + + t.Run("info", func(t *testing.T) { + dir := t.TempDir() + blobstortest.TestInfo(t, func(t *testing.T) common.Storage { + return helper(t, dir) + }, Type, dir) + }) +} + +func TestControl(t *testing.T) { + const maxObjectSize = 2048 + + newStore := func(t *testing.T) common.Storage { + return New(WithPath(t.TempDir())) + } + + blobstortest.TestControl(t, newStore, 1024, maxObjectSize) +} diff --git a/pkg/local_object_storage/blobstor/badgerstore/get.go b/pkg/local_object_storage/blobstor/badgerstore/get.go new file mode 100644 index 000000000..ba028e3c2 --- /dev/null +++ b/pkg/local_object_storage/blobstor/badgerstore/get.go @@ -0,0 +1,107 @@ +package badgerstore + +import ( + "context" + "encoding/hex" + "fmt" + "strconv" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + "github.com/dgraph-io/badger/v4" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// Get implements common.Storage. +func (s *Store) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.Get", + trace.WithAttributes( + attribute.String("path", s.cfg.db.Dir), + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + attribute.Bool("raw", prm.Raw), + )) + defer span.End() + + data, err := s.getObjectData(prm.Address) + if err != nil { + return common.GetRes{}, err + } + + data, err = s.cfg.compression.Decompress(data) + if err != nil { + return common.GetRes{}, fmt.Errorf("could not decompress object data: %w", err) + } + + obj := objectSDK.New() + if err := obj.Unmarshal(data); err != nil { + return common.GetRes{}, fmt.Errorf("could not unmarshal the object: %w", err) + } + + return common.GetRes{Object: obj, RawData: data}, nil +} + +// GetRange implements common.Storage. +func (s *Store) GetRange(ctx context.Context, prm common.GetRangePrm) (common.GetRangeRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.GetRange", + trace.WithAttributes( + attribute.String("path", s.cfg.db.Dir), + attribute.String("address", prm.Address.EncodeToString()), + attribute.String("storage_id", hex.EncodeToString(prm.StorageID)), + attribute.String("offset", strconv.FormatUint(prm.Range.GetOffset(), 10)), + attribute.String("length", strconv.FormatUint(prm.Range.GetLength(), 10)), + )) + defer span.End() + + data, err := s.getObjectData(prm.Address) + if err != nil { + return common.GetRangeRes{}, err + } + + data, err = s.cfg.compression.Decompress(data) + if err != nil { + return common.GetRangeRes{}, fmt.Errorf("could not decompress object data: %w", err) + } + + obj := objectSDK.New() + if err := obj.Unmarshal(data); err != nil { + return common.GetRangeRes{}, fmt.Errorf("could not unmarshal the object: %w", err) + } + + from := prm.Range.GetOffset() + to := from + prm.Range.GetLength() + payload := obj.Payload() + + if pLen := uint64(len(payload)); to < from || pLen < from || pLen < to { + return common.GetRangeRes{}, logicerr.Wrap(new(apistatus.ObjectOutOfRange)) + } + + return common.GetRangeRes{ + Data: payload[from:to], + }, nil +} + +func (s *Store) getObjectData(addr oid.Address) ([]byte, error) { + var data []byte + tx := s.db.NewTransaction(false) + defer tx.Discard() + + item, err := tx.Get(key(addr)) + if err != nil { + if err == badger.ErrKeyNotFound { + return nil, logicerr.Wrap(new(apistatus.ObjectNotFound)) + } + return nil, err + } + + data, err = item.ValueCopy(nil) + if err != nil { + return nil, err + } + return data, nil +} diff --git a/pkg/local_object_storage/blobstor/badgerstore/iterate.go b/pkg/local_object_storage/blobstor/badgerstore/iterate.go new file mode 100644 index 000000000..8a1a1570d --- /dev/null +++ b/pkg/local_object_storage/blobstor/badgerstore/iterate.go @@ -0,0 +1,113 @@ +package badgerstore + +import ( + "bytes" + "context" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "github.com/dgraph-io/badger/v4" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// Iterate implements common.Storage. +func (s *Store) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.Iterate", + trace.WithAttributes( + attribute.String("path", s.cfg.db.Dir), + attribute.Bool("ignore_errors", prm.IgnoreErrors), + )) + defer span.End() + + var last []byte + opts := badger.DefaultIteratorOptions + batch := make([]keyValue, 0, opts.PrefetchSize) + opts.PrefetchSize++ // to skip last + for { + select { + case <-ctx.Done(): + return common.IterateRes{}, ctx.Err() + default: + } + + batch = batch[:0] + err := s.db.View(func(tx *badger.Txn) error { + it := tx.NewIterator(opts) + defer it.Close() + + for it.Seek(last); it.Valid(); it.Next() { + if bytes.Equal(last, it.Item().Key()) { + continue + } + + var kv keyValue + var err error + kv.key = it.Item().KeyCopy(nil) + kv.value, err = it.Item().ValueCopy(nil) + if err != nil { + if prm.IgnoreErrors { + continue + } + return err + } + batch = append(batch, kv) + last = kv.key + if len(batch) == opts.PrefetchSize-1 { + break + } + } + return nil + }) + if err != nil { + return common.IterateRes{}, err + } + + select { + case <-ctx.Done(): + return common.IterateRes{}, ctx.Err() + default: + } + + if len(batch) == 0 { + break + } + if err := s.iterateBatch(batch, prm); err != nil { + return common.IterateRes{}, err + } + } + + return common.IterateRes{}, nil +} + +func (s *Store) iterateBatch(batch []keyValue, prm common.IteratePrm) error { + for _, kv := range batch { + addr, err := address(kv.key) + if err != nil { + if prm.IgnoreErrors { + continue + } + } + data, err := s.cfg.compression.Decompress(kv.value) + if err != nil { + if prm.IgnoreErrors { + continue + } + return fmt.Errorf("could not decompress object data: %w", err) + } + + if err := prm.Handler(common.IterationElement{ + Address: addr, + ObjectData: data, + StorageID: defaultStorageID, + }); err != nil { + return err + } + } + return nil +} + +type keyValue struct { + key, value []byte +} diff --git a/pkg/local_object_storage/blobstor/badgerstore/keys.go b/pkg/local_object_storage/blobstor/badgerstore/keys.go new file mode 100644 index 000000000..6a5869d43 --- /dev/null +++ b/pkg/local_object_storage/blobstor/badgerstore/keys.go @@ -0,0 +1,33 @@ +package badgerstore + +import ( + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +const ( + keyLength = 64 + objectIDOffset = 32 +) + +func key(add oid.Address) []byte { + res := make([]byte, keyLength) + add.Container().Encode(res) + add.Object().Encode(res[objectIDOffset:]) + return res +} + +func address(k []byte) (oid.Address, error) { + var res oid.Address + var containerID cid.ID + var objectID oid.ID + if err := containerID.Decode(k[:objectIDOffset]); err != nil { + return res, err + } + if err := objectID.Decode(k[objectIDOffset:]); err != nil { + return res, err + } + res.SetContainer(containerID) + res.SetObject(objectID) + return res, nil +} diff --git a/pkg/local_object_storage/blobstor/badgerstore/put.go b/pkg/local_object_storage/blobstor/badgerstore/put.go new file mode 100644 index 000000000..213f3a4f1 --- /dev/null +++ b/pkg/local_object_storage/blobstor/badgerstore/put.go @@ -0,0 +1,40 @@ +package badgerstore + +import ( + "context" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +var defaultStorageID = []byte("badger") + +// Put implements common.Storage. +func (s *Store) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.Put", + trace.WithAttributes( + attribute.String("path", s.cfg.db.Dir), + attribute.String("address", prm.Address.EncodeToString()), + attribute.Bool("dont_compress", prm.DontCompress), + )) + defer span.End() + + if s.readOnly() { + return common.PutRes{}, common.ErrReadOnly + } + + if !prm.DontCompress { + prm.RawData = s.cfg.compression.Compress(prm.RawData) + } + + tx := s.db.NewTransaction(true) + defer tx.Discard() + + err := tx.Set(key(prm.Address), prm.RawData) + if err != nil { + return common.PutRes{}, err + } + return common.PutRes{StorageID: defaultStorageID}, tx.Commit() +} diff --git a/pkg/local_object_storage/blobstor/badgerstore/store.go b/pkg/local_object_storage/blobstor/badgerstore/store.go new file mode 100644 index 000000000..7c1040bf5 --- /dev/null +++ b/pkg/local_object_storage/blobstor/badgerstore/store.go @@ -0,0 +1,71 @@ +package badgerstore + +import ( + "context" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "github.com/dgraph-io/badger/v4" +) + +const ( + Type = "badgerstore" +) + +var _ common.Storage = (*Store)(nil) + +type Store struct { + cfg *cfg + db *badger.DB + + modeMtx *sync.Mutex // protects fields in group below + opened bool + gcCancel context.CancelFunc + + wg *sync.WaitGroup +} + +// New returns new Store instance with opts applied. +func New(opts ...Option) *Store { + s := &Store{ + cfg: defaultCfg(), + modeMtx: &sync.Mutex{}, + wg: &sync.WaitGroup{}, + } + for _, opt := range opts { + opt(s.cfg) + } + return s +} + +// Compressor implements common.Storage. +func (s *Store) Compressor() *compression.Config { + return s.cfg.compression +} + +// Path implements common.Storage. +func (s *Store) Path() string { + return s.cfg.db.Dir +} + +// SetCompressor implements common.Storage. +func (s *Store) SetCompressor(cc *compression.Config) { + s.cfg.compression = cc +} + +// SetParentID implements common.Storage. +func (*Store) SetParentID(parentID string) {} + +// SetReportErrorFunc implements common.Storage. +func (*Store) SetReportErrorFunc(func(string, error)) {} + +// Type implements common.Storage. +func (*Store) Type() string { + return Type +} + +// Rebuild implements common.Storage. +func (*Store) Rebuild(context.Context, common.RebuildPrm) (common.RebuildRes, error) { + return common.RebuildRes{}, nil +} diff --git a/pkg/local_object_storage/blobstor/perf_test.go b/pkg/local_object_storage/blobstor/perf_test.go index 72079acba..bbee6f58a 100644 --- a/pkg/local_object_storage/blobstor/perf_test.go +++ b/pkg/local_object_storage/blobstor/perf_test.go @@ -5,6 +5,7 @@ import ( "fmt" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/badgerstore" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" @@ -77,6 +78,14 @@ var storages = []storage{ ) }, }, + { + desc: "badger", + create: func(dir string) common.Storage { + return badgerstore.New( + badgerstore.WithPath(dir), + ) + }, + }, } func BenchmarkSubstorageReadPerf(b *testing.B) { -- 2.45.3 From 13cda5de8e45993dcfe52c081cfbb1ac14053632 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 26 Oct 2023 14:31:15 +0300 Subject: [PATCH 2/7] [#645] metrics: Add badgerstore metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 5 + .../blobstor/badgerstore/config.go | 9 ++ .../blobstor/badgerstore/control.go | 2 + .../blobstor/badgerstore/delete.go | 17 +++- .../blobstor/badgerstore/exists.go | 10 ++ .../blobstor/badgerstore/get.go | 26 ++++- .../blobstor/badgerstore/iterate.go | 9 ++ .../blobstor/badgerstore/metrics.go | 31 ++++++ .../blobstor/badgerstore/put.go | 9 ++ .../blobstor/badgerstore/store.go | 4 +- .../metrics/badgerstore.go | 74 ++++++++++++++ pkg/metrics/badgerstore.go | 98 +++++++++++++++++++ pkg/metrics/consts.go | 1 + pkg/metrics/node.go | 6 ++ 14 files changed, 294 insertions(+), 7 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/badgerstore/metrics.go create mode 100644 pkg/local_object_storage/metrics/badgerstore.go create mode 100644 pkg/metrics/badgerstore.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 7a3b6aa10..ea521f26b 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -970,6 +970,11 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { badgerstore.WithMemTablesCount(sRead.memTablesCount), badgerstore.WithValueLogSize(sRead.valueLogFileSize), } + if c.metricsCollector != nil { + badgerStoreOpts = append(badgerStoreOpts, + badgerstore.WithMetrics( + lsmetrics.NewBadgerStoreMetrics(sRead.path, c.metricsCollector.BadgerStoreMetrics()))) + } ss = append(ss, blobstor.SubStorage{ Storage: badgerstore.New(badgerStoreOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { diff --git a/pkg/local_object_storage/blobstor/badgerstore/config.go b/pkg/local_object_storage/blobstor/badgerstore/config.go index 72a9f51bb..886d8024d 100644 --- a/pkg/local_object_storage/blobstor/badgerstore/config.go +++ b/pkg/local_object_storage/blobstor/badgerstore/config.go @@ -16,6 +16,7 @@ type cfg struct { db badger.Options gcTimeout time.Duration gcDiscardRatio float64 + metrics Metrics } type Option func(*cfg) @@ -56,6 +57,7 @@ func defaultCfg() *cfg { db: opts, gcTimeout: 10 * time.Minute, gcDiscardRatio: 0.2, // for 1GB vLog file GC will perform only if around 200MB could be free + metrics: &noopMetrics{}, } } @@ -115,3 +117,10 @@ func WithValueLogSize(sz int64) Option { c.db.ValueLogFileSize = sz } } + +// WithMetrics sets metrics. +func WithMetrics(m Metrics) Option { + return func(c *cfg) { + c.metrics = m + } +} diff --git a/pkg/local_object_storage/blobstor/badgerstore/control.go b/pkg/local_object_storage/blobstor/badgerstore/control.go index 2b808d97b..815fa6a52 100644 --- a/pkg/local_object_storage/blobstor/badgerstore/control.go +++ b/pkg/local_object_storage/blobstor/badgerstore/control.go @@ -29,6 +29,7 @@ func (s *Store) Close() error { return err } s.opened = false + s.cfg.metrics.Close() return nil } @@ -85,6 +86,7 @@ func (s *Store) Open(readOnly bool) error { return err } s.opened = true + s.cfg.metrics.SetMode(readOnly) return nil } diff --git a/pkg/local_object_storage/blobstor/badgerstore/delete.go b/pkg/local_object_storage/blobstor/badgerstore/delete.go index 0bad88117..34fcf986f 100644 --- a/pkg/local_object_storage/blobstor/badgerstore/delete.go +++ b/pkg/local_object_storage/blobstor/badgerstore/delete.go @@ -2,6 +2,7 @@ package badgerstore import ( "context" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -14,6 +15,13 @@ import ( // Delete implements common.Storage. func (s *Store) Delete(ctx context.Context, prm common.DeletePrm) (common.DeleteRes, error) { + success := false + startedAt := time.Now() + + defer func() { + s.cfg.metrics.Delete(time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.Delete", trace.WithAttributes( attribute.String("path", s.cfg.db.Dir), @@ -36,9 +44,12 @@ func (s *Store) Delete(ctx context.Context, prm common.DeletePrm) (common.Delete return common.DeleteRes{}, err } - err = tx.Delete(key(prm.Address)) - if err != nil { + if err = tx.Delete(key(prm.Address)); err != nil { return common.DeleteRes{}, err } - return common.DeleteRes{}, tx.Commit() + if err = tx.Commit(); err != nil { + return common.DeleteRes{}, err + } + success = true + return common.DeleteRes{}, nil } diff --git a/pkg/local_object_storage/blobstor/badgerstore/exists.go b/pkg/local_object_storage/blobstor/badgerstore/exists.go index 5bf879f82..12f85801f 100644 --- a/pkg/local_object_storage/blobstor/badgerstore/exists.go +++ b/pkg/local_object_storage/blobstor/badgerstore/exists.go @@ -3,6 +3,7 @@ package badgerstore import ( "context" "encoding/hex" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -13,6 +14,13 @@ import ( // Exists implements common.Storage. func (s *Store) Exists(ctx context.Context, prm common.ExistsPrm) (common.ExistsRes, error) { + success := false + startedAt := time.Now() + + defer func() { + s.cfg.metrics.Exists(time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.Exists", trace.WithAttributes( attribute.String("path", s.cfg.db.Dir), @@ -27,10 +35,12 @@ func (s *Store) Exists(ctx context.Context, prm common.ExistsPrm) (common.Exists _, err := tx.Get(key(prm.Address)) if err != nil { if err == badger.ErrKeyNotFound { + success = true return common.ExistsRes{Exists: false}, nil } return common.ExistsRes{}, err } + success = true return common.ExistsRes{Exists: true}, nil } diff --git a/pkg/local_object_storage/blobstor/badgerstore/get.go b/pkg/local_object_storage/blobstor/badgerstore/get.go index ba028e3c2..c56ec86c0 100644 --- a/pkg/local_object_storage/blobstor/badgerstore/get.go +++ b/pkg/local_object_storage/blobstor/badgerstore/get.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "strconv" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr" @@ -19,6 +20,14 @@ import ( // Get implements common.Storage. func (s *Store) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, error) { + success := false + size := 0 + startedAt := time.Now() + + defer func() { + s.cfg.metrics.Get(time.Since(startedAt), size, success) + }() + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.Get", trace.WithAttributes( attribute.String("path", s.cfg.db.Dir), @@ -43,11 +52,21 @@ func (s *Store) Get(ctx context.Context, prm common.GetPrm) (common.GetRes, erro return common.GetRes{}, fmt.Errorf("could not unmarshal the object: %w", err) } + success = true + size = len(data) return common.GetRes{Object: obj, RawData: data}, nil } // GetRange implements common.Storage. func (s *Store) GetRange(ctx context.Context, prm common.GetRangePrm) (common.GetRangeRes, error) { + success := false + size := 0 + startedAt := time.Now() + + defer func() { + s.cfg.metrics.GetRange(time.Since(startedAt), size, success) + }() + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.GetRange", trace.WithAttributes( attribute.String("path", s.cfg.db.Dir), @@ -81,9 +100,10 @@ func (s *Store) GetRange(ctx context.Context, prm common.GetRangePrm) (common.Ge return common.GetRangeRes{}, logicerr.Wrap(new(apistatus.ObjectOutOfRange)) } - return common.GetRangeRes{ - Data: payload[from:to], - }, nil + res := common.GetRangeRes{Data: payload[from:to]} + success = true + size = len(res.Data) + return res, nil } func (s *Store) getObjectData(addr oid.Address) ([]byte, error) { diff --git a/pkg/local_object_storage/blobstor/badgerstore/iterate.go b/pkg/local_object_storage/blobstor/badgerstore/iterate.go index 8a1a1570d..476f79634 100644 --- a/pkg/local_object_storage/blobstor/badgerstore/iterate.go +++ b/pkg/local_object_storage/blobstor/badgerstore/iterate.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -14,6 +15,13 @@ import ( // Iterate implements common.Storage. func (s *Store) Iterate(ctx context.Context, prm common.IteratePrm) (common.IterateRes, error) { + success := false + startedAt := time.Now() + + defer func() { + s.cfg.metrics.Iterate(time.Since(startedAt), success) + }() + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.Iterate", trace.WithAttributes( attribute.String("path", s.cfg.db.Dir), @@ -78,6 +86,7 @@ func (s *Store) Iterate(ctx context.Context, prm common.IteratePrm) (common.Iter } } + success = true return common.IterateRes{}, nil } diff --git a/pkg/local_object_storage/blobstor/badgerstore/metrics.go b/pkg/local_object_storage/blobstor/badgerstore/metrics.go new file mode 100644 index 000000000..051b087f0 --- /dev/null +++ b/pkg/local_object_storage/blobstor/badgerstore/metrics.go @@ -0,0 +1,31 @@ +package badgerstore + +import "time" + +type Metrics interface { + SetParentID(parentID string) + + SetMode(readOnly bool) + Close() + + Delete(d time.Duration, success bool) + Exists(d time.Duration, success bool) + GetRange(d time.Duration, size int, success bool) + Get(d time.Duration, size int, success bool) + Iterate(d time.Duration, success bool) + Put(d time.Duration, size int, success bool) +} + +var _ Metrics = (*noopMetrics)(nil) + +type noopMetrics struct{} + +func (*noopMetrics) Close() {} +func (*noopMetrics) Delete(time.Duration, bool) {} +func (*noopMetrics) Exists(time.Duration, bool) {} +func (*noopMetrics) Get(time.Duration, int, bool) {} +func (*noopMetrics) GetRange(time.Duration, int, bool) {} +func (*noopMetrics) Iterate(time.Duration, bool) {} +func (*noopMetrics) Put(time.Duration, int, bool) {} +func (*noopMetrics) SetMode(bool) {} +func (*noopMetrics) SetParentID(string) {} diff --git a/pkg/local_object_storage/blobstor/badgerstore/put.go b/pkg/local_object_storage/blobstor/badgerstore/put.go index 213f3a4f1..fa8147e0a 100644 --- a/pkg/local_object_storage/blobstor/badgerstore/put.go +++ b/pkg/local_object_storage/blobstor/badgerstore/put.go @@ -2,6 +2,7 @@ package badgerstore import ( "context" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common" "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" @@ -13,6 +14,14 @@ var defaultStorageID = []byte("badger") // Put implements common.Storage. func (s *Store) Put(ctx context.Context, prm common.PutPrm) (common.PutRes, error) { + success := false + size := 0 + startedAt := time.Now() + + defer func() { + s.cfg.metrics.Put(time.Since(startedAt), size, success) + }() + _, span := tracing.StartSpanFromContext(ctx, "BadgerStore.Put", trace.WithAttributes( attribute.String("path", s.cfg.db.Dir), diff --git a/pkg/local_object_storage/blobstor/badgerstore/store.go b/pkg/local_object_storage/blobstor/badgerstore/store.go index 7c1040bf5..372d3d498 100644 --- a/pkg/local_object_storage/blobstor/badgerstore/store.go +++ b/pkg/local_object_storage/blobstor/badgerstore/store.go @@ -55,7 +55,9 @@ func (s *Store) SetCompressor(cc *compression.Config) { } // SetParentID implements common.Storage. -func (*Store) SetParentID(parentID string) {} +func (s *Store) SetParentID(parentID string) { + s.cfg.metrics.SetParentID(parentID) +} // SetReportErrorFunc implements common.Storage. func (*Store) SetReportErrorFunc(func(string, error)) {} diff --git a/pkg/local_object_storage/metrics/badgerstore.go b/pkg/local_object_storage/metrics/badgerstore.go new file mode 100644 index 000000000..41a8111ac --- /dev/null +++ b/pkg/local_object_storage/metrics/badgerstore.go @@ -0,0 +1,74 @@ +package metrics + +import ( + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/badgerstore" + metrics_impl "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" +) + +func NewBadgerStoreMetrics(path string, m metrics_impl.BadgerStoreMetrics) badgerstore.Metrics { + return &badgerStoreMetrics{ + path: path, + m: m, + } +} + +type badgerStoreMetrics struct { + path, shardID string + m metrics_impl.BadgerStoreMetrics +} + +// Close implements badgerstore.Metrics. +func (m *badgerStoreMetrics) Close() { + m.m.Close(m.shardID, m.path) +} + +// Delete implements badgerstore.Metrics. +func (m *badgerStoreMetrics) Delete(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Delete", d, success) +} + +// Exists implements badgerstore.Metrics. +func (m *badgerStoreMetrics) Exists(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Exists", d, success) +} + +// Get implements badgerstore.Metrics. +func (m *badgerStoreMetrics) Get(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Get", d, success) + if success { + m.m.AddGet(m.shardID, m.path, size) + } +} + +// GetRange implements badgerstore.Metrics. +func (m *badgerStoreMetrics) GetRange(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "GetRange", d, success) + if success { + m.m.AddGet(m.shardID, m.path, size) + } +} + +// Iterate implements badgerstore.Metrics. +func (m *badgerStoreMetrics) Iterate(d time.Duration, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Iterate", d, success) +} + +// Put implements badgerstore.Metrics. +func (m *badgerStoreMetrics) Put(d time.Duration, size int, success bool) { + m.m.MethodDuration(m.shardID, m.path, "Put", d, success) + if success { + m.m.AddPut(m.shardID, m.path, size) + } +} + +// SetMode implements badgerstore.Metrics. +func (m *badgerStoreMetrics) SetMode(readOnly bool) { + m.m.SetMode(m.shardID, m.path, readOnly) +} + +// SetParentID implements badgerstore.Metrics. +func (m *badgerStoreMetrics) SetParentID(parentID string) { + m.shardID = parentID +} diff --git a/pkg/metrics/badgerstore.go b/pkg/metrics/badgerstore.go new file mode 100644 index 000000000..4f18f58e9 --- /dev/null +++ b/pkg/metrics/badgerstore.go @@ -0,0 +1,98 @@ +package metrics + +import ( + "strconv" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type BadgerStoreMetrics interface { + SetMode(shardID, path string, readOnly bool) + Close(shardID, path string) + MethodDuration(shardID, path string, method string, d time.Duration, success bool) + AddPut(shardID, path string, size int) + AddGet(shardID, path string, size int) +} + +var _ BadgerStoreMetrics = (*badgerStoreMetrics)(nil) + +type badgerStoreMetrics struct { + mode *shardIDPathModeValue + reqDuration *prometheus.HistogramVec + put *prometheus.CounterVec + get *prometheus.CounterVec +} + +func newbadgerStoreMetrics() *badgerStoreMetrics { + return &badgerStoreMetrics{ + mode: newShardIDPathMode(badgerStoreSubSystem, "mode", "BadgerStore mode"), + reqDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: badgerStoreSubSystem, + Name: "request_duration_seconds", + Help: "Accumulated BadgerStore request process duration", + }, []string{shardIDLabel, pathLabel, successLabel, methodLabel}), + put: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: badgerStoreSubSystem, + Name: "put_bytes", + Help: "Accumulated payload size written to BadgerStore", + }, []string{shardIDLabel, pathLabel}), + get: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: badgerStoreSubSystem, + Name: "get_bytes", + Help: "Accumulated payload size read from BadgerStore", + }, []string{shardIDLabel, pathLabel}), + } +} + +// AddGet implements BadgerStoreMetrics. +func (b *badgerStoreMetrics) AddGet(shardID string, path string, size int) { + b.get.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +// AddPut implements BadgerStoreMetrics. +func (b *badgerStoreMetrics) AddPut(shardID string, path string, size int) { + b.put.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }).Add(float64(size)) +} + +// Close implements BadgerStoreMetrics. +func (b *badgerStoreMetrics) Close(shardID string, path string) { + b.mode.SetMode(shardID, path, closedMode) + b.reqDuration.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + b.get.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) + b.put.DeletePartialMatch(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + }) +} + +// MethodDuration implements BadgerStoreMetrics. +func (b *badgerStoreMetrics) MethodDuration(shardID string, path string, method string, d time.Duration, success bool) { + b.reqDuration.With(prometheus.Labels{ + shardIDLabel: shardID, + pathLabel: path, + successLabel: strconv.FormatBool(success), + methodLabel: method, + }).Observe(d.Seconds()) +} + +// SetMode implements BadgerStoreMetrics. +func (b *badgerStoreMetrics) SetMode(shardID string, path string, readOnly bool) { + b.mode.SetMode(shardID, path, modeFromBool(readOnly)) +} diff --git a/pkg/metrics/consts.go b/pkg/metrics/consts.go index f3b83a2e5..054368208 100644 --- a/pkg/metrics/consts.go +++ b/pkg/metrics/consts.go @@ -21,6 +21,7 @@ const ( writeCacheSubsystem = "writecache" grpcServerSubsystem = "grpc_server" policerSubsystem = "policer" + badgerStoreSubSystem = "badgerstore" successLabel = "success" shardIDLabel = "shard_id" diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 0dd86d90e..6a275d539 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -20,6 +20,7 @@ type NodeMetrics struct { metabase *metabaseMetrics pilorama *piloramaMetrics grpc *grpcServerMetrics + badgerStore *badgerStoreMetrics policer *policerMetrics morphClient *morphClientMetrics morphCache *morphCacheMetrics @@ -49,6 +50,7 @@ func NewNodeMetrics() *NodeMetrics { morphClient: newMorphClientMetrics(), morphCache: newMorphCacheMetrics(namespace), log: logger.NewLogMetrics(namespace), + badgerStore: newbadgerStoreMetrics(), } } @@ -116,3 +118,7 @@ func (m *NodeMetrics) MorphCacheMetrics() MorphCacheMetrics { func (m *NodeMetrics) LogMetrics() logger.LogMetrics { return m.log } + +func (m *NodeMetrics) BadgerStoreMetrics() BadgerStoreMetrics { + return m.badgerStore +} -- 2.45.3 From 9c10844eb0d8b9bc90e9e53f44ca4978472adfee Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Fri, 27 Oct 2023 17:52:22 +0300 Subject: [PATCH 3/7] [#645] config: Resolve funlen linter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 147 ++++++++++++++++++++----------------- 1 file changed, 80 insertions(+), 67 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index ea521f26b..b04409e7c 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -185,21 +185,21 @@ type subStorageCfg struct { noSync bool // blobovnicza-specific - size uint64 - width uint64 - leafWidth uint64 - openedCacheSize int - initWorkerCount int - initInAdvance bool - - // badgerstore-specific - indexCacheSize int64 - memTablesCount int - compactorsCount int - gcInterval time.Duration - gcDiscardRatio float64 - valueLogFileSize int64 + size uint64 + width uint64 + leafWidth uint64 + openedCacheSize int + initWorkerCount int + initInAdvance bool rebuildDropTimeout time.Duration + + // badgerstore-specific + indexCacheSize int64 + memTablesCount int + compactorsCount int + gcInterval time.Duration + gcDiscardRatio float64 + valueLogFileSize int64 } // readConfig fills applicationConfiguration with raw configuration values @@ -909,50 +909,15 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { for _, sRead := range shCfg.subStorages { switch sRead.typ { case blobovniczatree.Type: - blobTreeOpts := []blobovniczatree.Option{ - blobovniczatree.WithRootPath(sRead.path), - blobovniczatree.WithPermissions(sRead.perm), - blobovniczatree.WithBlobovniczaSize(sRead.size), - blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), - blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), - blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), - blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), - blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), - blobovniczatree.WithInitInAdvance(sRead.initInAdvance), - blobovniczatree.WithWaitBeforeDropDB(sRead.rebuildDropTimeout), - blobovniczatree.WithLogger(c.log), - blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit), - } - - if c.metricsCollector != nil { - blobTreeOpts = append(blobTreeOpts, - blobovniczatree.WithMetrics( - lsmetrics.NewBlobovniczaTreeMetrics(sRead.path, c.metricsCollector.BlobobvnizcaTreeMetrics()), - ), - ) - } + blobovniczaTreeOpts := c.getBlobovniczaTreeOpts(sRead) ss = append(ss, blobstor.SubStorage{ - Storage: blobovniczatree.NewBlobovniczaTree(blobTreeOpts...), + Storage: blobovniczatree.NewBlobovniczaTree(blobovniczaTreeOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < shCfg.smallSizeObjectLimit }, }) case fstree.Type: - fstreeOpts := []fstree.Option{ - fstree.WithPath(sRead.path), - fstree.WithPerm(sRead.perm), - fstree.WithDepth(sRead.depth), - fstree.WithNoSync(sRead.noSync), - fstree.WithLogger(c.log), - } - if c.metricsCollector != nil { - fstreeOpts = append(fstreeOpts, - fstree.WithMetrics( - lsmetrics.NewFSTreeMetricsWithoutShardID(sRead.path, c.metricsCollector.FSTree()), - ), - ) - } - + fstreeOpts := c.getFSTreeOpts(sRead) ss = append(ss, blobstor.SubStorage{ Storage: fstree.New(fstreeOpts...), Policy: func(_ *objectSDK.Object, _ []byte) bool { @@ -960,21 +925,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { }, }) case badgerstore.Type: - badgerStoreOpts := []badgerstore.Option{ - badgerstore.WithPath(sRead.path), - badgerstore.WithPermissions(sRead.perm), - badgerstore.WithCompactorsCount(sRead.compactorsCount), - badgerstore.WithGCDiscardRatio(sRead.gcDiscardRatio), - badgerstore.WithGCInterval(sRead.gcInterval), - badgerstore.WithIndexCacheSize(sRead.indexCacheSize), - badgerstore.WithMemTablesCount(sRead.memTablesCount), - badgerstore.WithValueLogSize(sRead.valueLogFileSize), - } - if c.metricsCollector != nil { - badgerStoreOpts = append(badgerStoreOpts, - badgerstore.WithMetrics( - lsmetrics.NewBadgerStoreMetrics(sRead.path, c.metricsCollector.BadgerStoreMetrics()))) - } + badgerStoreOpts := c.getBadgerStoreOpts(sRead) ss = append(ss, blobstor.SubStorage{ Storage: badgerstore.New(badgerStoreOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { @@ -989,6 +940,68 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { return ss } +func (c *cfg) getBadgerStoreOpts(sRead subStorageCfg) []badgerstore.Option { + badgerStoreOpts := []badgerstore.Option{ + badgerstore.WithPath(sRead.path), + badgerstore.WithPermissions(sRead.perm), + badgerstore.WithCompactorsCount(sRead.compactorsCount), + badgerstore.WithGCDiscardRatio(sRead.gcDiscardRatio), + badgerstore.WithGCInterval(sRead.gcInterval), + badgerstore.WithIndexCacheSize(sRead.indexCacheSize), + badgerstore.WithMemTablesCount(sRead.memTablesCount), + badgerstore.WithValueLogSize(sRead.valueLogFileSize), + } + if c.metricsCollector != nil { + badgerStoreOpts = append(badgerStoreOpts, + badgerstore.WithMetrics( + lsmetrics.NewBadgerStoreMetrics(sRead.path, c.metricsCollector.BadgerStoreMetrics()))) + } + return badgerStoreOpts +} + +func (c *cfg) getFSTreeOpts(sRead subStorageCfg) []fstree.Option { + fstreeOpts := []fstree.Option{ + fstree.WithPath(sRead.path), + fstree.WithPerm(sRead.perm), + fstree.WithDepth(sRead.depth), + fstree.WithNoSync(sRead.noSync), + fstree.WithLogger(c.log), + } + if c.metricsCollector != nil { + fstreeOpts = append(fstreeOpts, + fstree.WithMetrics( + lsmetrics.NewFSTreeMetricsWithoutShardID(sRead.path, c.metricsCollector.FSTree()), + ), + ) + } + return fstreeOpts +} + +func (c *cfg) getBlobovniczaTreeOpts(sRead subStorageCfg) []blobovniczatree.Option { + blobTreeOpts := []blobovniczatree.Option{ + blobovniczatree.WithRootPath(sRead.path), + blobovniczatree.WithPermissions(sRead.perm), + blobovniczatree.WithBlobovniczaSize(sRead.size), + blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), + blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), + blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), + blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), + blobovniczatree.WithInitInAdvance(sRead.initInAdvance), + blobovniczatree.WithLogger(c.log), + blobovniczatree.WithWaitBeforeDropDB(sRead.rebuildDropTimeout), + } + + if c.metricsCollector != nil { + blobTreeOpts = append(blobTreeOpts, + blobovniczatree.WithMetrics( + lsmetrics.NewBlobovniczaTreeMetrics(sRead.path, c.metricsCollector.BlobobvnizcaTreeMetrics()), + ), + ) + } + return blobTreeOpts +} + func (c *cfg) getShardOpts(shCfg shardCfg) shardOptsWithID { writeCacheOpts := c.getWriteCacheOpts(shCfg) piloramaOpts := c.getPiloramaOpts(shCfg) -- 2.45.3 From 597a8cb5d19e5003493d0b3a7a235fab457cf19b Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 30 Oct 2023 14:07:28 +0300 Subject: [PATCH 4/7] [#645] badgerstore: Add logger Signed-off-by: Dmitrii Stepanov --- internal/logs/logs.go | 1 + .../blobstor/badgerstore/config.go | 11 +++++++++++ .../blobstor/badgerstore/control.go | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 64e5f771e..b6b9792c2 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -526,6 +526,7 @@ const ( RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" FailedToCountWritecacheItems = "failed to count writecache items" AttemtToCloseAlreadyClosedBlobovnicza = "attempt to close an already closed blobovnicza" + BadgerStoreGCFailed = "failed to run GC on badgerstore" FailedToGetContainerCounters = "failed to get container counters values" FailedToRebuildBlobstore = "failed to rebuild blobstore" BlobstoreRebuildStarted = "blobstore rebuild started" diff --git a/pkg/local_object_storage/blobstor/badgerstore/config.go b/pkg/local_object_storage/blobstor/badgerstore/config.go index 886d8024d..21bdc1f91 100644 --- a/pkg/local_object_storage/blobstor/badgerstore/config.go +++ b/pkg/local_object_storage/blobstor/badgerstore/config.go @@ -6,8 +6,10 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/dgraph-io/badger/v4" "github.com/dgraph-io/badger/v4/options" + "go.uber.org/zap" ) type cfg struct { @@ -17,6 +19,7 @@ type cfg struct { gcTimeout time.Duration gcDiscardRatio float64 metrics Metrics + logger *logger.Logger } type Option func(*cfg) @@ -58,6 +61,7 @@ func defaultCfg() *cfg { gcTimeout: 10 * time.Minute, gcDiscardRatio: 0.2, // for 1GB vLog file GC will perform only if around 200MB could be free metrics: &noopMetrics{}, + logger: &logger.Logger{Logger: zap.L()}, } } @@ -124,3 +128,10 @@ func WithMetrics(m Metrics) Option { c.metrics = m } } + +// WithLogger sets logger. +func WithLogger(l *logger.Logger) Option { + return func(c *cfg) { + c.logger = l + } +} diff --git a/pkg/local_object_storage/blobstor/badgerstore/control.go b/pkg/local_object_storage/blobstor/badgerstore/control.go index 815fa6a52..9ebcc6aee 100644 --- a/pkg/local_object_storage/blobstor/badgerstore/control.go +++ b/pkg/local_object_storage/blobstor/badgerstore/control.go @@ -5,8 +5,10 @@ import ( "errors" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util" "github.com/dgraph-io/badger/v4" + "go.uber.org/zap" ) var errStoreMustBeOpenedBeforeInit = errors.New("store must be opened before initialization") @@ -63,6 +65,8 @@ func (s *Store) startGC() { case <-t.C: if err := s.db.RunValueLogGC(s.cfg.gcDiscardRatio); err == nil { _ = s.db.RunValueLogGC(s.cfg.gcDiscardRatio) // see https://dgraph.io/docs/badger/get-started/#garbage-collection + } else { + s.cfg.logger.Error(logs.BadgerStoreGCFailed, zap.Error(err), zap.String("path", s.cfg.db.Dir)) } } }() -- 2.45.3 From f0d72870dec9f995ecabafaa6d89c50ce466545e Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 27 Dec 2023 14:57:33 +0300 Subject: [PATCH 5/7] [#645] go.mod: Add badger Signed-off-by: Dmitrii Stepanov --- go.mod | 9 ++++++++ go.sum | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/go.mod b/go.mod index f940b306a..b5edc145c 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/cheggaaa/pb v1.0.29 github.com/chzyer/readline v1.5.1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc + github.com/dgraph-io/badger/v4 v4.2.0 github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 github.com/google/uuid v1.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 @@ -67,12 +68,18 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.1.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/flatbuffers v1.12.1 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.1 // indirect @@ -103,6 +110,7 @@ require ( github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20240122090917-ef99a7a9e33f // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.46.0 // indirect @@ -115,6 +123,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/twmb/murmur3 v1.1.8 // indirect github.com/urfave/cli v1.22.14 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0 // indirect diff --git a/go.sum b/go.sum index 7f7e84398..2d69b24a0 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= code.gitea.io/sdk/gitea v0.17.1 h1:3jCPOG2ojbl8AcfaUCRYLT5MUcBMFwS0OSK2mA5Zok8= code.gitea.io/sdk/gitea v0.17.1/go.mod h1:aCnBqhHpoEWA180gMbaCtdX9Pl6BWBAuuP2miadoTNM= git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240327095603-491a47e7fe24 h1:uIkl0mKWwDICUZTbNWZ38HLYDBI9rMgdAhYQWZ0C9iQ= @@ -20,6 +21,7 @@ git.frostfs.info/TrueCloudLab/tzhash v1.8.0 h1:UFMnUIk0Zh17m8rjGHJMqku2hCgaXDqjq git.frostfs.info/TrueCloudLab/tzhash v1.8.0/go.mod h1:dhY+oy274hV8wGvGL4MwwMpdL3GYvaX1a8GQZQHvlF8= git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02 h1:HeY8n27VyPRQe49l/fzyVMkWEB2fsLJYKp64pwA7tz4= git.frostfs.info/TrueCloudLab/zapjournald v0.0.0-20240124114243-cb2e66427d02/go.mod h1:rQFJJdEOV7KbbMtQYR2lNfiZk+ONRDJSbMCTWxKt8Fw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= @@ -29,6 +31,8 @@ github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJR github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= @@ -42,6 +46,8 @@ github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObk github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.12.2-0.20231222162921-eb75782795d2 h1:tYj5Ydh5D7Xg2R1tJnoG36Yta7NVB8C0vx36oPA3Bbw= @@ -57,6 +63,18 @@ github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454Wv github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BMXYYRWTLOJKlh+lOBt6nUQgXAfB7oVIQt5cNreqSLI= @@ -75,14 +93,25 @@ github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -90,14 +119,20 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -215,12 +250,14 @@ github.com/paulmach/orb v0.11.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/En github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= @@ -256,12 +293,14 @@ github.com/ssgreg/journald v1.0.0/go.mod h1:RUckwmTM8ghGWPslq2+ZBZzbb9/2KgjzYZ4J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -290,6 +329,8 @@ go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk= go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= @@ -320,19 +361,28 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -343,7 +393,9 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -353,6 +405,7 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -374,6 +427,7 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -396,6 +450,10 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -408,11 +466,21 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -420,7 +488,10 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= @@ -443,6 +514,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= -- 2.45.3 From 22f3b51f69c88708cbb129b0622442c65eb39af2 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 15 Feb 2024 12:32:21 +0300 Subject: [PATCH 6/7] [#645] config: Add `storage_engine` parameter for blobovnicza Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 91 +++++++++++-------- .../engine/shard/blobstor/storage/config.go | 7 ++ cmd/frostfs-node/validate.go | 10 +- docs/storage-node-configuration.md | 42 ++++++--- 4 files changed, 98 insertions(+), 52 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index b04409e7c..2fd9d37e9 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -90,6 +90,15 @@ const maxMsgSize = 4 << 20 // transport msg limit 4 MiB // for each contract listener. const notificationHandlerPoolSize = 10 +const ( + storageTypeBlobovnicza = "blobovnicza" + storageTypeFStree = "fstree" + + storageEngineUnspecified = "" + storageEngineBBolt = "bbolt" + storageEngineBadger = "badger" +) + // applicationConfiguration reads and stores component-specific configuration // values. It should not store any application helpers structs (pointers to shared // structs). @@ -183,6 +192,7 @@ type subStorageCfg struct { perm fs.FileMode depth uint64 noSync bool + engine string // blobovnicza-specific size uint64 @@ -310,31 +320,36 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.typ = storagesCfg[i].Type() sCfg.path = storagesCfg[i].Path() sCfg.perm = storagesCfg[i].Perm() + sCfg.engine = storagesCfg[i].StorageEngine() switch storagesCfg[i].Type() { - case blobovniczatree.Type: - sub := blobovniczaconfig.From((*config.Config)(storagesCfg[i])) + case storageTypeBlobovnicza: + if sCfg.engine == storageEngineUnspecified || sCfg.engine == storageEngineBBolt { + sub := blobovniczaconfig.From((*config.Config)(storagesCfg[i])) - sCfg.size = sub.Size() - sCfg.depth = sub.ShallowDepth() - sCfg.width = sub.ShallowWidth() - sCfg.leafWidth = sub.LeafWidth() - sCfg.openedCacheSize = sub.OpenedCacheSize() - sCfg.initWorkerCount = sub.InitWorkerCount() - sCfg.initInAdvance = sub.InitInAdvance() - sCfg.rebuildDropTimeout = sub.RebuildDropTimeout() - case fstree.Type: + sCfg.size = sub.Size() + sCfg.depth = sub.ShallowDepth() + sCfg.width = sub.ShallowWidth() + sCfg.leafWidth = sub.LeafWidth() + sCfg.openedCacheSize = sub.OpenedCacheSize() + sCfg.initWorkerCount = sub.InitWorkerCount() + sCfg.initInAdvance = sub.InitInAdvance() + sCfg.rebuildDropTimeout = sub.RebuildDropTimeout() + } else if sCfg.engine == storageEngineBadger { + sub := badgerstoreconfig.From((*config.Config)(storagesCfg[i])) + sCfg.indexCacheSize = sub.IndexCacheSize() + sCfg.memTablesCount = sub.MemTablesCount() + sCfg.compactorsCount = sub.CompactorsCount() + sCfg.gcInterval = sub.GCInterval() + sCfg.gcDiscardRatio = sub.GCDiscardRatio() + sCfg.valueLogFileSize = sub.ValueLogFileSize() + } else { + return fmt.Errorf("invalid storage engine: %s", sCfg.engine) + } + case storageTypeFStree: sub := fstreeconfig.From((*config.Config)(storagesCfg[i])) sCfg.depth = sub.Depth() sCfg.noSync = sub.NoSync() - case badgerstore.Type: - sub := badgerstoreconfig.From((*config.Config)(storagesCfg[i])) - sCfg.indexCacheSize = sub.IndexCacheSize() - sCfg.memTablesCount = sub.MemTablesCount() - sCfg.compactorsCount = sub.CompactorsCount() - sCfg.gcInterval = sub.GCInterval() - sCfg.gcDiscardRatio = sub.GCDiscardRatio() - sCfg.valueLogFileSize = sub.ValueLogFileSize() default: return fmt.Errorf("invalid storage type: %s", storagesCfg[i].Type()) } @@ -908,28 +923,30 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { var ss []blobstor.SubStorage for _, sRead := range shCfg.subStorages { switch sRead.typ { - case blobovniczatree.Type: - blobovniczaTreeOpts := c.getBlobovniczaTreeOpts(sRead) - ss = append(ss, blobstor.SubStorage{ - Storage: blobovniczatree.NewBlobovniczaTree(blobovniczaTreeOpts...), - Policy: func(_ *objectSDK.Object, data []byte) bool { - return uint64(len(data)) < shCfg.smallSizeObjectLimit - }, - }) - case fstree.Type: + case storageTypeBlobovnicza: + if sRead.engine == storageEngineUnspecified || sRead.engine == storageEngineBBolt { + blobovniczaTreeOpts := c.getBlobovniczaTreeOpts(sRead) + ss = append(ss, blobstor.SubStorage{ + Storage: blobovniczatree.NewBlobovniczaTree(blobovniczaTreeOpts...), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return uint64(len(data)) < shCfg.smallSizeObjectLimit + }, + }) + } else if sRead.engine == storageEngineBadger { + badgerStoreOpts := c.getBadgerStoreOpts(sRead) + ss = append(ss, blobstor.SubStorage{ + Storage: badgerstore.New(badgerStoreOpts...), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return uint64(len(data)) < shCfg.smallSizeObjectLimit + }, + }) + } + case storageTypeFStree: fstreeOpts := c.getFSTreeOpts(sRead) ss = append(ss, blobstor.SubStorage{ Storage: fstree.New(fstreeOpts...), - Policy: func(_ *objectSDK.Object, _ []byte) bool { - return true - }, - }) - case badgerstore.Type: - badgerStoreOpts := c.getBadgerStoreOpts(sRead) - ss = append(ss, blobstor.SubStorage{ - Storage: badgerstore.New(badgerStoreOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { - return uint64(len(data)) < shCfg.smallSizeObjectLimit + return true }, }) default: diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go index e83c69de2..28c73e3f7 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go @@ -53,3 +53,10 @@ func (x *Config) Perm() fs.FileMode { return fs.FileMode(p) } + +// StorageEngine returns storage engine. +func (x *Config) StorageEngine() string { + return config.String( + (*config.Config)(x), + "storage_engine") +} diff --git a/cmd/frostfs-node/validate.go b/cmd/frostfs-node/validate.go index dde895692..48759d960 100644 --- a/cmd/frostfs-node/validate.go +++ b/cmd/frostfs-node/validate.go @@ -9,9 +9,6 @@ import ( shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard" loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger" treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/badgerstore" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" ) @@ -61,7 +58,12 @@ func validateConfig(c *config.Config) error { } for i := range blobstor { switch blobstor[i].Type() { - case fstree.Type, blobovniczatree.Type, badgerstore.Type: + case storageTypeBlobovnicza: + storageEngine := blobstor[i].StorageEngine() + if storageEngine != storageEngineUnspecified && storageEngine != storageEngineBBolt && storageEngine != storageEngineBadger { + return fmt.Errorf("unexpected storage engine: %s (shard %d)", storageEngine, shardNum) + } + case storageTypeFStree: default: return fmt.Errorf("unexpected storage type: %s (shard %d)", blobstor[i].Type(), shardNum) } diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 02ead3020..00996138e 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -227,17 +227,37 @@ blobstor: | `depth` | `int` | `4` | File-system tree depth. | #### `blobovnicza` type options -| Parameter | Type | Default value | Description | -| ----------------------- | ---------- | ------------- | --------------------------------------------------------------------- | -| `path` | `string` | | Path to the root of the blobstor. | -| `perm` | file mode | `0660` | Default permission for created files and directories. | -| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | -| `depth` | `int` | `2` | Blobovnicza tree depth. | -| `width` | `int` | `16` | Blobovnicza tree width. | -| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | -| `init_worker_count` | `int` | `5` | Maximum number of concurrent initialization workers. | -| `init_in_advance` | `bool` | `false` | If `true`, than all the blobovnicza files will be created on startup. | -| `rebuild_drop_timeout` | `duration` | `10s` | Timeout before drop empty blobovnicza file during rebuild. | +| Parameter | Type | Default value | Description | +| ---------------- | --------- | ------------- | ------------------------------------------------------| +| `path` | `string` | | Path to the root of the blobstor. | +| `perm` | file mode | `0660` | Default permission for created files and directories. | +| `storage_engine` | `string` | | Storage engine for blobovnicza. `bbolt` or `badger`. | + + +##### `bbolt` engine options + +| Parameter | Type | Default value | Description | +| ----------------------- | ---------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | +| `depth` | `int` | `2` | Blobovnicza tree depth. | +| `width` | `int` | `16` | Blobovnicza tree width. | +| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | +| `opened_cache_ttl` | `duration` | `0` | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. | +| `init_worker_count` | `int` | `5` | Maximum number of concurrent initialization workers. | +| `init_in_advance` | `bool` | `false` | If `true`, than all the blobovnicza files will be created on startup. | +| `rebuild_drop_timeout` | `duration` | `10s` | Timeout before drop empty blobovnicza file during rebuild. | + +##### `badger` engine options + +| Parameter | Type | Default value | Description | +| --------------------- | ---------- | ------------- | ----------------------------------------------------------------------------------------------------------------- | +| `index_cache_size` | `size` | `256 MB` | How much memory should be used by badger table indices. | +| `mem_tables_count` | `int` | `32` | Maximum number of tables to keep in memory before stalling. | +| `compactors_count` | `int` | `64` | Number of compaction workers to run concurrently. | +| `gc_interval` | `duration` | `10 m` | Delay between garbage collector runs. | +| `gc_discard_percent` | `uint` | `20` | Garbage collector will rewrite value log file if it can discard at least `gc_discard_percent` space of that file. | +| `value_log_file_size` | `size` | `1 GB` | Maximum size of single value log file. | + | ### `gc` subsection -- 2.45.3 From ec81c1bcfd6904b2223634c421f59e552d46adbe Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 9 Apr 2024 17:08:42 +0300 Subject: [PATCH 7/7] [#645] config: Resolve unused linter Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 2fd9d37e9..8c4541a7a 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -945,7 +945,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { fstreeOpts := c.getFSTreeOpts(sRead) ss = append(ss, blobstor.SubStorage{ Storage: fstree.New(fstreeOpts...), - Policy: func(_ *objectSDK.Object, data []byte) bool { + Policy: func(_ *objectSDK.Object, _ []byte) bool { return true }, }) -- 2.45.3