From fba92a3d61b3157eb6b448d795216f670763cb9c Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 15 Feb 2024 12:32:21 +0300 Subject: [PATCH] [#645] config: Add `storage_engine` parameter for blobovnicza Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 91 +++++++++++-------- .../engine/shard/blobstor/storage/config.go | 7 ++ cmd/frostfs-node/validate.go | 10 +- docs/storage-node-configuration.md | 42 ++++++--- 4 files changed, 98 insertions(+), 52 deletions(-) diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index a5812c60..dfd162be 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -90,6 +90,15 @@ const maxMsgSize = 4 << 20 // transport msg limit 4 MiB // for each contract listener. const notificationHandlerPoolSize = 10 +const ( + storageTypeBlobovnicza = "blobovnicza" + storageTypeFStree = "fstree" + + storageEngineUnspecified = "" + storageEngineBBolt = "bbolt" + storageEngineBadger = "badger" +) + // applicationConfiguration reads and stores component-specific configuration // values. It should not store any application helpers structs (pointers to shared // structs). @@ -183,6 +192,7 @@ type subStorageCfg struct { perm fs.FileMode depth uint64 noSync bool + engine string // blobovnicza-specific size uint64 @@ -310,31 +320,36 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.typ = storagesCfg[i].Type() sCfg.path = storagesCfg[i].Path() sCfg.perm = storagesCfg[i].Perm() + sCfg.engine = storagesCfg[i].StorageEngine() switch storagesCfg[i].Type() { - case blobovniczatree.Type: - sub := blobovniczaconfig.From((*config.Config)(storagesCfg[i])) + case storageTypeBlobovnicza: + if sCfg.engine == storageEngineUnspecified || sCfg.engine == storageEngineBBolt { + sub := blobovniczaconfig.From((*config.Config)(storagesCfg[i])) - sCfg.size = sub.Size() - sCfg.depth = sub.ShallowDepth() - sCfg.width = sub.ShallowWidth() - sCfg.leafWidth = sub.LeafWidth() - sCfg.openedCacheSize = sub.OpenedCacheSize() - sCfg.initWorkerCount = sub.InitWorkerCount() - sCfg.initInAdvance = sub.InitInAdvance() - sCfg.rebuildDropTimeout = sub.RebuildDropTimeout() - case fstree.Type: + sCfg.size = sub.Size() + sCfg.depth = sub.ShallowDepth() + sCfg.width = sub.ShallowWidth() + sCfg.leafWidth = sub.LeafWidth() + sCfg.openedCacheSize = sub.OpenedCacheSize() + sCfg.initWorkerCount = sub.InitWorkerCount() + sCfg.initInAdvance = sub.InitInAdvance() + sCfg.rebuildDropTimeout = sub.RebuildDropTimeout() + } else if sCfg.engine == storageEngineBadger { + sub := badgerstoreconfig.From((*config.Config)(storagesCfg[i])) + sCfg.indexCacheSize = sub.IndexCacheSize() + sCfg.memTablesCount = sub.MemTablesCount() + sCfg.compactorsCount = sub.CompactorsCount() + sCfg.gcInterval = sub.GCInterval() + sCfg.gcDiscardRatio = sub.GCDiscardRatio() + sCfg.valueLogFileSize = sub.ValueLogFileSize() + } else { + return fmt.Errorf("invalid storage engine: %s", sCfg.engine) + } + case storageTypeFStree: sub := fstreeconfig.From((*config.Config)(storagesCfg[i])) sCfg.depth = sub.Depth() sCfg.noSync = sub.NoSync() - case badgerstore.Type: - sub := badgerstoreconfig.From((*config.Config)(storagesCfg[i])) - sCfg.indexCacheSize = sub.IndexCacheSize() - sCfg.memTablesCount = sub.MemTablesCount() - sCfg.compactorsCount = sub.CompactorsCount() - sCfg.gcInterval = sub.GCInterval() - sCfg.gcDiscardRatio = sub.GCDiscardRatio() - sCfg.valueLogFileSize = sub.ValueLogFileSize() default: return fmt.Errorf("invalid storage type: %s", storagesCfg[i].Type()) } @@ -908,28 +923,30 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { var ss []blobstor.SubStorage for _, sRead := range shCfg.subStorages { switch sRead.typ { - case blobovniczatree.Type: - blobovniczaTreeOpts := c.getBlobovniczaTreeOpts(sRead) - ss = append(ss, blobstor.SubStorage{ - Storage: blobovniczatree.NewBlobovniczaTree(blobovniczaTreeOpts...), - Policy: func(_ *objectSDK.Object, data []byte) bool { - return uint64(len(data)) < shCfg.smallSizeObjectLimit - }, - }) - case fstree.Type: + case storageTypeBlobovnicza: + if sRead.engine == storageEngineUnspecified || sRead.engine == storageEngineBBolt { + blobovniczaTreeOpts := c.getBlobovniczaTreeOpts(sRead) + ss = append(ss, blobstor.SubStorage{ + Storage: blobovniczatree.NewBlobovniczaTree(blobovniczaTreeOpts...), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return uint64(len(data)) < shCfg.smallSizeObjectLimit + }, + }) + } else if sRead.engine == storageEngineBadger { + badgerStoreOpts := c.getBadgerStoreOpts(sRead) + ss = append(ss, blobstor.SubStorage{ + Storage: badgerstore.New(badgerStoreOpts...), + Policy: func(_ *objectSDK.Object, data []byte) bool { + return uint64(len(data)) < shCfg.smallSizeObjectLimit + }, + }) + } + case storageTypeFStree: fstreeOpts := c.getFSTreeOpts(sRead) ss = append(ss, blobstor.SubStorage{ Storage: fstree.New(fstreeOpts...), - Policy: func(_ *objectSDK.Object, _ []byte) bool { - return true - }, - }) - case badgerstore.Type: - badgerStoreOpts := c.getBadgerStoreOpts(sRead) - ss = append(ss, blobstor.SubStorage{ - Storage: badgerstore.New(badgerStoreOpts...), Policy: func(_ *objectSDK.Object, data []byte) bool { - return uint64(len(data)) < shCfg.smallSizeObjectLimit + return true }, }) default: diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go index e83c69de..28c73e3f 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go @@ -53,3 +53,10 @@ func (x *Config) Perm() fs.FileMode { return fs.FileMode(p) } + +// StorageEngine returns storage engine. +func (x *Config) StorageEngine() string { + return config.String( + (*config.Config)(x), + "storage_engine") +} diff --git a/cmd/frostfs-node/validate.go b/cmd/frostfs-node/validate.go index dde89569..48759d96 100644 --- a/cmd/frostfs-node/validate.go +++ b/cmd/frostfs-node/validate.go @@ -9,9 +9,6 @@ import ( shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard" loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger" treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/badgerstore" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" ) @@ -61,7 +58,12 @@ func validateConfig(c *config.Config) error { } for i := range blobstor { switch blobstor[i].Type() { - case fstree.Type, blobovniczatree.Type, badgerstore.Type: + case storageTypeBlobovnicza: + storageEngine := blobstor[i].StorageEngine() + if storageEngine != storageEngineUnspecified && storageEngine != storageEngineBBolt && storageEngine != storageEngineBadger { + return fmt.Errorf("unexpected storage engine: %s (shard %d)", storageEngine, shardNum) + } + case storageTypeFStree: default: return fmt.Errorf("unexpected storage type: %s (shard %d)", blobstor[i].Type(), shardNum) } diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 02ead302..00996138 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -227,17 +227,37 @@ blobstor: | `depth` | `int` | `4` | File-system tree depth. | #### `blobovnicza` type options -| Parameter | Type | Default value | Description | -| ----------------------- | ---------- | ------------- | --------------------------------------------------------------------- | -| `path` | `string` | | Path to the root of the blobstor. | -| `perm` | file mode | `0660` | Default permission for created files and directories. | -| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | -| `depth` | `int` | `2` | Blobovnicza tree depth. | -| `width` | `int` | `16` | Blobovnicza tree width. | -| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | -| `init_worker_count` | `int` | `5` | Maximum number of concurrent initialization workers. | -| `init_in_advance` | `bool` | `false` | If `true`, than all the blobovnicza files will be created on startup. | -| `rebuild_drop_timeout` | `duration` | `10s` | Timeout before drop empty blobovnicza file during rebuild. | +| Parameter | Type | Default value | Description | +| ---------------- | --------- | ------------- | ------------------------------------------------------| +| `path` | `string` | | Path to the root of the blobstor. | +| `perm` | file mode | `0660` | Default permission for created files and directories. | +| `storage_engine` | `string` | | Storage engine for blobovnicza. `bbolt` or `badger`. | + + +##### `bbolt` engine options + +| Parameter | Type | Default value | Description | +| ----------------------- | ---------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `size` | `size` | `1 G` | Maximum size of a single blobovnicza | +| `depth` | `int` | `2` | Blobovnicza tree depth. | +| `width` | `int` | `16` | Blobovnicza tree width. | +| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. | +| `opened_cache_ttl` | `duration` | `0` | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. | +| `init_worker_count` | `int` | `5` | Maximum number of concurrent initialization workers. | +| `init_in_advance` | `bool` | `false` | If `true`, than all the blobovnicza files will be created on startup. | +| `rebuild_drop_timeout` | `duration` | `10s` | Timeout before drop empty blobovnicza file during rebuild. | + +##### `badger` engine options + +| Parameter | Type | Default value | Description | +| --------------------- | ---------- | ------------- | ----------------------------------------------------------------------------------------------------------------- | +| `index_cache_size` | `size` | `256 MB` | How much memory should be used by badger table indices. | +| `mem_tables_count` | `int` | `32` | Maximum number of tables to keep in memory before stalling. | +| `compactors_count` | `int` | `64` | Number of compaction workers to run concurrently. | +| `gc_interval` | `duration` | `10 m` | Delay between garbage collector runs. | +| `gc_discard_percent` | `uint` | `20` | Garbage collector will rewrite value log file if it can discard at least `gc_discard_percent` space of that file. | +| `value_log_file_size` | `size` | `1 GB` | Maximum size of single value log file. | + | ### `gc` subsection