TrueCloudLab
/
frostfs-node
18
1
Fork 20
Code Issues 87 Pull Requests 5 Actions Releases Activity

[#645] config: Add `storage_engine` parameter for blobovnicza
DCO action / DCO (pull_request) Successful in 3m16s Details
Tests and linters / Staticcheck (pull_request) Failing after 4m0s Details
Vulncheck / Vulncheck (pull_request) Failing after 3m59s Details
Tests and linters / Lint (pull_request) Failing after 4m42s Details
Build / Build Components (1.21) (pull_request) Failing after 4m11s Details
Build / Build Components (1.20) (pull_request) Failing after 4m16s Details
Tests and linters / gopls check (pull_request) Failing after 6m6s Details
Tests and linters / Tests (1.21) (pull_request) Failing after 7m43s Details
Tests and linters / Tests (1.20) (pull_request) Failing after 8m5s Details
Tests and linters / Tests with -race (pull_request) Failing after 8m11s Details 

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
Dmitrii Stepanov 2024-02-15 12:32:21 +03:00
parent 1ffe73a67e
commit f5e7a7cffb
4 changed files with 93 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
cmd/frostfs-node/config.go
View File

@ -90,6 +90,15 @@ const maxMsgSize = 4 << 20 // transport msg limit 4 MiB
// for each contract listener.
const notificationHandlerPoolSize = 10
const (
storageTypeBlobovnicza = "blobovnicza"
storageTypeFStree = "fstree"
storageEngineUnspecified = ""
storageEngineBBolt = "bbolt"
storageEngineBadger = "badger"
)
// applicationConfiguration reads and stores component-specific configuration
// values. It should not store any application helpers structs (pointers to shared
// structs).
@ -183,6 +192,7 @@ type subStorageCfg struct {
perm fs.FileMode
depth uint64
noSync bool
engine string
// blobovnicza-specific
size uint64
@ -310,31 +320,36 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol
sCfg.typ = storagesCfg[i].Type()
sCfg.path = storagesCfg[i].Path()
sCfg.perm = storagesCfg[i].Perm()
sCfg.engine = storagesCfg[i].StorageEngine()
switch storagesCfg[i].Type() {
case blobovniczatree.Type:
sub := blobovniczaconfig.From((*config.Config)(storagesCfg[i]))
case storageTypeBlobovnicza:
if sCfg.engine == storageEngineUnspecified || sCfg.engine == storageEngineBBolt {
sub := blobovniczaconfig.From((*config.Config)(storagesCfg[i]))
sCfg.size = sub.Size()
sCfg.depth = sub.ShallowDepth()
sCfg.width = sub.ShallowWidth()
sCfg.leafWidth = sub.LeafWidth()
sCfg.openedCacheSize = sub.OpenedCacheSize()
sCfg.initWorkerCount = sub.InitWorkerCount()
sCfg.initInAdvance = sub.InitInAdvance()
sCfg.rebuildDropTimeout = sub.RebuildDropTimeout()
case fstree.Type:
sCfg.size = sub.Size()
sCfg.depth = sub.ShallowDepth()
sCfg.width = sub.ShallowWidth()
sCfg.leafWidth = sub.LeafWidth()
sCfg.openedCacheSize = sub.OpenedCacheSize()
sCfg.initWorkerCount = sub.InitWorkerCount()
sCfg.initInAdvance = sub.InitInAdvance()
sCfg.rebuildDropTimeout = sub.RebuildDropTimeout()
} else if sCfg.engine == storageEngineBadger {
sub := badgerstoreconfig.From((*config.Config)(storagesCfg[i]))
sCfg.indexCacheSize = sub.IndexCacheSize()
sCfg.memTablesCount = sub.MemTablesCount()
sCfg.compactorsCount = sub.CompactorsCount()
sCfg.gcInterval = sub.GCInterval()
sCfg.gcDiscardRatio = sub.GCDiscardRatio()
sCfg.valueLogFileSize = sub.ValueLogFileSize()
} else {
return fmt.Errorf("invalid storage engine: %s", sCfg.engine)
}
case storageTypeFStree:
sub := fstreeconfig.From((*config.Config)(storagesCfg[i]))
sCfg.depth = sub.Depth()
sCfg.noSync = sub.NoSync()
case badgerstore.Type:
sub := badgerstoreconfig.From((*config.Config)(storagesCfg[i]))
sCfg.indexCacheSize = sub.IndexCacheSize()
sCfg.memTablesCount = sub.MemTablesCount()
sCfg.compactorsCount = sub.CompactorsCount()
sCfg.gcInterval = sub.GCInterval()
sCfg.gcDiscardRatio = sub.GCDiscardRatio()
sCfg.valueLogFileSize = sub.ValueLogFileSize()
default:
return fmt.Errorf("invalid storage type: %s", storagesCfg[i].Type())
}
@ -908,28 +923,30 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage {
var ss []blobstor.SubStorage
for _, sRead := range shCfg.subStorages {
switch sRead.typ {
case blobovniczatree.Type:
blobovniczaTreeOpts := c.getBlobovniczaTreeOpts(sRead)
ss = append(ss, blobstor.SubStorage{
Storage: blobovniczatree.NewBlobovniczaTree(blobovniczaTreeOpts...),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) < shCfg.smallSizeObjectLimit
},
})
case fstree.Type:
case storageTypeBlobovnicza:
if sRead.engine == storageEngineUnspecified || sRead.engine == storageEngineBBolt {
blobovniczaTreeOpts := c.getBlobovniczaTreeOpts(sRead)
ss = append(ss, blobstor.SubStorage{
Storage: blobovniczatree.NewBlobovniczaTree(blobovniczaTreeOpts...),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) < shCfg.smallSizeObjectLimit
},
})
} else if sRead.engine == storageEngineBadger {
badgerStoreOpts := c.getBadgerStoreOpts(sRead)
ss = append(ss, blobstor.SubStorage{
Storage: badgerstore.New(badgerStoreOpts...),
Policy: func(_ *objectSDK.Object, _ []byte) bool {
return uint64(len(data)) < shCfg.smallSizeObjectLimit
},
})
}
case storageTypeFStree:
fstreeOpts := c.getFSTreeOpts(sRead)
ss = append(ss, blobstor.SubStorage{
Storage: fstree.New(fstreeOpts...),
Policy: func(_ *objectSDK.Object, _ []byte) bool {
return true
},
})
case badgerstore.Type:
badgerStoreOpts := c.getBadgerStoreOpts(sRead)
ss = append(ss, blobstor.SubStorage{
Storage: badgerstore.New(badgerStoreOpts...),
Policy: func(_ *objectSDK.Object, data []byte) bool {
return uint64(len(data)) < shCfg.smallSizeObjectLimit
return true
},
})
default:

7
cmd/frostfs-node/config/engine/shard/blobstor/storage/config.go
View File

@ -53,3 +53,10 @@ func (x *Config) Perm() fs.FileMode {
return fs.FileMode(p)
}
// StorageEngine returns storage engine.
func (x *Config) StorageEngine() string {
return config.String(
(*config.Config)(x),
"storage_engine")
}

10
cmd/frostfs-node/validate.go
View File

@ -9,9 +9,6 @@ import (
shardconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard"
loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger"
treeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/tree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/badgerstore"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
)
@ -61,7 +58,12 @@ func validateConfig(c *config.Config) error {
}
for i := range blobstor {
switch blobstor[i].Type() {
case fstree.Type, blobovniczatree.Type, badgerstore.Type:
case storageTypeBlobovnicza:
storageEngine := blobstor[i].StorageEngine()
if storageEngine != storageEngineUnspecified && storageEngine != storageEngineBBolt && storageEngine != storageEngineBadger {
return fmt.Errorf("unexpected storage engine: %s (shard %d)", storageEngine, shardNum)
}
case storageTypeFStree:
default:
return fmt.Errorf("unexpected storage type: %s (shard %d)", blobstor[i].Type(), shardNum)
}

33
docs/storage-node-configuration.md
View File

@ -231,13 +231,32 @@ blobstor:
| ----------------------- | ---------- | ------------- | --------------------------------------------------------------------- |
| `path` | `string` | | Path to the root of the blobstor. |
| `perm` | file mode | `0660` | Default permission for created files and directories. |
| `size` | `size` | `1 G` | Maximum size of a single blobovnicza |
| `depth` | `int` | `2` | Blobovnicza tree depth. |
| `width` | `int` | `16` | Blobovnicza tree width. |
| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. |
| `init_worker_count` | `int` | `5` | Maximum number of concurrent initialization workers. |
| `init_in_advance` | `bool` | `false` | If `true`, than all the blobovnicza files will be created on startup. |
| `rebuild_drop_timeout` | `duration` | `10s` | Timeout before drop empty blobovnicza file during rebuild. |
##### `bbolt` engine options
| Parameter | Type | Default value | Description |
| ----------------------- | ---------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `size` | `size` | `1 G` | Maximum size of a single blobovnicza |
| `depth` | `int` | `2` | Blobovnicza tree depth. |
| `width` | `int` | `16` | Blobovnicza tree width. |
| `opened_cache_capacity` | `int` | `16` | Maximum number of simultaneously opened blobovniczas. |
| `opened_cache_ttl` | `duration` | `0` | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. |
| `init_worker_count` | `int` | `5` | Maximum number of concurrent initialization workers. |
| `init_in_advance` | `bool` | `false` | If `true`, than all the blobovnicza files will be created on startup. |
| `rebuild_drop_timeout` | `duration` | `10s` | Timeout before drop empty blobovnicza file during rebuild. |
##### `badger` engine options
| Parameter | Type | Default value | Description |
| --------------------- | ---------- | ------------- | ----------------------------------------------------------------------------------------------------------------- |
| `index_cache_size` | `size` | `256 MB` | How much memory should be used by badger table indices. |
| `mem_tables_count` | `int` | `32` | Maximum number of tables to keep in memory before stalling. |
| `compactors_count` | `int` | `64` | Number of compaction workers to run concurrently. |
| `gc_interval` | `duration` | `10 m` | Delay between garbage collector runs. |
| `gc_discard_percent` | `uint` | `20` | Garbage collector will rewrite value log file if it can discard at least `gc_discard_percent` space of that file. |
| `value_log_file_size` | `size` | `1 GB` | Maximum size of single value log file. |
|
### `gc` subsection