Blobovnicza tree rebuild (support) #703
9 changed files with 46 additions and 18 deletions
|
@ -178,6 +178,7 @@ type subStorageCfg struct {
|
|||
leafWidth uint64
|
||||
openedCacheSize int
|
||||
initWorkerCount int
|
||||
initInAdvance bool
|
||||
}
|
||||
|
||||
// readConfig fills applicationConfiguration with raw configuration values
|
||||
|
@ -295,6 +296,7 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol
|
|||
sCfg.leafWidth = sub.LeafWidth()
|
||||
sCfg.openedCacheSize = sub.OpenedCacheSize()
|
||||
sCfg.initWorkerCount = sub.InitWorkerCount()
|
||||
sCfg.initInAdvance = sub.InitInAdvance()
|
||||
case fstree.Type:
|
||||
sub := fstreeconfig.From((*config.Config)(storagesCfg[i]))
|
||||
sCfg.depth = sub.Depth()
|
||||
|
@ -786,6 +788,7 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage {
|
|||
blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth),
|
||||
blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize),
|
||||
blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount),
|
||||
blobovniczatree.WithInitInAdvance(sRead.initInAdvance),
|
||||
blobovniczatree.WithLogger(c.log),
|
||||
blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit),
|
||||
}
|
||||
|
|
|
@ -98,6 +98,7 @@ func TestEngineSection(t *testing.T) {
|
|||
require.EqualValues(t, 50, blz.OpenedCacheSize())
|
||||
require.EqualValues(t, 10, blz.LeafWidth())
|
||||
require.EqualValues(t, 10, blz.InitWorkerCount())
|
||||
require.EqualValues(t, true, blz.InitInAdvance())
|
||||
|
||||
require.Equal(t, "tmp/0/blob", ss[1].Path())
|
||||
require.EqualValues(t, 0o644, ss[1].Perm())
|
||||
|
|
|
@ -131,3 +131,13 @@ func (x *Config) InitWorkerCount() int {
|
|||
|
||||
return InitWorkerCountDefault
|
||||
}
|
||||
|
||||
// InitInAdvance returns the value of "init_in_advance" config parameter.
|
||||
//
|
||||
// Returns False if the value is not defined or invalid.
|
||||
func (x *Config) InitInAdvance() bool {
|
||||
return config.BoolSafe(
|
||||
(*config.Config)(x),
|
||||
"init_in_advance",
|
||||
)
|
||||
}
|
||||
|
|
|
@ -125,6 +125,7 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4
|
|||
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
|
||||
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10
|
||||
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10
|
||||
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE
|
||||
### FSTree config
|
||||
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree
|
||||
FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH=tmp/0/blob
|
||||
|
|
|
@ -172,7 +172,8 @@
|
|||
"width": 4,
|
||||
"opened_cache_capacity": 50,
|
||||
"leaf_width": 10,
|
||||
"init_worker_count": 10
|
||||
"init_worker_count": 10,
|
||||
"init_in_advance": true
|
||||
},
|
||||
{
|
||||
"type": "fstree",
|
||||
|
|
|
@ -184,6 +184,7 @@ storage:
|
|||
- type: blobovnicza
|
||||
path: tmp/0/blob/blobovnicza
|
||||
init_worker_count: 10 #count of workers to initialize blobovniczas
|
||||
init_in_advance: true
|
||||
- type: fstree
|
||||
path: tmp/0/blob # blobstor path
|
||||
|
||||
|
|
|
@ -82,27 +82,29 @@ func (b *Blobovniczas) initializeDBs(ctx context.Context) error {
|
|||
return err
|
||||
}
|
||||
|
||||
err = b.iterateSortedLeaves(egCtx, nil, func(p string) (bool, error) {
|
||||
if _, found := visited[p]; found {
|
||||
return false, nil
|
||||
}
|
||||
eg.Go(func() error {
|
||||
shBlz := b.getBlobovniczaWithoutCaching(p)
|
||||
_, err := shBlz.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
if b.createDBInAdvance {
|
||||
err = b.iterateSortedLeaves(egCtx, nil, func(p string) (bool, error) {
|
||||
if _, found := visited[p]; found {
|
||||
return false, nil
|
||||
}
|
||||
defer shBlz.Close()
|
||||
eg.Go(func() error {
|
||||
shBlz := b.getBlobovniczaWithoutCaching(p)
|
||||
_, err := shBlz.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer shBlz.Close()
|
||||
|
||||
|
||||
b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p))
|
||||
return nil
|
||||
b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p))
|
||||
return nil
|
||||
})
|
||||
return false, nil
|
||||
})
|
||||
return false, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
_ = eg.Wait()
|
||||
return err
|
||||
if err != nil {
|
||||
_ = eg.Wait()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return eg.Wait()
|
||||
|
|
|
@ -16,6 +16,7 @@ func TestIterateSortedLeavesAndDBPathsAreSame(t *testing.T) {
|
|||
WithBlobovniczaShallowWidth(5),
|
||||
WithRootPath(t.TempDir()),
|
||||
)
|
||||
blz.createDBInAdvance = true
|
||||
require.NoError(t, blz.Open(false))
|
||||
require.NoError(t, blz.Init())
|
||||
defer func() {
|
||||
|
|
|
@ -26,6 +26,7 @@ type cfg struct {
|
|||
waitBeforeDropDB time.Duration
|
||||
blzInitWorkerCount int
|
||||
blzMoveBatchSize int
|
||||
createDBInAdvance bool
|
||||
}
|
||||
|
||||
type Option func(*cfg)
|
||||
|
@ -139,3 +140,10 @@ func WithInitWorkerCount(v int) Option {
|
|||
c.blzInitWorkerCount = v
|
||||
}
|
||||
}
|
||||
|
||||
// WithInitInAdvance returns an option to create blobovnicza tree DB's in advance.
|
||||
func WithInitInAdvance(v bool) Option {
|
||||
return func(c *cfg) {
|
||||
c.createDBInAdvance = v
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue
SuccessfullyInitialized
, but we only callOpen
withoutInit
, is this expected?shBlz
is wrapper overblobovnicza
, soshBlz.Open()
calls blobovnicza'sOpen()
andInit()
.