diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d78a90cfc..362fabb14 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -183,13 +183,13 @@ type subStorageCfg struct { noSync bool // blobovnicza-specific - size uint64 - width uint64 - leafWidth uint64 - openedCacheSize int - initWorkerCount int - initInAdvance bool - rebuildDropTimeout time.Duration + size uint64 + width uint64 + openedCacheSize int + initWorkerCount int + rebuildDropTimeout time.Duration + openedCacheTTL time.Duration + openedCacheExpInterval time.Duration } // readConfig fills applicationConfiguration with raw configuration values @@ -308,10 +308,8 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol sCfg.size = sub.Size() sCfg.depth = sub.ShallowDepth() sCfg.width = sub.ShallowWidth() - sCfg.leafWidth = sub.LeafWidth() sCfg.openedCacheSize = sub.OpenedCacheSize() sCfg.initWorkerCount = sub.InitWorkerCount() - sCfg.initInAdvance = sub.InitInAdvance() sCfg.rebuildDropTimeout = sub.RebuildDropTimeout() case fstree.Type: sub := fstreeconfig.From((*config.Config)(storagesCfg[i])) @@ -897,10 +895,8 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage { blobovniczatree.WithBlobovniczaSize(sRead.size), blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), - blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth), blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount), - blobovniczatree.WithInitInAdvance(sRead.initInAdvance), blobovniczatree.WithWaitBeforeDropDB(sRead.rebuildDropTimeout), blobovniczatree.WithLogger(c.log), blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit), diff --git a/cmd/frostfs-node/config/engine/config_test.go b/cmd/frostfs-node/config/engine/config_test.go index 86c938309..c78cf2837 100644 --- a/cmd/frostfs-node/config/engine/config_test.go +++ b/cmd/frostfs-node/config/engine/config_test.go @@ -98,7 +98,6 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 1, blz.ShallowDepth()) require.EqualValues(t, 4, blz.ShallowWidth()) require.EqualValues(t, 50, blz.OpenedCacheSize()) - require.EqualValues(t, 10, blz.LeafWidth()) require.EqualValues(t, 10, blz.InitWorkerCount()) require.EqualValues(t, true, blz.InitInAdvance()) require.EqualValues(t, 30*time.Second, blz.RebuildDropTimeout()) @@ -150,7 +149,6 @@ func TestEngineSection(t *testing.T) { require.EqualValues(t, 1, blz.ShallowDepth()) require.EqualValues(t, 4, blz.ShallowWidth()) require.EqualValues(t, 50, blz.OpenedCacheSize()) - require.EqualValues(t, 10, blz.LeafWidth()) require.EqualValues(t, blobovniczaconfig.InitWorkerCountDefault, blz.InitWorkerCount()) require.EqualValues(t, blobovniczaconfig.RebuildDropTimeoutDefault, blz.RebuildDropTimeout()) diff --git a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go index 9619a1027..bf26570cd 100644 --- a/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go +++ b/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go @@ -111,16 +111,6 @@ func (x *Config) BoltDB() *boltdbconfig.Config { return (*boltdbconfig.Config)(x) } -// LeafWidth returns the value of "leaf_width" config parameter. -// -// Returns 0 if the value is not a positive number. -func (x *Config) LeafWidth() uint64 { - return config.UintSafe( - (*config.Config)(x), - "leaf_width", - ) -} - // InitWorkerCount returns the value of "init_worker_count" config parameter. // // Returns InitWorkerCountDefault if the value is not a positive number. diff --git a/config/example/node.env b/config/example/node.env index 976c42629..2006e5800 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -127,7 +127,6 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 -FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_REBUILD_DROP_TIMEOUT=30s @@ -177,7 +176,6 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50 -FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10 ### FSTree config FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH=tmp/1/blob diff --git a/config/example/node.json b/config/example/node.json index 648fb77b0..8b385b126 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -175,7 +175,6 @@ "depth": 1, "width": 4, "opened_cache_capacity": 50, - "leaf_width": 10, "init_worker_count": 10, "init_in_advance": true, "rebuild_drop_timeout": "30s" @@ -227,8 +226,7 @@ "size": 4194304, "depth": 1, "width": 4, - "opened_cache_capacity": 50, - "leaf_width": 10 + "opened_cache_capacity": 50 }, { "type": "fstree", diff --git a/config/example/node.yaml b/config/example/node.yaml index 2dcf7c4d9..86fd1e2a1 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -151,7 +151,6 @@ storage: depth: 1 # max depth of object tree storage in key-value DB width: 4 # max width of object tree storage in key-value DB opened_cache_capacity: 50 # maximum number of opened database files - leaf_width: 10 # max count of key-value DB on leafs of object tree storage - perm: 0644 # permissions for blobstor files(directories: +x for current user and group) depth: 5 # max depth of object tree storage in FS diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/active.go b/pkg/local_object_storage/blobstor/blobovniczatree/active.go index 0e3497383..603c6abe3 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/active.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/active.go @@ -37,17 +37,17 @@ type activeDBManager struct { closed bool dbManager *dbManager - leafWidth uint64 + rootPath string } -func newActiveDBManager(dbManager *dbManager, leafWidth uint64) *activeDBManager { +func newActiveDBManager(dbManager *dbManager, rootPath string) *activeDBManager { return &activeDBManager{ levelToActiveDBGuard: &sync.RWMutex{}, levelToActiveDB: make(map[string]*sharedDB), levelLock: utilSync.NewKeyLocker[string](), dbManager: dbManager, - leafWidth: leafWidth, + rootPath: rootPath, } } @@ -144,30 +144,25 @@ func (m *activeDBManager) updateAndGetActive(lvlPath string) (*activeDB, error) } func (m *activeDBManager) getNextSharedDB(lvlPath string) (*sharedDB, error) { - var idx uint64 - var iterCount uint64 + var nextActiveDBIdx uint64 hasActive, currentIdx := m.hasActiveDB(lvlPath) if hasActive { - idx = (currentIdx + 1) % m.leafWidth - } - - var next *sharedDB - - for iterCount < m.leafWidth { - path := filepath.Join(lvlPath, u64ToHexStringExt(idx)) - shDB := m.dbManager.GetByPath(path) - db, err := shDB.Open() // open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close() + nextActiveDBIdx = currentIdx + 1 + } else { + hasDBs, maxIdx, err := getBlobovniczaMaxIndex(filepath.Join(m.rootPath, lvlPath)) if err != nil { return nil, err } - if db.IsFull() { - shDB.Close() - } else { - next = shDB - break + if hasDBs { + nextActiveDBIdx = maxIdx } - idx = (idx + 1) % m.leafWidth - iterCount++ + } + + path := filepath.Join(lvlPath, u64ToHexStringExt(nextActiveDBIdx)) + next := m.dbManager.GetByPath(path) + _, err := next.Open() // open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close() + if err != nil { + return nil, err } previous, updated := m.replace(lvlPath, next) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go index be51509b6..ef2155974 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go @@ -2,6 +2,7 @@ package blobovniczatree import ( "errors" + "os" "strconv" "strings" "sync" @@ -80,12 +81,8 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) { opts[i](&blz.cfg) } - if blz.blzLeafWidth == 0 { - blz.blzLeafWidth = blz.blzShallowWidth - } - blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log) - blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth) + blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.rootPath) blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager) blz.deleteProtectedObjects = newAddressMap() blz.dbFilesGuard = &sync.RWMutex{} @@ -122,6 +119,32 @@ func u64FromHexString(str string) uint64 { return v } +func getBlobovniczaMaxIndex(directory string) (bool, uint64, error) { + entries, err := os.ReadDir(directory) + if os.IsNotExist(err) { // non initialized tree + return false, 0, nil + } + if err != nil { + return false, 0, err + } + if len(entries) == 0 { + return false, 0, nil + } + var hasDBs bool + var maxIdx uint64 + for _, e := range entries { + if e.IsDir() { + continue + } + hasDBs = true + current := u64FromHexString(e.Name()) + if current > maxIdx { + maxIdx = current + } + } + return hasDBs, maxIdx, nil +} + // Type is blobovniczatree storage type used in logs and configuration. const Type = "blobovnicza" diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go index 92014fd55..af3d9e720 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/iterate.go @@ -130,7 +130,14 @@ func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, cur isLeafLevel := uint64(len(curPath)) == b.blzShallowDepth levelWidth := b.blzShallowWidth if isLeafLevel { - levelWidth = b.blzLeafWidth + hasDBs, maxIdx, err := getBlobovniczaMaxIndex(filepath.Join(append([]string{b.rootPath}, curPath...)...)) + if err != nil { + return false, err + } + levelWidth = 0 + if hasDBs { + levelWidth = maxIdx + 1 + } } indices := indexSlice(levelWidth) diff --git a/pkg/local_object_storage/blobstor/blobovniczatree/option.go b/pkg/local_object_storage/blobstor/blobovniczatree/option.go index d36074dc8..bb9b3b99c 100644 --- a/pkg/local_object_storage/blobstor/blobovniczatree/option.go +++ b/pkg/local_object_storage/blobstor/blobovniczatree/option.go @@ -18,7 +18,6 @@ type cfg struct { openedCacheSize int blzShallowDepth uint64 blzShallowWidth uint64 - blzLeafWidth uint64 compression *compression.Config blzOpts []blobovnicza.Option reportError func(string, error) // reportError is the function called when encountering disk errors. @@ -75,12 +74,6 @@ func WithBlobovniczaShallowWidth(width uint64) Option { } } -func WithBlobovniczaLeafWidth(w uint64) Option { - return func(c *cfg) { - c.blzLeafWidth = w - } -} - func WithBlobovniczaShallowDepth(depth uint64) Option { return func(c *cfg) { c.blzShallowDepth = depth