[#9999] blobovnicza: Drop leaf limit

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

cmd/frostfs-node/config.go

@@ -183,13 +183,13 @@ type subStorageCfg struct {
 	noSync bool
 
 	// blobovnicza-specific
-	size               uint64
-	width              uint64
-	leafWidth          uint64
-	openedCacheSize    int
-	initWorkerCount    int
-	initInAdvance      bool
-	rebuildDropTimeout time.Duration
+	size                   uint64
+	width                  uint64
+	openedCacheSize        int
+	initWorkerCount        int
+	rebuildDropTimeout     time.Duration
+	openedCacheTTL         time.Duration
+	openedCacheExpInterval time.Duration
 }
 
 // readConfig fills applicationConfiguration with raw configuration values
@@ -308,10 +308,8 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol
 			sCfg.size = sub.Size()
 			sCfg.depth = sub.ShallowDepth()
 			sCfg.width = sub.ShallowWidth()
-			sCfg.leafWidth = sub.LeafWidth()
 			sCfg.openedCacheSize = sub.OpenedCacheSize()
 			sCfg.initWorkerCount = sub.InitWorkerCount()
-			sCfg.initInAdvance = sub.InitInAdvance()
 			sCfg.rebuildDropTimeout = sub.RebuildDropTimeout()
 		case fstree.Type:
 			sub := fstreeconfig.From((*config.Config)(storagesCfg[i]))
@@ -897,10 +895,8 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage {
 				blobovniczatree.WithBlobovniczaSize(sRead.size),
 				blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth),
 				blobovniczatree.WithBlobovniczaShallowWidth(sRead.width),
-				blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth),
 				blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize),
 				blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount),
-				blobovniczatree.WithInitInAdvance(sRead.initInAdvance),
 				blobovniczatree.WithWaitBeforeDropDB(sRead.rebuildDropTimeout),
 				blobovniczatree.WithLogger(c.log),
 				blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit),

cmd/frostfs-node/config/engine/config_test.go

@@ -98,7 +98,6 @@ func TestEngineSection(t *testing.T) {
 				require.EqualValues(t, 1, blz.ShallowDepth())
 				require.EqualValues(t, 4, blz.ShallowWidth())
 				require.EqualValues(t, 50, blz.OpenedCacheSize())
-				require.EqualValues(t, 10, blz.LeafWidth())
 				require.EqualValues(t, 10, blz.InitWorkerCount())
 				require.EqualValues(t, true, blz.InitInAdvance())
 				require.EqualValues(t, 30*time.Second, blz.RebuildDropTimeout())
@@ -150,7 +149,6 @@ func TestEngineSection(t *testing.T) {
 				require.EqualValues(t, 1, blz.ShallowDepth())
 				require.EqualValues(t, 4, blz.ShallowWidth())
 				require.EqualValues(t, 50, blz.OpenedCacheSize())
-				require.EqualValues(t, 10, blz.LeafWidth())
 				require.EqualValues(t, blobovniczaconfig.InitWorkerCountDefault, blz.InitWorkerCount())
 				require.EqualValues(t, blobovniczaconfig.RebuildDropTimeoutDefault, blz.RebuildDropTimeout())
 

cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go

@@ -111,16 +111,6 @@ func (x *Config) BoltDB() *boltdbconfig.Config {
 	return (*boltdbconfig.Config)(x)
 }
 
-// LeafWidth returns the value of "leaf_width" config parameter.
-//
-// Returns 0 if the value is not a positive number.
-func (x *Config) LeafWidth() uint64 {
-	return config.UintSafe(
-		(*config.Config)(x),
-		"leaf_width",
-	)
-}
-
 // InitWorkerCount returns the value of "init_worker_count" config parameter.
 //
 // Returns InitWorkerCountDefault if the value is not a positive number.

config/example/node.env

@@ -127,7 +127,6 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_SIZE=4194304
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
-FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_REBUILD_DROP_TIMEOUT=30s
@@ -177,7 +176,6 @@ FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_SIZE=4194304
 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_DEPTH=1
 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_WIDTH=4
 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
-FROSTFS_STORAGE_SHARD_1_BLOBSTOR_0_LEAF_WIDTH=10
 ### FSTree config
 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_TYPE=fstree
 FROSTFS_STORAGE_SHARD_1_BLOBSTOR_1_PATH=tmp/1/blob

config/example/node.json

@@ -175,7 +175,6 @@
             "depth": 1,
             "width": 4,
             "opened_cache_capacity": 50,
-            "leaf_width": 10,
             "init_worker_count": 10,
             "init_in_advance": true,
             "rebuild_drop_timeout": "30s"
@@ -227,8 +226,7 @@
             "size": 4194304,
             "depth": 1,
             "width": 4,
-            "opened_cache_capacity": 50,
-            "leaf_width": 10
+            "opened_cache_capacity": 50
           },
           {
             "type": "fstree",

config/example/node.yaml

@@ -151,7 +151,6 @@ storage:
           depth: 1  # max depth of object tree storage in key-value DB
           width: 4   # max width of object tree storage in key-value DB
           opened_cache_capacity: 50  # maximum number of opened database files
-          leaf_width: 10 # max count of key-value DB on leafs of object tree storage
         - perm: 0644  # permissions for blobstor files(directories: +x for current user and group)
           depth: 5  # max depth of object tree storage in FS
 

pkg/local_object_storage/blobstor/blobovniczatree/active.go

@@ -37,17 +37,17 @@ type activeDBManager struct {
 	closed               bool
 
 	dbManager *dbManager
-	leafWidth uint64
+	rootPath  string
 }
 
-func newActiveDBManager(dbManager *dbManager, leafWidth uint64) *activeDBManager {
+func newActiveDBManager(dbManager *dbManager, rootPath string) *activeDBManager {
 	return &activeDBManager{
 		levelToActiveDBGuard: &sync.RWMutex{},
 		levelToActiveDB:      make(map[string]*sharedDB),
 		levelLock:            utilSync.NewKeyLocker[string](),
 
 		dbManager: dbManager,
-		leafWidth: leafWidth,
+		rootPath:  rootPath,
 	}
 }
 
@@ -144,30 +144,25 @@ func (m *activeDBManager) updateAndGetActive(lvlPath string) (*activeDB, error)
 }
 
 func (m *activeDBManager) getNextSharedDB(lvlPath string) (*sharedDB, error) {
-	var idx uint64
-	var iterCount uint64
+	var nextActiveDBIdx uint64
 	hasActive, currentIdx := m.hasActiveDB(lvlPath)
 	if hasActive {
-		idx = (currentIdx + 1) % m.leafWidth
-	}
-
-	var next *sharedDB
-
-	for iterCount < m.leafWidth {
-		path := filepath.Join(lvlPath, u64ToHexStringExt(idx))
-		shDB := m.dbManager.GetByPath(path)
-		db, err := shDB.Open() // open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close()
+		nextActiveDBIdx = currentIdx + 1
+	} else {
+		hasDBs, maxIdx, err := getBlobovniczaMaxIndex(filepath.Join(m.rootPath, lvlPath))
 		if err != nil {
 			return nil, err
 		}
-		if db.IsFull() {
-			shDB.Close()
-		} else {
-			next = shDB
-			break
+		if hasDBs {
+			nextActiveDBIdx = maxIdx
 		}
-		idx = (idx + 1) % m.leafWidth
-		iterCount++
+	}
+
+	path := filepath.Join(lvlPath, u64ToHexStringExt(nextActiveDBIdx))
+	next := m.dbManager.GetByPath(path)
+	_, err := next.Open() // open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close()
+	if err != nil {
+		return nil, err
 	}
 
 	previous, updated := m.replace(lvlPath, next)

pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go

@@ -2,6 +2,7 @@ package blobovniczatree
 
 import (
 	"errors"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -80,12 +81,8 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) {
 		opts[i](&blz.cfg)
 	}
 
-	if blz.blzLeafWidth == 0 {
-		blz.blzLeafWidth = blz.blzShallowWidth
-	}
-
 	blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log)
-	blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth)
+	blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.rootPath)
 	blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager)
 	blz.deleteProtectedObjects = newAddressMap()
 	blz.dbFilesGuard = &sync.RWMutex{}
@@ -122,6 +119,32 @@ func u64FromHexString(str string) uint64 {
 	return v
 }
 
+func getBlobovniczaMaxIndex(directory string) (bool, uint64, error) {
+	entries, err := os.ReadDir(directory)
+	if os.IsNotExist(err) { // non initialized tree
+		return false, 0, nil
+	}
+	if err != nil {
+		return false, 0, err
+	}
+	if len(entries) == 0 {
+		return false, 0, nil
+	}
+	var hasDBs bool
+	var maxIdx uint64
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		hasDBs = true
+		current := u64FromHexString(e.Name())
+		if current > maxIdx {
+			maxIdx = current
+		}
+	}
+	return hasDBs, maxIdx, nil
+}
+
 // Type is blobovniczatree storage type used in logs and configuration.
 const Type = "blobovnicza"
 

pkg/local_object_storage/blobstor/blobovniczatree/iterate.go

@@ -130,7 +130,14 @@ func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, cur
 	isLeafLevel := uint64(len(curPath)) == b.blzShallowDepth
 	levelWidth := b.blzShallowWidth
 	if isLeafLevel {
-		levelWidth = b.blzLeafWidth
+		hasDBs, maxIdx, err := getBlobovniczaMaxIndex(filepath.Join(append([]string{b.rootPath}, curPath...)...))
+		if err != nil {
+			return false, err
+		}
+		levelWidth = 0
+		if hasDBs {
+			levelWidth = maxIdx + 1
+		}
 	}
 	indices := indexSlice(levelWidth)
 

pkg/local_object_storage/blobstor/blobovniczatree/option.go

@@ -18,7 +18,6 @@ type cfg struct {
 	openedCacheSize    int
 	blzShallowDepth    uint64
 	blzShallowWidth    uint64
-	blzLeafWidth       uint64
 	compression        *compression.Config
 	blzOpts            []blobovnicza.Option
 	reportError        func(string, error) // reportError is the function called when encountering disk errors.
@@ -75,12 +74,6 @@ func WithBlobovniczaShallowWidth(width uint64) Option {
 	}
 }
 
-func WithBlobovniczaLeafWidth(w uint64) Option {
-	return func(c *cfg) {
-		c.blzLeafWidth = w
-	}
-}
-
 func WithBlobovniczaShallowDepth(depth uint64) Option {
 	return func(c *cfg) {
 		c.blzShallowDepth = depth