[#1226] blobovniczatree: Drop init in advance option

To make blobovniczatree unlimited.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

cmd/frostfs-node/config.go

@@ -191,7 +191,6 @@ type subStorageCfg struct {
 	leafWidth              uint64
 	openedCacheSize        int
 	initWorkerCount        int
-	initInAdvance          bool
 	rebuildDropTimeout     time.Duration
 	openedCacheTTL         time.Duration
 	openedCacheExpInterval time.Duration
@@ -319,7 +318,6 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol
 			sCfg.openedCacheTTL = sub.OpenedCacheTTL()
 			sCfg.openedCacheExpInterval = sub.OpenedCacheExpInterval()
 			sCfg.initWorkerCount = sub.InitWorkerCount()
-			sCfg.initInAdvance = sub.InitInAdvance()
 			sCfg.rebuildDropTimeout = sub.RebuildDropTimeout()
 		case fstree.Type:
 			sub := fstreeconfig.From((*config.Config)(storagesCfg[i]))
@@ -911,7 +909,6 @@ func (c *cfg) getSubstorageOpts(ctx context.Context, shCfg shardCfg) []blobstor.
 				blobovniczatree.WithOpenedCacheTTL(sRead.openedCacheTTL),
 				blobovniczatree.WithOpenedCacheExpInterval(sRead.openedCacheExpInterval),
 				blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount),
-				blobovniczatree.WithInitInAdvance(sRead.initInAdvance),
 				blobovniczatree.WithWaitBeforeDropDB(sRead.rebuildDropTimeout),
 				blobovniczatree.WithLogger(c.log),
 				blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit),

cmd/frostfs-node/config/engine/config_test.go

@@ -102,7 +102,6 @@ func TestEngineSection(t *testing.T) {
 				require.EqualValues(t, 30*time.Second, blz.OpenedCacheExpInterval())
 				require.EqualValues(t, 10, blz.LeafWidth())
 				require.EqualValues(t, 10, blz.InitWorkerCount())
-				require.EqualValues(t, true, blz.InitInAdvance())
 				require.EqualValues(t, 30*time.Second, blz.RebuildDropTimeout())
 
 				require.Equal(t, "tmp/0/blob", ss[1].Path())

cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go

@@ -175,16 +175,6 @@ func (x *Config) InitWorkerCount() int {
 	return InitWorkerCountDefault
 }
 
-// InitInAdvance returns the value of "init_in_advance" config parameter.
-//
-// Returns False if the value is not defined or invalid.
-func (x *Config) InitInAdvance() bool {
-	return config.BoolSafe(
-		(*config.Config)(x),
-		"init_in_advance",
-	)
-}
-
 // RebuildDropTimeout returns the value of "rebuild_drop_timeout" config parameter.
 //
 // Returns RebuildDropTimeoutDefault if the value is not defined or invalid.

config/example/node.env

@@ -128,7 +128,6 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_TTL=1m
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_EXP_INTERVAL=30s
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10
-FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_REBUILD_DROP_TIMEOUT=30s
 ### FSTree config
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree

config/example/node.json

@@ -176,7 +176,6 @@
             "opened_cache_exp_interval": "30s",
             "leaf_width": 10,
             "init_worker_count": 10,
-            "init_in_advance": true,
             "rebuild_drop_timeout": "30s"
           },
           {

config/example/node.yaml

@@ -189,7 +189,6 @@ storage:
         - type: blobovnicza
           path: tmp/0/blob/blobovnicza
           init_worker_count: 10 #count of workers to initialize blobovniczas
-          init_in_advance: true
           rebuild_drop_timeout: 30s # timeout before drop single blobovnicza
           opened_cache_ttl: 1m
           opened_cache_exp_interval: 30s

docs/storage-node-configuration.md

@@ -242,7 +242,6 @@ blobstor:
 | `opened_cache_ttl`          | `duration` | `0`           | TTL in cache for opened blobovniczas(disabled by default). In case of heavy random-read and 10 shards each with 10_000 databases and accessing 400 objects per-second we will access each db approximately once per ((10 * 10_000 / 400) = 250 seconds <= 300 seconds = 5 min). Also take in mind that in this scenario they will probably be closed earlier because of the cache capacity, so bigger values are likely to be of no use. |
 | `opened_cache_exp_interval` | `duration` | `15s`         | Cache cleanup interval for expired blobovnicza's.                                                                                                                                                                                                                                                                                                                                                                                        |
 | `init_worker_count`         | `int`      | `5`           | Maximum number of concurrent initialization workers.                                                                                                                                                                                                                                                                                                                                                                                     |
-| `init_in_advance`           | `bool`     | `false`       | If `true`, than all the blobovnicza files will be created on startup.                                                                                                                                                                                                                                                                                                                                                                    |
 | `rebuild_drop_timeout`      | `duration` | `10s`         | Timeout before drop empty blobovnicza file during rebuild.                                                                                                                                                                                                                                                                                                                                                                               |
 
 ### `gc` subsection

pkg/local_object_storage/blobstor/blobovniczatree/control.go

@@ -83,31 +83,6 @@ func (b *Blobovniczas) initializeDBs(ctx context.Context) error {
 		return err
 	}
 
-	if b.createDBInAdvance {
-		err = b.iterateSortedLeaves(egCtx, nil, func(p string) (bool, error) {
-			if _, found := visited[p]; found {
-				return false, nil
-			}
-			eg.Go(func() error {
-				shBlz := b.getBlobovniczaWithoutCaching(p)
-				_, err := shBlz.Open()
-				if err != nil {
-					return err
-				}
-				defer shBlz.Close()
-
-				b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p))
-				return nil
-			})
-			return false, nil
-		})
-
-		if err != nil {
-			_ = eg.Wait()
-			return err
-		}
-	}
-
 	return eg.Wait()
 }
 

pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go

@@ -1,44 +0,0 @@
-package blobovniczatree
-
-import (
-	"context"
-	"testing"
-
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
-	oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
-	"github.com/stretchr/testify/require"
-)
-
-func TestIterateSortedLeavesAndDBPathsAreSame(t *testing.T) {
-	t.Parallel()
-
-	blz := NewBlobovniczaTree(
-		context.Background(),
-		WithBlobovniczaShallowDepth(3),
-		WithBlobovniczaShallowWidth(5),
-		WithRootPath(t.TempDir()),
-	)
-	blz.createDBInAdvance = true
-	require.NoError(t, blz.Open(mode.ComponentReadWrite))
-	require.NoError(t, blz.Init())
-	defer func() {
-		require.NoError(t, blz.Close())
-	}()
-
-	addr := oidtest.Address()
-
-	var leaves []string
-	var dbPaths []string
-
-	blz.iterateSortedLeaves(context.Background(), &addr, func(s string) (bool, error) {
-		leaves = append(leaves, s)
-		return false, nil
-	})
-
-	blz.iterateSortedDBPaths(context.Background(), addr, func(s string) (bool, error) {
-		dbPaths = append(dbPaths, s)
-		return false, nil
-	})
-
-	require.Equal(t, leaves, dbPaths)
-}

pkg/local_object_storage/blobstor/blobovniczatree/option.go

@@ -26,7 +26,6 @@ type cfg struct {
 	waitBeforeDropDB   time.Duration
 	blzInitWorkerCount int
 	blzMoveBatchSize   int
-	createDBInAdvance  bool
 	// TTL for blobovnicza's cache
 	openedCacheTTL time.Duration
 	// Interval for deletion expired blobovnicza's
@@ -160,10 +159,3 @@ func WithInitWorkerCount(v int) Option {
 		c.blzInitWorkerCount = v
 	}
 }
-
-// WithInitInAdvance returns an option to create blobovnicza tree DB's in advance.
-func WithInitInAdvance(v bool) Option {
-	return func(c *cfg) {
-		c.createDBInAdvance = v
-	}
-}