[#1071] Revert "[#446] engine: Move to read-only on blobstor errors"

This reverts commit 69df0d21c2.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>

CHANGELOG.md

@@ -1,7 +1,7 @@
 # Changelog
 Changelog for FrostFS Node
 
-## [Unreleased]
+## [v0.37.0] - 2023-12-07 - Academy of Sciences
 
 ### Added
 - Support impersonate bearer token (#229)
@@ -11,9 +11,12 @@ Changelog for FrostFS Node
 - Set extra wallets on SIGHUP for ir (#125)
 - Writecache metrics (#312)
 - Add tree service metrics (#370)
+- Add `frostfs-cli control ir remove-container` (#733)
 
 ### Changed
 - `frostfs-cli util locode generate` is now much faster (#309)
+- Send bootstrap query every epoch in `frostfs-node` (#691)
+
 ### Fixed
 - Take network settings into account during netmap contract update (#100)
 - Read config files from dir even if config file not provided via `--config` for node (#238)
@@ -21,6 +24,8 @@ Changelog for FrostFS Node
 - Tree service panic in its internal client cache (#322)
 - Iterate over endpoints when create ws client in morph's constructor (#304)
 - Delete complex objects with GC (#332)
+- Connection leaks after netmap address updates (#674)
+- Count writecache metrics properly (#725)
 
 ### Removed
 ### Updated

VERSION

@@ -1 +1 @@
-v0.36.0
+v0.37.0

cmd/frostfs-cli/modules/control/ir.go

@@ -12,8 +12,10 @@ func initControlIRCmd() {
 	irCmd.AddCommand(tickEpochCmd)
 	irCmd.AddCommand(removeNodeCmd)
 	irCmd.AddCommand(irHealthCheckCmd)
+	irCmd.AddCommand(removeContainerCmd)
 
 	initControlIRTickEpochCmd()
 	initControlIRRemoveNodeCmd()
 	initControlIRHealthCheckCmd()
+	initControlIRRemoveContainerCmd()
 }

cmd/frostfs-cli/modules/control/ir_remove_container.go

@@ -0,0 +1,94 @@
+package control
+
+import (
+	"errors"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs"
+	rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
+	commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
+	ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
+	ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server"
+	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
+	"github.com/spf13/cobra"
+)
+
+const (
+	ownerFlag = "owner"
+)
+
+var removeContainerCmd = &cobra.Command{
+	Use:   "remove-container",
+	Short: "Schedules a container removal",
+	Long: `Schedules a container removal via a notary request.
+Container data will be deleted asynchronously by policer.
+To check removal status "frostfs-cli container list" command can be used.`,
+	Run: removeContainer,
+}
+
+func initControlIRRemoveContainerCmd() {
+	initControlFlags(removeContainerCmd)
+
+	flags := removeContainerCmd.Flags()
+	flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage)
+	flags.String(ownerFlag, "", "Container owner's wallet address.")
+	removeContainerCmd.MarkFlagsMutuallyExclusive(commonflags.CIDFlag, ownerFlag)
+}
+
+func removeContainer(cmd *cobra.Command, _ []string) {
+	req := prepareRemoveContainerRequest(cmd)
+
+	pk := key.Get(cmd)
+	c := getClient(cmd, pk)
+
+	commonCmd.ExitOnErr(cmd, "could not sign request: %w", ircontrolsrv.SignMessage(pk, req))
+
+	var resp *ircontrol.RemoveContainerResponse
+	err := c.ExecRaw(func(client *rawclient.Client) error {
+		var err error
+		resp, err = ircontrol.RemoveContainer(client, req)
+		return err
+	})
+	commonCmd.ExitOnErr(cmd, "failed to execute request: %w", err)
+
+	verifyResponse(cmd, resp.GetSignature(), resp.GetBody())
+
+	if len(req.GetBody().GetContainerId()) > 0 {
+		cmd.Println("Container scheduled to removal")
+	} else {
+		cmd.Println("User containers sheduled to removal")
+	}
+}
+
+func prepareRemoveContainerRequest(cmd *cobra.Command) *ircontrol.RemoveContainerRequest {
+	req := &ircontrol.RemoveContainerRequest{
+		Body: &ircontrol.RemoveContainerRequest_Body{},
+	}
+
+	cidStr, err := cmd.Flags().GetString(commonflags.CIDFlag)
+	commonCmd.ExitOnErr(cmd, "failed to get cid: ", err)
+
+	ownerStr, err := cmd.Flags().GetString(ownerFlag)
+	commonCmd.ExitOnErr(cmd, "failed to get owner: ", err)
+
+	if len(ownerStr) == 0 && len(cidStr) == 0 {
+		commonCmd.ExitOnErr(cmd, "invalid usage: %w", errors.New("neither owner's wallet address nor container ID are specified"))
+	}
+
+	if len(ownerStr) > 0 {
+		var owner user.ID
+		commonCmd.ExitOnErr(cmd, "invalid owner ID: %w", owner.DecodeString(ownerStr))
+		var ownerID refs.OwnerID
+		owner.WriteToV2(&ownerID)
+		req.Body.Owner = ownerID.StableMarshal(nil)
+	}
+
+	if len(cidStr) > 0 {
+		var containerID cid.ID
+		commonCmd.ExitOnErr(cmd, "invalid container ID: %w", containerID.DecodeString(cidStr))
+		req.Body.ContainerId = containerID[:]
+	}
+	return req
+}

cmd/frostfs-cli/modules/object/nodes.go

@@ -34,7 +34,7 @@ type objectNodesInfo struct {
 	containerID       cid.ID
 	objectID          oid.ID
 	relatedObjectIDs  []oid.ID
-	isLock           bool
+	isLockOrTombstone bool
 }
 
 type boolError struct {
@@ -103,7 +103,7 @@ func getObjectInfo(cmd *cobra.Command, cnrID cid.ID, objID oid.ID, cli *client.C
 		return &objectNodesInfo{
 			containerID:       cnrID,
 			objectID:          objID,
-			isLock:      res.Header().Type() == objectSDK.TypeLock,
+			isLockOrTombstone: res.Header().Type() == objectSDK.TypeLock || res.Header().Type() == objectSDK.TypeTombstone,
 		}
 	}
 
@@ -191,7 +191,7 @@ func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placemen
 		numOfReplicas := placementPolicy.ReplicaNumberByIndex(repIdx)
 		var nodeIdx uint32
 		for _, n := range rep {
-			if !objInfo.isLock && nodeIdx == numOfReplicas { //lock object should be on all container nodes
+			if !objInfo.isLockOrTombstone && nodeIdx == numOfReplicas { // lock and tombstone objects should be on all container nodes
 				break
 			}
 			nodes[n.Hash()] = n
@@ -213,7 +213,8 @@ func getRequiredPlacement(cmd *cobra.Command, objInfo *objectNodesInfo, placemen
 }
 
 func getActualPlacement(cmd *cobra.Command, netmap *netmapSDK.NetMap, requiredPlacement map[uint64]netmapSDK.NodeInfo,
-	pk *ecdsa.PrivateKey, objInfo *objectNodesInfo) map[uint64]boolError {
+	pk *ecdsa.PrivateKey, objInfo *objectNodesInfo,
+) map[uint64]boolError {
 	result := make(map[uint64]boolError)
 	resultMtx := &sync.Mutex{}
 

cmd/frostfs-ir/config.go

@@ -7,6 +7,7 @@ import (
 
 	configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
 	"github.com/spf13/viper"
 	"go.uber.org/zap"
 )
@@ -67,6 +68,7 @@ func watchForSignal(cancel func()) {
 				if err != nil {
 					log.Error(logs.FrostFSNodeConfigurationReading, zap.Error(err))
 				}
+				client.CleanInvScript = cfg.GetBool("morph.cleaninvscript")
 				log.Info(logs.FrostFSNodeConfigurationHasBeenReloadedSuccessfully)
 			case syscall.SIGTERM, syscall.SIGINT:
 				log.Info(logs.FrostFSNodeTerminationSignalHasBeenReceivedStopping)

cmd/frostfs-ir/main.go

@@ -10,6 +10,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/misc"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
 	"github.com/spf13/viper"
 	"go.uber.org/zap"
@@ -60,6 +61,7 @@ func main() {
 	var err error
 	cfg, err = newConfig()
 	exitErr(err)
+	client.CleanInvScript = cfg.GetBool("morph.cleaninvscript")
 
 	logPrm.MetricsNamespace = "frostfs_ir"
 	err = logPrm.SetLevelString(

cmd/frostfs-lens/internal/meta/inspect.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	common "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-lens/internal"
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/blobovniczatree"
 	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
 	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@@ -40,7 +40,7 @@ func inspectFunc(cmd *cobra.Command, _ []string) {
 	common.ExitOnErr(cmd, common.Errf("could not check if the obj is small: %w", err))
 
 	if id := resStorageID.StorageID(); id != nil {
-		cmd.Printf("Object storageID: %s\n\n", blobovnicza.NewIDFromBytes(id).String())
+		cmd.Printf("Object storageID: %s\n\n", blobovniczatree.NewIDFromBytes(id).Path())
 	} else {
 		cmd.Printf("Object does not contain storageID\n\n")
 	}

cmd/frostfs-node/cache.go

@@ -6,13 +6,11 @@ import (
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
-	cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
 	putsvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/object/put"
 	utilSync "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/sync"
 	apistatus "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client/status"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
 	netmapSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
 	lru "github.com/hashicorp/golang-lru/v2"
 )
 
@@ -244,117 +242,6 @@ func (s *lruNetmapSource) Epoch() (uint64, error) {
 	return s.netState.CurrentEpoch(), nil
 }
 
-// wrapper over TTL cache of values read from the network
-// that implements container lister.
-type ttlContainerLister struct {
-	inner  *ttlNetCache[string, *cacheItemContainerList]
-	client *cntClient.Client
-}
-
-// value type for ttlNetCache used by ttlContainerLister.
-type cacheItemContainerList struct {
-	// protects list from concurrent add/remove ops
-	mtx sync.RWMutex
-	// actual list of containers owner by the particular user
-	list []cid.ID
-}
-
-func newCachedContainerLister(c *cntClient.Client, ttl time.Duration) ttlContainerLister {
-	const containerListerCacheSize = 100
-
-	lruCnrListerCache := newNetworkTTLCache(containerListerCacheSize, ttl, func(strID string) (*cacheItemContainerList, error) {
-		var id *user.ID
-
-		if strID != "" {
-			id = new(user.ID)
-
-			err := id.DecodeString(strID)
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		list, err := c.ContainersOf(id)
-		if err != nil {
-			return nil, err
-		}
-
-		return &cacheItemContainerList{
-			list: list,
-		}, nil
-	})
-
-	return ttlContainerLister{inner: lruCnrListerCache, client: c}
-}
-
-// List returns list of container IDs from the cache. If list is missing in the
-// cache or expired, then it returns container IDs from side chain and updates
-// the cache.
-func (s ttlContainerLister) List(id *user.ID) ([]cid.ID, error) {
-	if id == nil {
-		return s.client.ContainersOf(nil)
-	}
-
-	item, err := s.inner.get(id.EncodeToString())
-	if err != nil {
-		return nil, err
-	}
-
-	item.mtx.RLock()
-	res := make([]cid.ID, len(item.list))
-	copy(res, item.list)
-	item.mtx.RUnlock()
-
-	return res, nil
-}
-
-// updates cached list of owner's containers: cnr is added if flag is true, otherwise it's removed.
-// Concurrent calls can lead to some races:
-//   - two parallel additions to missing owner's cache can lead to only one container to be cached
-//   - async cache value eviction can lead to idle addition
-//
-// All described race cases aren't critical since cache values expire anyway, we just try
-// to increase cache actuality w/o huge overhead on synchronization.
-func (s *ttlContainerLister) update(owner user.ID, cnr cid.ID, add bool) {
-	strOwner := owner.EncodeToString()
-
-	val, ok := s.inner.cache.Peek(strOwner)
-	if !ok {
-		// we could cache the single cnr but in this case we will disperse
-		// with the Sidechain a lot
-		return
-	}
-
-	if s.inner.ttl <= time.Since(val.t) {
-		return
-	}
-
-	item := val.v
-
-	item.mtx.Lock()
-	{
-		found := false
-
-		for i := range item.list {
-			if found = item.list[i].Equals(cnr); found {
-				if !add {
-					item.list = append(item.list[:i], item.list[i+1:]...)
-					// if list became empty we don't remove the value from the cache
-					// since empty list is a correct value, and we don't want to insta
-					// re-request it from the Sidechain
-				}
-
-				break
-			}
-		}
-
-		if add && !found {
-			item.list = append(item.list, cnr)
-		}
-	}
-	item.mtx.Unlock()
-}
-
 type cachedIRFetcher struct {
 	*ttlNetCache[struct{}, [][]byte]
 }

cmd/frostfs-node/config.go

@@ -24,6 +24,7 @@ import (
 	blobovniczaconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza"
 	fstreeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/engine/shard/blobstor/fstree"
 	loggerconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/logger"
+	morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph"
 	nodeconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/node"
 	objectconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/object"
 	replicatorconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/replicator"
@@ -101,6 +102,7 @@ type applicationConfiguration struct {
 		shardPoolSize  uint32
 		shards         []shardCfg
 		lowMem         bool
+		rebuildWorkers uint32
 	}
 }
 
@@ -176,6 +178,8 @@ type subStorageCfg struct {
 	width           uint64
 	leafWidth       uint64
 	openedCacheSize int
+	initWorkerCount int
+	initInAdvance   bool
 }
 
 // readConfig fills applicationConfiguration with raw configuration values
@@ -207,6 +211,10 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error {
 	a.EngineCfg.errorThreshold = engineconfig.ShardErrorThreshold(c)
 	a.EngineCfg.shardPoolSize = engineconfig.ShardPoolSize(c)
 	a.EngineCfg.lowMem = engineconfig.EngineLowMemoryConsumption(c)
+	a.EngineCfg.rebuildWorkers = engineconfig.EngineRebuildWorkersCount(c)
+
+	// Kludge
+	client.CleanInvScript = morphconfig.CleanInvocationScript(c)
 
 	return engineconfig.IterateShards(c, false, func(sc *shardconfig.Config) error { return a.updateShardConfig(c, sc) })
 }
@@ -291,6 +299,8 @@ func (a *applicationConfiguration) setShardStorageConfig(newConfig *shardCfg, ol
 			sCfg.width = sub.ShallowWidth()
 			sCfg.leafWidth = sub.LeafWidth()
 			sCfg.openedCacheSize = sub.OpenedCacheSize()
+			sCfg.initWorkerCount = sub.InitWorkerCount()
+			sCfg.initInAdvance = sub.InitInAdvance()
 		case fstree.Type:
 			sub := fstreeconfig.From((*config.Config)(storagesCfg[i]))
 			sCfg.depth = sub.Depth()
@@ -346,7 +356,6 @@ type internals struct {
 	healthStatus *atomic.Int32
 	// is node under maintenance
 	isMaintenance atomic.Bool
-	alreadyBootstraped bool
 }
 
 // starts node's maintenance.
@@ -686,13 +695,14 @@ func initCfgObject(appCfg *config.Config) cfgObject {
 }
 
 func (c *cfg) engineOpts() []engine.Option {
-	opts := make([]engine.Option, 0, 4)
+	var opts []engine.Option
 
 	opts = append(opts,
 		engine.WithShardPoolSize(c.EngineCfg.shardPoolSize),
 		engine.WithErrorThreshold(c.EngineCfg.errorThreshold),
 		engine.WithLogger(c.log),
 		engine.WithLowMemoryConsumption(c.EngineCfg.lowMem),
+		engine.WithRebuildWorkersCount(c.EngineCfg.rebuildWorkers),
 	)
 
 	if c.metricsCollector != nil {
@@ -781,7 +791,10 @@ func (c *cfg) getSubstorageOpts(shCfg shardCfg) []blobstor.SubStorage {
 				blobovniczatree.WithBlobovniczaShallowWidth(sRead.width),
 				blobovniczatree.WithBlobovniczaLeafWidth(sRead.leafWidth),
 				blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize),
+				blobovniczatree.WithInitWorkerCount(sRead.initWorkerCount),
+				blobovniczatree.WithInitInAdvance(sRead.initInAdvance),
 				blobovniczatree.WithLogger(c.log),
+				blobovniczatree.WithObjectSizeLimit(shCfg.smallSizeObjectLimit),
 			}
 
 			if c.metricsCollector != nil {
@@ -1149,9 +1162,8 @@ func (c *cfg) shutdown() {
 	c.setHealthStatus(control.HealthStatus_SHUTTING_DOWN)
 
 	c.ctxCancel()
-	c.done <- struct{}{}
+	close(c.done)
 	for i := range c.closers {
 		c.closers[len(c.closers)-1-i].fn()
 	}
-	close(c.internalErr)
 }

cmd/frostfs-node/config/engine/config.go

@@ -15,6 +15,9 @@ const (
 	// ShardPoolSizeDefault is a default value of routine pool size per-shard to
 	// process object PUT operations in a storage engine.
 	ShardPoolSizeDefault = 20
+	// RebuildWorkersCountDefault is a default value of the workers count to
+	// process storage rebuild operations in a storage engine.
+	RebuildWorkersCountDefault = 100
 )
 
 // ErrNoShardConfigured is returned when at least 1 shard is required but none are found.
@@ -88,3 +91,11 @@ func ShardErrorThreshold(c *config.Config) uint32 {
 func EngineLowMemoryConsumption(c *config.Config) bool {
 	return config.BoolSafe(c.Sub(subsection), "low_mem")
 }
+
+// EngineRebuildWorkersCount returns value of "rebuild_workers_count" config parmeter from "storage" section.
+func EngineRebuildWorkersCount(c *config.Config) uint32 {
+	if v := config.Uint32Safe(c.Sub(subsection), "rebuild_workers_count"); v > 0 {
+		return v
+	}
+	return RebuildWorkersCountDefault
+}

cmd/frostfs-node/config/engine/config_test.go

@@ -38,15 +38,17 @@ func TestEngineSection(t *testing.T) {
 		require.EqualValues(t, 0, engineconfig.ShardErrorThreshold(empty))
 		require.EqualValues(t, engineconfig.ShardPoolSizeDefault, engineconfig.ShardPoolSize(empty))
 		require.EqualValues(t, mode.ReadWrite, shardconfig.From(empty).Mode())
+		require.EqualValues(t, engineconfig.RebuildWorkersCountDefault, engineconfig.EngineRebuildWorkersCount(empty))
 	})
 
 	const path = "../../../../config/example/node"
 
-	var fileConfigTest = func(c *config.Config) {
+	fileConfigTest := func(c *config.Config) {
 		num := 0
 
 		require.EqualValues(t, 100, engineconfig.ShardErrorThreshold(c))
 		require.EqualValues(t, 15, engineconfig.ShardPoolSize(c))
+		require.EqualValues(t, uint32(1000), engineconfig.EngineRebuildWorkersCount(c))
 
 		err := engineconfig.IterateShards(c, true, func(sc *shardconfig.Config) error {
 			defer func() {
@@ -78,7 +80,7 @@ func TestEngineSection(t *testing.T) {
 				require.EqualValues(t, 3221225472, wc.SizeLimit())
 
 				require.Equal(t, "tmp/0/meta", meta.Path())
-				require.Equal(t, fs.FileMode(0644), meta.BoltDB().Perm())
+				require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm())
 				require.Equal(t, 100, meta.BoltDB().MaxBatchSize())
 				require.Equal(t, 10*time.Millisecond, meta.BoltDB().MaxBatchDelay())
 
@@ -89,15 +91,17 @@ func TestEngineSection(t *testing.T) {
 				require.Equal(t, 2, len(ss))
 				blz := blobovniczaconfig.From((*config.Config)(ss[0]))
 				require.Equal(t, "tmp/0/blob/blobovnicza", ss[0].Path())
-				require.EqualValues(t, 0644, blz.BoltDB().Perm())
+				require.EqualValues(t, 0o644, blz.BoltDB().Perm())
 				require.EqualValues(t, 4194304, blz.Size())
 				require.EqualValues(t, 1, blz.ShallowDepth())
 				require.EqualValues(t, 4, blz.ShallowWidth())
 				require.EqualValues(t, 50, blz.OpenedCacheSize())
 				require.EqualValues(t, 10, blz.LeafWidth())
+				require.EqualValues(t, 10, blz.InitWorkerCount())
+				require.EqualValues(t, true, blz.InitInAdvance())
 
 				require.Equal(t, "tmp/0/blob", ss[1].Path())
-				require.EqualValues(t, 0644, ss[1].Perm())
+				require.EqualValues(t, 0o644, ss[1].Perm())
 
 				fst := fstreeconfig.From((*config.Config)(ss[1]))
 				require.EqualValues(t, 5, fst.Depth())
@@ -112,7 +116,7 @@ func TestEngineSection(t *testing.T) {
 				require.Equal(t, mode.ReadOnly, sc.Mode())
 			case 1:
 				require.Equal(t, "tmp/1/blob/pilorama.db", pl.Path())
-				require.Equal(t, fs.FileMode(0644), pl.Perm())
+				require.Equal(t, fs.FileMode(0o644), pl.Perm())
 				require.True(t, pl.NoSync())
 				require.Equal(t, 5*time.Millisecond, pl.MaxBatchDelay())
 				require.Equal(t, 100, pl.MaxBatchSize())
@@ -127,7 +131,7 @@ func TestEngineSection(t *testing.T) {
 				require.EqualValues(t, 4294967296, wc.SizeLimit())
 
 				require.Equal(t, "tmp/1/meta", meta.Path())
-				require.Equal(t, fs.FileMode(0644), meta.BoltDB().Perm())
+				require.Equal(t, fs.FileMode(0o644), meta.BoltDB().Perm())
 				require.Equal(t, 200, meta.BoltDB().MaxBatchSize())
 				require.Equal(t, 20*time.Millisecond, meta.BoltDB().MaxBatchDelay())
 
@@ -144,9 +148,10 @@ func TestEngineSection(t *testing.T) {
 				require.EqualValues(t, 4, blz.ShallowWidth())
 				require.EqualValues(t, 50, blz.OpenedCacheSize())
 				require.EqualValues(t, 10, blz.LeafWidth())
+				require.EqualValues(t, blobovniczaconfig.InitWorkerCountDefault, blz.InitWorkerCount())
 
 				require.Equal(t, "tmp/1/blob", ss[1].Path())
-				require.EqualValues(t, 0644, ss[1].Perm())
+				require.EqualValues(t, 0o644, ss[1].Perm())
 
 				fst := fstreeconfig.From((*config.Config)(ss[1]))
 				require.EqualValues(t, 5, fst.Depth())

cmd/frostfs-node/config/engine/shard/blobstor/blobovnicza/config.go

@@ -22,6 +22,9 @@ const (
 
 	// OpenedCacheSizeDefault is a default cache size of opened Blobovnicza's.
 	OpenedCacheSizeDefault = 16
+
+	// InitWorkerCountDefault is a default workers count to initialize Blobovnicza's.
+	InitWorkerCountDefault = 5
 )
 
 // From wraps config section into Config.
@@ -112,3 +115,29 @@ func (x *Config) LeafWidth() uint64 {
 		"leaf_width",
 	)
 }
+
+// InitWorkersCount returns the value of "init_worker_count" config parameter.
+//
+// Returns InitWorkerCountDefault if the value is not a positive number.
+func (x *Config) InitWorkerCount() int {
+	d := config.IntSafe(
+		(*config.Config)(x),
+		"init_worker_count",
+	)
+
+	if d > 0 {
+		return int(d)
+	}
+
+	return InitWorkerCountDefault
+}
+
+// InitInAdvance returns the value of "init_in_advance" config parameter.
+//
+// Returns False if the value is not defined or invalid.
+func (x *Config) InitInAdvance() bool {
+	return config.BoolSafe(
+		(*config.Config)(x),
+		"init_in_advance",
+	)
+}

cmd/frostfs-node/config/morph/config.go

@@ -97,3 +97,8 @@ func SwitchInterval(c *config.Config) time.Duration {
 
 	return SwitchIntervalDefault
 }
+
+// CleanInvocationScript this is a kludge purely for update to work.
+func CleanInvocationScript(c *config.Config) bool {
+	return config.Bool(c.Sub(subsection), "cleaninvscript")
+}

cmd/frostfs-node/container.go

@@ -3,44 +3,22 @@ package main
 import (
 	"bytes"
 	"context"
-	"crypto/ecdsa"
-	"crypto/sha256"
-	"errors"
-	"fmt"
-	"strconv"
 
-	containerV2 "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container"
 	containerGRPC "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container/grpc"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/client"
 	containerCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container"
-	netmapCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine"
 	cntClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event"
 	containerEvent "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event/container"
 	containerTransportGRPC "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/network/transport/container/grpc"
 	containerService "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container"
-	loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller"
-	loadroute "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/route"
-	placementrouter "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/route/placement"
-	loadstorage "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/storage"
 	containerMorph "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/morph"
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
-	apiClient "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
-	containerSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
 	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user"
 	"go.uber.org/zap"
 )
 
-const (
+func initContainerService(_ context.Context, c *cfg) {
-	startEstimationNotifyEvent = "StartEstimation"
-	stopEstimationNotifyEvent  = "StopEstimation"
-)
-
-func initContainerService(ctx context.Context, c *cfg) {
 	// container wrapper that tries to invoke notary
 	// requests if chain is configured so
 	wrap, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0, cntClient.TryNotary())
@@ -52,44 +30,10 @@ func initContainerService(ctx context.Context, c *cfg) {
 
 	cnrRdr, cnrWrt := configureEACLAndContainerSources(c, wrap, cnrSrc)
 
-	loadAccumulator := loadstorage.New(loadstorage.Prm{})
-
-	loadPlacementBuilder := &loadPlacementBuilder{
-		log:    c.log,
-		nmSrc:  c.netMapSource,
-		cnrSrc: cnrSrc,
-	}
-
-	routeBuilder := placementrouter.New(placementrouter.Prm{
-		PlacementBuilder: loadPlacementBuilder,
-	})
-
-	loadRouter := loadroute.New(
-		loadroute.Prm{
-			LocalServerInfo: c,
-			RemoteWriterProvider: &remoteLoadAnnounceProvider{
-				key:             &c.key.PrivateKey,
-				netmapKeys:      c,
-				clientCache:     c.bgClientCache,
-				deadEndProvider: loadcontroller.SimpleWriterProvider(loadAccumulator),
-			},
-			Builder: routeBuilder,
-		},
-		loadroute.WithLogger(c.log),
-	)
-
-	setLoadController(ctx, c, loadRouter, loadAccumulator)
-
 	server := containerTransportGRPC.New(
 		containerService.NewSignService(
 			&c.key.PrivateKey,
-			&usedSpaceService{
+			containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc),
-				Server:               containerService.NewExecutionService(containerMorph.NewExecutor(cnrRdr, cnrWrt), c.respSvc),
-				loadWriterProvider:   loadRouter,
-				loadPlacementBuilder: loadPlacementBuilder,
-				routeBuilder:         routeBuilder,
-				cfg:                  c,
-			},
 		),
 	)
 
@@ -119,7 +63,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c
 		// use RPC node as source of Container contract items (with caching)
 		cachedContainerStorage := newCachedContainerStorage(cnrSrc, c.cfgMorph.cacheTTL)
 		cachedEACLStorage := newCachedEACLStorage(eACLFetcher, c.cfgMorph.cacheTTL)
-		cachedContainerLister := newCachedContainerLister(client, c.cfgMorph.cacheTTL)
 
 		subscribeToContainerCreation(c, func(e event.Event) {
 			ev := e.(containerEvent.PutSuccess)
@@ -130,7 +73,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c
 			//  creation success are most commonly tracked by polling GET op.
 			cnr, err := cnrSrc.Get(ev.ID)
 			if err == nil {
-				cachedContainerLister.update(cnr.Value.Owner(), ev.ID, true)
 				cachedContainerStorage.containerCache.set(ev.ID, cnr, nil)
 			} else {
 				// unlike removal, we expect successful receive of the container
@@ -149,15 +91,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c
 		subscribeToContainerRemoval(c, func(e event.Event) {
 			ev := e.(containerEvent.DeleteSuccess)
 
-			// read owner of the removed container in order to update the listing cache.
-			// It's strange to read already removed container, but we can successfully hit
-			// the cache.
-			// TODO: use owner directly from the event after neofs-contract#256 will become resolved
-			cnr, err := cachedContainerStorage.Get(ev.ID)
-			if err == nil {
-				cachedContainerLister.update(cnr.Value.Owner(), ev.ID, false)
-			}
-
 			cachedContainerStorage.handleRemoval(ev.ID)
 			c.log.Debug(logs.FrostFSNodeContainerRemovalEventsReceipt,
 				zap.Stringer("id", ev.ID),
@@ -167,7 +100,7 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c
 		c.cfgObject.eaclSource = cachedEACLStorage
 		c.cfgObject.cnrSource = cachedContainerStorage
 
-		cnrRdr.lister = cachedContainerLister
+		cnrRdr.lister = client
 		cnrRdr.eacl = c.cfgObject.eaclSource
 		cnrRdr.src = c.cfgObject.cnrSource
 
@@ -178,50 +111,6 @@ func configureEACLAndContainerSources(c *cfg, client *cntClient.Client, cnrSrc c
 	return cnrRdr, cnrWrt
 }
 
-func setLoadController(ctx context.Context, c *cfg, loadRouter *loadroute.Router, loadAccumulator *loadstorage.Storage) {
-	pubKey := c.key.PublicKey().Bytes()
-
-	// container wrapper that always sends non-notary
-	// requests
-	wrapperNoNotary, err := cntClient.NewFromMorph(c.cfgMorph.client, c.cfgContainer.scriptHash, 0)
-	fatalOnErr(err)
-
-	resultWriter := &morphLoadWriter{
-		log:            c.log,
-		cnrMorphClient: wrapperNoNotary,
-		key:            pubKey,
-	}
-
-	localMetrics := &localStorageLoad{
-		log:    c.log,
-		engine: c.cfgObject.cfgLocalStorage.localStorage,
-	}
-
-	ctrl := loadcontroller.New(
-		loadcontroller.Prm{
-			LocalMetrics:            loadcontroller.SimpleIteratorProvider(localMetrics),
-			AnnouncementAccumulator: loadcontroller.SimpleIteratorProvider(loadAccumulator),
-			LocalAnnouncementTarget: loadRouter,
-			ResultReceiver:          loadcontroller.SimpleWriterProvider(resultWriter),
-		},
-		loadcontroller.WithLogger(c.log),
-	)
-
-	setContainerNotificationParser(c, startEstimationNotifyEvent, containerEvent.ParseStartEstimation)
-	addContainerAsyncNotificationHandler(c, startEstimationNotifyEvent, func(ev event.Event) {
-		ctrl.Start(ctx, loadcontroller.StartPrm{
-			Epoch: ev.(containerEvent.StartEstimation).Epoch(),
-		})
-	})
-
-	setContainerNotificationParser(c, stopEstimationNotifyEvent, containerEvent.ParseStopEstimation)
-	addContainerAsyncNotificationHandler(c, stopEstimationNotifyEvent, func(ev event.Event) {
-		ctrl.Stop(ctx, loadcontroller.StopPrm{
-			Epoch: ev.(containerEvent.StopEstimation).Epoch(),
-		})
-	})
-}
-
 // addContainerNotificationHandler adds handler that will be executed synchronously.
 func addContainerNotificationHandler(c *cfg, sTyp string, h event.Handler) {
 	typ := event.TypeFromString(sTyp)
@@ -284,219 +173,6 @@ func setContainerNotificationParser(c *cfg, sTyp string, p event.NotificationPar
 	c.cfgContainer.parsers[typ] = p
 }
 
-type morphLoadWriter struct {
-	log *logger.Logger
-
-	cnrMorphClient *cntClient.Client
-
-	key []byte
-}
-
-func (w *morphLoadWriter) Put(a containerSDK.SizeEstimation) error {
-	w.log.Debug(logs.FrostFSNodeSaveUsedSpaceAnnouncementInContract,
-		zap.Uint64("epoch", a.Epoch()),
-		zap.Stringer("cid", a.Container()),
-		zap.Uint64("size", a.Value()),
-	)
-
-	prm := cntClient.AnnounceLoadPrm{}
-
-	prm.SetAnnouncement(a)
-	prm.SetReporter(w.key)
-
-	return w.cnrMorphClient.AnnounceLoad(prm)
-}
-
-func (*morphLoadWriter) Close(context.Context) error {
-	return nil
-}
-
-type nopLoadWriter struct{}
-
-func (nopLoadWriter) Put(containerSDK.SizeEstimation) error {
-	return nil
-}
-
-func (nopLoadWriter) Close(context.Context) error {
-	return nil
-}
-
-type remoteLoadAnnounceProvider struct {
-	key *ecdsa.PrivateKey
-
-	netmapKeys netmapCore.AnnouncedKeys
-
-	clientCache interface {
-		Get(client.NodeInfo) (client.MultiAddressClient, error)
-	}
-
-	deadEndProvider loadcontroller.WriterProvider
-}
-
-func (r *remoteLoadAnnounceProvider) InitRemote(srv loadcontroller.ServerInfo) (loadcontroller.WriterProvider, error) {
-	if srv == nil {
-		return r.deadEndProvider, nil
-	}
-
-	if r.netmapKeys.IsLocalKey(srv.PublicKey()) {
-		// if local => return no-op writer
-		return loadcontroller.SimpleWriterProvider(new(nopLoadWriter)), nil
-	}
-
-	var info client.NodeInfo
-
-	err := client.NodeInfoFromRawNetmapElement(&info, srv)
-	if err != nil {
-		return nil, fmt.Errorf("parse client node info: %w", err)
-	}
-
-	c, err := r.clientCache.Get(info)
-	if err != nil {
-		return nil, fmt.Errorf("could not initialize API client: %w", err)
-	}
-
-	return &remoteLoadAnnounceWriterProvider{
-		client: c,
-	}, nil
-}
-
-type remoteLoadAnnounceWriterProvider struct {
-	client client.Client
-}
-
-func (p *remoteLoadAnnounceWriterProvider) InitWriter([]loadcontroller.ServerInfo) (loadcontroller.Writer, error) {
-	return &remoteLoadAnnounceWriter{
-		client: p.client,
-	}, nil
-}
-
-type remoteLoadAnnounceWriter struct {
-	client client.Client
-
-	buf []containerSDK.SizeEstimation
-}
-
-func (r *remoteLoadAnnounceWriter) Put(a containerSDK.SizeEstimation) error {
-	r.buf = append(r.buf, a)
-
-	return nil
-}
-
-func (r *remoteLoadAnnounceWriter) Close(ctx context.Context) error {
-	cliPrm := apiClient.PrmAnnounceSpace{
-		Announcements: r.buf,
-	}
-
-	_, err := r.client.ContainerAnnounceUsedSpace(ctx, cliPrm)
-	return err
-}
-
-type loadPlacementBuilder struct {
-	log *logger.Logger
-
-	nmSrc netmapCore.Source
-
-	cnrSrc containerCore.Source
-}
-
-func (l *loadPlacementBuilder) BuildPlacement(epoch uint64, cnr cid.ID) ([][]netmap.NodeInfo, error) {
-	cnrNodes, nm, err := l.buildPlacement(epoch, cnr)
-	if err != nil {
-		return nil, err
-	}
-
-	const pivotPrefix = "load_announcement_"
-
-	pivot := []byte(
-		pivotPrefix + strconv.FormatUint(epoch, 10),
-	)
-
-	placement, err := nm.PlacementVectors(cnrNodes, pivot)
-	if err != nil {
-		return nil, fmt.Errorf("could not build placement vectors: %w", err)
-	}
-
-	return placement, nil
-}
-
-func (l *loadPlacementBuilder) buildPlacement(epoch uint64, idCnr cid.ID) ([][]netmap.NodeInfo, *netmap.NetMap, error) {
-	cnr, err := l.cnrSrc.Get(idCnr)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	nm, err := l.nmSrc.GetNetMapByEpoch(epoch)
-	if err != nil {
-		return nil, nil, fmt.Errorf("could not get network map: %w", err)
-	}
-
-	binCnr := make([]byte, sha256.Size)
-	idCnr.Encode(binCnr)
-
-	cnrNodes, err := nm.ContainerNodes(cnr.Value.PlacementPolicy(), binCnr)
-	if err != nil {
-		return nil, nil, fmt.Errorf("could not build container nodes: %w", err)
-	}
-
-	return cnrNodes, nm, nil
-}
-
-type localStorageLoad struct {
-	log *logger.Logger
-
-	engine *engine.StorageEngine
-}
-
-func (d *localStorageLoad) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontroller.UsedSpaceHandler) error {
-	idList, err := engine.ListContainers(context.TODO(), d.engine)
-	if err != nil {
-		return fmt.Errorf("list containers on engine failure: %w", err)
-	}
-
-	for i := range idList {
-		sz, err := engine.ContainerSize(d.engine, idList[i])
-		if err != nil {
-			d.log.Debug(logs.FrostFSNodeFailedToCalculateContainerSizeInStorageEngine,
-				zap.Stringer("cid", idList[i]),
-				zap.String("error", err.Error()),
-			)
-
-			continue
-		}
-
-		d.log.Debug(logs.FrostFSNodeContainerSizeInStorageEngineCalculatedSuccessfully,
-			zap.Uint64("size", sz),
-			zap.Stringer("cid", idList[i]),
-		)
-
-		var a containerSDK.SizeEstimation
-		a.SetContainer(idList[i])
-		a.SetValue(sz)
-
-		if f != nil && !f(a) {
-			continue
-		}
-
-		if err := h(a); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-type usedSpaceService struct {
-	containerService.Server
-
-	loadWriterProvider loadcontroller.WriterProvider
-
-	loadPlacementBuilder *loadPlacementBuilder
-
-	routeBuilder loadroute.Builder
-
-	cfg *cfg
-}
-
 func (c *cfg) PublicKey() []byte {
 	return nodeKeyFromNetmap(c)
 }
@@ -517,125 +193,6 @@ func (c *cfg) ExternalAddresses() []string {
 	return c.cfgNodeInfo.localInfo.ExternalAddresses()
 }
 
-func (c *usedSpaceService) PublicKey() []byte {
-	return nodeKeyFromNetmap(c.cfg)
-}
-
-func (c *usedSpaceService) IterateAddresses(f func(string) bool) {
-	c.cfg.iterateNetworkAddresses(f)
-}
-
-func (c *usedSpaceService) NumberOfAddresses() int {
-	return c.cfg.addressNum()
-}
-
-func (c *usedSpaceService) ExternalAddresses() []string {
-	return c.cfg.ExternalAddresses()
-}
-
-func (c *usedSpaceService) AnnounceUsedSpace(ctx context.Context, req *containerV2.AnnounceUsedSpaceRequest) (*containerV2.AnnounceUsedSpaceResponse, error) {
-	var passedRoute []loadcontroller.ServerInfo
-
-	for hdr := req.GetVerificationHeader(); hdr != nil; hdr = hdr.GetOrigin() {
-		passedRoute = append(passedRoute, &containerOnlyKeyRemoteServerInfo{
-			key: hdr.GetBodySignature().GetKey(),
-		})
-	}
-
-	for left, right := 0, len(passedRoute)-1; left < right; left, right = left+1, right-1 {
-		passedRoute[left], passedRoute[right] = passedRoute[right], passedRoute[left]
-	}
-
-	passedRoute = append(passedRoute, c)
-
-	w, err := c.loadWriterProvider.InitWriter(passedRoute)
-	if err != nil {
-		return nil, fmt.Errorf("could not initialize container's used space writer: %w", err)
-	}
-
-	var est containerSDK.SizeEstimation
-
-	for _, aV2 := range req.GetBody().GetAnnouncements() {
-		err = est.ReadFromV2(aV2)
-		if err != nil {
-			return nil, fmt.Errorf("invalid size announcement: %w", err)
-		}
-
-		if err := c.processLoadValue(ctx, est, passedRoute, w); err != nil {
-			return nil, err
-		}
-	}
-
-	respBody := new(containerV2.AnnounceUsedSpaceResponseBody)
-
-	resp := new(containerV2.AnnounceUsedSpaceResponse)
-	resp.SetBody(respBody)
-
-	c.cfg.respSvc.SetMeta(resp)
-
-	return resp, nil
-}
-
-var errNodeOutsideContainer = errors.New("node outside the container")
-
-type containerOnlyKeyRemoteServerInfo struct {
-	key []byte
-}
-
-func (i *containerOnlyKeyRemoteServerInfo) PublicKey() []byte {
-	return i.key
-}
-
-func (*containerOnlyKeyRemoteServerInfo) IterateAddresses(func(string) bool) {
-}
-
-func (*containerOnlyKeyRemoteServerInfo) NumberOfAddresses() int {
-	return 0
-}
-
-func (*containerOnlyKeyRemoteServerInfo) ExternalAddresses() []string {
-	return nil
-}
-
-func (l *loadPlacementBuilder) isNodeFromContainerKey(epoch uint64, cnr cid.ID, key []byte) (bool, error) {
-	cnrNodes, _, err := l.buildPlacement(epoch, cnr)
-	if err != nil {
-		return false, err
-	}
-
-	for i := range cnrNodes {
-		for j := range cnrNodes[i] {
-			if bytes.Equal(cnrNodes[i][j].PublicKey(), key) {
-				return true, nil
-			}
-		}
-	}
-
-	return false, nil
-}
-
-func (c *usedSpaceService) processLoadValue(_ context.Context, a containerSDK.SizeEstimation,
-	route []loadcontroller.ServerInfo, w loadcontroller.Writer) error {
-	fromCnr, err := c.loadPlacementBuilder.isNodeFromContainerKey(a.Epoch(), a.Container(), route[0].PublicKey())
-	if err != nil {
-		return fmt.Errorf("could not verify that the sender belongs to the container: %w", err)
-	} else if !fromCnr {
-		return errNodeOutsideContainer
-	}
-
-	err = loadroute.CheckRoute(c.routeBuilder, a, route)
-	if err != nil {
-		return fmt.Errorf("wrong route of container's used space value: %w", err)
-	}
-
-	err = w.Put(a)
-	if err != nil {
-		return fmt.Errorf("could not write container's used space value: %w", err)
-	}
-
-	return nil
-}
-
 // implements interface required by container service provided by morph executor.
 type morphContainerReader struct {
 	eacl containerCore.EACLSource

cmd/frostfs-node/main.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"sync"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
@@ -148,9 +149,22 @@ func wait(c *cfg) {
 
 	<-c.done // graceful shutdown
 
+	drain := &sync.WaitGroup{}
+	drain.Add(1)
+	go func() {
+		defer drain.Done()
+		for err := range c.internalErr {
+			c.log.Warn(logs.FrostFSNodeInternalApplicationError,
+				zap.String("message", err.Error()))
+		}
+	}()
+
 	c.log.Debug(logs.FrostFSNodeWaitingForAllProcessesToStop)
 
 	c.wg.Wait()
+
+	close(c.internalErr)
+	drain.Wait()
 }
 
 func (c *cfg) onShutdown(f func()) {

cmd/frostfs-node/netmap.go

@@ -179,16 +179,9 @@ func addNewEpochNotificationHandlers(c *cfg) {
 			return
 		}
 
-		n := ev.(netmapEvent.NewEpoch).EpochNumber()
+		if err := c.bootstrap(); err != nil {
-
-		const reBootstrapInterval = 2
-
-		if (n-c.cfgNetmap.startEpoch)%reBootstrapInterval == 0 {
-			err := c.bootstrap()
-			if err != nil {
 			c.log.Warn(logs.FrostFSNodeCantSendRebootstrapTx, zap.Error(err))
 		}
-		}
 	})
 
 	addNewEpochAsyncNotificationHandler(c, func(ev event.Event) {
@@ -227,10 +220,6 @@ func bootstrapNode(c *cfg) {
 			c.log.Info(logs.FrostFSNodeNodeIsUnderMaintenanceSkipInitialBootstrap)
 			return
 		}
-		if c.alreadyBootstraped {
-			c.log.Info(logs.NetmapNodeAlreadyInCandidateListOnlineSkipInitialBootstrap)
-			return
-		}
 		err := c.bootstrap()
 		fatalOnErrDetails("bootstrap error", err)
 	}
@@ -263,7 +252,7 @@ func initNetmapState(c *cfg) {
 	fatalOnErrDetails("could not initialize current epoch number", err)
 
 	var ni *netmapSDK.NodeInfo
-	ni, c.alreadyBootstraped, err = c.netmapInitLocalNodeState(epoch)
+	ni, err = c.netmapInitLocalNodeState(epoch)
 	fatalOnErrDetails("could not init network state", err)
 
 	stateWord := nodeState(ni)
@@ -282,13 +271,6 @@ func initNetmapState(c *cfg) {
 	c.handleLocalNodeInfo(ni)
 }
 
-func sameNodeInfo(a, b *netmapSDK.NodeInfo) bool {
-	// Suboptimal, but we do this once on the node startup.
-	rawA := a.Marshal()
-	rawB := b.Marshal()
-	return bytes.Equal(rawA, rawB)
-}
-
 func nodeState(ni *netmapSDK.NodeInfo) string {
 	if ni != nil {
 		switch {
@@ -303,29 +285,27 @@ func nodeState(ni *netmapSDK.NodeInfo) string {
 	return "undefined"
 }
 
-func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, bool, error) {
+func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) {
 	nmNodes, err := c.cfgNetmap.wrapper.GetCandidates()
 	if err != nil {
-		return nil, false, err
+		return nil, err
 	}
 
 	var candidate *netmapSDK.NodeInfo
-	alreadyBootstraped := false
 	for i := range nmNodes {
 		if bytes.Equal(nmNodes[i].PublicKey(), c.binPublicKey) {
 			candidate = &nmNodes[i]
-			alreadyBootstraped = candidate.IsOnline() && sameNodeInfo(&c.cfgNodeInfo.localInfo, candidate)
 			break
 		}
 	}
 
 	node, err := c.netmapLocalNodeState(epoch)
 	if err != nil {
-		return nil, false, err
+		return nil, err
 	}
 
 	if candidate == nil {
-		return node, false, nil
+		return node, nil
 	}
 
 	nmState := nodeState(node)
@@ -337,7 +317,7 @@ func (c *cfg) netmapInitLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, bool,
 			zap.String("netmap", nmState),
 			zap.String("candidate", candidateState))
 	}
-	return candidate, alreadyBootstraped, nil
+	return candidate, nil
 }
 
 func (c *cfg) netmapLocalNodeState(epoch uint64) (*netmapSDK.NodeInfo, error) {

config/example/node.env

@@ -92,6 +92,7 @@ FROSTFS_OBJECT_DELETE_TOMBSTONE_LIFETIME=10
 # Storage engine section
 FROSTFS_STORAGE_SHARD_POOL_SIZE=15
 FROSTFS_STORAGE_SHARD_RO_ERROR_THRESHOLD=100
+FROSTFS_STORAGE_REBUILD_WORKERS_COUNT=1000
 ## 0 shard
 ### Flag to refill Metabase from BlobStor
 FROSTFS_STORAGE_SHARD_0_RESYNC_METABASE=false
@@ -123,6 +124,8 @@ FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_DEPTH=1
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_WIDTH=4
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_OPENED_CACHE_CAPACITY=50
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_LEAF_WIDTH=10
+FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_WORKER_COUNT=10
+FROSTFS_STORAGE_SHARD_0_BLOBSTOR_0_INIT_IN_ADVANCE=TRUE
 ### FSTree config
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_TYPE=fstree
 FROSTFS_STORAGE_SHARD_0_BLOBSTOR_1_PATH=tmp/0/blob

config/example/node.json

@@ -137,6 +137,7 @@
   "storage": {
     "shard_pool_size": 15,
     "shard_ro_error_threshold": 100,
+    "rebuild_workers_count": 1000,
     "shard": {
       "0": {
         "mode": "read-only",
@@ -170,7 +171,9 @@
             "depth": 1,
             "width": 4,
             "opened_cache_capacity": 50,
-            "leaf_width": 10
+            "leaf_width": 10,
+            "init_worker_count": 10,
+            "init_in_advance": true
           },
           {
             "type": "fstree",

config/example/node.yaml

@@ -116,6 +116,7 @@ storage:
   # note: shard configuration can be omitted for relay node (see `node.relay`)
   shard_pool_size: 15 # size of per-shard worker pools used for PUT operations
   shard_ro_error_threshold: 100 # amount of errors to occur before shard is made read-only (default: 0, ignore errors)
+  rebuild_workers_count: 1000 # count of rebuild storage concurrent workers
 
   shard:
     default: # section with the default shard parameters
@@ -182,6 +183,8 @@ storage:
       blobstor:
         - type: blobovnicza
           path: tmp/0/blob/blobovnicza
+          init_worker_count: 10 #count of workers to initialize blobovniczas
+          init_in_advance: true
         - type: fstree
           path: tmp/0/blob  # blobstor path
 

go.mod

@@ -3,10 +3,10 @@ module git.frostfs.info/TrueCloudLab/frostfs-node
 go 1.20
 
 require (
-	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.0
+	git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20231121085847-241a9f1ad0a4
 	git.frostfs.info/TrueCloudLab/frostfs-contract v0.18.0
 	git.frostfs.info/TrueCloudLab/frostfs-observability v0.0.0-20230531082742-c97d21411eb6
-	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230911122224-ac8fc6d4400c
+	git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230928142024-84b9d29fc98c
 	git.frostfs.info/TrueCloudLab/hrw v1.2.1
 	git.frostfs.info/TrueCloudLab/tzhash v1.8.0
 	github.com/cheggaaa/pb v1.0.29

internal/logs/logs.go

@@ -512,4 +512,37 @@ const (
 	RuntimeSoftMemoryDefinedWithGOMEMLIMIT                                  = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped"
 	FailedToCountWritecacheItems                                            = "failed to count writecache items"
 	AttemtToCloseAlreadyClosedBlobovnicza                                   = "attempt to close an already closed blobovnicza"
+	FailedToRebuildBlobstore                                                = "failed to rebuild blobstore"
+	BlobstoreRebuildStarted                                                 = "blobstore rebuild started"
+	BlobstoreRebuildCompletedSuccessfully                                   = "blobstore rebuild completed successfully"
+	BlobstoreRebuildStopped                                                 = "blobstore rebuild stopped"
+	BlobovniczaTreeFixingFileExtensions                                     = "fixing blobovnicza tree file extensions..."
+	BlobovniczaTreeFixingFileExtensionsCompletedSuccessfully                = "fixing blobovnicza tree file extensions completed successfully"
+	BlobovniczaTreeFixingFileExtensionsFailed                               = "failed to fix blobovnicza tree file extensions"
+	BlobovniczaTreeFixingFileExtensionForFile                               = "fixing blobovnicza file extension..."
+	BlobovniczaTreeFixingFileExtensionCompletedSuccessfully                 = "fixing blobovnicza file extension completed successfully"
+	BlobovniczaTreeFixingFileExtensionFailed                                = "failed to fix blobovnicza file extension"
+	BlobstorRebuildFailedToRebuildStorages                                  = "failed to rebuild storages"
+	BlobstorRebuildRebuildStoragesCompleted                                 = "storages rebuild completed"
+	BlobovniczaTreeCollectingDBToRebuild                                    = "collecting blobovniczas to rebuild..."
+	BlobovniczaTreeCollectingDBToRebuildFailed                              = "collecting blobovniczas to rebuild failed"
+	BlobovniczaTreeCollectingDBToRebuildSuccess                             = "collecting blobovniczas to rebuild completed successfully"
+	BlobovniczaTreeRebuildingBlobovnicza                                    = "rebuilding blobovnicza..."
+	BlobovniczaTreeRebuildingBlobovniczaFailed                              = "rebuilding blobovnicza failed"
+	BlobovniczaTreeRebuildingBlobovniczaSuccess                             = "rebuilding blobovnicza completed successfully"
+	BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza                   = "could not put move info to source blobovnicza"
+	BlobovniczatreeCouldNotUpdateStorageID                                  = "could not update storage ID"
+	BlobovniczatreeCouldNotDropMoveInfo                                     = "could not drop move info from source blobovnicza"
+	BlobovniczatreeCouldNotDeleteFromSource                                 = "could not delete object from source blobovnicza"
+	BlobovniczaTreeCompletingPreviousRebuild                                = "completing previous rebuild if failed..."
+	BlobovniczaTreeCompletedPreviousRebuildSuccess                          = "previous rebuild completed successfully"
+	BlobovniczaTreeCompletedPreviousRebuildFailed                           = "failed to complete previous rebuild"
+	BlobovniczatreeCouldNotCheckExistenceInSourceDB                         = "could not check object existence in source blobovnicza"
+	BlobovniczatreeCouldNotCheckExistenceInTargetDB                         = "could not check object existence in target blobovnicza"
+	BlobovniczatreeCouldNotGetObjectFromSourceDB                            = "could not get object from source blobovnicza"
+	BlobovniczatreeCouldNotPutObjectToTargetDB                              = "could not put object to target blobovnicza"
+	BlobovniczaSavingCountersToMeta                                         = "saving counters to blobovnicza's meta..."
+	BlobovniczaSavingCountersToMetaSuccess                                  = "saving counters to blobovnicza's meta completed successfully"
+	BlobovniczaSavingCountersToMetaFailed                                   = "saving counters to blobovnicza's meta failed"
+	ObjectRemovalFailureExistsInWritecache                                  = "can't remove object: object must be flushed from writecache"
 )

pkg/innerring/initialization.go

@@ -343,7 +343,7 @@ func (s *Server) initGRPCServer(cfg *viper.Viper) error {
 	p.SetPrivateKey(*s.key)
 	p.SetHealthChecker(s)
 
-	controlSvc := controlsrv.New(p, s.netmapClient,
+	controlSvc := controlsrv.New(p, s.netmapClient, s.containerClient,
 		controlsrv.WithAllowedKeys(authKeys),
 	)
 
@@ -389,6 +389,7 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) {
 	if err != nil {
 		return nil, err
 	}
+	s.containerClient = result.CnrClient
 
 	s.netmapClient, err = nmClient.NewFromMorph(s.morphClient, s.contracts.netmap, fee, nmClient.TryNotary(), nmClient.AsAlphabet())
 	if err != nil {

pkg/innerring/innerring.go

@@ -16,6 +16,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
 	balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container"
 	nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/subscriber"
@@ -56,6 +57,7 @@ type (
 		balanceClient   *balanceClient.Client
 		netmapClient    *nmClient.Client
 		persistate      *state.PersistentStorage
+		containerClient *container.Client
 
 		// metrics
 		irMetrics *metrics.InnerRingServiceMetrics

pkg/innerring/processors/netmap/cleanup_table.go

@@ -24,6 +24,8 @@ type (
 		epochStamp
 
 		binNodeInfo []byte
+
+		maintenance bool
 	}
 )
 
@@ -58,6 +60,7 @@ func (c *cleanupTable) update(snapshot netmap.NetMap, now uint64) {
 		}
 
 		access.binNodeInfo = binNodeInfo
+		access.maintenance = nmNodes[i].IsMaintenance()
 
 		newMap[keyString] = access
 	}
@@ -105,7 +108,7 @@ func (c *cleanupTable) forEachRemoveCandidate(epoch uint64, f func(string) error
 	defer c.Unlock()
 
 	for keyString, access := range c.lastAccess {
-		if epoch-access.epoch > c.threshold {
+		if !access.maintenance && epoch-access.epoch > c.threshold {
 			access.removeFlag = true // set remove flag
 			c.lastAccess[keyString] = access
 

pkg/innerring/processors/netmap/cleanup_table_test.go

@@ -124,6 +124,21 @@ func TestCleanupTable(t *testing.T) {
 				}))
 			require.EqualValues(t, len(infos)-1, cnt)
 		})
+
+		t.Run("skip maintenance nodes", func(t *testing.T) {
+			cnt := 0
+			infos[1].SetMaintenance()
+			key := netmap.StringifyPublicKey(infos[1])
+			c.update(networkMap, 5)
+
+			require.NoError(t,
+				c.forEachRemoveCandidate(5, func(s string) error {
+					cnt++
+					require.NotEqual(t, s, key)
+					return nil
+				}))
+			require.EqualValues(t, len(infos)-1, cnt)
+		})
 	})
 }
 

pkg/local_object_storage/blobovnicza/control.go

@@ -104,17 +104,42 @@ func (b *Blobovnicza) Init() error {
 func (b *Blobovnicza) initializeCounters() error {
 	var size uint64
 	var items uint64
+	var sizeExists bool
+	var itemsCountExists bool
+
 	err := b.boltDB.View(func(tx *bbolt.Tx) error {
-		return b.iterateAllBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) {
+		size, sizeExists = hasDataSize(tx)
-			keysN := uint64(b.Stats().KeyN)
+		items, itemsCountExists = hasItemsCount(tx)
-			size += keysN * upper
+
-			items += keysN
+		if sizeExists && itemsCountExists {
-			return false, nil
+			return nil
+		}
+
+		return b.iterateAllDataBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) {
+			return false, b.ForEach(func(k, v []byte) error {
+				size += uint64(len(k) + len(v))
+				items++
+				return nil
+			})
 		})
 	})
 	if err != nil {
 		return fmt.Errorf("can't determine DB size: %w", err)
 	}
+	if (!sizeExists || !itemsCountExists) && !b.boltOptions.ReadOnly {
+		b.log.Debug(logs.BlobovniczaSavingCountersToMeta, zap.Uint64("size", size), zap.Uint64("items", items))
+		if err := b.boltDB.Update(func(tx *bbolt.Tx) error {
+			if err := saveDataSize(tx, size); err != nil {
+				return err
+			}
+			return saveItemsCount(tx, items)
+		}); err != nil {
+			b.log.Debug(logs.BlobovniczaSavingCountersToMetaFailed, zap.Uint64("size", size), zap.Uint64("items", items))
+			return fmt.Errorf("can't save blobovnicza's size and items count: %w", err)
+		}
+		b.log.Debug(logs.BlobovniczaSavingCountersToMetaSuccess, zap.Uint64("size", size), zap.Uint64("items", items))
+	}
+
 	b.dataSize.Store(size)
 	b.itemsCount.Store(items)
 	b.metrics.AddOpenBlobovniczaSize(size)

pkg/local_object_storage/blobovnicza/delete.go

@@ -49,9 +49,10 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err
 	var sizeUpperBound uint64
 	var sizeLowerBound uint64
 	var dataSize uint64
+	var recordSize uint64
 
 	err := b.boltDB.Update(func(tx *bbolt.Tx) error {
-		return b.iterateAllBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) {
+		err := b.iterateAllDataBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) {
 			objData := buck.Get(addrKey)
 			if objData == nil {
 				// object is not in bucket => continue iterating
@@ -60,9 +61,27 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err
 			dataSize = uint64(len(objData))
 			sizeLowerBound = lower
 			sizeUpperBound = upper
+			recordSize = dataSize + uint64(len(addrKey))
 			found = true
 			return true, buck.Delete(addrKey)
 		})
+		if err != nil {
+			return err
+		}
+		if found {
+			return updateMeta(tx, func(count, size uint64) (uint64, uint64) {
+				if count > 0 {
+					count--
+				}
+				if size >= recordSize {
+					size -= recordSize
+				} else {
+					size = 0
+				}
+				return count, size
+			})
+		}
+		return nil
 	})
 
 	if err == nil && !found {
@@ -74,7 +93,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err
 			zap.String("binary size", stringifyByteSize(dataSize)),
 			zap.String("range", stringifyBounds(sizeLowerBound, sizeUpperBound)),
 		)
-		b.itemDeleted(sizeUpperBound)
+		b.itemDeleted(recordSize)
 	}
 
 	return DeleteRes{}, err

pkg/local_object_storage/blobovnicza/exists.go

@@ -26,7 +26,10 @@ func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error
 	addrKey := addressKey(addr)
 
 	err := b.boltDB.View(func(tx *bbolt.Tx) error {
-		return tx.ForEach(func(_ []byte, buck *bbolt.Bucket) error {
+		return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error {
+			if isNonDataBucket(bucketName) {
+				return nil
+			}
 			exists = buck.Get(addrKey) != nil
 			if exists {
 				return errInterruptForEach

pkg/local_object_storage/blobovnicza/get.go

@@ -57,7 +57,11 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) {
 	)
 
 	if err := b.boltDB.View(func(tx *bbolt.Tx) error {
-		return tx.ForEach(func(_ []byte, buck *bbolt.Bucket) error {
+		return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error {
+			if isNonDataBucket(bucketName) {
+				return nil
+			}
+
 			data = buck.Get(addrKey)
 			if data == nil {
 				return nil

pkg/local_object_storage/blobovnicza/iterate.go

@@ -12,11 +12,11 @@ import (
 	"go.opentelemetry.io/otel/trace"
 )
 
-// iterateAllBuckets iterates all buckets in db
+// iterateAllDataBuckets iterates all buckets in db
 //
 // If the maximum size of the object (b.objSizeLimit) has been changed to lower value,
 // then there may be more buckets than the current limit of the object size.
-func (b *Blobovnicza) iterateAllBuckets(tx *bbolt.Tx, f func(uint64, uint64, *bbolt.Bucket) (bool, error)) error {
+func (b *Blobovnicza) iterateAllDataBuckets(tx *bbolt.Tx, f func(uint64, uint64, *bbolt.Bucket) (bool, error)) error {
 	return b.iterateBucketKeys(false, func(lower uint64, upper uint64, key []byte) (bool, error) {
 		buck := tx.Bucket(key)
 		if buck == nil {
@@ -139,7 +139,10 @@ func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes,
 	var elem IterationElement
 
 	if err := b.boltDB.View(func(tx *bbolt.Tx) error {
-		return tx.ForEach(func(name []byte, buck *bbolt.Bucket) error {
+		return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error {
+			if isNonDataBucket(bucketName) {
+				return nil
+			}
 			return buck.ForEach(func(k, v []byte) error {
 				select {
 				case <-ctx.Done():

pkg/local_object_storage/blobovnicza/meta.go

@@ -0,0 +1,104 @@
+package blobovnicza
+
+import (
+	"bytes"
+	"encoding/binary"
+
+	"go.etcd.io/bbolt"
+)
+
+const (
+	dataSizeAndItemsCountBufLength = 8
+)
+
+var (
+	metaBucketName = []byte("META")
+	dataSizeKey    = []byte("data_size")
+	itemsCountKey  = []byte("items_count")
+)
+
+func isNonDataBucket(bucketName []byte) bool {
+	return bytes.Equal(bucketName, incompletedMoveBucketName) || bytes.Equal(bucketName, metaBucketName)
+}
+
+func hasDataSize(tx *bbolt.Tx) (uint64, bool) {
+	b := tx.Bucket(metaBucketName)
+	if b == nil {
+		return 0, false
+	}
+	v := b.Get(dataSizeKey)
+	if v == nil {
+		return 0, false
+	}
+	if len(v) != dataSizeAndItemsCountBufLength {
+		return 0, false
+	}
+	return binary.LittleEndian.Uint64(v), true
+}
+
+func hasItemsCount(tx *bbolt.Tx) (uint64, bool) {
+	b := tx.Bucket(metaBucketName)
+	if b == nil {
+		return 0, false
+	}
+	v := b.Get(itemsCountKey)
+	if v == nil {
+		return 0, false
+	}
+	if len(v) != dataSizeAndItemsCountBufLength {
+		return 0, false
+	}
+	return binary.LittleEndian.Uint64(v), true
+}
+
+func saveDataSize(tx *bbolt.Tx, size uint64) error {
+	b, err := tx.CreateBucketIfNotExists(metaBucketName)
+	if err != nil {
+		return err
+	}
+	buf := make([]byte, dataSizeAndItemsCountBufLength)
+	binary.LittleEndian.PutUint64(buf, size)
+	return b.Put(dataSizeKey, buf)
+}
+
+func saveItemsCount(tx *bbolt.Tx, count uint64) error {
+	b, err := tx.CreateBucketIfNotExists(metaBucketName)
+	if err != nil {
+		return err
+	}
+	buf := make([]byte, dataSizeAndItemsCountBufLength)
+	binary.LittleEndian.PutUint64(buf, count)
+	return b.Put(itemsCountKey, buf)
+}
+
+func updateMeta(tx *bbolt.Tx, updateValues func(count, size uint64) (uint64, uint64)) error {
+	b, err := tx.CreateBucketIfNotExists(metaBucketName)
+	if err != nil {
+		return err
+	}
+
+	var count uint64
+	var size uint64
+
+	v := b.Get(itemsCountKey)
+	if v != nil {
+		count = binary.LittleEndian.Uint64(v)
+	}
+
+	v = b.Get(dataSizeKey)
+	if v != nil {
+		size = binary.LittleEndian.Uint64(v)
+	}
+
+	count, size = updateValues(count, size)
+
+	sizeBuf := make([]byte, dataSizeAndItemsCountBufLength)
+	binary.LittleEndian.PutUint64(sizeBuf, size)
+	if err := b.Put(dataSizeKey, sizeBuf); err != nil {
+		return err
+	}
+
+	countBuf := make([]byte, dataSizeAndItemsCountBufLength)
+	binary.LittleEndian.PutUint64(countBuf, count)
+	return b.Put(itemsCountKey, countBuf)
+}

pkg/local_object_storage/blobovnicza/move.go

@@ -0,0 +1,108 @@
+package blobovnicza
+
+import (
+	"context"
+	"fmt"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"go.etcd.io/bbolt"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
+
+var incompletedMoveBucketName = []byte("INCOMPLETED_MOVE")
+
+type MoveInfo struct {
+	Address         oid.Address
+	TargetStorageID []byte
+}
+
+func (b *Blobovnicza) PutMoveInfo(ctx context.Context, prm MoveInfo) error {
+	_, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.PutMoveInfo",
+		trace.WithAttributes(
+			attribute.String("path", b.path),
+			attribute.String("address", prm.Address.EncodeToString()),
+			attribute.String("target_storage_id", string(prm.TargetStorageID)),
+		))
+	defer span.End()
+
+	key := addressKey(prm.Address)
+
+	return b.boltDB.Update(func(tx *bbolt.Tx) error {
+		bucket, err := tx.CreateBucketIfNotExists(incompletedMoveBucketName)
+		if err != nil {
+			return err
+		}
+
+		if err := bucket.Put(key, prm.TargetStorageID); err != nil {
+			return fmt.Errorf("(%T) failed to save move info: %w", b, err)
+		}
+
+		return nil
+	})
+}
+
+func (b *Blobovnicza) DropMoveInfo(ctx context.Context, address oid.Address) error {
+	_, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.DropMoveInfo",
+		trace.WithAttributes(
+			attribute.String("path", b.path),
+			attribute.String("address", address.EncodeToString()),
+		))
+	defer span.End()
+
+	key := addressKey(address)
+
+	return b.boltDB.Update(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(incompletedMoveBucketName)
+		if bucket == nil {
+			return nil
+		}
+
+		if err := bucket.Delete(key); err != nil {
+			return fmt.Errorf("(%T) failed to drop move info: %w", b, err)
+		}
+
+		c := bucket.Cursor()
+		k, v := c.First()
+		bucketEmpty := k == nil && v == nil
+		if bucketEmpty {
+			return tx.DeleteBucket(incompletedMoveBucketName)
+		}
+
+		return nil
+	})
+}
+
+func (b *Blobovnicza) ListMoveInfo(ctx context.Context) ([]MoveInfo, error) {
+	_, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.ListMoveInfo",
+		trace.WithAttributes(
+			attribute.String("path", b.path),
+		))
+	defer span.End()
+
+	var result []MoveInfo
+	if err := b.boltDB.View(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(incompletedMoveBucketName)
+		if bucket == nil {
+			return nil
+		}
+		return bucket.ForEach(func(k, v []byte) error {
+			var addr oid.Address
+			storageID := make([]byte, len(v))
+			if err := addressFromKey(&addr, k); err != nil {
+				return err
+			}
+			copy(storageID, v)
+			result = append(result, MoveInfo{
+				Address:         addr,
+				TargetStorageID: storageID,
+			})
+			return nil
+		})
+	}); err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}

pkg/local_object_storage/blobovnicza/put.go

@@ -57,8 +57,9 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) {
 	defer span.End()
 
 	sz := uint64(len(prm.objData))
-	bucketName, upperBound := bucketForSize(sz)
+	bucketName := bucketForSize(sz)
 	key := addressKey(prm.addr)
+	recordSize := sz + uint64(len(key))
 
 	err := b.boltDB.Batch(func(tx *bbolt.Tx) error {
 		buck := tx.Bucket(bucketName)
@@ -74,10 +75,12 @@ func (b *Blobovnicza) Put(ctx context.Context, prm PutPrm) (PutRes, error) {
 			return fmt.Errorf("(%T) could not save object in bucket: %w", b, err)
 		}
 
-		return nil
+		return updateMeta(tx, func(count, size uint64) (uint64, uint64) {
+			return count + 1, size + recordSize
+		})
 	})
 	if err == nil {
-		b.itemAdded(upperBound)
+		b.itemAdded(recordSize)
 	}
 
 	return PutRes{}, err

pkg/local_object_storage/blobovnicza/sizes.go

@@ -29,9 +29,8 @@ func bucketKeyFromBounds(upperBound uint64) []byte {
 	return buf[:ln]
 }
 
-func bucketForSize(sz uint64) ([]byte, uint64) {
+func bucketForSize(sz uint64) []byte {
-	upperBound := upperPowerOfTwo(sz)
+	return bucketKeyFromBounds(upperPowerOfTwo(sz))
-	return bucketKeyFromBounds(upperBound), upperBound
 }
 
 func upperPowerOfTwo(v uint64) uint64 {

pkg/local_object_storage/blobovnicza/sizes_test.go

@@ -34,7 +34,7 @@ func TestSizes(t *testing.T) {
 			upperBound: 4 * firstBucketBound,
 		},
 	} {
-		key, _ := bucketForSize(item.sz)
+		key := bucketForSize(item.sz)
 		require.Equal(t, bucketKeyFromBounds(item.upperBound), key)
 	}
 }

pkg/local_object_storage/blobstor/blobovniczatree/active.go

@@ -21,8 +21,8 @@ func (db *activeDB) Close() {
 	db.shDB.Close()
 }
 
-func (db *activeDB) Path() string {
+func (db *activeDB) SystemPath() string {
-	return db.shDB.Path()
+	return db.shDB.SystemPath()
 }
 
 // activeDBManager manages active blobovnicza instances (that is, those that are being used for Put).
@@ -154,7 +154,7 @@ func (m *activeDBManager) getNextSharedDB(lvlPath string) (*sharedDB, error) {
 	var next *sharedDB
 
 	for iterCount < m.leafWidth {
-		path := filepath.Join(lvlPath, u64ToHexString(idx))
+		path := filepath.Join(lvlPath, u64ToHexStringExt(idx))
 		shDB := m.dbManager.GetByPath(path)
 		db, err := shDB.Open() //open db to hold active DB open, will be closed if db is full, after m.replace or by activeDBManager.Close()
 		if err != nil {
@@ -192,7 +192,7 @@ func (m *activeDBManager) hasActiveDB(lvlPath string) (bool, uint64) {
 	if !ok {
 		return false, 0
 	}
-	return true, u64FromHexString(filepath.Base(db.Path()))
+	return true, u64FromHexString(filepath.Base(db.SystemPath()))
 }
 
 func (m *activeDBManager) replace(lvlPath string, shDB *sharedDB) (*sharedDB, bool) {

pkg/local_object_storage/blobstor/blobovniczatree/blobovnicza.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
+	"strings"
+	"sync"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
@@ -57,12 +59,19 @@ type Blobovniczas struct {
 	commondbManager        *dbManager
 	activeDBManager        *activeDBManager
 	dbCache                *dbCache
+	deleteProtectedObjects *addressMap
+	dbFilesGuard           *sync.RWMutex
+	rebuildGuard           *sync.RWMutex
 }
 
 var _ common.Storage = (*Blobovniczas)(nil)
 
 var errPutFailed = errors.New("could not save the object in any blobovnicza")
 
+const (
+	dbExtension = ".db"
+)
+
 // NewBlobovniczaTree returns new instance of blobovniczas tree.
 func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) {
 	blz = new(Blobovniczas)
@@ -76,9 +85,12 @@ func NewBlobovniczaTree(opts ...Option) (blz *Blobovniczas) {
 		blz.blzLeafWidth = blz.blzShallowWidth
 	}
 
-	blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.blzLeafWidth, blz.readOnly, blz.metrics.Blobovnicza(), blz.log)
+	blz.commondbManager = newDBManager(blz.rootPath, blz.blzOpts, blz.readOnly, blz.metrics.Blobovnicza(), blz.log)
 	blz.activeDBManager = newActiveDBManager(blz.commondbManager, blz.blzLeafWidth)
 	blz.dbCache = newDBCache(blz.openedCacheSize, blz.commondbManager)
+	blz.deleteProtectedObjects = newAddressMap()
+	blz.dbFilesGuard = &sync.RWMutex{}
+	blz.rebuildGuard = &sync.RWMutex{}
 
 	return blz
 }
@@ -94,14 +106,16 @@ func addressHash(addr *oid.Address, path string) uint64 {
 	return hrw.StringHash(a + path)
 }
 
-// converts uint64 to hex string.
 func u64ToHexString(ind uint64) string {
 	return strconv.FormatUint(ind, 16)
 }
 
-// converts uint64 hex string to uint64.
+func u64ToHexStringExt(ind uint64) string {
+	return strconv.FormatUint(ind, 16) + dbExtension
+}
+
 func u64FromHexString(str string) uint64 {
-	v, err := strconv.ParseUint(str, 16, 64)
+	v, err := strconv.ParseUint(strings.TrimSuffix(str, dbExtension), 16, 64)
 	if err != nil {
 		panic(fmt.Sprintf("blobovnicza name is not an index %s", str))
 	}

pkg/local_object_storage/blobstor/blobovniczatree/cache.go

@@ -15,8 +15,9 @@ import (
 type dbCache struct {
 	cacheGuard *sync.RWMutex
 	cache      simplelru.LRUCache[string, *sharedDB]
-	pathLock   *utilSync.KeyLocker[string]
+	pathLock   *utilSync.KeyLocker[string] // the order of locks is important: pathLock first, cacheGuard second
 	closed     bool
+	nonCached  map[string]struct{}
 
 	dbManager *dbManager
 }
@@ -34,6 +35,7 @@ func newDBCache(size int, dbManager *dbManager) *dbCache {
 		cache:      cache,
 		dbManager:  dbManager,
 		pathLock:   utilSync.NewKeyLocker[string](),
+		nonCached:  make(map[string]struct{}),
 	}
 }
 
@@ -59,6 +61,27 @@ func (c *dbCache) GetOrCreate(path string) *sharedDB {
 	return c.create(path)
 }
 
+func (c *dbCache) EvictAndMarkNonCached(path string) {
+	c.pathLock.Lock(path)
+	defer c.pathLock.Unlock(path)
+
+	c.cacheGuard.Lock()
+	defer c.cacheGuard.Unlock()
+
+	c.cache.Remove(path)
+	c.nonCached[path] = struct{}{}
+}
+
+func (c *dbCache) RemoveFromNonCached(path string) {
+	c.pathLock.Lock(path)
+	defer c.pathLock.Unlock(path)
+
+	c.cacheGuard.Lock()
+	defer c.cacheGuard.Unlock()
+
+	delete(c.nonCached, path)
+}
+
 func (c *dbCache) getExisted(path string) *sharedDB {
 	c.cacheGuard.Lock()
 	defer c.cacheGuard.Unlock()
@@ -94,7 +117,9 @@ func (c *dbCache) put(path string, db *sharedDB) bool {
 	c.cacheGuard.Lock()
 	defer c.cacheGuard.Unlock()
 
-	if !c.closed {
+	_, isNonCached := c.nonCached[path]
+
+	if !isNonCached && !c.closed {
 		c.cache.Add(path, db)
 		return true
 	}

pkg/local_object_storage/blobstor/blobovniczatree/control.go

@@ -2,15 +2,24 @@ package blobovniczatree
 
 import (
 	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util"
 	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
 )
 
+var errFailedToChangeExtensionReadOnly = errors.New("failed to change blobovnicza extension: read only mode")
+
 // Open opens blobovnicza tree.
 func (b *Blobovniczas) Open(readOnly bool) error {
 	b.readOnly = readOnly
 	b.metrics.SetMode(readOnly)
+	b.metrics.SetRebuildStatus(rebuildStatusNotStarted)
 	b.openManagers()
 	return nil
 }
@@ -21,33 +30,95 @@ func (b *Blobovniczas) Open(readOnly bool) error {
 func (b *Blobovniczas) Init() error {
 	b.log.Debug(logs.BlobovniczatreeInitializingBlobovniczas)
 
+	b.log.Debug(logs.BlobovniczaTreeFixingFileExtensions)
+	if err := b.addDBExtensionToDBs(b.rootPath, 0); err != nil {
+		b.log.Error(logs.BlobovniczaTreeFixingFileExtensionsFailed, zap.Error(err))
+		return err
+	}
+	b.log.Debug(logs.BlobovniczaTreeFixingFileExtensionsCompletedSuccessfully)
+
 	if b.readOnly {
 		b.log.Debug(logs.BlobovniczatreeReadonlyModeSkipBlobovniczasInitialization)
 		return nil
 	}
 
-	return b.iterateLeaves(context.TODO(), func(p string) (bool, error) {
+	return b.initializeDBs(context.TODO())
+}
+
+func (b *Blobovniczas) initializeDBs(ctx context.Context) error {
+	err := util.MkdirAllX(b.rootPath, b.perm)
+	if err != nil {
+		return err
+	}
+
+	eg, egCtx := errgroup.WithContext(ctx)
+	eg.SetLimit(b.blzInitWorkerCount)
+	visited := make(map[string]struct{})
+	err = b.iterateExistingDBPaths(egCtx, func(p string) (bool, error) {
+		visited[p] = struct{}{}
+		eg.Go(func() error {
+			shBlz := b.getBlobovniczaWithoutCaching(p)
+			blz, err := shBlz.Open()
+			if err != nil {
+				return err
+			}
+			defer shBlz.Close()
+
+			moveInfo, err := blz.ListMoveInfo(egCtx)
+			if err != nil {
+				return err
+			}
+			for _, move := range moveInfo {
+				b.deleteProtectedObjects.Add(move.Address)
+			}
+
+			b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p))
+			return nil
+		})
+		return false, nil
+	})
+	if err != nil {
+		_ = eg.Wait()
+		return err
+	}
+
+	if b.createDBInAdvance {
+		err = b.iterateSortedLeaves(egCtx, nil, func(p string) (bool, error) {
+			if _, found := visited[p]; found {
+				return false, nil
+			}
+			eg.Go(func() error {
 				shBlz := b.getBlobovniczaWithoutCaching(p)
 				_, err := shBlz.Open()
 				if err != nil {
-			return true, err
+					return err
 				}
 				defer shBlz.Close()
 
 				b.log.Debug(logs.BlobovniczatreeBlobovniczaSuccessfullyInitializedClosing, zap.String("id", p))
+				return nil
+			})
 			return false, nil
 		})
+
+		if err != nil {
+			_ = eg.Wait()
+			return err
+		}
+	}
+
+	return eg.Wait()
 }
 
 func (b *Blobovniczas) openManagers() {
-	b.commondbManager.Open() //order important
+	b.commondbManager.Open() // order important
 	b.activeDBManager.Open()
 	b.dbCache.Open()
 }
 
 // Close implements common.Storage.
 func (b *Blobovniczas) Close() error {
-	b.dbCache.Close() //order important
+	b.dbCache.Close() // order important
 	b.activeDBManager.Close()
 	b.commondbManager.Close()
 
@@ -64,3 +135,37 @@ func (b *Blobovniczas) getBlobovnicza(p string) *sharedDB {
 func (b *Blobovniczas) getBlobovniczaWithoutCaching(p string) *sharedDB {
 	return b.commondbManager.GetByPath(p)
 }
+
+func (b *Blobovniczas) addDBExtensionToDBs(path string, depth uint64) error {
+	entries, err := os.ReadDir(path)
+	if os.IsNotExist(err) && depth == 0 {
+		return nil
+	}
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			if err := b.addDBExtensionToDBs(filepath.Join(path, entry.Name()), depth+1); err != nil {
+				return err
+			}
+			continue
+		}
+
+		if strings.HasSuffix(entry.Name(), dbExtension) {
+			continue
+		}
+		if b.readOnly {
+			return errFailedToChangeExtensionReadOnly
+		}
+
+		sourcePath := filepath.Join(path, entry.Name())
+		targetPath := filepath.Join(path, entry.Name()+dbExtension)
+		b.log.Debug(logs.BlobovniczaTreeFixingFileExtensionForFile, zap.String("source", sourcePath), zap.String("target", targetPath))
+		if err := os.Rename(sourcePath, targetPath); err != nil {
+			b.log.Error(logs.BlobovniczaTreeFixingFileExtensionFailed, zap.String("source", sourcePath), zap.String("target", targetPath), zap.Error(err))
+			return err
+		}
+		b.log.Debug(logs.BlobovniczaTreeFixingFileExtensionCompletedSuccessfully, zap.String("source", sourcePath), zap.String("target", targetPath))
+	}
+
+	return nil
+}

pkg/local_object_storage/blobstor/blobovniczatree/control_test.go

@@ -0,0 +1,189 @@
+package blobovniczatree
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDBExtensionFix(t *testing.T) {
+	root := t.TempDir()
+	createTestTree(t, 0, 2, 3, root)
+	t.Run("adds suffix if not exists", func(t *testing.T) {
+		openAndCloseTestTree(t, 2, 3, root)
+		validateTestTree(t, root)
+	})
+
+	t.Run("not adds second suffix if exists", func(t *testing.T) {
+		openAndCloseTestTree(t, 2, 3, root)
+		validateTestTree(t, root)
+	})
+}
+
+func createTestTree(t *testing.T, currentDepth, depth, width uint64, path string) {
+	if currentDepth == depth {
+		var w uint64
+		for ; w < width; w++ {
+			dbPath := filepath.Join(path, u64ToHexString(w))
+			b := blobovnicza.New(blobovnicza.WithPath(dbPath))
+			require.NoError(t, b.Open())
+			require.NoError(t, b.Init())
+			require.NoError(t, b.Close())
+		}
+		return
+	}
+
+	var w uint64
+	for ; w < width; w++ {
+		createTestTree(t, currentDepth+1, depth, width, filepath.Join(path, u64ToHexString(w)))
+	}
+}
+
+func openAndCloseTestTree(t *testing.T, depth, width uint64, path string) {
+	blz := NewBlobovniczaTree(
+		WithBlobovniczaShallowDepth(depth),
+		WithBlobovniczaShallowWidth(width),
+		WithRootPath(path),
+	)
+	require.NoError(t, blz.Open(false))
+	require.NoError(t, blz.Init())
+	require.NoError(t, blz.Close())
+}
+
+func validateTestTree(t *testing.T, path string) {
+	entries, err := os.ReadDir(path)
+	require.NoError(t, err)
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			validateTestTree(t, filepath.Join(path, entry.Name()))
+		} else {
+			require.True(t, strings.HasSuffix(entry.Name(), dbExtension))
+			require.False(t, strings.HasSuffix(strings.TrimSuffix(entry.Name(), dbExtension), dbExtension))
+		}
+	}
+}
+
+func TestObjectsAvailableAfterDepthAndWidthEdit(t *testing.T) {
+	t.Parallel()
+
+	rootDir := t.TempDir()
+
+	blz := NewBlobovniczaTree(
+		WithBlobovniczaShallowDepth(3),
+		WithBlobovniczaShallowWidth(5),
+		WithRootPath(rootDir),
+	)
+
+	require.NoError(t, blz.Open(false))
+	require.NoError(t, blz.Init())
+
+	obj35 := blobstortest.NewObject(10 * 1024)
+	addr35 := objectCore.AddressOf(obj35)
+	raw, err := obj35.Marshal()
+	require.NoError(t, err)
+
+	pRes35, err := blz.Put(context.Background(), common.PutPrm{
+		Address: addr35,
+		Object:  obj35,
+		RawData: raw,
+	})
+	require.NoError(t, err)
+
+	gRes, err := blz.Get(context.Background(), common.GetPrm{
+		Address:   addr35,
+		StorageID: pRes35.StorageID,
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, obj35, gRes.Object)
+
+	gRes, err = blz.Get(context.Background(), common.GetPrm{
+		Address: addr35,
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, obj35, gRes.Object)
+
+	require.NoError(t, blz.Close())
+
+	// change depth and width
+	blz = NewBlobovniczaTree(
+		WithBlobovniczaShallowDepth(5),
+		WithBlobovniczaShallowWidth(2),
+		WithRootPath(rootDir),
+	)
+
+	require.NoError(t, blz.Open(false))
+	require.NoError(t, blz.Init())
+
+	gRes, err = blz.Get(context.Background(), common.GetPrm{
+		Address:   addr35,
+		StorageID: pRes35.StorageID,
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, obj35, gRes.Object)
+
+	gRes, err = blz.Get(context.Background(), common.GetPrm{
+		Address: addr35,
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, obj35, gRes.Object)
+
+	obj52 := blobstortest.NewObject(10 * 1024)
+	addr52 := objectCore.AddressOf(obj52)
+	raw, err = obj52.Marshal()
+	require.NoError(t, err)
+
+	pRes52, err := blz.Put(context.Background(), common.PutPrm{
+		Address: addr52,
+		Object:  obj52,
+		RawData: raw,
+	})
+	require.NoError(t, err)
+
+	require.NoError(t, blz.Close())
+
+	// change depth and width back
+	blz = NewBlobovniczaTree(
+		WithBlobovniczaShallowDepth(3),
+		WithBlobovniczaShallowWidth(5),
+		WithRootPath(rootDir),
+	)
+	require.NoError(t, blz.Open(false))
+	require.NoError(t, blz.Init())
+
+	gRes, err = blz.Get(context.Background(), common.GetPrm{
+		Address:   addr35,
+		StorageID: pRes35.StorageID,
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, obj35, gRes.Object)
+
+	gRes, err = blz.Get(context.Background(), common.GetPrm{
+		Address: addr35,
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, obj35, gRes.Object)
+
+	gRes, err = blz.Get(context.Background(), common.GetPrm{
+		Address:   addr52,
+		StorageID: pRes52.StorageID,
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, obj52, gRes.Object)
+
+	gRes, err = blz.Get(context.Background(), common.GetPrm{
+		Address: addr52,
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, obj52, gRes.Object)
+
+	require.NoError(t, blz.Close())
+}

pkg/local_object_storage/blobstor/blobovniczatree/delete.go

@@ -3,6 +3,7 @@ package blobovniczatree
 import (
 	"context"
 	"encoding/hex"
+	"errors"
 	"time"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
@@ -17,6 +18,8 @@ import (
 	"go.uber.org/zap"
 )
 
+var errObjectIsDeleteProtected = errors.New("object is delete protected")
+
 // Delete deletes object from blobovnicza tree.
 //
 // If blobocvnicza ID is specified, only this blobovnicza is processed.
@@ -42,12 +45,22 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co
 		return common.DeleteRes{}, common.ErrReadOnly
 	}
 
+	if b.rebuildGuard.TryRLock() {
+		defer b.rebuildGuard.RUnlock()
+	} else {
+		return common.DeleteRes{}, errRebuildInProgress
+	}
+
+	if b.deleteProtectedObjects.Contains(prm.Address) {
+		return common.DeleteRes{}, errObjectIsDeleteProtected
+	}
+
 	var bPrm blobovnicza.DeletePrm
 	bPrm.SetAddress(prm.Address)
 
 	if prm.StorageID != nil {
-		id := blobovnicza.NewIDFromBytes(prm.StorageID)
+		id := NewIDFromBytes(prm.StorageID)
-		shBlz := b.getBlobovnicza(id.String())
+		shBlz := b.getBlobovnicza(id.Path())
 		blz, err := shBlz.Open()
 		if err != nil {
 			return res, err
@@ -62,7 +75,7 @@ func (b *Blobovniczas) Delete(ctx context.Context, prm common.DeletePrm) (res co
 
 	objectFound := false
 
-	err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) {
+	err = b.iterateSortedDBPaths(ctx, prm.Address, func(p string) (bool, error) {
 		res, err = b.deleteObjectFromLevel(ctx, bPrm, p)
 		if err != nil {
 			if !client.IsErrObjectNotFound(err) {

pkg/local_object_storage/blobstor/blobovniczatree/exists.go

@@ -35,8 +35,8 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common
 	defer span.End()
 
 	if prm.StorageID != nil {
-		id := blobovnicza.NewIDFromBytes(prm.StorageID)
+		id := NewIDFromBytes(prm.StorageID)
-		shBlz := b.getBlobovnicza(id.String())
+		shBlz := b.getBlobovnicza(id.Path())
 		blz, err := shBlz.Open()
 		if err != nil {
 			return common.ExistsRes{}, err
@@ -50,7 +50,7 @@ func (b *Blobovniczas) Exists(ctx context.Context, prm common.ExistsPrm) (common
 	var gPrm blobovnicza.GetPrm
 	gPrm.SetAddress(prm.Address)
 
-	err := b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) {
+	err := b.iterateSortedDBPaths(ctx, prm.Address, func(p string) (bool, error) {
 		_, err := b.getObjectFromLevel(ctx, gPrm, p)
 		if err != nil {
 			if !client.IsErrObjectNotFound(err) {

pkg/local_object_storage/blobstor/blobovniczatree/exists_test.go

@@ -55,7 +55,7 @@ func TestExistsInvalidStorageID(t *testing.T) {
 
 		// An invalid boltdb file is created so that it returns an error when opened
 		require.NoError(t, os.MkdirAll(filepath.Join(dir, relBadFileDir), os.ModePerm))
-		require.NoError(t, os.WriteFile(filepath.Join(dir, relBadFileDir, badFileName), []byte("not a boltdb file content"), 0777))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, relBadFileDir, badFileName+".db"), []byte("not a boltdb file content"), 0777))
 
 		res, err := b.Exists(context.Background(), common.ExistsPrm{Address: addr, StorageID: []byte(filepath.Join(relBadFileDir, badFileName))})
 		require.Error(t, err)

pkg/local_object_storage/blobstor/blobovniczatree/get.go

@@ -46,8 +46,8 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G
 	bPrm.SetAddress(prm.Address)
 
 	if prm.StorageID != nil {
-		id := blobovnicza.NewIDFromBytes(prm.StorageID)
+		id := NewIDFromBytes(prm.StorageID)
-		shBlz := b.getBlobovnicza(id.String())
+		shBlz := b.getBlobovnicza(id.Path())
 		blz, err := shBlz.Open()
 		if err != nil {
 			return res, err
@@ -62,7 +62,7 @@ func (b *Blobovniczas) Get(ctx context.Context, prm common.GetPrm) (res common.G
 		return res, err
 	}
 
-	err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) {
+	err = b.iterateSortedDBPaths(ctx, prm.Address, func(p string) (bool, error) {
 		res, err = b.getObjectFromLevel(ctx, bPrm, p)
 		if err != nil {
 			if !client.IsErrObjectNotFound(err) {

pkg/local_object_storage/blobstor/blobovniczatree/get_range.go

@@ -45,8 +45,8 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re
 	defer span.End()
 
 	if prm.StorageID != nil {
-		id := blobovnicza.NewIDFromBytes(prm.StorageID)
+		id := NewIDFromBytes(prm.StorageID)
-		shBlz := b.getBlobovnicza(id.String())
+		shBlz := b.getBlobovnicza(id.Path())
 		blz, err := shBlz.Open()
 		if err != nil {
 			return common.GetRangeRes{}, err
@@ -63,7 +63,7 @@ func (b *Blobovniczas) GetRange(ctx context.Context, prm common.GetRangePrm) (re
 
 	objectFound := false
 
-	err = b.iterateSortedLeaves(ctx, &prm.Address, func(p string) (bool, error) {
+	err = b.iterateSortedDBPaths(ctx, prm.Address, func(p string) (bool, error) {
 		res, err = b.getRangeFromLevel(ctx, prm, p)
 		if err != nil {
 			outOfBounds := isErrOutOfRange(err)

pkg/local_object_storage/blobstor/blobovniczatree/id.go

@@ -1,4 +1,4 @@
-package blobovnicza
+package blobovniczatree
 
 // ID represents Blobovnicza identifier.
 type ID []byte
@@ -8,8 +8,8 @@ func NewIDFromBytes(v []byte) *ID {
 	return (*ID)(&v)
 }
 
-func (id ID) String() string {
+func (id ID) Path() string {
-	return string(id)
+	return string(id) + dbExtension
 }
 
 func (id ID) Bytes() []byte {

pkg/local_object_storage/blobstor/blobovniczatree/iterate.go

@@ -3,7 +3,9 @@ package blobovniczatree
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
@@ -50,7 +52,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm
 				return prm.Handler(common.IterationElement{
 					Address:    elem.Address(),
 					ObjectData: data,
-					StorageID:  []byte(p),
+					StorageID:  []byte(strings.TrimSuffix(p, dbExtension)),
 				})
 			}
 			return prm.LazyHandler(elem.Address(), func() ([]byte, error) {
@@ -67,7 +69,7 @@ func (b *Blobovniczas) Iterate(ctx context.Context, prm common.IteratePrm) (comm
 
 // iterator over all Blobovniczas in unsorted order. Break on f's error return.
 func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors bool, f func(string, *blobovnicza.Blobovnicza) error) error {
-	return b.iterateLeaves(ctx, func(p string) (bool, error) {
+	return b.iterateExistingDBPaths(ctx, func(p string) (bool, error) {
 		shBlz := b.getBlobovnicza(p)
 		blz, err := shBlz.Open()
 		if err != nil {
@@ -84,7 +86,9 @@ func (b *Blobovniczas) iterateBlobovniczas(ctx context.Context, ignoreErrors boo
 	})
 }
 
-// iterator over the paths of Blobovniczas sorted by weight.
+// iterateSortedLeaves iterates over the paths of Blobovniczas sorted by weight.
+//
+// Uses depth, width and leaf width for iteration.
 func (b *Blobovniczas) iterateSortedLeaves(ctx context.Context, addr *oid.Address, f func(string) (bool, error)) error {
 	_, err := b.iterateSorted(
 		ctx,
@@ -124,7 +128,9 @@ func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, cur
 	}
 	indices := indexSlice(levelWidth)
 
+	if !isLeafLevel {
 		hrw.SortSliceByValue(indices, addressHash(addr, filepath.Join(curPath...)))
+	}
 
 	exec := uint64(len(curPath)) == execDepth
 
@@ -134,10 +140,16 @@ func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, cur
 			return false, ctx.Err()
 		default:
 		}
+
+		lastPart := u64ToHexString(indices[i])
+		if isLeafLevel {
+			lastPart = u64ToHexStringExt(indices[i])
+		}
+
 		if i == 0 {
-			curPath = append(curPath, u64ToHexString(indices[i]))
+			curPath = append(curPath, lastPart)
 		} else {
-			curPath[len(curPath)-1] = u64ToHexString(indices[i])
+			curPath[len(curPath)-1] = lastPart
 		}
 
 		if exec {
@@ -156,9 +168,110 @@ func (b *Blobovniczas) iterateSorted(ctx context.Context, addr *oid.Address, cur
 	return false, nil
 }
 
-// iterator over the paths of Blobovniczas in random order.
+// iterateExistingDBPaths iterates over the paths of Blobovniczas without any order.
-func (b *Blobovniczas) iterateLeaves(ctx context.Context, f func(string) (bool, error)) error {
+//
-	return b.iterateSortedLeaves(ctx, nil, f)
+// Uses existed blobovnicza files for iteration.
+func (b *Blobovniczas) iterateExistingDBPaths(ctx context.Context, f func(string) (bool, error)) error {
+	b.dbFilesGuard.RLock()
+	defer b.dbFilesGuard.RUnlock()
+
+	_, err := b.iterateExistingDBPathsDFS(ctx, "", f)
+	return err
+}
+
+func (b *Blobovniczas) iterateExistingDBPathsDFS(ctx context.Context, path string, f func(string) (bool, error)) (bool, error) {
+	sysPath := filepath.Join(b.rootPath, path)
+	entries, err := os.ReadDir(sysPath)
+	if os.IsNotExist(err) && b.readOnly && path == "" { //non initialized tree in read only mode
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	for _, entry := range entries {
+		select {
+		case <-ctx.Done():
+			return false, ctx.Err()
+		default:
+		}
+		if entry.IsDir() {
+			stop, err := b.iterateExistingDBPathsDFS(ctx, filepath.Join(path, entry.Name()), f)
+			if err != nil {
+				return false, err
+			}
+			if stop {
+				return true, nil
+			}
+		} else {
+			stop, err := f(filepath.Join(path, entry.Name()))
+			if err != nil {
+				return false, err
+			}
+			if stop {
+				return true, nil
+			}
+		}
+	}
+	return false, nil
+}
+
+func (b *Blobovniczas) iterateSortedDBPaths(ctx context.Context, addr oid.Address, f func(string) (bool, error)) error {
+	b.dbFilesGuard.RLock()
+	defer b.dbFilesGuard.RUnlock()
+
+	_, err := b.iterateSordedDBPathsInternal(ctx, "", addr, f)
+	return err
+}
+
+func (b *Blobovniczas) iterateSordedDBPathsInternal(ctx context.Context, path string, addr oid.Address, f func(string) (bool, error)) (bool, error) {
+	sysPath := filepath.Join(b.rootPath, path)
+	entries, err := os.ReadDir(sysPath)
+	if os.IsNotExist(err) && b.readOnly && path == "" { //non initialized tree in read only mode
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	var dbIdxs []uint64
+	var dirIdxs []uint64
+
+	for _, entry := range entries {
+		idx := u64FromHexString(entry.Name())
+		if entry.IsDir() {
+			dirIdxs = append(dirIdxs, idx)
+		} else {
+			dbIdxs = append(dbIdxs, idx)
+		}
+	}
+
+	if len(dbIdxs) > 0 {
+		for _, dbIdx := range dbIdxs {
+			dbPath := filepath.Join(path, u64ToHexStringExt(dbIdx))
+			stop, err := f(dbPath)
+			if err != nil {
+				return false, err
+			}
+			if stop {
+				return true, nil
+			}
+		}
+	}
+
+	if len(dirIdxs) > 0 {
+		hrw.SortSliceByValue(dirIdxs, addressHash(&addr, path))
+		for _, dirIdx := range dirIdxs {
+			dirPath := filepath.Join(path, u64ToHexString(dirIdx))
+			stop, err := b.iterateSordedDBPathsInternal(ctx, dirPath, addr, f)
+			if err != nil {
+				return false, err
+			}
+			if stop {
+				return true, nil
+			}
+		}
+	}
+
+	return false, nil
 }
 
 // makes slice of uint64 values from 0 to number-1.

pkg/local_object_storage/blobstor/blobovniczatree/iterate_test.go

@@ -0,0 +1,42 @@
+package blobovniczatree
+
+import (
+	"context"
+	"testing"
+
+	oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIterateSortedLeavesAndDBPathsAreSame(t *testing.T) {
+	t.Parallel()
+
+	blz := NewBlobovniczaTree(
+		WithBlobovniczaShallowDepth(3),
+		WithBlobovniczaShallowWidth(5),
+		WithRootPath(t.TempDir()),
+	)
+	blz.createDBInAdvance = true
+	require.NoError(t, blz.Open(false))
+	require.NoError(t, blz.Init())
+	defer func() {
+		require.NoError(t, blz.Close())
+	}()
+
+	addr := oidtest.Address()
+
+	var leaves []string
+	var dbPaths []string
+
+	blz.iterateSortedLeaves(context.Background(), &addr, func(s string) (bool, error) {
+		leaves = append(leaves, s)
+		return false, nil
+	})
+
+	blz.iterateSortedDBPaths(context.Background(), addr, func(s string) (bool, error) {
+		dbPaths = append(dbPaths, s)
+		return false, nil
+	})
+
+	require.Equal(t, leaves, dbPaths)
+}

pkg/local_object_storage/blobstor/blobovniczatree/manager.go

@@ -1,7 +1,9 @@
 package blobovniczatree
 
 import (
+	"errors"
 	"fmt"
+	"os"
 	"path/filepath"
 	"sync"
 	"sync/atomic"
@@ -12,9 +14,13 @@ import (
 	"go.uber.org/zap"
 )
 
+var (
+	errClosingClosedBlobovnicza = errors.New("closing closed blobovnicza is not allowed")
+)
+
 // sharedDB is responsible for opening and closing a file of single blobovnicza.
 type sharedDB struct {
-	guard    *sync.RWMutex
+	cond     *sync.Cond
 	blcza    *blobovnicza.Blobovnicza
 	refCount uint32
 
@@ -30,8 +36,9 @@ type sharedDB struct {
 func newSharedDB(options []blobovnicza.Option, path string, readOnly bool,
 	metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlag *atomic.Bool, log *logger.Logger) *sharedDB {
 	return &sharedDB{
-		guard: &sync.RWMutex{},
+		cond: &sync.Cond{
-
+			L: &sync.RWMutex{},
+		},
 		options:       options,
 		path:          path,
 		readOnly:      readOnly,
@@ -47,8 +54,8 @@ func (b *sharedDB) Open() (*blobovnicza.Blobovnicza, error) {
 		return nil, errClosed
 	}
 
-	b.guard.Lock()
+	b.cond.L.Lock()
-	defer b.guard.Unlock()
+	defer b.cond.L.Unlock()
 
 	if b.refCount > 0 {
 		b.refCount++
@@ -76,11 +83,12 @@ func (b *sharedDB) Open() (*blobovnicza.Blobovnicza, error) {
 }
 
 func (b *sharedDB) Close() {
-	b.guard.Lock()
+	b.cond.L.Lock()
-	defer b.guard.Unlock()
+	defer b.cond.L.Unlock()
 
 	if b.refCount == 0 {
 		b.log.Error(logs.AttemtToCloseAlreadyClosedBlobovnicza, zap.String("id", b.path))
+		b.cond.Broadcast()
 		return
 	}
 
@@ -98,32 +106,108 @@ func (b *sharedDB) Close() {
 	}
 
 	b.refCount--
+	if b.refCount == 1 {
+		b.cond.Broadcast()
+	}
 }
 
-func (b *sharedDB) Path() string {
+func (b *sharedDB) CloseAndRemoveFile() error {
+	b.cond.L.Lock()
+	if b.refCount > 1 {
+		b.cond.Wait()
+	}
+	defer b.cond.L.Unlock()
+
+	if b.refCount == 0 {
+		return errClosingClosedBlobovnicza
+	}
+
+	if err := b.blcza.Close(); err != nil {
+		b.log.Error(logs.BlobovniczatreeCouldNotCloseBlobovnicza,
+			zap.String("id", b.path),
+			zap.String("error", err.Error()),
+		)
+		return fmt.Errorf("failed to close blobovnicza (path = %s): %w", b.path, err)
+	}
+
+	b.refCount = 0
+	b.blcza = nil
+	b.openDBCounter.Dec()
+
+	return os.Remove(b.path)
+}
+
+func (b *sharedDB) SystemPath() string {
 	return b.path
 }
 
 // levelDbManager stores pointers of the sharedDB's for the leaf directory of the blobovnicza tree.
 type levelDbManager struct {
-	databases []*sharedDB
+	dbMtx     *sync.RWMutex
+	databases map[uint64]*sharedDB
+
+	options       []blobovnicza.Option
+	path          string
+	readOnly      bool
+	metrics       blobovnicza.Metrics
+	openDBCounter *openDBCounter
+	closedFlag    *atomic.Bool
+	log           *logger.Logger
 }
 
-func newLevelDBManager(width uint64, options []blobovnicza.Option, rootPath string, lvlPath string,
+func newLevelDBManager(options []blobovnicza.Option, rootPath string, lvlPath string,
-	readOnly bool, metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlog *atomic.Bool, log *logger.Logger) *levelDbManager {
+	readOnly bool, metrics blobovnicza.Metrics, openDBCounter *openDBCounter, closedFlag *atomic.Bool, log *logger.Logger) *levelDbManager {
 	result := &levelDbManager{
-		databases: make([]*sharedDB, width),
+		databases: make(map[uint64]*sharedDB),
-	}
+		dbMtx:     &sync.RWMutex{},
-	for idx := uint64(0); idx < width; idx++ {
+
-		result.databases[idx] = newSharedDB(options, filepath.Join(rootPath, lvlPath, u64ToHexString(idx)), readOnly, metrics, openDBCounter, closedFlog, log)
+		options:       options,
+		path:          filepath.Join(rootPath, lvlPath),
+		readOnly:      readOnly,
+		metrics:       metrics,
+		openDBCounter: openDBCounter,
+		closedFlag:    closedFlag,
+		log:           log,
 	}
 	return result
 }
 
 func (m *levelDbManager) GetByIndex(idx uint64) *sharedDB {
+	res := m.getDBIfExists(idx)
+	if res != nil {
+		return res
+	}
+	return m.getOrCreateDB(idx)
+}
+
+func (m *levelDbManager) getDBIfExists(idx uint64) *sharedDB {
+	m.dbMtx.RLock()
+	defer m.dbMtx.RUnlock()
+
 	return m.databases[idx]
 }
 
+func (m *levelDbManager) getOrCreateDB(idx uint64) *sharedDB {
+	m.dbMtx.Lock()
+	defer m.dbMtx.Unlock()
+
+	db := m.databases[idx]
+	if db != nil {
+		return db
+	}
+
+	db = newSharedDB(m.options, filepath.Join(m.path, u64ToHexStringExt(idx)), m.readOnly, m.metrics, m.openDBCounter, m.closedFlag, m.log)
+	m.databases[idx] = db
+	return db
+}
+
+func (m *levelDbManager) hasAnyDB() bool {
+	m.dbMtx.RLock()
+	defer m.dbMtx.RUnlock()
+
+	return len(m.databases) > 0
+}
+
 // dbManager manages the opening and closing of blobovnicza instances.
 //
 // The blobovnicza opens at the first request, closes after the last request.
@@ -137,17 +221,15 @@ type dbManager struct {
 	options  []blobovnicza.Option
 	readOnly bool
 	metrics  blobovnicza.Metrics
-	leafWidth uint64
 	log      *logger.Logger
 }
 
-func newDBManager(rootPath string, options []blobovnicza.Option, leafWidth uint64, readOnly bool, metrics blobovnicza.Metrics, log *logger.Logger) *dbManager {
+func newDBManager(rootPath string, options []blobovnicza.Option, readOnly bool, metrics blobovnicza.Metrics, log *logger.Logger) *dbManager {
 	return &dbManager{
 		rootPath:            rootPath,
 		options:             options,
 		readOnly:            readOnly,
 		metrics:             metrics,
-		leafWidth:           leafWidth,
 		levelToManager:      make(map[string]*levelDbManager),
 		levelToManagerGuard: &sync.RWMutex{},
 		log:                 log,
@@ -163,6 +245,17 @@ func (m *dbManager) GetByPath(path string) *sharedDB {
 	return levelManager.GetByIndex(curIndex)
 }
 
+func (m *dbManager) CleanResources(path string) {
+	lvlPath := filepath.Dir(path)
+
+	m.levelToManagerGuard.Lock()
+	defer m.levelToManagerGuard.Unlock()
+
+	if result, ok := m.levelToManager[lvlPath]; ok && !result.hasAnyDB() {
+		delete(m.levelToManager, lvlPath)
+	}
+}
+
 func (m *dbManager) Open() {
 	m.closedFlag.Store(false)
 }
@@ -195,7 +288,7 @@ func (m *dbManager) getOrCreateLevelManager(lvlPath string) *levelDbManager {
 		return result
 	}
 
-	result := newLevelDBManager(m.leafWidth, m.options, m.rootPath, lvlPath, m.readOnly, m.metrics, m.dbCounter, m.closedFlag, m.log)
+	result := newLevelDBManager(m.options, m.rootPath, lvlPath, m.readOnly, m.metrics, m.dbCounter, m.closedFlag, m.log)
 	m.levelToManager[lvlPath] = result
 	return result
 }

pkg/local_object_storage/blobstor/blobovniczatree/metrics.go

@@ -6,6 +6,13 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
 )
 
+const (
+	rebuildStatusNotStarted = "not_started"
+	rebuildStatusRunning    = "running"
+	rebuildStatusCompleted  = "completed"
+	rebuildStatusFailed     = "failed"
+)
+
 type Metrics interface {
 	Blobovnicza() blobovnicza.Metrics
 
@@ -14,6 +21,10 @@ type Metrics interface {
 	SetMode(readOnly bool)
 	Close()
 
+	SetRebuildStatus(status string)
+	ObjectMoved(d time.Duration)
+	SetRebuildPercent(value uint32)
+
 	Delete(d time.Duration, success, withStorageID bool)
 	Exists(d time.Duration, success, withStorageID bool)
 	GetRange(d time.Duration, size int, success, withStorageID bool)
@@ -27,6 +38,9 @@ type noopMetrics struct{}
 func (m *noopMetrics) SetParentID(string)                      {}
 func (m *noopMetrics) SetMode(bool)                            {}
 func (m *noopMetrics) Close()                                  {}
+func (m *noopMetrics) SetRebuildStatus(string)                 {}
+func (m *noopMetrics) SetRebuildPercent(uint32)                {}
+func (m *noopMetrics) ObjectMoved(time.Duration)               {}
 func (m *noopMetrics) Delete(time.Duration, bool, bool)        {}
 func (m *noopMetrics) Exists(time.Duration, bool, bool)        {}
 func (m *noopMetrics) GetRange(time.Duration, int, bool, bool) {}

pkg/local_object_storage/blobstor/blobovniczatree/option.go

@@ -2,6 +2,7 @@ package blobovniczatree
 
 import (
 	"io/fs"
+	"time"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/compression"
@@ -20,18 +21,24 @@ type cfg struct {
 	blzLeafWidth       uint64
 	compression        *compression.Config
 	blzOpts            []blobovnicza.Option
-	// reportError is the function called when encountering disk errors.
+	reportError        func(string, error) // reportError is the function called when encountering disk errors.
-	reportError func(string, error)
 	metrics            Metrics
+	waitBeforeDropDB   time.Duration
+	blzInitWorkerCount int
+	blzMoveBatchSize   int
+	createDBInAdvance  bool
 }
 
 type Option func(*cfg)
 
 const (
-	defaultPerm            = 0700
+	defaultPerm               = 0o700
 	defaultOpenedCacheSize    = 50
 	defaultBlzShallowDepth    = 2
 	defaultBlzShallowWidth    = 16
+	defaultWaitBeforeDropDB   = 10 * time.Second
+	defaultBlzInitWorkerCount = 5
+	defaulBlzMoveBatchSize    = 10000
 )
 
 func initConfig(c *cfg) {
@@ -43,6 +50,9 @@ func initConfig(c *cfg) {
 		blzShallowWidth:    defaultBlzShallowWidth,
 		reportError:        func(string, error) {},
 		metrics:            &noopMetrics{},
+		waitBeforeDropDB:   defaultWaitBeforeDropDB,
+		blzInitWorkerCount: defaultBlzInitWorkerCount,
+		blzMoveBatchSize:   defaulBlzMoveBatchSize,
 	}
 }
 
@@ -106,3 +116,34 @@ func WithMetrics(m Metrics) Option {
 		c.metrics = m
 	}
 }
+
+func WithWaitBeforeDropDB(t time.Duration) Option {
+	return func(c *cfg) {
+		c.waitBeforeDropDB = t
+	}
+}
+
+func WithMoveBatchSize(v int) Option {
+	return func(c *cfg) {
+		c.blzMoveBatchSize = v
+	}
+}
+
+// WithInitWorkersCount sets maximum workers count to init blobovnicza tree.
+//
+// Negative or zero value means no limit.
+func WithInitWorkerCount(v int) Option {
+	if v <= 0 {
+		v = -1
+	}
+	return func(c *cfg) {
+		c.blzInitWorkerCount = v
+	}
+}
+
+// WithInitInAdvance returns an option to create blobovnicza tree DB's in advance.
+func WithInitInAdvance(v bool) Option {
+	return func(c *cfg) {
+		c.createDBInAdvance = v
+	}
+}

pkg/local_object_storage/blobstor/blobovniczatree/put.go

@@ -69,7 +69,7 @@ func (b *Blobovniczas) Put(ctx context.Context, prm common.PutPrm) (common.PutRe
 
 type putIterator struct {
 	B       *Blobovniczas
-	ID      *blobovnicza.ID
+	ID      *ID
 	AllFull bool
 	PutPrm  blobovnicza.PutPrm
 }
@@ -101,15 +101,15 @@ func (i *putIterator) iterate(ctx context.Context, lvlPath string) (bool, error)
 			i.B.reportError(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err)
 		} else {
 			i.B.log.Debug(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza,
-				zap.String("path", active.Path()),
+				zap.String("path", active.SystemPath()),
 				zap.String("error", err.Error()))
 		}
 
 		return false, nil
 	}
 
-	idx := u64FromHexString(filepath.Base(active.Path()))
+	idx := u64FromHexString(filepath.Base(active.SystemPath()))
-	i.ID = blobovnicza.NewIDFromBytes([]byte(filepath.Join(lvlPath, u64ToHexString(idx))))
+	i.ID = NewIDFromBytes([]byte(filepath.Join(lvlPath, u64ToHexString(idx))))
 
 	return true, nil
 }

pkg/local_object_storage/blobstor/blobovniczatree/rebuild.go

@@ -0,0 +1,478 @@
+package blobovniczatree
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	errRebuildInProgress = errors.New("rebuild is in progress, the operation cannot be performed")
+	errBatchFull         = errors.New("batch full")
+)
+
+func (b *Blobovniczas) Rebuild(ctx context.Context, prm common.RebuildPrm) (common.RebuildRes, error) {
+	if b.readOnly {
+		return common.RebuildRes{}, common.ErrReadOnly
+	}
+
+	b.metrics.SetRebuildStatus(rebuildStatusRunning)
+	b.metrics.SetRebuildPercent(0)
+	success := true
+	defer func() {
+		if success {
+			b.metrics.SetRebuildStatus(rebuildStatusCompleted)
+		} else {
+			b.metrics.SetRebuildStatus(rebuildStatusFailed)
+		}
+	}()
+
+	b.rebuildGuard.Lock()
+	defer b.rebuildGuard.Unlock()
+
+	var res common.RebuildRes
+
+	b.log.Debug(logs.BlobovniczaTreeCompletingPreviousRebuild)
+	completedPreviosMoves, err := b.completeIncompletedMove(ctx, prm.MetaStorage)
+	res.ObjectsMoved += completedPreviosMoves
+	if err != nil {
+		b.log.Warn(logs.BlobovniczaTreeCompletedPreviousRebuildFailed, zap.Error(err))
+		success = false
+		return res, err
+	}
+	b.log.Debug(logs.BlobovniczaTreeCompletedPreviousRebuildSuccess)
+
+	b.log.Debug(logs.BlobovniczaTreeCollectingDBToRebuild)
+	dbsToMigrate, err := b.getDBsToRebuild(ctx)
+	if err != nil {
+		b.log.Warn(logs.BlobovniczaTreeCollectingDBToRebuildFailed, zap.Error(err))
+		success = false
+		return res, err
+	}
+
+	b.log.Info(logs.BlobovniczaTreeCollectingDBToRebuildSuccess, zap.Int("blobovniczas_to_rebuild", len(dbsToMigrate)))
+	res, err = b.migrateDBs(ctx, dbsToMigrate, prm, res)
+	if err != nil {
+		success = false
+	}
+	return res, err
+}
+
+func (b *Blobovniczas) migrateDBs(ctx context.Context, dbs []string, prm common.RebuildPrm, res common.RebuildRes) (common.RebuildRes, error) {
+	var completedDBCount uint32
+	for _, db := range dbs {
+		b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovnicza, zap.String("path", db))
+		movedObjects, err := b.rebuildDB(ctx, db, prm.MetaStorage, prm.WorkerLimiter)
+		res.ObjectsMoved += movedObjects
+		if err != nil {
+			b.log.Warn(logs.BlobovniczaTreeRebuildingBlobovniczaFailed, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects), zap.Error(err))
+			return res, err
+		}
+		b.log.Debug(logs.BlobovniczaTreeRebuildingBlobovniczaSuccess, zap.String("path", db), zap.Uint64("moved_objects_count", movedObjects))
+		res.FilesRemoved++
+		completedDBCount++
+		b.metrics.SetRebuildPercent((100 * completedDBCount) / uint32(len(dbs)))
+	}
+	b.metrics.SetRebuildPercent(100)
+	return res, nil
+}
+
+func (b *Blobovniczas) getDBsToRebuild(ctx context.Context) ([]string, error) {
+	dbsToMigrate := make(map[string]struct{})
+	if err := b.iterateExistingDBPaths(ctx, func(s string) (bool, error) {
+		dbsToMigrate[s] = struct{}{}
+		return false, nil
+	}); err != nil {
+		return nil, err
+	}
+	if err := b.iterateSortedLeaves(ctx, nil, func(s string) (bool, error) {
+		delete(dbsToMigrate, s)
+		return false, nil
+	}); err != nil {
+		return nil, err
+	}
+	result := make([]string, 0, len(dbsToMigrate))
+	for db := range dbsToMigrate {
+		result = append(result, db)
+	}
+	return result, nil
+}
+
+func (b *Blobovniczas) rebuildDB(ctx context.Context, path string, meta common.MetaStorage, limiter common.ConcurrentWorkersLimiter) (uint64, error) {
+	shDB := b.getBlobovnicza(path)
+	blz, err := shDB.Open()
+	if err != nil {
+		return 0, err
+	}
+	shDBClosed := false
+	defer func() {
+		if shDBClosed {
+			return
+		}
+		shDB.Close()
+	}()
+
+	migratedObjects, err := b.moveObjects(ctx, blz, shDB.SystemPath(), meta, limiter)
+	if err != nil {
+		return migratedObjects, err
+	}
+	shDBClosed, err = b.dropDB(ctx, path, shDB)
+	return migratedObjects, err
+}
+
+func (b *Blobovniczas) moveObjects(ctx context.Context, blz *blobovnicza.Blobovnicza, blzPath string, meta common.MetaStorage, limiter common.ConcurrentWorkersLimiter) (uint64, error) {
+	var result atomic.Uint64
+	batch := make(map[oid.Address][]byte)
+
+	var prm blobovnicza.IteratePrm
+	prm.DecodeAddresses()
+	prm.SetHandler(func(ie blobovnicza.IterationElement) error {
+		batch[ie.Address()] = bytes.Clone(ie.ObjectData())
+		if len(batch) == b.blzMoveBatchSize {
+			return errBatchFull
+		}
+		return nil
+	})
+
+	for {
+		_, err := blz.Iterate(ctx, prm)
+		if err != nil && !errors.Is(err, errBatchFull) {
+			return result.Load(), err
+		}
+
+		if len(batch) == 0 {
+			break
+		}
+
+		eg, egCtx := errgroup.WithContext(ctx)
+
+		for addr, data := range batch {
+			addr := addr
+			data := data
+
+			if err := limiter.AcquireWorkSlot(egCtx); err != nil {
+				_ = eg.Wait()
+				return result.Load(), err
+			}
+			eg.Go(func() error {
+				defer limiter.ReleaseWorkSlot()
+				err := b.moveObject(egCtx, blz, blzPath, addr, data, meta)
+				if err == nil {
+					result.Add(1)
+				}
+				return err
+			})
+		}
+		if err := eg.Wait(); err != nil {
+			return result.Load(), err
+		}
+
+		batch = make(map[oid.Address][]byte)
+	}
+
+	return result.Load(), nil
+}
+
+func (b *Blobovniczas) moveObject(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string,
+	addr oid.Address, data []byte, metaStore common.MetaStorage) error {
+	startedAt := time.Now()
+	defer func() {
+		b.metrics.ObjectMoved(time.Since(startedAt))
+	}()
+	it := &moveIterator{
+		B:             b,
+		ID:            nil,
+		AllFull:       true,
+		Address:       addr,
+		ObjectData:    data,
+		MetaStore:     metaStore,
+		Source:        source,
+		SourceSysPath: sourcePath,
+	}
+
+	if err := b.iterateDeepest(ctx, addr, func(lvlPath string) (bool, error) { return it.tryMoveToLvl(ctx, lvlPath) }); err != nil {
+		return err
+	} else if it.ID == nil {
+		if it.AllFull {
+			return common.ErrNoSpace
+		}
+		return errPutFailed
+	}
+	return nil
+}
+
+func (b *Blobovniczas) dropDB(ctx context.Context, path string, shDb *sharedDB) (bool, error) {
+	select {
+	case <-ctx.Done():
+		return false, ctx.Err()
+	case <-time.After(b.waitBeforeDropDB): // to complete requests with old storage ID
+	}
+
+	b.dbCache.EvictAndMarkNonCached(path)
+	defer b.dbCache.RemoveFromNonCached(path)
+
+	b.dbFilesGuard.Lock()
+	defer b.dbFilesGuard.Unlock()
+
+	if err := shDb.CloseAndRemoveFile(); err != nil {
+		return false, err
+	}
+	b.commondbManager.CleanResources(path)
+	if err := b.dropDirectoryIfEmpty(filepath.Dir(path)); err != nil {
+		return true, err
+	}
+	return true, nil
+}
+
+func (b *Blobovniczas) dropDirectoryIfEmpty(path string) error {
+	if path == "." {
+		return nil
+	}
+
+	sysPath := filepath.Join(b.rootPath, path)
+	entries, err := os.ReadDir(sysPath)
+	if err != nil {
+		return err
+	}
+	if len(entries) > 0 {
+		return nil
+	}
+	if err := os.Remove(sysPath); err != nil {
+		return err
+	}
+	return b.dropDirectoryIfEmpty(filepath.Dir(path))
+}
+
+func (b *Blobovniczas) completeIncompletedMove(ctx context.Context, metaStore common.MetaStorage) (uint64, error) {
+	var count uint64
+	return count, b.iterateExistingDBPaths(ctx, func(s string) (bool, error) {
+		shDB := b.getBlobovnicza(s)
+		blz, err := shDB.Open()
+		if err != nil {
+			return true, err
+		}
+		defer shDB.Close()
+
+		incompletedMoves, err := blz.ListMoveInfo(ctx)
+		if err != nil {
+			return true, err
+		}
+
+		for _, move := range incompletedMoves {
+			if err := b.performMove(ctx, blz, shDB.SystemPath(), move, metaStore); err != nil {
+				return true, err
+			}
+			count++
+		}
+
+		return false, nil
+	})
+}
+
+func (b *Blobovniczas) performMove(ctx context.Context, source *blobovnicza.Blobovnicza, sourcePath string,
+	move blobovnicza.MoveInfo, metaStore common.MetaStorage) error {
+	targetDB := b.getBlobovnicza(NewIDFromBytes(move.TargetStorageID).Path())
+	target, err := targetDB.Open()
+	if err != nil {
+		return err
+	}
+	defer targetDB.Close()
+
+	existsInSource := true
+	var gPrm blobovnicza.GetPrm
+	gPrm.SetAddress(move.Address)
+	gRes, err := source.Get(ctx, gPrm)
+	if err != nil {
+		if client.IsErrObjectNotFound(err) {
+			existsInSource = false
+		} else {
+			b.log.Warn(logs.BlobovniczatreeCouldNotCheckExistenceInTargetDB, zap.Error(err))
+			return err
+		}
+	}
+
+	if !existsInSource { //object was deleted by Rebuild, need to delete move info
+		if err = source.DropMoveInfo(ctx, move.Address); err != nil {
+			b.log.Warn(logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", sourcePath), zap.Error(err))
+			return err
+		}
+		b.deleteProtectedObjects.Delete(move.Address)
+		return nil
+	}
+
+	existsInTarget, err := target.Exists(ctx, move.Address)
+	if err != nil {
+		b.log.Warn(logs.BlobovniczatreeCouldNotCheckExistenceInTargetDB, zap.Error(err))
+		return err
+	}
+
+	if !existsInTarget {
+		var putPrm blobovnicza.PutPrm
+		putPrm.SetAddress(move.Address)
+		putPrm.SetMarshaledObject(gRes.Object())
+		_, err = target.Put(ctx, putPrm)
+		if err != nil {
+			b.log.Warn(logs.BlobovniczatreeCouldNotPutObjectToTargetDB, zap.String("path", targetDB.SystemPath()), zap.Error(err))
+			return err
+		}
+	}
+
+	if err = metaStore.UpdateStorageID(ctx, move.Address, move.TargetStorageID); err != nil {
+		b.log.Warn(logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err), zap.Stringer("address", move.Address))
+		if !client.IsErrObjectNotFound(err) {
+			return err
+		}
+	}
+
+	var deletePrm blobovnicza.DeletePrm
+	deletePrm.SetAddress(move.Address)
+	if _, err = source.Delete(ctx, deletePrm); err != nil {
+		b.log.Warn(logs.BlobovniczatreeCouldNotDeleteFromSource, zap.String("path", sourcePath), zap.Error(err))
+		return err
+	}
+
+	if err = source.DropMoveInfo(ctx, move.Address); err != nil {
+		b.log.Warn(logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", sourcePath), zap.Error(err))
+		return err
+	}
+
+	b.deleteProtectedObjects.Delete(move.Address)
+	return nil
+}
+
+type moveIterator struct {
+	B             *Blobovniczas
+	ID            *ID
+	AllFull       bool
+	Address       oid.Address
+	ObjectData    []byte
+	MetaStore     common.MetaStorage
+	Source        *blobovnicza.Blobovnicza
+	SourceSysPath string
+}
+
+func (i *moveIterator) tryMoveToLvl(ctx context.Context, lvlPath string) (bool, error) {
+	target, err := i.B.activeDBManager.GetOpenedActiveDBForLevel(lvlPath)
+	if err != nil {
+		if !isLogical(err) {
+			i.B.reportError(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, err)
+		} else {
+			i.B.log.Warn(logs.BlobovniczatreeCouldNotGetActiveBlobovnicza, zap.Error(err))
+		}
+		return false, nil
+	}
+
+	if target == nil {
+		i.B.log.Warn(logs.BlobovniczatreeBlobovniczaOverflowed, zap.String("level", lvlPath))
+		return false, nil
+	}
+	defer target.Close()
+
+	i.AllFull = false
+
+	targetIDx := u64FromHexString(filepath.Base(target.SystemPath()))
+	targetStorageID := NewIDFromBytes([]byte(filepath.Join(lvlPath, u64ToHexString(targetIDx))))
+
+	if err = i.Source.PutMoveInfo(ctx, blobovnicza.MoveInfo{
+		Address:         i.Address,
+		TargetStorageID: targetStorageID.Bytes(),
+	}); err != nil {
+		if !isLogical(err) {
+			i.B.reportError(logs.BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza, err)
+		} else {
+			i.B.log.Warn(logs.BlobovniczatreeCouldNotPutMoveInfoToSourceBlobovnicza, zap.String("path", i.SourceSysPath), zap.Error(err))
+		}
+		return true, nil
+	}
+	i.B.deleteProtectedObjects.Add(i.Address)
+
+	var putPrm blobovnicza.PutPrm
+	putPrm.SetAddress(i.Address)
+	putPrm.SetMarshaledObject(i.ObjectData)
+
+	_, err = target.Blobovnicza().Put(ctx, putPrm)
+	if err != nil {
+		if !isLogical(err) {
+			i.B.reportError(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, err)
+		} else {
+			i.B.log.Warn(logs.BlobovniczatreeCouldNotPutObjectToActiveBlobovnicza, zap.String("path", target.SystemPath()), zap.Error(err))
+		}
+		return true, nil
+	}
+
+	if err = i.MetaStore.UpdateStorageID(ctx, i.Address, targetStorageID.Bytes()); err != nil {
+		i.B.log.Warn(logs.BlobovniczatreeCouldNotUpdateStorageID, zap.Error(err), zap.Stringer("address", i.Address))
+		return true, nil
+	}
+
+	var deletePrm blobovnicza.DeletePrm
+	deletePrm.SetAddress(i.Address)
+	if _, err = i.Source.Delete(ctx, deletePrm); err != nil {
+		if !isLogical(err) {
+			i.B.reportError(logs.BlobovniczatreeCouldNotDeleteFromSource, err)
+		} else {
+			i.B.log.Warn(logs.BlobovniczatreeCouldNotDeleteFromSource, zap.String("path", i.SourceSysPath), zap.Error(err))
+		}
+		return true, nil
+	}
+
+	if err = i.Source.DropMoveInfo(ctx, i.Address); err != nil {
+		if !isLogical(err) {
+			i.B.reportError(logs.BlobovniczatreeCouldNotDropMoveInfo, err)
+		} else {
+			i.B.log.Warn(logs.BlobovniczatreeCouldNotDropMoveInfo, zap.String("path", i.SourceSysPath), zap.Error(err))
+		}
+		return true, nil
+	}
+	i.B.deleteProtectedObjects.Delete(i.Address)
+
+	i.ID = targetStorageID
+	return true, nil
+}
+
+type addressMap struct {
+	data  map[oid.Address]struct{}
+	guard *sync.RWMutex
+}
+
+func newAddressMap() *addressMap {
+	return &addressMap{
+		data:  make(map[oid.Address]struct{}),
+		guard: &sync.RWMutex{},
+	}
+}
+
+func (m *addressMap) Add(address oid.Address) {
+	m.guard.Lock()
+	defer m.guard.Unlock()
+
+	m.data[address] = struct{}{}
+}
+
+func (m *addressMap) Delete(address oid.Address) {
+	m.guard.Lock()
+	defer m.guard.Unlock()
+
+	delete(m.data, address)
+}
+
+func (m *addressMap) Contains(address oid.Address) bool {
+	m.guard.RLock()
+	defer m.guard.RUnlock()
+
+	_, contains := m.data[address]
+	return contains
+}

pkg/local_object_storage/blobstor/blobovniczatree/rebuild_failover_test.go

@@ -0,0 +1,195 @@
+package blobovniczatree
+
+import (
+	"bytes"
+	"context"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobovnicza"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
+	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
+	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRebuildFailover(t *testing.T) {
+	t.Parallel()
+
+	t.Run("only move info saved", testRebuildFailoverOnlyMoveInfoSaved)
+
+	t.Run("object saved to target", testRebuildFailoverObjectSavedToTarget)
+
+	t.Run("object deleted from source", testRebuildFailoverObjectDeletedFromSource)
+}
+
+func testRebuildFailoverOnlyMoveInfoSaved(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+
+	blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db")))
+	require.NoError(t, blz.Open())
+	require.NoError(t, blz.Init())
+
+	obj := blobstortest.NewObject(1024)
+	data, err := obj.Marshal()
+	require.NoError(t, err)
+
+	var pPrm blobovnicza.PutPrm
+	pPrm.SetAddress(object.AddressOf(obj))
+	pPrm.SetMarshaledObject(data)
+	_, err = blz.Put(context.Background(), pPrm)
+	require.NoError(t, err)
+
+	require.NoError(t, blz.PutMoveInfo(context.Background(), blobovnicza.MoveInfo{
+		Address:         object.AddressOf(obj),
+		TargetStorageID: []byte("0/0/0"),
+	}))
+
+	require.NoError(t, blz.Close())
+
+	testRebuildFailoverValidate(t, dir, obj, true)
+}
+
+func testRebuildFailoverObjectSavedToTarget(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+
+	blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db")))
+	require.NoError(t, blz.Open())
+	require.NoError(t, blz.Init())
+
+	obj := blobstortest.NewObject(1024)
+	data, err := obj.Marshal()
+	require.NoError(t, err)
+
+	var pPrm blobovnicza.PutPrm
+	pPrm.SetAddress(object.AddressOf(obj))
+	pPrm.SetMarshaledObject(data)
+	_, err = blz.Put(context.Background(), pPrm)
+	require.NoError(t, err)
+
+	require.NoError(t, blz.PutMoveInfo(context.Background(), blobovnicza.MoveInfo{
+		Address:         object.AddressOf(obj),
+		TargetStorageID: []byte("0/0/0"),
+	}))
+
+	require.NoError(t, blz.Close())
+
+	blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db")))
+	require.NoError(t, blz.Open())
+	require.NoError(t, blz.Init())
+
+	_, err = blz.Put(context.Background(), pPrm)
+	require.NoError(t, err)
+
+	require.NoError(t, blz.Close())
+
+	testRebuildFailoverValidate(t, dir, obj, true)
+}
+
+func testRebuildFailoverObjectDeletedFromSource(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+
+	blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db")))
+	require.NoError(t, blz.Open())
+	require.NoError(t, blz.Init())
+
+	obj := blobstortest.NewObject(1024)
+	data, err := obj.Marshal()
+	require.NoError(t, err)
+
+	require.NoError(t, blz.PutMoveInfo(context.Background(), blobovnicza.MoveInfo{
+		Address:         object.AddressOf(obj),
+		TargetStorageID: []byte("0/0/0"),
+	}))
+
+	require.NoError(t, blz.Close())
+
+	blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db")))
+	require.NoError(t, blz.Open())
+	require.NoError(t, blz.Init())
+
+	var pPrm blobovnicza.PutPrm
+	pPrm.SetAddress(object.AddressOf(obj))
+	pPrm.SetMarshaledObject(data)
+	_, err = blz.Put(context.Background(), pPrm)
+	require.NoError(t, err)
+
+	require.NoError(t, blz.Close())
+
+	testRebuildFailoverValidate(t, dir, obj, false)
+}
+
+func testRebuildFailoverValidate(t *testing.T, dir string, obj *objectSDK.Object, mustUpdateStorageID bool) {
+	b := NewBlobovniczaTree(
+		WithLogger(test.NewLogger(t, true)),
+		WithObjectSizeLimit(2048),
+		WithBlobovniczaShallowWidth(2),
+		WithBlobovniczaShallowDepth(2),
+		WithRootPath(dir),
+		WithBlobovniczaSize(100*1024*1024),
+		WithWaitBeforeDropDB(0),
+		WithOpenedCacheSize(1000))
+	require.NoError(t, b.Open(false))
+	require.NoError(t, b.Init())
+
+	var dPrm common.DeletePrm
+	dPrm.Address = object.AddressOf(obj)
+	dPrm.StorageID = []byte("0/0/1")
+	_, err := b.Delete(context.Background(), dPrm)
+	require.ErrorIs(t, err, errObjectIsDeleteProtected)
+
+	metaStub := &storageIDUpdateStub{
+		storageIDs: make(map[oid.Address][]byte),
+		guard:      &sync.Mutex{},
+	}
+	rRes, err := b.Rebuild(context.Background(), common.RebuildPrm{
+		MetaStorage:   metaStub,
+		WorkerLimiter: &rebuildLimiterStub{},
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint64(1), rRes.ObjectsMoved)
+	require.Equal(t, uint64(0), rRes.FilesRemoved)
+
+	require.NoError(t, b.Close())
+
+	blz := blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "1.db")))
+	require.NoError(t, blz.Open())
+	require.NoError(t, blz.Init())
+
+	moveInfo, err := blz.ListMoveInfo(context.Background())
+	require.NoError(t, err)
+	require.Equal(t, 0, len(moveInfo))
+
+	var gPrm blobovnicza.GetPrm
+	gPrm.SetAddress(object.AddressOf(obj))
+	_, err = blz.Get(context.Background(), gPrm)
+	require.True(t, client.IsErrObjectNotFound(err))
+
+	require.NoError(t, blz.Close())
+
+	blz = blobovnicza.New(blobovnicza.WithPath(filepath.Join(dir, "0", "0", "0.db")))
+	require.NoError(t, blz.Open())
+	require.NoError(t, blz.Init())
+
+	moveInfo, err = blz.ListMoveInfo(context.Background())
+	require.NoError(t, err)
+	require.Equal(t, 0, len(moveInfo))
+
+	gRes, err := blz.Get(context.Background(), gPrm)
+	require.NoError(t, err)
+	require.True(t, len(gRes.Object()) > 0)
+
+	if mustUpdateStorageID {
+		require.True(t, bytes.Equal([]byte("0/0/0"), metaStub.storageIDs[object.AddressOf(obj)]))
+	}
+
+	require.NoError(t, blz.Close())
+}

pkg/local_object_storage/blobstor/blobovniczatree/rebuild_test.go

@@ -0,0 +1,145 @@
+package blobovniczatree
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/internal/blobstortest"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
+)
+
+func TestBlobovniczaTreeRebuild(t *testing.T) {
+	t.Parallel()
+
+	t.Run("width increased", func(t *testing.T) {
+		t.Parallel()
+		testBlobovniczaTreeRebuildHelper(t, 2, 2, 2, 3, false)
+	})
+
+	t.Run("width reduced", func(t *testing.T) {
+		t.Parallel()
+		testBlobovniczaTreeRebuildHelper(t, 2, 2, 2, 1, true)
+	})
+
+	t.Run("depth increased", func(t *testing.T) {
+		t.Parallel()
+		testBlobovniczaTreeRebuildHelper(t, 1, 2, 2, 2, true)
+	})
+
+	t.Run("depth reduced", func(t *testing.T) {
+		t.Parallel()
+		testBlobovniczaTreeRebuildHelper(t, 2, 2, 1, 2, true)
+	})
+}
+
+func testBlobovniczaTreeRebuildHelper(t *testing.T, sourceDepth, sourceWidth, targetDepth, targetWidth uint64, shouldMigrate bool) {
+	dir := t.TempDir()
+	b := NewBlobovniczaTree(
+		WithLogger(test.NewLogger(t, true)),
+		WithObjectSizeLimit(2048),
+		WithBlobovniczaShallowWidth(sourceWidth),
+		WithBlobovniczaShallowDepth(sourceDepth),
+		WithRootPath(dir),
+		WithBlobovniczaSize(100*1024*1024),
+		WithWaitBeforeDropDB(0),
+		WithOpenedCacheSize(1000),
+		WithMoveBatchSize(3))
+	require.NoError(t, b.Open(false))
+	require.NoError(t, b.Init())
+
+	eg, egCtx := errgroup.WithContext(context.Background())
+	storageIDs := make(map[oid.Address][]byte)
+	storageIDsGuard := &sync.Mutex{}
+	for i := 0; i < 1000; i++ {
+		eg.Go(func() error {
+			obj := blobstortest.NewObject(1024)
+			data, err := obj.Marshal()
+			if err != nil {
+				return err
+			}
+			var prm common.PutPrm
+			prm.Address = object.AddressOf(obj)
+			prm.RawData = data
+			res, err := b.Put(egCtx, prm)
+			if err != nil {
+				return err
+			}
+			storageIDsGuard.Lock()
+			storageIDs[prm.Address] = res.StorageID
+			storageIDsGuard.Unlock()
+			return nil
+		})
+	}
+
+	require.NoError(t, eg.Wait())
+	require.NoError(t, b.Close())
+
+	b = NewBlobovniczaTree(
+		WithLogger(test.NewLogger(t, true)),
+		WithObjectSizeLimit(2048),
+		WithBlobovniczaShallowWidth(targetWidth),
+		WithBlobovniczaShallowDepth(targetDepth),
+		WithRootPath(dir),
+		WithBlobovniczaSize(100*1024*1024),
+		WithWaitBeforeDropDB(0),
+		WithOpenedCacheSize(1000),
+		WithMoveBatchSize(3))
+	require.NoError(t, b.Open(false))
+	require.NoError(t, b.Init())
+
+	for addr, storageID := range storageIDs {
+		var gPrm common.GetPrm
+		gPrm.Address = addr
+		gPrm.StorageID = storageID
+		_, err := b.Get(context.Background(), gPrm)
+		require.NoError(t, err)
+	}
+
+	metaStub := &storageIDUpdateStub{
+		storageIDs: storageIDs,
+		guard:      &sync.Mutex{},
+	}
+	var rPrm common.RebuildPrm
+	rPrm.MetaStorage = metaStub
+	rPrm.WorkerLimiter = &rebuildLimiterStub{}
+	rRes, err := b.Rebuild(context.Background(), rPrm)
+	require.NoError(t, err)
+	dataMigrated := rRes.ObjectsMoved > 0 || rRes.FilesRemoved > 0 || metaStub.updatedCount > 0
+	require.Equal(t, shouldMigrate, dataMigrated)
+
+	for addr, storageID := range storageIDs {
+		var gPrm common.GetPrm
+		gPrm.Address = addr
+		gPrm.StorageID = storageID
+		_, err := b.Get(context.Background(), gPrm)
+		require.NoError(t, err)
+	}
+
+	require.NoError(t, b.Close())
+}
+
+type storageIDUpdateStub struct {
+	guard        *sync.Mutex
+	storageIDs   map[oid.Address][]byte
+	updatedCount uint64
+}
+
+func (s *storageIDUpdateStub) UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error {
+	s.guard.Lock()
+	defer s.guard.Unlock()
+
+	s.storageIDs[addr] = storageID
+	s.updatedCount++
+	return nil
+}
+
+type rebuildLimiterStub struct{}
+
+func (s *rebuildLimiterStub) AcquireWorkSlot(context.Context) error { return nil }
+func (s *rebuildLimiterStub) ReleaseWorkSlot()                      {}

pkg/local_object_storage/blobstor/common/rebuild.go

@@ -0,0 +1,26 @@
+package common
+
+import (
+	"context"
+
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+)
+
+type RebuildRes struct {
+	ObjectsMoved uint64
+	FilesRemoved uint64
+}
+
+type RebuildPrm struct {
+	MetaStorage   MetaStorage
+	WorkerLimiter ConcurrentWorkersLimiter
+}
+
+type MetaStorage interface {
+	UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error
+}
+
+type ConcurrentWorkersLimiter interface {
+	AcquireWorkSlot(ctx context.Context) error
+	ReleaseWorkSlot()
+}

pkg/local_object_storage/blobstor/common/storage.go

@@ -30,4 +30,5 @@ type Storage interface {
 	Put(context.Context, PutPrm) (PutRes, error)
 	Delete(context.Context, DeletePrm) (DeleteRes, error)
 	Iterate(context.Context, IteratePrm) (IterateRes, error)
+	Rebuild(context.Context, RebuildPrm) (RebuildRes, error)
 }

pkg/local_object_storage/blobstor/fstree/fstree.go

@@ -570,3 +570,7 @@ func (t *FSTree) SetReportErrorFunc(_ func(string, error)) {
 func (t *FSTree) SetParentID(parentID string) {
 	t.metrics.SetParentID(parentID)
 }
+
+func (t *FSTree) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) {
+	return common.RebuildRes{}, nil
+}

pkg/local_object_storage/blobstor/memstore/memstore.go

@@ -166,3 +166,7 @@ func (s *memstoreImpl) Iterate(_ context.Context, req common.IteratePrm) (common
 	}
 	return common.IterateRes{}, nil
 }
+
+func (s *memstoreImpl) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) {
+	return common.RebuildRes{}, nil
+}

pkg/local_object_storage/blobstor/rebuild.go

@@ -0,0 +1,45 @@
+package blobstor
+
+import (
+	"context"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"go.uber.org/zap"
+)
+
+type StorageIDUpdate interface {
+	UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error
+}
+
+type ConcurrentWorkersLimiter interface {
+	AcquireWorkSlot(ctx context.Context) error
+	ReleaseWorkSlot()
+}
+
+func (b *BlobStor) Rebuild(ctx context.Context, upd StorageIDUpdate, limiter ConcurrentWorkersLimiter) error {
+	var summary common.RebuildRes
+	var rErr error
+	for _, storage := range b.storage {
+		res, err := storage.Storage.Rebuild(ctx, common.RebuildPrm{
+			MetaStorage:   upd,
+			WorkerLimiter: limiter,
+		})
+		summary.FilesRemoved += res.FilesRemoved
+		summary.ObjectsMoved += res.ObjectsMoved
+		if err != nil {
+			b.log.Error(logs.BlobstorRebuildFailedToRebuildStorages,
+				zap.String("failed_storage_path", storage.Storage.Path()),
+				zap.String("failed_storage_type", storage.Storage.Type()),
+				zap.Error(err))
+			rErr = err
+			break
+		}
+	}
+	b.log.Info(logs.BlobstorRebuildRebuildStoragesCompleted,
+		zap.Bool("success", rErr == nil),
+		zap.Uint64("total_files_removed", summary.FilesRemoved),
+		zap.Uint64("total_objects_moved", summary.ObjectsMoved))
+	return rErr
+}

pkg/local_object_storage/blobstor/teststore/teststore.go

@@ -229,3 +229,7 @@ func (s *TestStore) Iterate(ctx context.Context, req common.IteratePrm) (common.
 }
 
 func (s *TestStore) SetParentID(string) {}
+
+func (s *TestStore) Rebuild(_ context.Context, _ common.RebuildPrm) (common.RebuildRes, error) {
+	return common.RebuildRes{}, nil
+}

pkg/local_object_storage/engine/engine.go

@@ -7,7 +7,6 @@ import (
 	"sync/atomic"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
@@ -38,6 +37,7 @@ type StorageEngine struct {
 		err error
 	}
 	evacuateLimiter *evacuationLimiter
+	rebuildLimiter  *rebuildLimiter
 }
 
 type shardWrapper struct {
@@ -47,7 +47,6 @@ type shardWrapper struct {
 
 type setModeRequest struct {
 	sh         *shard.Shard
-	isMeta     bool
 	errorCount uint32
 }
 
@@ -73,7 +72,7 @@ func (e *StorageEngine) setModeLoop() {
 			if !ok {
 				inProgress[sid] = struct{}{}
 				go func() {
-					e.moveToDegraded(r.sh, r.errorCount, r.isMeta)
+					e.moveToDegraded(r.sh, r.errorCount)
 
 					mtx.Lock()
 					delete(inProgress, sid)
@@ -85,7 +84,7 @@ func (e *StorageEngine) setModeLoop() {
 	}
 }
 
-func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta bool) {
+func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32) {
 	sid := sh.ID()
 	log := e.log.With(
 		zap.Stringer("shard_id", sid),
@@ -94,23 +93,21 @@ func (e *StorageEngine) moveToDegraded(sh *shard.Shard, errCount uint32, isMeta
 	e.mtx.RLock()
 	defer e.mtx.RUnlock()
 
-	if isMeta {
 	err := sh.SetMode(mode.DegradedReadOnly)
-		if err == nil {
+	if err != nil {
-			log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold)
-			return
-		}
 		log.Error(logs.EngineFailedToMoveShardInDegradedreadonlyModeMovingToReadonly,
 			zap.Error(err))
-	}
 
-	err := sh.SetMode(mode.ReadOnly)
+		err = sh.SetMode(mode.ReadOnly)
 		if err != nil {
-		log.Error(logs.EngineFailedToMoveShardInReadonlyMode, zap.Error(err))
+			log.Error(logs.EngineFailedToMoveShardInReadonlyMode,
-		return
+				zap.Error(err))
-	}
+		} else {
-
 			log.Info(logs.EngineShardIsMovedInReadonlyModeDueToErrorThreshold)
+		}
+	} else {
+		log.Info(logs.EngineShardIsMovedInDegradedModeDueToErrorThreshold)
+	}
 }
 
 // reportShardErrorBackground increases shard error counter and logs an error.
@@ -174,13 +171,11 @@ func (e *StorageEngine) reportShardErrorWithFlags(
 		return
 	}
 
-	isMeta := errors.As(err, new(metaerr.Error))
 	if block {
-		e.moveToDegraded(sh, errCount, isMeta)
+		e.moveToDegraded(sh, errCount)
 	} else {
 		req := setModeRequest{
 			errorCount: errCount,
-			isMeta:     isMeta,
 			sh:         sh,
 		}
 
@@ -213,13 +208,15 @@ type cfg struct {
 	shardPoolSize uint32
 
 	lowMem bool
+
+	rebuildWorkersCount uint32
 }
 
 func defaultCfg() *cfg {
 	return &cfg{
 		log:                 &logger.Logger{Logger: zap.L()},
-
 		shardPoolSize:       20,
+		rebuildWorkersCount: 100,
 	}
 }
 
@@ -238,6 +235,7 @@ func New(opts ...Option) *StorageEngine {
 		closeCh:         make(chan struct{}),
 		setModeCh:       make(chan setModeRequest),
 		evacuateLimiter: &evacuationLimiter{},
+		rebuildLimiter:  newRebuildLimiter(c.rebuildWorkersCount),
 	}
 }
 
@@ -275,3 +273,10 @@ func WithLowMemoryConsumption(lowMemCons bool) Option {
 		c.lowMem = lowMemCons
 	}
 }
+
+// WithRebuildWorkersCount returns an option to set the count of concurrent rebuild workers.
+func WithRebuildWorkersCount(count uint32) Option {
+	return func(c *cfg) {
+		c.rebuildWorkersCount = count
+	}
+}

pkg/local_object_storage/engine/error_test.go

@@ -152,7 +152,7 @@ func TestErrorReporting(t *testing.T) {
 		for i := uint32(0); i < 2; i++ {
 			_, err = te.ng.Get(context.Background(), GetPrm{addr: object.AddressOf(obj)})
 			require.Error(t, err)
-			checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.ReadOnly)
+			checkShardState(t, te.ng, te.shards[0].id, errThreshold+i, mode.DegradedReadOnly)
 			checkShardState(t, te.ng, te.shards[1].id, 0, mode.ReadWrite)
 		}
 

pkg/local_object_storage/engine/rebuild_limiter.go

@@ -0,0 +1,26 @@
+package engine
+
+import "context"
+
+type rebuildLimiter struct {
+	semaphore chan struct{}
+}
+
+func newRebuildLimiter(workersCount uint32) *rebuildLimiter {
+	return &rebuildLimiter{
+		semaphore: make(chan struct{}, workersCount),
+	}
+}
+
+func (l *rebuildLimiter) AcquireWorkSlot(ctx context.Context) error {
+	select {
+	case l.semaphore <- struct{}{}:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (l *rebuildLimiter) ReleaseWorkSlot() {
+	<-l.semaphore
+}

pkg/local_object_storage/engine/shards.go

@@ -110,6 +110,7 @@ func (e *StorageEngine) createShard(ctx context.Context, opts []shard.Option) (*
 		shard.WithExpiredLocksCallback(e.processExpiredLocks),
 		shard.WithDeletedLockCallback(e.processDeletedLocks),
 		shard.WithReportErrorFunc(e.reportShardErrorBackground),
+		shard.WithRebuildWorkerLimiter(e.rebuildLimiter),
 	)...)
 
 	if err := sh.UpdateID(ctx); err != nil {

pkg/local_object_storage/engine/writecache.go

@@ -68,16 +68,10 @@ func (m *writeCacheMetrics) Get(d time.Duration, success bool, st writecache.Sto
 
 func (m *writeCacheMetrics) Delete(d time.Duration, success bool, st writecache.StorageType) {
 	m.metrics.AddMethodDuration(m.shardID, "Delete", success, d, st.String())
-	if success {
-		m.metrics.DecActualCount(m.shardID, st.String())
-	}
 }
 
 func (m *writeCacheMetrics) Put(d time.Duration, success bool, st writecache.StorageType) {
 	m.metrics.AddMethodDuration(m.shardID, "Put", success, d, st.String())
-	if success {
-		m.metrics.IncActualCount(m.shardID, st.String())
-	}
 }
 
 func (m *writeCacheMetrics) SetEstimateSize(db, fstree uint64) {
@@ -99,7 +93,6 @@ func (m *writeCacheMetrics) Flush(success bool, st writecache.StorageType) {
 }
 
 func (m *writeCacheMetrics) Evict(st writecache.StorageType) {
-	m.metrics.DecActualCount(m.shardID, st.String())
 	m.metrics.IncOperationCounter(m.shardID, "Evict", metrics.NullBool{}, st.String())
 }
 

pkg/local_object_storage/metabase/storage_id.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/util/logicerr"
 	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
 	"github.com/nspcc-dev/neo-go/pkg/util/slice"
@@ -107,7 +108,7 @@ func (db *DB) UpdateStorageID(prm UpdateStorageIDPrm) (res UpdateStorageIDRes, e
 
 	err = db.boltDB.Batch(func(tx *bbolt.Tx) error {
 		exists, err := db.exists(tx, prm.addr, currEpoch)
-		if err == nil && exists || errors.Is(err, ErrObjectIsExpired) {
+		if err == nil && exists || errors.As(err, new(logicerr.Logical)) {
 			err = updateStorageID(tx, prm.addr, prm.id)
 		}
 

pkg/local_object_storage/metrics/blobovnicza.go

@@ -42,6 +42,18 @@ func (m *blobovniczaTreeMetrics) Close() {
 	m.m.CloseBlobobvnizcaTree(m.shardID, m.path)
 }
 
+func (m *blobovniczaTreeMetrics) SetRebuildStatus(status string) {
+	m.m.BlobovniczaTreeRebuildStatus(m.shardID, m.path, status)
+}
+
+func (m *blobovniczaTreeMetrics) SetRebuildPercent(value uint32) {
+	m.m.BlobovniczaTreeRebuildPercent(m.shardID, m.path, value)
+}
+
+func (m *blobovniczaTreeMetrics) ObjectMoved(d time.Duration) {
+	m.m.BlobovniczaTreeObjectMoved(m.shardID, m.path, d)
+}
+
 func (m *blobovniczaTreeMetrics) Delete(d time.Duration, success, withStorageID bool) {
 	m.m.BlobobvnizcaTreeMethodDuration(m.shardID, m.path, "Delete", d, success, metrics_impl.NullBool{Valid: true, Bool: withStorageID})
 }

pkg/local_object_storage/shard/control.go

@@ -162,6 +162,9 @@ func (s *Shard) Init(ctx context.Context) error {
 
 	s.gc.init(ctx)
 
+	s.rb = newRebuilder(s.rebuildLimiter)
+	s.rb.Start(ctx, s.blobStor, s.metaBase, s.log)
+
 	return nil
 }
 
@@ -266,6 +269,9 @@ func (s *Shard) refillTombstoneObject(ctx context.Context, obj *objectSDK.Object
 
 // Close releases all Shard's components.
 func (s *Shard) Close() error {
+	if s.rb != nil {
+		s.rb.Stop(s.log)
+	}
 	components := []interface{ Close() error }{}
 
 	if s.pilorama != nil {
@@ -310,6 +316,11 @@ func (s *Shard) Reload(ctx context.Context, opts ...Option) error {
 	unlock := s.lockExclusive()
 	defer unlock()
 
+	s.rb.Stop(s.log)
+	defer func() {
+		s.rb.Start(ctx, s.blobStor, s.metaBase, s.log)
+	}()
+
 	ok, err := s.metaBase.Reload(c.metaOpts...)
 	if err != nil {
 		if errors.Is(err, meta.ErrDegradedMode) {

pkg/local_object_storage/shard/delete.go

@@ -2,12 +2,11 @@ package shard
 
 import (
 	"context"
-	"errors"
+	"fmt"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/common"
 	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/writecache"
 	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@@ -33,8 +32,7 @@ func (p *DeletePrm) SetAddresses(addr ...oid.Address) {
 	p.addr = append(p.addr, addr...)
 }
 
-// Delete removes data from the shard's writeCache, metaBase and
+// Delete removes data from the shard's metaBase and blobStor.
-// blobStor.
 func (s *Shard) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) {
 	ctx, span := tracing.StartSpanFromContext(ctx, "Shard.Delete",
 		trace.WithAttributes(
@@ -46,10 +44,10 @@ func (s *Shard) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) {
 	s.m.RLock()
 	defer s.m.RUnlock()
 
-	return s.delete(ctx, prm)
+	return s.delete(ctx, prm, false)
 }
 
-func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) {
+func (s *Shard) delete(ctx context.Context, prm DeletePrm, skipFailed bool) (DeleteRes, error) {
 	if s.info.Mode.ReadOnly() {
 		return DeleteRes{}, ErrReadOnlyMode
 	} else if s.info.Mode.NoMetabase() {
@@ -64,12 +62,25 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) {
 		default:
 		}
 
-		s.deleteObjectFromWriteCacheSafe(ctx, addr)
+		if err := s.validateWritecacheDoesntContainObject(ctx, addr); err != nil {
+			if skipFailed {
+				continue
+			}
+			return result, err
+		}
 
-		s.deleteFromBlobstorSafe(ctx, addr)
+		if err := s.deleteFromBlobstor(ctx, addr); err != nil {
+			if skipFailed {
+				continue
+			}
+			return result, err
+		}
 
 		if err := s.deleteFromMetabase(ctx, addr); err != nil {
-			return result, err // stop on metabase error ?
+			if skipFailed {
+				continue
+			}
+			return result, err
 		}
 		result.deleted++
 	}
@@ -77,16 +88,22 @@ func (s *Shard) delete(ctx context.Context, prm DeletePrm) (DeleteRes, error) {
 	return result, nil
 }
 
-func (s *Shard) deleteObjectFromWriteCacheSafe(ctx context.Context, addr oid.Address) {
+func (s *Shard) validateWritecacheDoesntContainObject(ctx context.Context, addr oid.Address) error {
-	if s.hasWriteCache() {
+	if !s.hasWriteCache() {
-		err := s.writeCache.Delete(ctx, addr)
+		return nil
-		if err != nil && !client.IsErrObjectNotFound(err) && !errors.Is(err, writecache.ErrReadOnly) {
-			s.log.Warn(logs.ShardCantDeleteObjectFromWriteCache, zap.Error(err))
 	}
+	_, err := s.writeCache.Head(ctx, addr)
+	if err == nil {
+		s.log.Warn(logs.ObjectRemovalFailureExistsInWritecache, zap.Stringer("object_address", addr))
+		return fmt.Errorf("object %s must be flushed from writecache", addr)
 	}
+	if client.IsErrObjectNotFound(err) {
+		return nil
+	}
+	return err
 }
 
-func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) {
+func (s *Shard) deleteFromBlobstor(ctx context.Context, addr oid.Address) error {
 	var sPrm meta.StorageIDPrm
 	sPrm.SetAddress(addr)
 
@@ -95,6 +112,7 @@ func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) {
 		s.log.Debug(logs.StorageIDRetrievalFailure,
 			zap.Stringer("object", addr),
 			zap.String("error", err.Error()))
+		return err
 	}
 	storageID := res.StorageID()
 
@@ -103,11 +121,13 @@ func (s *Shard) deleteFromBlobstorSafe(ctx context.Context, addr oid.Address) {
 	delPrm.StorageID = storageID
 
 	_, err = s.blobStor.Delete(ctx, delPrm)
-	if err != nil {
+	if err != nil && !client.IsErrObjectNotFound(err) {
 		s.log.Debug(logs.ObjectRemovalFailureBlobStor,
 			zap.Stringer("object_address", addr),
 			zap.String("error", err.Error()))
+		return err
 	}
+	return nil
 }
 
 func (s *Shard) deleteFromMetabase(ctx context.Context, addr oid.Address) error {

pkg/local_object_storage/shard/delete_test.go

@@ -52,13 +52,18 @@ func testShardDelete(t *testing.T, hasWriteCache bool) {
 		_, err = testGet(t, sh, getPrm, hasWriteCache)
 		require.NoError(t, err)
 
-		_, err = sh.Delete(context.TODO(), delPrm)
+		if hasWriteCache {
-		require.NoError(t, err)
-
 			require.Eventually(t, func() bool {
+				_, err = sh.Delete(context.Background(), delPrm)
+				return err == nil
+			}, 30*time.Second, 100*time.Millisecond)
+		} else {
+			_, err = sh.Delete(context.Background(), delPrm)
+			require.NoError(t, err)
+		}
+
 		_, err = sh.Get(context.Background(), getPrm)
-			return client.IsErrObjectNotFound(err)
+		require.True(t, client.IsErrObjectNotFound(err))
-		}, time.Second, 50*time.Millisecond)
 	})
 
 	t.Run("small object", func(t *testing.T) {
@@ -78,12 +83,17 @@ func testShardDelete(t *testing.T, hasWriteCache bool) {
 		_, err = sh.Get(context.Background(), getPrm)
 		require.NoError(t, err)
 
+		if hasWriteCache {
+			require.Eventually(t, func() bool {
+				_, err = sh.Delete(context.Background(), delPrm)
+				return err == nil
+			}, 10*time.Second, 100*time.Millisecond)
+		} else {
 			_, err = sh.Delete(context.Background(), delPrm)
 			require.NoError(t, err)
+		}
 
-		require.Eventually(t, func() bool {
 		_, err = sh.Get(context.Background(), getPrm)
-			return client.IsErrObjectNotFound(err)
+		require.True(t, client.IsErrObjectNotFound(err))
-		}, time.Second, 50*time.Millisecond)
 	})
 }

pkg/local_object_storage/shard/gc.go

@@ -297,7 +297,7 @@ func (s *Shard) removeGarbage(pctx context.Context) (result gcRunResult) {
 	deletePrm.SetAddresses(buf...)
 
 	// delete accumulated objects
-	res, err := s.delete(ctx, deletePrm)
+	res, err := s.delete(ctx, deletePrm, true)
 
 	result.deleted = res.deleted
 	result.failedToDelete = uint64(len(buf)) - res.deleted

pkg/local_object_storage/shard/rebuild_limiter.go

@@ -0,0 +1,13 @@
+package shard
+
+import "context"
+
+type RebuildWorkerLimiter interface {
+	AcquireWorkSlot(ctx context.Context) error
+	ReleaseWorkSlot()
+}
+
+type noopRebuildLimiter struct{}
+
+func (l *noopRebuildLimiter) AcquireWorkSlot(context.Context) error { return nil }
+func (l *noopRebuildLimiter) ReleaseWorkSlot()                      {}

pkg/local_object_storage/shard/rebuilder.go

@@ -0,0 +1,98 @@
+package shard
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
+	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
+	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"go.uber.org/zap"
+)
+
+type rebuilder struct {
+	mtx     *sync.Mutex
+	wg      *sync.WaitGroup
+	cancel  func()
+	limiter RebuildWorkerLimiter
+}
+
+func newRebuilder(l RebuildWorkerLimiter) *rebuilder {
+	return &rebuilder{
+		mtx:     &sync.Mutex{},
+		wg:      &sync.WaitGroup{},
+		cancel:  nil,
+		limiter: l,
+	}
+}
+
+func (r *rebuilder) Start(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *logger.Logger) {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+
+	r.start(ctx, bs, mb, log)
+}
+
+func (r *rebuilder) start(ctx context.Context, bs *blobstor.BlobStor, mb *meta.DB, log *logger.Logger) {
+	if r.cancel != nil {
+		r.stop(log)
+	}
+	ctx, cancel := context.WithCancel(ctx)
+	r.cancel = cancel
+	r.wg.Add(1)
+	go func() {
+		defer r.wg.Done()
+
+		log.Info(logs.BlobstoreRebuildStarted)
+		if err := bs.Rebuild(ctx, &mbStorageIDUpdate{mb: mb}, r.limiter); err != nil {
+			log.Warn(logs.FailedToRebuildBlobstore, zap.Error(err))
+		} else {
+			log.Info(logs.BlobstoreRebuildCompletedSuccessfully)
+		}
+	}()
+}
+
+func (r *rebuilder) Stop(log *logger.Logger) {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+
+	r.stop(log)
+}
+
+func (r *rebuilder) stop(log *logger.Logger) {
+	if r.cancel == nil {
+		return
+	}
+
+	r.cancel()
+	r.wg.Wait()
+	r.cancel = nil
+	log.Info(logs.BlobstoreRebuildStopped)
+}
+
+var errMBIsNotAvailable = errors.New("metabase is not available")
+
+type mbStorageIDUpdate struct {
+	mb *meta.DB
+}
+
+func (u *mbStorageIDUpdate) UpdateStorageID(ctx context.Context, addr oid.Address, storageID []byte) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+	}
+
+	if u.mb == nil {
+		return errMBIsNotAvailable
+	}
+
+	var prm meta.UpdateStorageIDPrm
+	prm.SetAddress(addr)
+	prm.SetStorageID(storageID)
+	_, err := u.mb.UpdateStorageID(prm)
+	return err
+}

pkg/local_object_storage/shard/shard.go

@@ -38,6 +38,8 @@ type Shard struct {
 
 	tsSource TombstoneSource
 
+	rb *rebuilder
+
 	gcCancel         atomic.Value
 	setModeRequested atomic.Bool
 }
@@ -121,6 +123,8 @@ type cfg struct {
 	metricsWriter MetricsWriter
 
 	reportErrorFunc func(selfID string, message string, err error)
+
+	rebuildLimiter RebuildWorkerLimiter
 }
 
 func defaultCfg() *cfg {
@@ -129,6 +133,7 @@ func defaultCfg() *cfg {
 		log:             &logger.Logger{Logger: zap.L()},
 		gcCfg:           defaultGCCfg(),
 		reportErrorFunc: func(string, string, error) {},
+		rebuildLimiter:  &noopRebuildLimiter{},
 	}
 }
 
@@ -366,6 +371,14 @@ func WithExpiredCollectorWorkersCount(count int) Option {
 	}
 }
 
+// WithRebuildWorkerLimiter return option to set concurrent
+// workers count of storage rebuild operation.
+func WithRebuildWorkerLimiter(l RebuildWorkerLimiter) Option {
+	return func(c *cfg) {
+		c.rebuildLimiter = l
+	}
+}
+
 func (s *Shard) fillInfo() {
 	s.cfg.info.MetaBaseInfo = s.metaBase.DumpInfo()
 	s.cfg.info.BlobStorInfo = s.blobStor.DumpInfo()

pkg/local_object_storage/writecache/writecachebadger/delete.go

@@ -59,7 +59,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error {
 					storagelog.OpField("db DELETE"),
 				)
 				deleted = true
-				c.objCounters.DecDB()
+				c.decDB()
 			}
 			return err
 		}

pkg/local_object_storage/writecache/writecachebadger/put.go

@@ -76,7 +76,7 @@ func (c *cache) put(obj objectInfo) error {
 			storagelog.StorageTypeField(wcStorageType),
 			storagelog.OpField("db PUT"),
 		)
-		c.objCounters.IncDB()
+		c.incDB()
 	}
 	return err
 }

pkg/local_object_storage/writecache/writecachebadger/state.go

@@ -55,3 +55,13 @@ func (c *cache) initCounters() error {
 
 	return nil
 }
+
+func (c *cache) incDB() {
+	c.objCounters.IncDB()
+	c.metrics.SetActualCounters(c.objCounters.DB(), 0)
+}
+
+func (c *cache) decDB() {
+	c.objCounters.DecDB()
+	c.metrics.SetActualCounters(c.objCounters.DB(), 0)
+}

pkg/local_object_storage/writecache/writecachebadger/storage.go

@@ -73,7 +73,7 @@ func (c *cache) deleteFromDB(keys []internalKey) []internalKey {
 	}
 
 	for i := 0; i < errorIndex; i++ {
-		c.objCounters.DecDB()
+		c.decDB()
 		c.metrics.Evict(writecache.StorageTypeDB)
 		storagelog.Write(c.log,
 			storagelog.AddressField(keys[i]),

pkg/local_object_storage/writecache/writecachebbolt/delete.go

@@ -83,6 +83,7 @@ func (c *cache) Delete(ctx context.Context, addr oid.Address) error {
 			storagelog.OpField("fstree DELETE"),
 		)
 		deleted = true
+		// counter changed by fstree
 		c.estimateCacheSize()
 	}
 	return metaerr.Wrap(err)

pkg/local_object_storage/writecache/writecachebbolt/flush.go

@@ -70,6 +70,7 @@ func (c *cache) runFlushLoop() {
 			case <-tt.C:
 				c.flushSmallObjects()
 				tt.Reset(defaultFlushInterval)
+				c.estimateCacheSize()
 			case <-c.closeCh:
 				return
 			}

pkg/local_object_storage/writecache/writecachebbolt/put.go

@@ -131,6 +131,7 @@ func (c *cache) putBig(ctx context.Context, addr string, prm common.PutPrm) erro
 		storagelog.StorageTypeField(wcStorageType),
 		storagelog.OpField("fstree PUT"),
 	)
+	// counter changed by fstree
 	c.estimateCacheSize()
 
 	return nil

pkg/local_object_storage/writecache/writecachebbolt/state.go

@@ -72,5 +72,6 @@ func (c *cache) initCounters() error {
 		return fmt.Errorf("could not read write-cache DB counter: %w", err)
 	}
 	c.objCounters.cDB.Store(inDB)
+	c.estimateCacheSize()
 	return nil
 }

pkg/local_object_storage/writecache/writecachebbolt/storage.go

@@ -73,7 +73,7 @@ func (c *cache) deleteFromDB(key string) {
 	err := c.db.Batch(func(tx *bbolt.Tx) error {
 		b := tx.Bucket(defaultBucket)
 		key := []byte(key)
-		recordDeleted = !recordDeleted && b.Get(key) != nil
+		recordDeleted = b.Get(key) != nil
 		return b.Delete(key)
 	})
 
@@ -122,6 +122,7 @@ func (c *cache) deleteFromDisk(ctx context.Context, keys []string) []string {
 				storagelog.OpField("fstree DELETE"),
 			)
 			c.metrics.Evict(writecache.StorageTypeFSTree)
+			// counter changed by fstree
 			c.estimateCacheSize()
 		}
 	}

pkg/metrics/blobovnicza.go

@@ -23,6 +23,10 @@ type BlobobvnizcaMetrics interface {
 
 	IncOpenBlobovniczaCount(shardID, path string)
 	DecOpenBlobovniczaCount(shardID, path string)
+
+	BlobovniczaTreeRebuildStatus(shardID, path, status string)
+	BlobovniczaTreeRebuildPercent(shardID, path string, value uint32)
+	BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration)
 }
 
 type blobovnicza struct {
@@ -33,6 +37,9 @@ type blobovnicza struct {
 	treeOpenSize           *prometheus.GaugeVec
 	treeOpenItems          *prometheus.GaugeVec
 	treeOpenCounter        *prometheus.GaugeVec
+	treeObjectMoveDuration *prometheus.HistogramVec
+	treeRebuildStatus      *shardIDPathModeValue
+	treeRebuildPercent     *prometheus.GaugeVec
 }
 
 func newBlobovnicza() *blobovnicza {
@@ -75,6 +82,19 @@ func newBlobovnicza() *blobovnicza {
 			Name:      "open_blobovnicza_count",
 			Help:      "Count of opened blobovniczas of Blobovnicza tree",
 		}, []string{shardIDLabel, pathLabel}),
+		treeObjectMoveDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: blobovniczaTreeSubSystem,
+			Name:      "object_move_duration_seconds",
+			Help:      "Accumulated Blobovnicza tree object move duration",
+		}, []string{shardIDLabel, pathLabel}),
+		treeRebuildStatus: newShardIDPathMode(blobovniczaTreeSubSystem, "rebuild_status", "Blobovnicza tree rebuild status"),
+		treeRebuildPercent: metrics.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: blobovniczaTreeSubSystem,
+			Name:      "rebuild_complete_percent",
+			Help:      "Percent of rebuild completeness",
+		}, []string{shardIDLabel, pathLabel}),
 	}
 }
 
@@ -96,6 +116,15 @@ func (b *blobovnicza) CloseBlobobvnizcaTree(shardID, path string) {
 		shardIDLabel: shardID,
 		pathLabel:    path,
 	})
+	b.treeObjectMoveDuration.DeletePartialMatch(prometheus.Labels{
+		shardIDLabel: shardID,
+		pathLabel:    path,
+	})
+	b.treeRebuildPercent.DeletePartialMatch(prometheus.Labels{
+		shardIDLabel: shardID,
+		pathLabel:    path,
+	})
+	b.treeRebuildStatus.SetMode(shardID, path, undefinedStatus)
 }
 
 func (b *blobovnicza) BlobobvnizcaTreeMethodDuration(shardID, path string, method string, d time.Duration, success bool, withStorageID NullBool) {
@@ -163,3 +192,21 @@ func (b *blobovnicza) SubOpenBlobovniczaItems(shardID, path string, items uint64
 		pathLabel:    path,
 	}).Sub(float64(items))
 }
+
+func (b *blobovnicza) BlobovniczaTreeRebuildStatus(shardID, path, status string) {
+	b.treeRebuildStatus.SetMode(shardID, path, status)
+}
+
+func (b *blobovnicza) BlobovniczaTreeObjectMoved(shardID, path string, d time.Duration) {
+	b.treeObjectMoveDuration.With(prometheus.Labels{
+		shardIDLabel: shardID,
+		pathLabel:    path,
+	}).Observe(d.Seconds())
+}
+
+func (b *blobovnicza) BlobovniczaTreeRebuildPercent(shardID, path string, value uint32) {
+	b.treeRebuildPercent.With(prometheus.Labels{
+		shardIDLabel: shardID,
+		pathLabel:    path,
+	}).Set(float64(value))
+}

pkg/metrics/consts.go

@@ -44,4 +44,5 @@ const (
 
 	failedToDeleteStatus = "failed_to_delete"
 	deletedStatus        = "deleted"
+	undefinedStatus      = "undefined"
 )

pkg/metrics/writecache.go

@@ -10,16 +10,10 @@ import (
 
 type WriteCacheMetrics interface {
 	AddMethodDuration(shardID string, method string, success bool, d time.Duration, storageType string)
-
-	IncActualCount(shardID string, storageType string)
-	DecActualCount(shardID string, storageType string)
 	SetActualCount(shardID string, count uint64, storageType string)
-
 	SetEstimateSize(shardID string, size uint64, storageType string)
 	SetMode(shardID string, mode string)
-
 	IncOperationCounter(shardID string, operation string, success NullBool, storageType string)
-
 	Close(shardID string)
 }
 
@@ -65,20 +59,6 @@ func (m *writeCacheMetrics) AddMethodDuration(shardID string, method string, suc
 	).Observe(d.Seconds())
 }
 
-func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) {
-	m.actualCount.With(prometheus.Labels{
-		shardIDLabel: shardID,
-		storageLabel: storageType,
-	}).Inc()
-}
-
-func (m *writeCacheMetrics) DecActualCount(shardID string, storageType string) {
-	m.actualCount.With(prometheus.Labels{
-		shardIDLabel: shardID,
-		storageLabel: storageType,
-	}).Dec()
-}
-
 func (m *writeCacheMetrics) SetActualCount(shardID string, count uint64, storageType string) {
 	m.actualCount.With(prometheus.Labels{
 		shardIDLabel: shardID,

pkg/morph/client/container/delete.go

@@ -67,7 +67,7 @@ func (d *DeletePrm) SetKey(key []byte) {
 //
 // If TryNotary is provided, calls notary contract.
 func (c *Client) Delete(p DeletePrm) error {
-	if len(p.signature) == 0 {
+	if len(p.signature) == 0 && !p.IsControl() {
 		return errNilArgument
 	}
 

pkg/morph/client/notary.go

@@ -67,6 +67,9 @@ const (
 
 var errUnexpectedItems = errors.New("invalid number of NEO VM arguments on stack")
 
+// This is a kludge purely for update to work.
+var CleanInvScript bool
+
 func defaultNotaryConfig(c *Client) *notaryCfg {
 	return &notaryCfg{
 		txValidTime:    defaultNotaryValidTime,
@@ -418,6 +421,20 @@ func (c *Client) NotarySignAndInvokeTX(mainTx *transaction.Transaction) error {
 		return err
 	}
 
+	if CleanInvScript {
+		// This is necessary to suppress this check on neo-go side:
+		// https://github.com/nspcc-dev/neo-go/blob/8ed6d97085d3229d4faf56a47bbd6cf73c132a76/pkg/services/notary/notary.go#L538
+		// This is a kludge purely for update to work.
+		for i := range mainTx.Signers {
+			for _, sig := range cosigners {
+				if mainTx.Signers[i].Account.Equals(sig.Account.ScriptHash()) {
+					mainTx.Scripts[i].InvocationScript = []byte{}
+					break
+				}
+			}
+		}
+	}
+
 	// Sign exactly the same transaction we've got from the received Notary request.
 	err = nAct.Sign(mainTx)
 	if err != nil {

pkg/morph/client/static.go

@@ -115,6 +115,11 @@ func (i *InvokePrmOptional) SetControlTX(b bool) {
 	i.controlTX = b
 }
 
+// IsControl gets whether a control transaction will be used.
+func (i *InvokePrmOptional) IsControl() bool {
+	return i.controlTX
+}
+
 // Invoke calls Invoke method of Client with static internal script hash and fee.
 // Supported args types are the same as in Client.
 //

pkg/services/container/announcement/load/controller/calls.go

@@ -1,307 +0,0 @@
-package loadcontroller
-
-import (
-	"context"
-
-	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
-	"go.uber.org/zap"
-)
-
-// StartPrm groups the required parameters of the Controller.Start method.
-type StartPrm struct {
-	// Epoch number by which you want to select
-	// the values of the used space of containers.
-	Epoch uint64
-}
-
-type commonContext struct {
-	epoch uint64
-
-	ctrl *Controller
-
-	log *logger.Logger
-}
-
-type announcer struct {
-	commonContext
-}
-
-// Start starts the processing of container.SizeEstimation values.
-//
-// Single Start operation overtakes all data from LocalMetrics to
-// LocalAnnouncementTarget (Controller's parameters).
-// No filter by epoch is used for the iterator, since it is expected
-// that the source of metrics does not track the change of epochs.
-//
-// Each call acquires an announcement context for an Epoch parameter.
-// At the very end of the operation, the context is released.
-func (c *Controller) Start(ctx context.Context, prm StartPrm) {
-	var announcer *announcer
-	// acquire announcement
-	ctx, announcer = c.acquireAnnouncement(ctx, prm)
-	if announcer == nil {
-		return
-	}
-
-	// finally stop and free the announcement
-	defer announcer.freeAnnouncement()
-
-	// announce local values
-	announcer.announce(ctx)
-}
-
-func (c *announcer) announce(ctx context.Context) {
-	c.log.Debug(logs.ControllerStartingToAnnounceTheValuesOfTheMetrics)
-
-	var (
-		metricsIterator Iterator
-		err             error
-	)
-
-	// initialize iterator over locally collected metrics
-	metricsIterator, err = c.ctrl.prm.LocalMetrics.InitIterator()
-	if err != nil {
-		c.log.Debug(logs.ControllerCouldNotInitializeIteratorOverLocallyCollectedMetrics,
-			zap.String("error", err.Error()),
-		)
-
-		return
-	}
-
-	// initialize target of local announcements
-	targetWriter, err := c.ctrl.prm.LocalAnnouncementTarget.InitWriter(nil)
-	if err != nil {
-		c.log.Debug(logs.ControllerCouldNotInitializeAnnouncementAccumulator,
-			zap.String("error", err.Error()),
-		)
-
-		return
-	}
-
-	// iterate over all collected metrics and write them to the target
-	err = metricsIterator.Iterate(
-		func(container.SizeEstimation) bool {
-			return true // local metrics don't know about epochs
-		},
-		func(a container.SizeEstimation) error {
-			a.SetEpoch(c.epoch) // set epoch explicitly
-			return targetWriter.Put(a)
-		},
-	)
-	if err != nil {
-		c.log.Debug(logs.ControllerIteratorOverLocallyCollectedMetricsAborted,
-			zap.String("error", err.Error()),
-		)
-
-		return
-	}
-
-	// finish writing
-	err = targetWriter.Close(ctx)
-	if err != nil {
-		c.log.Debug(logs.ControllerCouldNotFinishWritingLocalAnnouncements,
-			zap.String("error", err.Error()),
-		)
-
-		return
-	}
-
-	c.log.Debug(logs.ControllerTrustAnnouncementSuccessfullyFinished)
-}
-
-func (c *Controller) acquireAnnouncement(ctx context.Context, prm StartPrm) (context.Context, *announcer) {
-	started := true
-	c.announceMtx.Lock()
-	{
-		if cancel := c.mAnnounceCtx[prm.Epoch]; cancel == nil {
-			ctx, cancel = context.WithCancel(ctx)
-			c.mAnnounceCtx[prm.Epoch] = cancel
-			started = false
-		}
-	}
-	c.announceMtx.Unlock()
-
-	log := &logger.Logger{Logger: c.opts.log.With(
-		zap.Uint64("epoch", prm.Epoch),
-	)}
-
-	if started {
-		log.Debug(logs.ControllerAnnouncementIsAlreadyStarted)
-		return ctx, nil
-	}
-
-	return ctx, &announcer{
-		commonContext: commonContext{
-			epoch: prm.Epoch,
-			ctrl:  c,
-			log:   log,
-		},
-	}
-}
-
-func (c *commonContext) freeAnnouncement() {
-	var stopped bool
-
-	c.ctrl.announceMtx.Lock()
-
-	{
-		var cancel context.CancelFunc
-
-		cancel, stopped = c.ctrl.mAnnounceCtx[c.epoch]
-
-		if stopped {
-			cancel()
-			delete(c.ctrl.mAnnounceCtx, c.epoch)
-		}
-	}
-
-	c.ctrl.announceMtx.Unlock()
-
-	if stopped {
-		c.log.Debug(logs.ControllerAnnouncementSuccessfullyInterrupted)
-	} else {
-		c.log.Debug(logs.ControllerAnnouncementIsNotStartedOrAlreadyInterrupted)
-	}
-}
-
-// StopPrm groups the required parameters of the Controller.Stop method.
-type StopPrm struct {
-	// Epoch number the analysis of the values of which must be interrupted.
-	Epoch uint64
-}
-
-type reporter struct {
-	commonContext
-}
-
-// Stop interrupts the processing of container.SizeEstimation values.
-//
-// Single Stop operation releases an announcement context and overtakes
-// all data from AnnouncementAccumulator to ResultReceiver (Controller's
-// parameters). Only values for the specified Epoch parameter are processed.
-//
-// Each call acquires a report context for an Epoch parameter.
-// At the very end of the operation, the context is released.
-func (c *Controller) Stop(ctx context.Context, prm StopPrm) {
-	var reporter *reporter
-	ctx, reporter = c.acquireReport(ctx, prm)
-	if reporter == nil {
-		return
-	}
-
-	// finally stop and free reporting
-	defer reporter.freeReport()
-
-	// interrupt announcement
-	reporter.freeAnnouncement()
-
-	// report the estimations
-	reporter.report(ctx)
-}
-
-func (c *Controller) acquireReport(ctx context.Context, prm StopPrm) (context.Context, *reporter) {
-	started := true
-
-	c.reportMtx.Lock()
-	{
-		if cancel := c.mReportCtx[prm.Epoch]; cancel == nil {
-			ctx, cancel = context.WithCancel(ctx)
-			c.mReportCtx[prm.Epoch] = cancel
-			started = false
-		}
-	}
-
-	c.reportMtx.Unlock()
-
-	log := &logger.Logger{Logger: c.opts.log.With(
-		zap.Uint64("epoch", prm.Epoch),
-	)}
-
-	if started {
-		log.Debug(logs.ControllerReportIsAlreadyStarted)
-		return ctx, nil
-	}
-
-	return ctx, &reporter{
-		commonContext: commonContext{
-			epoch: prm.Epoch,
-			ctrl:  c,
-			log:   log,
-		},
-	}
-}
-
-func (c *commonContext) freeReport() {
-	var stopped bool
-
-	c.ctrl.reportMtx.Lock()
-
-	{
-		var cancel context.CancelFunc
-
-		cancel, stopped = c.ctrl.mReportCtx[c.epoch]
-
-		if stopped {
-			cancel()
-			delete(c.ctrl.mReportCtx, c.epoch)
-		}
-	}
-
-	c.ctrl.reportMtx.Unlock()
-
-	if stopped {
-		c.log.Debug(logs.ControllerAnnouncementSuccessfullyInterrupted)
-	} else {
-		c.log.Debug(logs.ControllerAnnouncementIsNotStartedOrAlreadyInterrupted)
-	}
-}
-
-func (c *reporter) report(ctx context.Context) {
-	var (
-		localIterator Iterator
-		err           error
-	)
-
-	// initialize iterator over locally accumulated announcements
-	localIterator, err = c.ctrl.prm.AnnouncementAccumulator.InitIterator()
-	if err != nil {
-		c.log.Debug(logs.ControllerCouldNotInitializeIteratorOverLocallyAccumulatedAnnouncements,
-			zap.String("error", err.Error()),
-		)
-
-		return
-	}
-
-	// initialize final destination of load estimations
-	resultWriter, err := c.ctrl.prm.ResultReceiver.InitWriter(nil)
-	if err != nil {
-		c.log.Debug(logs.ControllerCouldNotInitializeResultTarget,
-			zap.String("error", err.Error()),
-		)
-
-		return
-	}
-
-	// iterate over all accumulated announcements and write them to the target
-	err = localIterator.Iterate(
-		usedSpaceFilterEpochEQ(c.epoch),
-		resultWriter.Put,
-	)
-	if err != nil {
-		c.log.Debug(logs.ControllerIteratorOverLocalAnnouncementsAborted,
-			zap.String("error", err.Error()),
-		)
-
-		return
-	}
-
-	// finish writing
-	err = resultWriter.Close(ctx)
-	if err != nil {
-		c.log.Debug(logs.ControllerCouldNotFinishWritingLoadEstimations,
-			zap.String("error", err.Error()),
-		)
-	}
-}

pkg/services/container/announcement/load/controller/calls_test.go

@@ -1,192 +0,0 @@
-package loadcontroller_test
-
-import (
-	"context"
-	"math/rand"
-	"sync"
-	"testing"
-
-	loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
-	cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
-	"github.com/stretchr/testify/require"
-)
-
-type testAnnouncementStorage struct {
-	w loadcontroller.Writer
-
-	i loadcontroller.Iterator
-
-	mtx sync.RWMutex
-
-	m map[uint64][]container.SizeEstimation
-}
-
-func newTestStorage() *testAnnouncementStorage {
-	return &testAnnouncementStorage{
-		m: make(map[uint64][]container.SizeEstimation),
-	}
-}
-
-func (s *testAnnouncementStorage) InitIterator() (loadcontroller.Iterator, error) {
-	if s.i != nil {
-		return s.i, nil
-	}
-
-	return s, nil
-}
-
-func (s *testAnnouncementStorage) Iterate(f loadcontroller.UsedSpaceFilter, h loadcontroller.UsedSpaceHandler) error {
-	s.mtx.RLock()
-	defer s.mtx.RUnlock()
-
-	for _, v := range s.m {
-		for _, a := range v {
-			if f(a) {
-				if err := h(a); err != nil {
-					return err
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-func (s *testAnnouncementStorage) InitWriter([]loadcontroller.ServerInfo) (loadcontroller.Writer, error) {
-	if s.w != nil {
-		return s.w, nil
-	}
-
-	return s, nil
-}
-
-func (s *testAnnouncementStorage) Put(v container.SizeEstimation) error {
-	s.mtx.Lock()
-	s.m[v.Epoch()] = append(s.m[v.Epoch()], v)
-	s.mtx.Unlock()
-
-	return nil
-}
-
-func (s *testAnnouncementStorage) Close(context.Context) error {
-	return nil
-}
-
-func randAnnouncement() (a container.SizeEstimation) {
-	a.SetContainer(cidtest.ID())
-	a.SetValue(rand.Uint64())
-
-	return
-}
-
-func TestSimpleScenario(t *testing.T) {
-	// create storage to write final estimations
-	resultStorage := newTestStorage()
-
-	// create storages to accumulate announcements
-	accumulatingStorageN2 := newTestStorage()
-
-	// create storage of local metrics
-	localStorageN1 := newTestStorage()
-	localStorageN2 := newTestStorage()
-
-	// create 2 controllers:  1st writes announcements to 2nd, 2nd directly to final destination
-	ctrlN1 := loadcontroller.New(loadcontroller.Prm{
-		LocalMetrics:            localStorageN1,
-		AnnouncementAccumulator: newTestStorage(),
-		LocalAnnouncementTarget: &testAnnouncementStorage{
-			w: accumulatingStorageN2,
-		},
-		ResultReceiver: resultStorage,
-	})
-
-	ctrlN2 := loadcontroller.New(loadcontroller.Prm{
-		LocalMetrics:            localStorageN2,
-		AnnouncementAccumulator: accumulatingStorageN2,
-		LocalAnnouncementTarget: &testAnnouncementStorage{
-			w: resultStorage,
-		},
-		ResultReceiver: resultStorage,
-	})
-
-	const processEpoch uint64 = 10
-
-	const goodNum = 4
-
-	// create 2 random values for processing epoch and 1 for some different
-	announces := make([]container.SizeEstimation, 0, goodNum)
-
-	for i := 0; i < goodNum; i++ {
-		a := randAnnouncement()
-		a.SetEpoch(processEpoch)
-
-		announces = append(announces, a)
-	}
-
-	// store one half of "good" announcements to 1st metrics storage, another - to 2nd
-	// and "bad" to both
-	for i := 0; i < goodNum/2; i++ {
-		require.NoError(t, localStorageN1.Put(announces[i]))
-	}
-
-	for i := goodNum / 2; i < goodNum; i++ {
-		require.NoError(t, localStorageN2.Put(announces[i]))
-	}
-
-	wg := new(sync.WaitGroup)
-	wg.Add(2)
-
-	startPrm := loadcontroller.StartPrm{
-		Epoch: processEpoch,
-	}
-
-	// start both controllers
-	go func() {
-		ctrlN1.Start(context.Background(), startPrm)
-		wg.Done()
-	}()
-
-	go func() {
-		ctrlN2.Start(context.Background(), startPrm)
-		wg.Done()
-	}()
-
-	wg.Wait()
-	wg.Add(2)
-
-	stopPrm := loadcontroller.StopPrm{
-		Epoch: processEpoch,
-	}
-
-	// stop both controllers
-	go func() {
-		ctrlN1.Stop(context.Background(), stopPrm)
-		wg.Done()
-	}()
-
-	go func() {
-		ctrlN2.Stop(context.Background(), stopPrm)
-		wg.Done()
-	}()
-
-	wg.Wait()
-
-	// result target should contain all "good" announcements and shoult not container the "bad" one
-	var res []container.SizeEstimation
-
-	err := resultStorage.Iterate(
-		func(a container.SizeEstimation) bool {
-			return true
-		},
-		func(a container.SizeEstimation) error {
-			res = append(res, a)
-			return nil
-		},
-	)
-	require.NoError(t, err)
-
-	for i := range announces {
-		require.Contains(t, res, announces[i])
-	}
-}

pkg/services/container/announcement/load/controller/controller.go

@@ -1,94 +0,0 @@
-package loadcontroller
-
-import (
-	"context"
-	"fmt"
-	"sync"
-)
-
-// Prm groups the required parameters of the Controller's constructor.
-//
-// All values must comply with the requirements imposed on them.
-// Passing incorrect parameter values will result in constructor
-// failure (error or panic depending on the implementation).
-type Prm struct {
-	// Iterator over the used space values of the containers
-	// collected by the node locally.
-	LocalMetrics IteratorProvider
-
-	// Place of recording the local values of
-	// the used space of containers.
-	LocalAnnouncementTarget WriterProvider
-
-	// Iterator over the summarized used space scores
-	// from the various network participants.
-	AnnouncementAccumulator IteratorProvider
-
-	// Place of recording the final estimates of
-	// the used space of containers.
-	ResultReceiver WriterProvider
-}
-
-// Controller represents main handler for starting
-// and interrupting container volume estimation.
-//
-// It binds the interfaces of the local value stores
-// to the target storage points. Controller is abstracted
-// from the internal storage device and the network location
-// of the connecting components. At its core, it is a
-// high-level start-stop trigger for calculations.
-//
-// For correct operation, the controller must be created
-// using the constructor (New) based on the required parameters
-// and optional components. After successful creation,
-// the constructor is immediately ready to work through
-// API of external control of calculations and data transfer.
-type Controller struct {
-	prm Prm
-
-	opts *options
-
-	announceMtx  sync.Mutex
-	mAnnounceCtx map[uint64]context.CancelFunc
-
-	reportMtx  sync.Mutex
-	mReportCtx map[uint64]context.CancelFunc
-}
-
-const invalidPrmValFmt = "invalid parameter %s (%T):%v"
-
-func panicOnPrmValue(n string, v any) {
-	panic(fmt.Sprintf(invalidPrmValFmt, n, v, v))
-}
-
-// New creates a new instance of the Controller.
-//
-// Panics if at least one value of the parameters is invalid.
-//
-// The created Controller does not require additional
-// initialization and is completely ready for work.
-func New(prm Prm, opts ...Option) *Controller {
-	switch {
-	case prm.LocalMetrics == nil:
-		panicOnPrmValue("LocalMetrics", prm.LocalMetrics)
-	case prm.AnnouncementAccumulator == nil:
-		panicOnPrmValue("AnnouncementAccumulator", prm.AnnouncementAccumulator)
-	case prm.LocalAnnouncementTarget == nil:
-		panicOnPrmValue("LocalAnnouncementTarget", prm.LocalAnnouncementTarget)
-	case prm.ResultReceiver == nil:
-		panicOnPrmValue("ResultReceiver", prm.ResultReceiver)
-	}
-
-	o := defaultOpts()
-
-	for _, opt := range opts {
-		opt(o)
-	}
-
-	return &Controller{
-		prm:          prm,
-		opts:         o,
-		mAnnounceCtx: make(map[uint64]context.CancelFunc),
-		mReportCtx:   make(map[uint64]context.CancelFunc),
-	}
-}

pkg/services/container/announcement/load/controller/deps.go

@@ -1,103 +0,0 @@
-package loadcontroller
-
-import (
-	"context"
-
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
-)
-
-// UsedSpaceHandler describes the signature of the container.SizeEstimation
-// value handling function.
-//
-// Termination of processing without failures is usually signaled
-// with a zero error, while a specific value may describe the reason
-// for failure.
-type UsedSpaceHandler func(container.SizeEstimation) error
-
-// UsedSpaceFilter describes the signature of the function for
-// checking whether a value meets a certain criterion.
-//
-// Return of true means conformity, false - vice versa.
-type UsedSpaceFilter func(container.SizeEstimation) bool
-
-// Iterator is a group of methods provided by entity
-// which can iterate over a group of container.SizeEstimation values.
-type Iterator interface {
-	// Iterate must start an iterator over values that
-	// meet the filter criterion (returns true).
-	// For each such value should call a handler, the error
-	// of which should be directly returned from the method.
-	//
-	// Internal failures of the iterator are also signaled via
-	// an error. After a successful call to the last value
-	// handler, nil should be returned.
-	Iterate(UsedSpaceFilter, UsedSpaceHandler) error
-}
-
-// IteratorProvider is a group of methods provided
-// by entity which generates iterators over
-// container.SizeEstimation values.
-type IteratorProvider interface {
-	// InitIterator should return an initialized Iterator.
-	//
-	// Initialization problems are reported via error.
-	// If no error was returned, then the Iterator must not be nil.
-	//
-	// Implementations can have different logic for different
-	// contexts, so specific ones may document their own behavior.
-	InitIterator() (Iterator, error)
-}
-
-// Writer describes the interface for storing container.SizeEstimation values.
-//
-// This interface is provided by both local storage
-// of values and remote (wrappers over the RPC).
-type Writer interface {
-	// Put performs a write operation of container.SizeEstimation value
-	// and returns any error encountered.
-	//
-	// All values after the Close call must be flushed to the
-	// physical target. Implementations can cache values before
-	// Close operation.
-	//
-	// Put must not be called after Close.
-	Put(container.SizeEstimation) error
-
-	// Close exits with method-providing Writer.
-	//
-	// All cached values must be flushed before
-	// the Close's return.
-	//
-	// Methods must not be called after Close.
-	Close(ctx context.Context) error
-}
-
-// WriterProvider is a group of methods provided
-// by entity which generates keepers of
-// container.SizeEstimation values.
-type WriterProvider interface {
-	// InitWriter should return an initialized Writer.
-	//
-	// Initialization problems are reported via error.
-	// If no error was returned, then the Writer must not be nil.
-	InitWriter(route []ServerInfo) (Writer, error)
-}
-
-// ServerInfo describes a set of
-// characteristics of a point in a route.
-type ServerInfo interface {
-	// PublicKey returns public key of the node
-	// from the route in a binary representation.
-	PublicKey() []byte
-
-	// Iterates over network addresses of the node
-	// in the route. Breaks iterating on true return
-	// of the handler.
-	IterateAddresses(func(string) bool)
-
-	// Returns number of server's network addresses.
-	NumberOfAddresses() int
-
-	// ExternalAddresses returns external node's addresses.
-	ExternalAddresses() []string
-}

pkg/services/container/announcement/load/controller/opts.go

@@ -1,28 +0,0 @@
-package loadcontroller
-
-import (
-	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
-	"go.uber.org/zap"
-)
-
-// Option sets an optional parameter of Controller.
-type Option func(*options)
-
-type options struct {
-	log *logger.Logger
-}
-
-func defaultOpts() *options {
-	return &options{
-		log: &logger.Logger{Logger: zap.L()},
-	}
-}
-
-// WithLogger returns option to specify logging component.
-func WithLogger(l *logger.Logger) Option {
-	return func(o *options) {
-		if l != nil {
-			o.log = l
-		}
-	}
-}

pkg/services/container/announcement/load/controller/util.go

@@ -1,36 +0,0 @@
-package loadcontroller
-
-import (
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
-)
-
-func usedSpaceFilterEpochEQ(epoch uint64) UsedSpaceFilter {
-	return func(a container.SizeEstimation) bool {
-		return a.Epoch() == epoch
-	}
-}
-
-type storageWrapper struct {
-	w Writer
-	i Iterator
-}
-
-func (s storageWrapper) InitIterator() (Iterator, error) {
-	return s.i, nil
-}
-
-func (s storageWrapper) InitWriter([]ServerInfo) (Writer, error) {
-	return s.w, nil
-}
-
-func SimpleIteratorProvider(i Iterator) IteratorProvider {
-	return &storageWrapper{
-		i: i,
-	}
-}
-
-func SimpleWriterProvider(w Writer) WriterProvider {
-	return &storageWrapper{
-		w: w,
-	}
-}

pkg/services/container/announcement/load/route/calls.go

@@ -1,145 +0,0 @@
-package loadroute
-
-import (
-	"context"
-	"encoding/hex"
-	"sync"
-
-	"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
-	loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
-	"go.uber.org/zap"
-)
-
-// InitWriter initializes and returns Writer that sends each value to its next route point.
-//
-// If route is present, then it is taken into account,
-// and the value will be sent to its continuation. Otherwise, the route will be laid
-// from scratch and the value will be sent to its primary point.
-//
-// After building a list of remote points of the next leg of the route, the value is sent
-// sequentially to all of them. If any transmissions (even all) fail, an error will not
-// be returned.
-//
-// Close of the composed Writer calls Close method on each internal Writer generated in
-// runtime and never returns an error.
-//
-// Always returns nil error.
-func (r *Router) InitWriter(route []loadcontroller.ServerInfo) (loadcontroller.Writer, error) {
-	if len(route) == 0 {
-		route = []loadcontroller.ServerInfo{r.localSrvInfo}
-	}
-
-	return &loadWriter{
-		router:   r,
-		route:    route,
-		mRoute:   make(map[routeKey]*valuesRoute),
-		mServers: make(map[string]loadcontroller.Writer),
-	}, nil
-}
-
-type routeKey struct {
-	epoch uint64
-
-	cid string
-}
-
-type valuesRoute struct {
-	route []loadcontroller.ServerInfo
-
-	values []container.SizeEstimation
-}
-
-type loadWriter struct {
-	router *Router
-
-	route []loadcontroller.ServerInfo
-
-	routeMtx sync.RWMutex
-	mRoute   map[routeKey]*valuesRoute
-
-	mServers map[string]loadcontroller.Writer
-}
-
-func (w *loadWriter) Put(a container.SizeEstimation) error {
-	w.routeMtx.Lock()
-	defer w.routeMtx.Unlock()
-
-	key := routeKey{
-		epoch: a.Epoch(),
-		cid:   a.Container().EncodeToString(),
-	}
-
-	routeValues, ok := w.mRoute[key]
-	if !ok {
-		route, err := w.router.routeBuilder.NextStage(a, w.route)
-		if err != nil {
-			return err
-		} else if len(route) == 0 {
-			route = []loadcontroller.ServerInfo{nil}
-		}
-
-		routeValues = &valuesRoute{
-			route:  route,
-			values: []container.SizeEstimation{a},
-		}
-
-		w.mRoute[key] = routeValues
-	}
-
-	for _, remoteInfo := range routeValues.route {
-		var key string
-
-		if remoteInfo != nil {
-			key = hex.EncodeToString(remoteInfo.PublicKey())
-		}
-
-		remoteWriter, ok := w.mServers[key]
-		if !ok {
-			provider, err := w.router.remoteProvider.InitRemote(remoteInfo)
-			if err != nil {
-				w.router.log.Debug(logs.RouteCouldNotInitializeWriterProvider,
-					zap.String("error", err.Error()),
-				)
-
-				continue // best effort
-			}
-
-			remoteWriter, err = provider.InitWriter(w.route)
-			if err != nil {
-				w.router.log.Debug(logs.RouteCouldNotInitializeWriter,
-					zap.String("error", err.Error()),
-				)
-
-				continue // best effort
-			}
-
-			w.mServers[key] = remoteWriter
-		}
-
-		err := remoteWriter.Put(a)
-		if err != nil {
-			w.router.log.Debug(logs.RouteCouldNotPutTheValue,
-				zap.String("error", err.Error()),
-			)
-		}
-
-		// continue best effort
-	}
-
-	return nil
-}
-
-func (w *loadWriter) Close(ctx context.Context) error {
-	for key, wRemote := range w.mServers {
-		err := wRemote.Close(ctx)
-		if err != nil {
-			w.router.log.Debug(logs.RouteCouldNotCloseRemoteServerWriter,
-				zap.String("key", key),
-				zap.String("error", err.Error()),
-			)
-		}
-	}
-
-	return nil
-}

pkg/services/container/announcement/load/route/deps.go

@@ -1,31 +0,0 @@
-package loadroute
-
-import (
-	loadcontroller "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/container/announcement/load/controller"
-	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container"
-)
-
-// Builder groups methods to route values in the network.
-type Builder interface {
-	// NextStage must return next group of route points for the value a
-	// based on the passed route.
-	//
-	// Empty passed list means being at the starting point of the route.
-	//
-	// Must return empty list and no error if the endpoint of the route is reached.
-	// If there are more than one point to go and the last passed point is included
-	// in that list (means that point is the last point in one of the route groups),
-	// returned route must contain nil point that should be interpreted as signal to,
-	// among sending to other route points, save the announcement in that point.
-	NextStage(a container.SizeEstimation, passed []loadcontroller.ServerInfo) ([]loadcontroller.ServerInfo, error)
-}
-
-// RemoteWriterProvider describes the component
-// for sending values to a fixed route point.
-type RemoteWriterProvider interface {
-	// InitRemote must return WriterProvider to the route point
-	// corresponding to info.
-	//
-	// Nil info matches the end of the route.
-	InitRemote(info loadcontroller.ServerInfo) (loadcontroller.WriterProvider, error)
-}