[#661] blobovniczatree: Make Rebuild failover safe

Now move info stores in blobovnicza, so in case of failover
rebuild completes previous operation first.

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

pkg/local_object_storage/blobovnicza/control.go

@@ -105,7 +105,7 @@ func (b *Blobovnicza) initializeCounters() error {
 	var size uint64
 	var items uint64
 	err := b.boltDB.View(func(tx *bbolt.Tx) error {
-		return b.iterateAllBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) {
+		return b.iterateAllDataBuckets(tx, func(lower, upper uint64, b *bbolt.Bucket) (bool, error) {
 			keysN := uint64(b.Stats().KeyN)
 			size += keysN * upper
 			items += keysN

pkg/local_object_storage/blobovnicza/delete.go

@@ -51,7 +51,7 @@ func (b *Blobovnicza) Delete(ctx context.Context, prm DeletePrm) (DeleteRes, err
 	var dataSize uint64
 
 	err := b.boltDB.Update(func(tx *bbolt.Tx) error {
-		return b.iterateAllBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) {
+		return b.iterateAllDataBuckets(tx, func(lower, upper uint64, buck *bbolt.Bucket) (bool, error) {
 			objData := buck.Get(addrKey)
 			if objData == nil {
 				// object is not in bucket => continue iterating

pkg/local_object_storage/blobovnicza/exists.go

@@ -1,6 +1,7 @@
 package blobovnicza
 
 import (
+	"bytes"
 	"context"
 
 	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
@@ -24,7 +25,10 @@ func (b *Blobovnicza) Exists(ctx context.Context, addr oid.Address) (bool, error
 	addrKey := addressKey(addr)
 
 	err := b.boltDB.View(func(tx *bbolt.Tx) error {
-		return tx.ForEach(func(_ []byte, buck *bbolt.Bucket) error {
+		return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error {
+			if bytes.Equal(bucketName, incompletedMoveBucketName) {
+				return nil
+			}
 			exists = buck.Get(addrKey) != nil
 			if exists {
 				return errInterruptForEach

pkg/local_object_storage/blobovnicza/get.go

@@ -57,7 +57,11 @@ func (b *Blobovnicza) Get(ctx context.Context, prm GetPrm) (GetRes, error) {
 	)
 
 	if err := b.boltDB.View(func(tx *bbolt.Tx) error {
-		return tx.ForEach(func(_ []byte, buck *bbolt.Bucket) error {
+		return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error {
+			if bytes.Equal(bucketName, incompletedMoveBucketName) {
+				return nil
+			}
+
 			data = buck.Get(addrKey)
 			if data == nil {
 				return nil

pkg/local_object_storage/blobovnicza/iterate.go

@@ -1,6 +1,7 @@
 package blobovnicza
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"math"
@@ -12,11 +13,11 @@ import (
 	"go.opentelemetry.io/otel/trace"
 )
 
-// iterateAllBuckets iterates all buckets in db
+// iterateAllDataBuckets iterates all buckets in db
 //
 // If the maximum size of the object (b.objSizeLimit) has been changed to lower value,
 // then there may be more buckets than the current limit of the object size.
-func (b *Blobovnicza) iterateAllBuckets(tx *bbolt.Tx, f func(uint64, uint64, *bbolt.Bucket) (bool, error)) error {
+func (b *Blobovnicza) iterateAllDataBuckets(tx *bbolt.Tx, f func(uint64, uint64, *bbolt.Bucket) (bool, error)) error {
 	return b.iterateBucketKeys(false, func(lower uint64, upper uint64, key []byte) (bool, error) {
 		buck := tx.Bucket(key)
 		if buck == nil {
@@ -138,7 +139,10 @@ func (b *Blobovnicza) Iterate(ctx context.Context, prm IteratePrm) (IterateRes,
 	var elem IterationElement
 
 	if err := b.boltDB.View(func(tx *bbolt.Tx) error {
-		return tx.ForEach(func(name []byte, buck *bbolt.Bucket) error {
+		return tx.ForEach(func(bucketName []byte, buck *bbolt.Bucket) error {
+			if bytes.Equal(bucketName, incompletedMoveBucketName) {
+				return nil
+			}
 			return buck.ForEach(func(k, v []byte) error {
 				select {
 				case <-ctx.Done():

pkg/local_object_storage/blobovnicza/move.go

@@ -0,0 +1,108 @@
+package blobovnicza
+
+import (
+	"context"
+	"fmt"
+
+	"git.frostfs.info/TrueCloudLab/frostfs-observability/tracing"
+	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
+	"go.etcd.io/bbolt"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
+
+var incompletedMoveBucketName = []byte("INCOMPLETED_MOVE")
+
+type MoveInfo struct {
+	Address         oid.Address
+	TargetStorageID []byte
+}
+
+func (b *Blobovnicza) PutMoveInfo(ctx context.Context, prm MoveInfo) error {
+	_, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.PutMoveInfo",
+		trace.WithAttributes(
+			attribute.String("path", b.path),
+			attribute.String("address", prm.Address.EncodeToString()),
+			attribute.String("target_storage_id", string(prm.TargetStorageID)),
+		))
+	defer span.End()
+
+	key := addressKey(prm.Address)
+
+	return b.boltDB.Update(func(tx *bbolt.Tx) error {
+		bucket, err := tx.CreateBucketIfNotExists(incompletedMoveBucketName)
+		if err != nil {
+			return err
+		}
+
+		if err := bucket.Put(key, prm.TargetStorageID); err != nil {
+			return fmt.Errorf("(%T) failed to save move info: %w", b, err)
+		}
+
+		return nil
+	})
+}
+
+func (b *Blobovnicza) DropMoveInfo(ctx context.Context, address oid.Address) error {
+	_, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.DropMoveInfo",
+		trace.WithAttributes(
+			attribute.String("path", b.path),
+			attribute.String("address", address.EncodeToString()),
+		))
+	defer span.End()
+
+	key := addressKey(address)
+
+	return b.boltDB.Update(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(incompletedMoveBucketName)
+		if bucket == nil {
+			return nil
+		}
+
+		if err := bucket.Delete(key); err != nil {
+			return fmt.Errorf("(%T) failed to drop move info: %w", b, err)
+		}
+
+		c := bucket.Cursor()
+		k, v := c.First()
+		bucketEmpty := k == nil && v == nil
+		if bucketEmpty {
+			return tx.DeleteBucket(incompletedMoveBucketName)
+		}
+
+		return nil
+	})
+}
+
+func (b *Blobovnicza) ListMoveInfo(ctx context.Context) ([]MoveInfo, error) {
+	_, span := tracing.StartSpanFromContext(ctx, "Blobovnicza.ListMoveInfo",
+		trace.WithAttributes(
+			attribute.String("path", b.path),
+		))
+	defer span.End()
+
+	var result []MoveInfo
+	if err := b.boltDB.View(func(tx *bbolt.Tx) error {
+		bucket := tx.Bucket(incompletedMoveBucketName)
+		if bucket == nil {
+			return nil
+		}
+		return bucket.ForEach(func(k, v []byte) error {
+			var addr oid.Address
+			storageID := make([]byte, len(v))
+			if err := addressFromKey(&addr, k); err != nil {
+				return err
+			}
+			copy(storageID, v)
+			result = append(result, MoveInfo{
+				Address:         addr,
+				TargetStorageID: storageID,
+			})
+			return nil
+		})
+	}); err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}