pilorama: Support tree migration between forests

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>

pkg/local_object_storage/pilorama/migrate.go

@@ -0,0 +1,33 @@
+package pilorama
+
+import (
+	"context"
+
+	cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
+	"golang.org/x/sync/errgroup"
+)
+
+const defaultMigrateBatchSize = 100
+
+// Migrate migrates a single tree to another forest.
+// If the migration is interrupted in the middle, some garbage may be left in the target forest.
+func Migrate(ctx context.Context, from Forest, to Forest, cnr cidSDK.ID, treeID string) error {
+	eg, ctx := errgroup.WithContext(ctx)
+	eg.SetLimit(defaultMigrateBatchSize)
+
+	var height uint64
+	for {
+		op, err := from.TreeGetOpLog(ctx, cnr, treeID, height)
+		if err != nil {
+			_ = eg.Wait()
+			return err
+		}
+		if op.Time == 0 {
+			return eg.Wait()
+		}
+		eg.Go(func() error {
+			return to.TreeApply(ctx, cnr, treeID, &op, false)
+		})
+		height = op.Time + 1
+	}
+}

pkg/local_object_storage/pilorama/migrate_test.go

@@ -0,0 +1,91 @@
+package pilorama
+
+import (
+	"context"
+	"strconv"
+	"testing"
+
+	cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
+	cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+)
+
+func prepareTree(t testing.TB, f Forest, cnr cidSDK.ID, treeID string, count int) {
+	for i := 0; i < count; i++ {
+		_, err := f.TreeMove(context.Background(), CIDDescriptor{CID: cnr, Size: 1}, treeID, &Move{
+			Parent: RootID,
+			Child:  Node(i + 1),
+			Meta: Meta{
+				Items: []KeyValue{
+					{Key: AttributeFilename, Value: []byte(uuid.New().String())},
+					{Key: "Another key", Value: []byte(strconv.FormatInt(int64(i), 10))},
+				},
+			},
+		})
+		require.NoError(t, err)
+	}
+
+}
+
+func BenchmarkMigrate(b *testing.B) {
+	const (
+		count     = 1000
+		batchSize = 10
+	)
+
+	for i := range providers {
+		if providers[i].name == "inmemory" {
+			continue
+		}
+
+		from := providers[i].construct(b)
+		cnr := cidtest.ID()
+		treeID := "benchtree"
+		cons := providers[i].construct
+
+		// Do not use random tree here, to better interpret benchmark results.
+		// It is append-only log without deletions.
+		prepareTree(b, from, cnr, treeID, count)
+
+		b.Run(providers[i].name, func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				to := cons(b, WithMaxBatchSize(batchSize))
+				err := Migrate(context.Background(), from, to, cnr, treeID)
+				if err != nil {
+					b.Fatalf("migrate: %v", err)
+				}
+				if err := to.Close(); err != nil {
+					b.Fatalf("close: %v", err)
+				}
+			}
+		})
+	}
+}
+
+func TestMigrate(t *testing.T) {
+	for i := range providers {
+		t.Run(providers[i].name, func(t *testing.T) {
+			testMigrate(t, providers[i].construct)
+		})
+	}
+}
+
+func testMigrate(t *testing.T, cons func(testing.TB, ...Option) ForestStorage) {
+	const (
+		count  = 100
+		treeID = "sometree"
+	)
+	cnr := cidtest.ID()
+
+	from := cons(t)
+	ops := prepareRandomTree(count, count)
+	for i := range ops {
+		err := from.TreeApply(context.Background(), cnr, treeID, &ops[i], true)
+		require.NoError(t, err)
+	}
+
+	to := cons(t)
+	require.NoError(t, Migrate(context.Background(), from, to, cnr, treeID))
+	compareForests(t, from, to, cnr, treeID, count)
+}