diff --git a/pkg/local_object_storage/pilorama/migrate.go b/pkg/local_object_storage/pilorama/migrate.go new file mode 100644 index 000000000..ee6d50276 --- /dev/null +++ b/pkg/local_object_storage/pilorama/migrate.go @@ -0,0 +1,33 @@ +package pilorama + +import ( + "context" + + cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "golang.org/x/sync/errgroup" +) + +const defaultMigrateBatchSize = 100 + +// Migrate migrates a single tree to another forest. +// If the migration is interrupted in the middle, some garbage may be left in the target forest. +func Migrate(ctx context.Context, from Forest, to Forest, cnr cidSDK.ID, treeID string) error { + eg, ctx := errgroup.WithContext(ctx) + eg.SetLimit(defaultMigrateBatchSize) + + var height uint64 + for { + op, err := from.TreeGetOpLog(ctx, cnr, treeID, height) + if err != nil { + _ = eg.Wait() + return err + } + if op.Time == 0 { + return eg.Wait() + } + eg.Go(func() error { + return to.TreeApply(ctx, cnr, treeID, &op, false) + }) + height = op.Time + 1 + } +} diff --git a/pkg/local_object_storage/pilorama/migrate_test.go b/pkg/local_object_storage/pilorama/migrate_test.go new file mode 100644 index 000000000..c42736418 --- /dev/null +++ b/pkg/local_object_storage/pilorama/migrate_test.go @@ -0,0 +1,91 @@ +package pilorama + +import ( + "context" + "strconv" + "testing" + + cidSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func prepareTree(t testing.TB, f Forest, cnr cidSDK.ID, treeID string, count int) { + for i := 0; i < count; i++ { + _, err := f.TreeMove(context.Background(), CIDDescriptor{CID: cnr, Size: 1}, treeID, &Move{ + Parent: RootID, + Child: Node(i + 1), + Meta: Meta{ + Items: []KeyValue{ + {Key: AttributeFilename, Value: []byte(uuid.New().String())}, + {Key: "Another key", Value: []byte(strconv.FormatInt(int64(i), 10))}, + }, + }, + }) + require.NoError(t, err) + } + +} + +func BenchmarkMigrate(b *testing.B) { + const ( + count = 1000 + batchSize = 10 + ) + + for i := range providers { + if providers[i].name == "inmemory" { + continue + } + + from := providers[i].construct(b) + cnr := cidtest.ID() + treeID := "benchtree" + cons := providers[i].construct + + // Do not use random tree here, to better interpret benchmark results. + // It is append-only log without deletions. + prepareTree(b, from, cnr, treeID, count) + + b.Run(providers[i].name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + to := cons(b, WithMaxBatchSize(batchSize)) + err := Migrate(context.Background(), from, to, cnr, treeID) + if err != nil { + b.Fatalf("migrate: %v", err) + } + if err := to.Close(); err != nil { + b.Fatalf("close: %v", err) + } + } + }) + } +} + +func TestMigrate(t *testing.T) { + for i := range providers { + t.Run(providers[i].name, func(t *testing.T) { + testMigrate(t, providers[i].construct) + }) + } +} + +func testMigrate(t *testing.T, cons func(testing.TB, ...Option) ForestStorage) { + const ( + count = 100 + treeID = "sometree" + ) + cnr := cidtest.ID() + + from := cons(t) + ops := prepareRandomTree(count, count) + for i := range ops { + err := from.TreeApply(context.Background(), cnr, treeID, &ops[i], true) + require.NoError(t, err) + } + + to := cons(t) + require.NoError(t, Migrate(context.Background(), from, to, cnr, treeID)) + compareForests(t, from, to, cnr, treeID, count) +}