[#947] engine: Evacuate trees to remote nodes

Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>

pkg/local_object_storage/engine/evacuate_test.go

@@ -18,6 +18,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
 	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
+	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
 	cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
 	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
 	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
@@ -480,3 +481,87 @@ func TestEvacuateTreesLocal(t *testing.T) {
 		require.Equal(t, sourceOps, targetOps)
 	}
 }
+
+func TestEvacuateTreesRemote(t *testing.T) {
+	e, ids, _ := newEngineEvacuate(t, 2, 3)
+	defer func() {
+		require.NoError(t, e.Close(context.Background()))
+	}()
+
+	require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly))
+	require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly))
+
+	evacuatedTreeOps := make(map[string][]*pilorama.Move)
+	var prm EvacuateShardPrm
+	prm.ShardID = ids
+	prm.Scope = EvacuateScopeTrees
+	prm.TreeHandler = func(ctx context.Context, contID cid.ID, treeID string, f pilorama.Forest) (string, error) {
+		key := contID.String() + treeID
+		var height uint64
+		for {
+			op, err := f.TreeGetOpLog(ctx, contID, treeID, height)
+			require.NoError(t, err)
+
+			if op.Time == 0 {
+				return "", nil
+			}
+			evacuatedTreeOps[key] = append(evacuatedTreeOps[key], &op)
+			height = op.Time + 1
+		}
+	}
+
+	expectedShardIDs := make([]string, 0, len(ids))
+	for _, id := range ids {
+		expectedShardIDs = append(expectedShardIDs, id.String())
+	}
+
+	st, err := e.GetEvacuationState(context.Background())
+	require.NoError(t, err, "get init state failed")
+	require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state")
+	require.Equal(t, uint64(0), st.TreesEvacuated(), "invalid init count")
+	require.Nil(t, st.StartedAt(), "invalid init started at")
+	require.Nil(t, st.FinishedAt(), "invalid init finished at")
+	require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids")
+	require.Equal(t, "", st.ErrorMessage(), "invalid init error message")
+
+	res, err := e.Evacuate(context.Background(), prm)
+	require.NotNil(t, res, "sync evacuation must return not nil")
+	require.NoError(t, err, "evacuation failed")
+
+	st, err = e.GetEvacuationState(context.Background())
+	require.NoError(t, err, "get evacuation state failed")
+	require.Equal(t, EvacuateProcessStateCompleted, st.ProcessingStatus())
+
+	require.NoError(t, err, "get final state failed")
+	require.Equal(t, uint64(6), st.TreesTotal(), "invalid trees total count")
+	require.Equal(t, uint64(6), st.TreesEvacuated(), "invalid trees evacuated count")
+	require.Equal(t, uint64(0), st.TreesFailed(), "invalid trees failed count")
+	require.NotNil(t, st.StartedAt(), "invalid final started at")
+	require.NotNil(t, st.FinishedAt(), "invalid final finished at")
+	require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids")
+	require.Equal(t, "", st.ErrorMessage(), "invalid final error message")
+
+	expectedTreeOps := make(map[string][]*pilorama.Move)
+	for i := 0; i < len(e.shards); i++ {
+		sourceTrees, err := pilorama.TreeListAll(context.Background(), e.shards[ids[i].String()])
+		require.NoError(t, err, "list source trees failed")
+		require.Len(t, sourceTrees, 3)
+
+		for _, tr := range sourceTrees {
+			key := tr.CID.String() + tr.TreeID
+			var height uint64
+			for {
+				op, err := e.shards[ids[i].String()].TreeGetOpLog(context.Background(), tr.CID, tr.TreeID, height)
+				require.NoError(t, err)
+
+				if op.Time == 0 {
+					break
+				}
+				expectedTreeOps[key] = append(expectedTreeOps[key], &op)
+				height = op.Time + 1
+			}
+		}
+	}
+
+	require.Equal(t, expectedTreeOps, evacuatedTreeOps)
+}