frostfs-node/pkg/local_object_storage/engine/evacuate_test.go

package engine

import (
	"context"
	"errors"
	"fmt"
	"path/filepath"
	"strconv"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor/fstree"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil"
	meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/pilorama"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode"
	"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test"
	cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id"
	cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test"
	objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object"
	oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
	"github.com/stretchr/testify/require"
	"golang.org/x/sync/errgroup"
)

func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEngine, []*shard.ID, []*objectSDK.Object) {
	dir := t.TempDir()

	te := testNewEngine(t).
		setShardsNumOpts(t, shardNum, func(id int) []shard.Option {
			return []shard.Option{
				shard.WithLogger(test.NewLogger(t)),
				shard.WithBlobStorOptions(
					blobstor.WithStorages([]blobstor.SubStorage{{
						Storage: fstree.New(
							fstree.WithPath(filepath.Join(dir, strconv.Itoa(id))),
							fstree.WithDepth(1)),
					}})),
				shard.WithMetaBaseOptions(
					meta.WithPath(filepath.Join(dir, fmt.Sprintf("%d.metabase", id))),
					meta.WithPermissions(0o700),
					meta.WithEpochState(epochState{})),
				shard.WithPiloramaOptions(
					pilorama.WithPath(filepath.Join(dir, fmt.Sprintf("%d.pilorama", id))),
					pilorama.WithPerm(0o700),
				),
			}
		})
	e, ids := te.engine, te.shardIDs
	require.NoError(t, e.Open(context.Background()))
	require.NoError(t, e.Init(context.Background()))

	objects := make([]*objectSDK.Object, 0, objPerShard*len(ids))
	treeID := "version"
	meta := []pilorama.KeyValue{
		{Key: pilorama.AttributeVersion, Value: []byte("XXX")},
		{Key: pilorama.AttributeFilename, Value: []byte("file.txt")},
	}

	for _, sh := range ids {
		for range objPerShard {
			contID := cidtest.ID()
			obj := testutil.GenerateObjectWithCID(contID)
			objects = append(objects, obj)

			var putPrm shard.PutPrm
			putPrm.SetObject(obj)
			_, err := e.shards[sh.String()].Put(context.Background(), putPrm)
			require.NoError(t, err)

			_, err = e.shards[sh.String()].TreeAddByPath(context.Background(), pilorama.CIDDescriptor{CID: contID, Position: 0, Size: 1},
				treeID, pilorama.AttributeFilename, []string{"path", "to", "the", "file"}, meta)
			require.NoError(t, err)
		}
	}
	return e, ids, objects
}

func TestEvacuateShardObjects(t *testing.T) {
	t.Parallel()

	const objPerShard = 3

	e, ids, objects := newEngineEvacuate(t, 3, objPerShard)
	defer func() {
		require.NoError(t, e.Close(context.Background()))
	}()

	evacuateShardID := ids[2].String()

	checkHasObjects := func(t *testing.T) {
		for i := range objects {
			var prm GetPrm
			prm.WithAddress(objectCore.AddressOf(objects[i]))

			_, err := e.Get(context.Background(), prm)
			require.NoError(t, err)
		}
	}

	checkHasObjects(t)

	var prm EvacuateShardPrm
	prm.ShardID = ids[2:3]
	prm.Scope = EvacuateScopeObjects

	t.Run("must be read-only", func(t *testing.T) {
		res, err := e.Evacuate(context.Background(), prm)
		require.ErrorIs(t, err, ErrMustBeReadOnly)
		require.Equal(t, uint64(0), res.ObjectsEvacuated())
	})

	require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly))

	res, err := e.Evacuate(context.Background(), prm)
	require.NoError(t, err)
	require.Equal(t, uint64(objPerShard), res.ObjectsEvacuated())

	// We check that all objects are available both before and after shard removal.
	// First case is a real-world use-case. It ensures that an object can be put in presense
	// of all metabase checks/marks.
	// Second case ensures that all objects are indeed moved and available.
	checkHasObjects(t)

	// Objects on evacuated shards should be logically unavailable, but persisted on disk.
	// This is necessary to prevent removing it by policer in case of `REP 1` policy.
	for _, obj := range objects[len(objects)-objPerShard:] {
		var prmGet shard.GetPrm
		prmGet.SetAddress(objectCore.AddressOf(obj))
		_, err = e.shards[evacuateShardID].Get(context.Background(), prmGet)
		require.Error(t, err)

		prmGet.SkipEvacCheck(true)
		_, err = e.shards[evacuateShardID].Get(context.Background(), prmGet)
		require.NoError(t, err)

		var prmHead shard.HeadPrm
		prmHead.SetAddress(objectCore.AddressOf(obj))
		_, err = e.shards[evacuateShardID].Head(context.Background(), prmHead)
		require.Error(t, err)

		var existsPrm shard.ExistsPrm
		existsPrm.Address = objectCore.AddressOf(obj)
		_, err = e.shards[evacuateShardID].Exists(context.Background(), existsPrm)
		require.Error(t, err)

		var rngPrm shard.RngPrm
		rngPrm.SetAddress(objectCore.AddressOf(obj))
		_, err = e.shards[evacuateShardID].GetRange(context.Background(), rngPrm)
		require.Error(t, err)
	}

	// Calling it again is OK, but all objects are already moved, so no new PUTs should be done.
	res, err = e.Evacuate(context.Background(), prm)
	require.NoError(t, err)
	require.Equal(t, uint64(0), res.ObjectsEvacuated())

	checkHasObjects(t)

	e.mtx.Lock()
	delete(e.shards, evacuateShardID)
	delete(e.shardPools, evacuateShardID)
	e.mtx.Unlock()

	checkHasObjects(t)
}

func TestEvacuateObjectsNetwork(t *testing.T) {
	t.Parallel()

	errReplication := errors.New("handler error")

	acceptOneOf := func(objects []*objectSDK.Object, max uint64) func(context.Context, oid.Address, *objectSDK.Object) (bool, error) {
		var n atomic.Uint64
		return func(_ context.Context, addr oid.Address, obj *objectSDK.Object) (bool, error) {
			if n.Load() == max {
				return false, errReplication
			}

			n.Add(1)
			for i := range objects {
				if addr == objectCore.AddressOf(objects[i]) {
					require.Equal(t, objects[i], obj)
					return true, nil
				}
			}
			require.FailNow(t, "handler was called with an unexpected object: %s", addr)
			panic("unreachable")
		}
	}

	t.Run("single shard", func(t *testing.T) {
		t.Parallel()
		e, ids, objects := newEngineEvacuate(t, 1, 3)
		defer func() {
			require.NoError(t, e.Close(context.Background()))
		}()

		evacuateShardID := ids[0].String()

		require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly))

		var prm EvacuateShardPrm
		prm.ShardID = ids[0:1]
		prm.Scope = EvacuateScopeObjects

		res, err := e.Evacuate(context.Background(), prm)
		require.ErrorIs(t, err, errMustHaveTwoShards)
		require.Equal(t, uint64(0), res.ObjectsEvacuated())

		prm.ObjectsHandler = acceptOneOf(objects, 2)

		res, err = e.Evacuate(context.Background(), prm)
		require.ErrorIs(t, err, errReplication)
		require.Equal(t, uint64(2), res.ObjectsEvacuated())
	})
	t.Run("multiple shards, evacuate one", func(t *testing.T) {
		t.Parallel()
		e, ids, objects := newEngineEvacuate(t, 2, 3)
		defer func() {
			require.NoError(t, e.Close(context.Background()))
		}()

		require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly))
		require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly))

		var prm EvacuateShardPrm
		prm.ShardID = ids[1:2]
		prm.ObjectsHandler = acceptOneOf(objects, 2)
		prm.Scope = EvacuateScopeObjects

		res, err := e.Evacuate(context.Background(), prm)
		require.ErrorIs(t, err, errReplication)
		require.Equal(t, uint64(2), res.ObjectsEvacuated())

		t.Run("no errors", func(t *testing.T) {
			prm.ObjectsHandler = acceptOneOf(objects, 3)

			res, err := e.Evacuate(context.Background(), prm)
			require.NoError(t, err)
			require.Equal(t, uint64(3), res.ObjectsEvacuated())
		})
	})
	t.Run("multiple shards, evacuate many", func(t *testing.T) {
		t.Parallel()
		e, ids, objects := newEngineEvacuate(t, 4, 5)
		defer func() {
			require.NoError(t, e.Close(context.Background()))
		}()

		evacuateIDs := ids[0:3]

		var totalCount uint64
		for i := range evacuateIDs {
			res, err := e.shards[ids[i].String()].List(context.Background())
			require.NoError(t, err)

			totalCount += uint64(len(res.AddressList()))
		}

		for i := range ids {
			require.NoError(t, e.shards[ids[i].String()].SetMode(mode.ReadOnly))
		}

		var prm EvacuateShardPrm
		prm.ShardID = evacuateIDs
		prm.ObjectsHandler = acceptOneOf(objects, totalCount-1)
		prm.Scope = EvacuateScopeObjects

		res, err := e.Evacuate(context.Background(), prm)
		require.ErrorIs(t, err, errReplication)
		require.Equal(t, totalCount-1, res.ObjectsEvacuated())

		t.Run("no errors", func(t *testing.T) {
			prm.ObjectsHandler = acceptOneOf(objects, totalCount)

			res, err := e.Evacuate(context.Background(), prm)
			require.NoError(t, err)
			require.Equal(t, totalCount, res.ObjectsEvacuated())
		})
	})
}

func TestEvacuateCancellation(t *testing.T) {
	t.Parallel()
	e, ids, _ := newEngineEvacuate(t, 2, 3)
	defer func() {
		require.NoError(t, e.Close(context.Background()))
	}()

	require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly))
	require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly))

	var prm EvacuateShardPrm
	prm.ShardID = ids[1:2]
	prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) (bool, error) {
		select {
		case <-ctx.Done():
			return false, ctx.Err()
		default:
		}
		return true, nil
	}
	prm.Scope = EvacuateScopeObjects

	ctx, cancel := context.WithCancel(context.Background())
	cancel()

	res, err := e.Evacuate(ctx, prm)
	require.ErrorContains(t, err, "context canceled")
	require.Equal(t, uint64(0), res.ObjectsEvacuated())
}

func TestEvacuateCancellationByError(t *testing.T) {
	t.Parallel()
	e, ids, _ := newEngineEvacuate(t, 2, 10)
	defer func() {
		require.NoError(t, e.Close(context.Background()))
	}()

	require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly))
	require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly))

	var prm EvacuateShardPrm
	prm.ShardID = ids[1:2]
	var once atomic.Bool
	prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) (bool, error) {
		var err error
		flag := true
		if once.CompareAndSwap(false, true) {
			err = errors.New("test error")
			flag = false
		}
		return flag, err
	}
	prm.Scope = EvacuateScopeObjects
	prm.ObjectWorkerCount = 2
	prm.ContainerWorkerCount = 2

	_, err := e.Evacuate(context.Background(), prm)
	require.ErrorContains(t, err, "test error")
}

func TestEvacuateSingleProcess(t *testing.T) {
	e, ids, _ := newEngineEvacuate(t, 2, 3)
	defer func() {
		require.NoError(t, e.Close(context.Background()))
	}()

	require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly))
	require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly))

	blocker := make(chan interface{})
	running := make(chan interface{})

	var prm EvacuateShardPrm
	prm.ShardID = ids[1:2]
	prm.Scope = EvacuateScopeObjects
	prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) (bool, error) {
		select {
		case <-running:
		default:
			close(running)
		}
		<-blocker
		return true, nil
	}

	eg, egCtx := errgroup.WithContext(context.Background())
	eg.Go(func() error {
		res, err := e.Evacuate(egCtx, prm)
		require.NoError(t, err, "first evacuation failed")
		require.Equal(t, uint64(3), res.ObjectsEvacuated())
		return nil
	})
	eg.Go(func() error {
		<-running
		res, err := e.Evacuate(egCtx, prm)
		require.ErrorContains(t, err, "evacuate is already running for shard ids", "second evacuation not failed")
		require.Equal(t, uint64(0), res.ObjectsEvacuated())
		close(blocker)
		return nil
	})
	require.NoError(t, eg.Wait())
}

func TestEvacuateObjectsAsync(t *testing.T) {
	e, ids, _ := newEngineEvacuate(t, 2, 3)
	defer func() {
		require.NoError(t, e.Close(context.Background()))
	}()

	require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly))
	require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly))

	blocker := make(chan interface{})
	running := make(chan interface{})

	var prm EvacuateShardPrm
	prm.ShardID = ids[1:2]
	prm.Scope = EvacuateScopeObjects
	prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) (bool, error) {
		select {
		case <-running:
		default:
			close(running)
		}
		<-blocker
		return true, nil
	}

	st, err := e.GetEvacuationState(context.Background())
	require.NoError(t, err, "get init state failed")
	require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state")
	require.Equal(t, uint64(0), st.ObjectsEvacuated(), "invalid init count")
	require.Nil(t, st.StartedAt(), "invalid init started at")
	require.Nil(t, st.FinishedAt(), "invalid init finished at")
	require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids")
	require.Equal(t, "", st.ErrorMessage(), "invalid init error message")

	eg, egCtx := errgroup.WithContext(context.Background())
	eg.Go(func() error {
		res, err := e.Evacuate(egCtx, prm)
		require.NoError(t, err, "first evacuation failed")
		require.Equal(t, uint64(3), res.ObjectsEvacuated())
		return nil
	})

	<-running

	st, err = e.GetEvacuationState(context.Background())
	require.NoError(t, err, "get running state failed")
	require.Equal(t, EvacuateProcessStateRunning, st.ProcessingStatus(), "invalid running state")
	require.Equal(t, uint64(0), st.ObjectsEvacuated(), "invalid running count")
	require.NotNil(t, st.StartedAt(), "invalid running started at")
	require.Nil(t, st.FinishedAt(), "invalid init finished at")
	expectedShardIDs := make([]string, 0, 2)
	for _, id := range ids[1:2] {
		expectedShardIDs = append(expectedShardIDs, id.String())
	}
	require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid running shard ids")
	require.Equal(t, "", st.ErrorMessage(), "invalid init error message")

	require.Error(t, e.ResetEvacuationStatus(context.Background()))

	close(blocker)

	require.Eventually(t, func() bool {
		st, err = e.GetEvacuationState(context.Background())
		return st.ProcessingStatus() == EvacuateProcessStateCompleted
	}, 3*time.Second, 10*time.Millisecond, "invalid final state")

	require.NoError(t, err, "get final state failed")
	require.Equal(t, uint64(3), st.ObjectsEvacuated(), "invalid final count")
	require.NotNil(t, st.StartedAt(), "invalid final started at")
	require.NotNil(t, st.FinishedAt(), "invalid final finished at")
	require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids")
	require.Equal(t, "", st.ErrorMessage(), "invalid final error message")

	require.NoError(t, eg.Wait())

	require.NoError(t, e.ResetEvacuationStatus(context.Background()))
	st, err = e.GetEvacuationState(context.Background())
	require.NoError(t, err, "get state after reset failed")
	require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid state after reset")
	require.Equal(t, uint64(0), st.ObjectsEvacuated(), "invalid count after reset")
	require.Nil(t, st.StartedAt(), "invalid started at after reset")
	require.Nil(t, st.FinishedAt(), "invalid finished at after reset")
	require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid shard ids after reset")
	require.Equal(t, "", st.ErrorMessage(), "invalid error message after reset")
}

func TestEvacuateTreesLocal(t *testing.T) {
	e, ids, _ := newEngineEvacuate(t, 2, 3)
	defer func() {
		require.NoError(t, e.Close(context.Background()))
	}()

	require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly))

	var prm EvacuateShardPrm
	prm.ShardID = ids[0:1]
	prm.Scope = EvacuateScopeTrees

	expectedShardIDs := make([]string, 0, 1)
	for _, id := range ids[0:1] {
		expectedShardIDs = append(expectedShardIDs, id.String())
	}

	st, err := e.GetEvacuationState(context.Background())
	require.NoError(t, err, "get init state failed")
	require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state")
	require.Equal(t, uint64(0), st.TreesEvacuated(), "invalid init count")
	require.Nil(t, st.StartedAt(), "invalid init started at")
	require.Nil(t, st.FinishedAt(), "invalid init finished at")
	require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids")
	require.Equal(t, "", st.ErrorMessage(), "invalid init error message")

	res, err := e.Evacuate(context.Background(), prm)
	require.NotNil(t, res, "sync evacuation result must be not nil")
	require.NoError(t, err, "evacuation failed")

	st, err = e.GetEvacuationState(context.Background())
	require.NoError(t, err, "get evacuation state failed")
	require.Equal(t, EvacuateProcessStateCompleted, st.ProcessingStatus())

	require.Equal(t, uint64(3), st.TreesTotal(), "invalid trees total count")
	require.Equal(t, uint64(3), st.TreesEvacuated(), "invalid trees evacuated count")
	require.Equal(t, uint64(0), st.TreesFailed(), "invalid trees failed count")
	require.NotNil(t, st.StartedAt(), "invalid final started at")
	require.NotNil(t, st.FinishedAt(), "invalid final finished at")
	require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids")
	require.Equal(t, "", st.ErrorMessage(), "invalid final error message")

	sourceTrees, err := pilorama.TreeListAll(context.Background(), e.shards[ids[0].String()])
	require.NoError(t, err, "list source trees failed")
	require.Len(t, sourceTrees, 3)

	for _, tr := range sourceTrees {
		exists, err := e.shards[ids[1].String()].TreeExists(context.Background(), tr.CID, tr.TreeID)
		require.NoError(t, err, "failed to check tree existance")
		require.True(t, exists, "tree doesn't exists on target shard")

		var height uint64
		var sourceOps []pilorama.Move
		for {
			op, err := e.shards[ids[0].String()].TreeGetOpLog(context.Background(), tr.CID, tr.TreeID, height)
			require.NoError(t, err)
			if op.Time == 0 {
				break
			}
			sourceOps = append(sourceOps, op)
			height = op.Time + 1
		}

		height = 0
		var targetOps []pilorama.Move
		for {
			op, err := e.shards[ids[1].String()].TreeGetOpLog(context.Background(), tr.CID, tr.TreeID, height)
			require.NoError(t, err)
			if op.Time == 0 {
				break
			}
			targetOps = append(targetOps, op)
			height = op.Time + 1
		}

		require.Equal(t, sourceOps, targetOps)
	}
}

func TestEvacuateTreesRemote(t *testing.T) {
	e, ids, _ := newEngineEvacuate(t, 2, 3)
	defer func() {
		require.NoError(t, e.Close(context.Background()))
	}()

	require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly))
	require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly))

	mutex := sync.Mutex{}
	evacuatedTreeOps := make(map[string][]*pilorama.Move)
	var prm EvacuateShardPrm
	prm.ShardID = ids
	prm.Scope = EvacuateScopeTrees
	prm.TreeHandler = func(ctx context.Context, contID cid.ID, treeID string, f pilorama.Forest) (bool, string, error) {
		key := contID.String() + treeID
		var height uint64
		for {
			op, err := f.TreeGetOpLog(ctx, contID, treeID, height)
			require.NoError(t, err)

			if op.Time == 0 {
				return true, "", nil
			}
			mutex.Lock()
			evacuatedTreeOps[key] = append(evacuatedTreeOps[key], &op)
			mutex.Unlock()
			height = op.Time + 1
		}
	}

	expectedShardIDs := make([]string, 0, len(ids))
	for _, id := range ids {
		expectedShardIDs = append(expectedShardIDs, id.String())
	}

	st, err := e.GetEvacuationState(context.Background())
	require.NoError(t, err, "get init state failed")
	require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state")
	require.Equal(t, uint64(0), st.TreesEvacuated(), "invalid init count")
	require.Nil(t, st.StartedAt(), "invalid init started at")
	require.Nil(t, st.FinishedAt(), "invalid init finished at")
	require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids")
	require.Equal(t, "", st.ErrorMessage(), "invalid init error message")

	res, err := e.Evacuate(context.Background(), prm)
	require.NotNil(t, res, "sync evacuation must return not nil")
	require.NoError(t, err, "evacuation failed")

	st, err = e.GetEvacuationState(context.Background())
	require.NoError(t, err, "get evacuation state failed")
	require.Equal(t, EvacuateProcessStateCompleted, st.ProcessingStatus())

	require.NoError(t, err, "get final state failed")
	require.Equal(t, uint64(6), st.TreesTotal(), "invalid trees total count")
	require.Equal(t, uint64(6), st.TreesEvacuated(), "invalid trees evacuated count")
	require.Equal(t, uint64(0), st.TreesFailed(), "invalid trees failed count")
	require.NotNil(t, st.StartedAt(), "invalid final started at")
	require.NotNil(t, st.FinishedAt(), "invalid final finished at")
	require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids")
	require.Equal(t, "", st.ErrorMessage(), "invalid final error message")

	expectedTreeOps := make(map[string][]*pilorama.Move)
	for i := range len(e.shards) {
		sourceTrees, err := pilorama.TreeListAll(context.Background(), e.shards[ids[i].String()])
		require.NoError(t, err, "list source trees failed")
		require.Len(t, sourceTrees, 3)

		for _, tr := range sourceTrees {
			key := tr.CID.String() + tr.TreeID
			var height uint64
			for {
				op, err := e.shards[ids[i].String()].TreeGetOpLog(context.Background(), tr.CID, tr.TreeID, height)
				require.NoError(t, err)

				if op.Time == 0 {
					break
				}
				expectedTreeOps[key] = append(expectedTreeOps[key], &op)
				height = op.Time + 1
			}
		}
	}

	require.Equal(t, expectedTreeOps, evacuatedTreeOps)
}