forked from TrueCloudLab/frostfs-node
[#329] node: Make evacuate async
Now it's possible to run evacuate shard in async. Also only one evacuate process can be in progress. Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
parent
100b1b5128
commit
e4889e06ba
11 changed files with 669 additions and 37 deletions
|
|
@ -7,6 +7,7 @@ import (
|
|||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
objectCore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/object"
|
||||
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/blobstor"
|
||||
|
|
@ -21,6 +22,7 @@ import (
|
|||
oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap/zaptest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
func newEngineEvacuate(t *testing.T, shardNum int, objPerShard int) (*StorageEngine, []*shard.ID, []*objectSDK.Object) {
|
||||
|
|
@ -103,14 +105,14 @@ func TestEvacuateShard(t *testing.T) {
|
|||
t.Run("must be read-only", func(t *testing.T) {
|
||||
res, err := e.Evacuate(context.Background(), prm)
|
||||
require.ErrorIs(t, err, ErrMustBeReadOnly)
|
||||
require.Equal(t, 0, res.Count())
|
||||
require.Equal(t, uint64(0), res.Evacuated())
|
||||
})
|
||||
|
||||
require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly))
|
||||
|
||||
res, err := e.Evacuate(context.Background(), prm)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, objPerShard, res.count)
|
||||
require.Equal(t, uint64(objPerShard), res.Evacuated())
|
||||
|
||||
// We check that all objects are available both before and after shard removal.
|
||||
// First case is a real-world use-case. It ensures that an object can be put in presense
|
||||
|
|
@ -121,7 +123,7 @@ func TestEvacuateShard(t *testing.T) {
|
|||
// Calling it again is OK, but all objects are already moved, so no new PUTs should be done.
|
||||
res, err = e.Evacuate(context.Background(), prm)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, res.count)
|
||||
require.Equal(t, uint64(0), res.Evacuated())
|
||||
|
||||
checkHasObjects(t)
|
||||
|
||||
|
|
@ -138,8 +140,8 @@ func TestEvacuateNetwork(t *testing.T) {
|
|||
|
||||
var errReplication = errors.New("handler error")
|
||||
|
||||
acceptOneOf := func(objects []*objectSDK.Object, max int) func(context.Context, oid.Address, *objectSDK.Object) error {
|
||||
var n int
|
||||
acceptOneOf := func(objects []*objectSDK.Object, max uint64) func(context.Context, oid.Address, *objectSDK.Object) error {
|
||||
var n uint64
|
||||
return func(_ context.Context, addr oid.Address, obj *objectSDK.Object) error {
|
||||
if n == max {
|
||||
return errReplication
|
||||
|
|
@ -169,13 +171,13 @@ func TestEvacuateNetwork(t *testing.T) {
|
|||
|
||||
res, err := e.Evacuate(context.Background(), prm)
|
||||
require.ErrorIs(t, err, errMustHaveTwoShards)
|
||||
require.Equal(t, 0, res.Count())
|
||||
require.Equal(t, uint64(0), res.Evacuated())
|
||||
|
||||
prm.handler = acceptOneOf(objects, 2)
|
||||
|
||||
res, err = e.Evacuate(context.Background(), prm)
|
||||
require.ErrorIs(t, err, errReplication)
|
||||
require.Equal(t, 2, res.Count())
|
||||
require.Equal(t, uint64(2), res.Evacuated())
|
||||
})
|
||||
t.Run("multiple shards, evacuate one", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
|
@ -190,14 +192,14 @@ func TestEvacuateNetwork(t *testing.T) {
|
|||
|
||||
res, err := e.Evacuate(context.Background(), prm)
|
||||
require.ErrorIs(t, err, errReplication)
|
||||
require.Equal(t, 2, res.Count())
|
||||
require.Equal(t, uint64(2), res.Evacuated())
|
||||
|
||||
t.Run("no errors", func(t *testing.T) {
|
||||
prm.handler = acceptOneOf(objects, 3)
|
||||
|
||||
res, err := e.Evacuate(context.Background(), prm)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 3, res.Count())
|
||||
require.Equal(t, uint64(3), res.Evacuated())
|
||||
})
|
||||
})
|
||||
t.Run("multiple shards, evacuate many", func(t *testing.T) {
|
||||
|
|
@ -205,12 +207,12 @@ func TestEvacuateNetwork(t *testing.T) {
|
|||
e, ids, objects := newEngineEvacuate(t, 4, 5)
|
||||
evacuateIDs := ids[0:3]
|
||||
|
||||
var totalCount int
|
||||
var totalCount uint64
|
||||
for i := range evacuateIDs {
|
||||
res, err := e.shards[ids[i].String()].List()
|
||||
require.NoError(t, err)
|
||||
|
||||
totalCount += len(res.AddressList())
|
||||
totalCount += uint64(len(res.AddressList()))
|
||||
}
|
||||
|
||||
for i := range ids {
|
||||
|
|
@ -223,14 +225,14 @@ func TestEvacuateNetwork(t *testing.T) {
|
|||
|
||||
res, err := e.Evacuate(context.Background(), prm)
|
||||
require.ErrorIs(t, err, errReplication)
|
||||
require.Equal(t, totalCount-1, res.Count())
|
||||
require.Equal(t, totalCount-1, res.Evacuated())
|
||||
|
||||
t.Run("no errors", func(t *testing.T) {
|
||||
prm.handler = acceptOneOf(objects, totalCount)
|
||||
|
||||
res, err := e.Evacuate(context.Background(), prm)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, totalCount, res.Count())
|
||||
require.Equal(t, totalCount, res.Evacuated())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
@ -258,5 +260,114 @@ func TestEvacuateCancellation(t *testing.T) {
|
|||
|
||||
res, err := e.Evacuate(ctx, prm)
|
||||
require.ErrorContains(t, err, "context canceled")
|
||||
require.Equal(t, 0, res.Count())
|
||||
require.Equal(t, uint64(0), res.Evacuated())
|
||||
}
|
||||
|
||||
func TestEvacuateSingleProcess(t *testing.T) {
|
||||
e, ids, _ := newEngineEvacuate(t, 2, 3)
|
||||
|
||||
require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly))
|
||||
require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly))
|
||||
|
||||
blocker := make(chan interface{})
|
||||
running := make(chan interface{})
|
||||
|
||||
var prm EvacuateShardPrm
|
||||
prm.shardID = ids[1:2]
|
||||
prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error {
|
||||
select {
|
||||
case <-running:
|
||||
default:
|
||||
close(running)
|
||||
}
|
||||
<-blocker
|
||||
return nil
|
||||
}
|
||||
|
||||
eg, egCtx := errgroup.WithContext(context.Background())
|
||||
eg.Go(func() error {
|
||||
res, err := e.Evacuate(egCtx, prm)
|
||||
require.NoError(t, err, "first evacuation failed")
|
||||
require.Equal(t, uint64(3), res.Evacuated())
|
||||
return nil
|
||||
})
|
||||
eg.Go(func() error {
|
||||
<-running
|
||||
res, err := e.Evacuate(egCtx, prm)
|
||||
require.ErrorContains(t, err, "evacuate is already running for shard ids", "second evacuation not failed")
|
||||
require.Equal(t, uint64(0), res.Evacuated())
|
||||
close(blocker)
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, eg.Wait())
|
||||
}
|
||||
|
||||
func TestEvacuateAsync(t *testing.T) {
|
||||
e, ids, _ := newEngineEvacuate(t, 2, 3)
|
||||
|
||||
require.NoError(t, e.shards[ids[0].String()].SetMode(mode.ReadOnly))
|
||||
require.NoError(t, e.shards[ids[1].String()].SetMode(mode.ReadOnly))
|
||||
|
||||
blocker := make(chan interface{})
|
||||
running := make(chan interface{})
|
||||
|
||||
var prm EvacuateShardPrm
|
||||
prm.shardID = ids[1:2]
|
||||
prm.handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error {
|
||||
select {
|
||||
case <-running:
|
||||
default:
|
||||
close(running)
|
||||
}
|
||||
<-blocker
|
||||
return nil
|
||||
}
|
||||
|
||||
st, err := e.GetEvacuationState(context.Background())
|
||||
require.NoError(t, err, "get init state failed")
|
||||
require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state")
|
||||
require.Equal(t, uint64(0), st.Evacuated(), "invalid init count")
|
||||
require.Nil(t, st.StartedAt(), "invalid init started at")
|
||||
require.Nil(t, st.FinishedAt(), "invalid init finished at")
|
||||
require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids")
|
||||
require.Equal(t, "", st.ErrorMessage(), "invalid init error message")
|
||||
|
||||
eg, egCtx := errgroup.WithContext(context.Background())
|
||||
eg.Go(func() error {
|
||||
res, err := e.Evacuate(egCtx, prm)
|
||||
require.NoError(t, err, "first evacuation failed")
|
||||
require.Equal(t, uint64(3), res.Evacuated())
|
||||
return nil
|
||||
})
|
||||
|
||||
<-running
|
||||
|
||||
st, err = e.GetEvacuationState(context.Background())
|
||||
require.NoError(t, err, "get running state failed")
|
||||
require.Equal(t, EvacuateProcessStateRunning, st.ProcessingStatus(), "invalid running state")
|
||||
require.Equal(t, uint64(0), st.Evacuated(), "invalid running count")
|
||||
require.NotNil(t, st.StartedAt(), "invalid running started at")
|
||||
require.Nil(t, st.FinishedAt(), "invalid init finished at")
|
||||
expectedShardIDs := make([]string, 0, 2)
|
||||
for _, id := range ids[1:2] {
|
||||
expectedShardIDs = append(expectedShardIDs, id.String())
|
||||
}
|
||||
require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid running shard ids")
|
||||
require.Equal(t, "", st.ErrorMessage(), "invalid init error message")
|
||||
|
||||
close(blocker)
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
st, err = e.GetEvacuationState(context.Background())
|
||||
return st.ProcessingStatus() == EvacuateProcessStateCompleted
|
||||
}, 3*time.Second, 10*time.Millisecond, "invalid final state")
|
||||
|
||||
require.NoError(t, err, "get final state failed")
|
||||
require.Equal(t, uint64(3), st.Evacuated(), "invalid final count")
|
||||
require.NotNil(t, st.StartedAt(), "invalid final started at")
|
||||
require.NotNil(t, st.FinishedAt(), "invalid final finished at")
|
||||
require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids")
|
||||
require.Equal(t, "", st.ErrorMessage(), "invalid final error message")
|
||||
|
||||
require.NoError(t, eg.Wait())
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue