From 15d853ea22115c5345cbd0a616d2879b06f883e0 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Mon, 5 Feb 2024 17:48:43 +0300 Subject: [PATCH] [#947] controlSvc: Return tree evacuation stat Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 142 ++++++++++++------ .../engine/evacuate_limiter.go | 37 ++++- .../engine/evacuate_test.go | 48 +++--- pkg/services/control/server/convert.go | 21 +-- pkg/services/control/server/evacuate.go | 10 +- pkg/services/control/server/evacuate_async.go | 10 +- pkg/services/control/service.pb.go | Bin 234031 -> 236122 bytes pkg/services/control/service.proto | 15 +- pkg/services/control/service_frostfs.pb.go | Bin 107736 -> 108123 bytes 9 files changed, 178 insertions(+), 105 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index ad432e40..3853c4e3 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -58,62 +58,93 @@ func (s EvacuateScope) String() string { // EvacuateShardPrm represents parameters for the EvacuateShard operation. type EvacuateShardPrm struct { - ShardID []*shard.ID - Handler func(context.Context, oid.Address, *objectSDK.Object) error - IgnoreErrors bool - Async bool - Scope EvacuateScope + ShardID []*shard.ID + ObjectsHandler func(context.Context, oid.Address, *objectSDK.Object) error + IgnoreErrors bool + Async bool + Scope EvacuateScope } // EvacuateShardRes represents result of the EvacuateShard operation. type EvacuateShardRes struct { - evacuated *atomic.Uint64 - total *atomic.Uint64 - failed *atomic.Uint64 - skipped *atomic.Uint64 + objEvacuated *atomic.Uint64 + objTotal *atomic.Uint64 + objFailed *atomic.Uint64 + objSkipped *atomic.Uint64 + + trEvacuated *atomic.Uint64 + trTotal *atomic.Uint64 + trFailed *atomic.Uint64 } // NewEvacuateShardRes creates new EvacuateShardRes instance. func NewEvacuateShardRes() *EvacuateShardRes { return &EvacuateShardRes{ - evacuated: new(atomic.Uint64), - total: new(atomic.Uint64), - failed: new(atomic.Uint64), - skipped: new(atomic.Uint64), + objEvacuated: new(atomic.Uint64), + objTotal: new(atomic.Uint64), + objFailed: new(atomic.Uint64), + objSkipped: new(atomic.Uint64), + trEvacuated: new(atomic.Uint64), + trTotal: new(atomic.Uint64), + trFailed: new(atomic.Uint64), } } -// Evacuated returns amount of evacuated objects. +// ObjectsEvacuated returns amount of evacuated objects. // Objects for which handler returned no error are also assumed evacuated. -func (p *EvacuateShardRes) Evacuated() uint64 { +func (p *EvacuateShardRes) ObjectsEvacuated() uint64 { if p == nil { return 0 } - return p.evacuated.Load() + return p.objEvacuated.Load() } -// Total returns total count objects to evacuate. -func (p *EvacuateShardRes) Total() uint64 { +// ObjectsTotal returns total count objects to evacuate. +func (p *EvacuateShardRes) ObjectsTotal() uint64 { if p == nil { return 0 } - return p.total.Load() + return p.objTotal.Load() } -// Failed returns count of failed objects to evacuate. -func (p *EvacuateShardRes) Failed() uint64 { +// ObjectsFailed returns count of failed objects to evacuate. +func (p *EvacuateShardRes) ObjectsFailed() uint64 { if p == nil { return 0 } - return p.failed.Load() + return p.objFailed.Load() } -// Skipped returns count of skipped objects. -func (p *EvacuateShardRes) Skipped() uint64 { +// ObjectsSkipped returns count of skipped objects. +func (p *EvacuateShardRes) ObjectsSkipped() uint64 { if p == nil { return 0 } - return p.skipped.Load() + return p.objSkipped.Load() +} + +// TreesEvacuated returns amount of evacuated trees. +func (p *EvacuateShardRes) TreesEvacuated() uint64 { + if p == nil { + return 0 + } + return p.trEvacuated.Load() +} + +// TreesTotal returns total count trees to evacuate. +func (p *EvacuateShardRes) TreesTotal() uint64 { + if p == nil { + return 0 + } + return p.trTotal.Load() +} + +// TreesFailed returns count of failed trees to evacuate. +func (p *EvacuateShardRes) TreesFailed() uint64 { + if p == nil { + return 0 + } + return p.trFailed.Load() } // DeepCopy returns deep copy of result instance. @@ -123,16 +154,22 @@ func (p *EvacuateShardRes) DeepCopy() *EvacuateShardRes { } res := &EvacuateShardRes{ - evacuated: new(atomic.Uint64), - total: new(atomic.Uint64), - failed: new(atomic.Uint64), - skipped: new(atomic.Uint64), + objEvacuated: new(atomic.Uint64), + objTotal: new(atomic.Uint64), + objFailed: new(atomic.Uint64), + objSkipped: new(atomic.Uint64), + trEvacuated: new(atomic.Uint64), + trTotal: new(atomic.Uint64), + trFailed: new(atomic.Uint64), } - res.evacuated.Store(p.evacuated.Load()) - res.total.Store(p.total.Load()) - res.failed.Store(p.failed.Load()) - res.skipped.Store(p.skipped.Load()) + res.objEvacuated.Store(p.objEvacuated.Load()) + res.objTotal.Store(p.objTotal.Load()) + res.objFailed.Store(p.objFailed.Load()) + res.objSkipped.Store(p.objSkipped.Load()) + res.trTotal.Store(p.trTotal.Load()) + res.trEvacuated.Store(p.trEvacuated.Load()) + res.trFailed.Store(p.trFailed.Load()) return res } @@ -168,7 +205,7 @@ func (e *StorageEngine) Evacuate(ctx context.Context, prm EvacuateShardPrm) (*Ev )) defer span.End() - shards, weights, err := e.getActualShards(shardIDs, prm.Handler != nil) + shards, weights, err := e.getActualShards(shardIDs, prm) if err != nil { return nil, err } @@ -216,6 +253,7 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p attribute.StringSlice("shardIDs", shardIDs), attribute.Bool("async", prm.Async), attribute.Bool("ignoreErrors", prm.IgnoreErrors), + attribute.Stringer("scope", prm.Scope), )) defer func() { @@ -224,18 +262,19 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p }() e.log.Info(logs.EngineStartedShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) err = e.getTotalObjectsCount(ctx, shardsToEvacuate, res) if err != nil { e.log.Error(logs.EngineShardsEvacuationFailedToCount, zap.Strings("shard_ids", shardIDs), zap.Error(err), evacuationOperationLogField, - zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) return err } for _, shardID := range shardIDs { if err = e.evacuateShard(ctx, shardID, prm, res, shards, weights, shardsToEvacuate); err != nil { - e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField) + e.log.Error(logs.EngineFinishedWithErrorShardsEvacuation, zap.Error(err), zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, + zap.String("trace_id", tracingPkg.GetTraceID(ctx)), zap.Stringer("scope", prm.Scope)) return err } } @@ -243,10 +282,13 @@ func (e *StorageEngine) evacuateShards(ctx context.Context, shardIDs []string, p e.log.Info(logs.EngineFinishedSuccessfullyShardsEvacuation, zap.Strings("shard_ids", shardIDs), evacuationOperationLogField, - zap.Uint64("total", res.Total()), - zap.Uint64("evacuated", res.Evacuated()), - zap.Uint64("failed", res.Failed()), - zap.Uint64("skipped", res.Skipped()), + zap.Uint64("total_objects", res.ObjectsTotal()), + zap.Uint64("evacuated_objects", res.ObjectsEvacuated()), + zap.Uint64("failed_objects", res.ObjectsFailed()), + zap.Uint64("skipped_objects", res.ObjectsSkipped()), + zap.Uint64("total_trees", res.TreesTotal()), + zap.Uint64("evacuated_trees", res.TreesEvacuated()), + zap.Uint64("failed_trees", res.TreesFailed()), ) return nil } @@ -263,7 +305,7 @@ func (e *StorageEngine) getTotalObjectsCount(ctx context.Context, shardsToEvacua } return err } - res.total.Add(cnt) + res.objTotal.Add(cnt) } return nil } @@ -307,7 +349,7 @@ func (e *StorageEngine) evacuateShard(ctx context.Context, shardID string, prm E return nil } -func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) ([]pooledShard, []float64, error) { +func (e *StorageEngine) getActualShards(shardIDs []string, prm EvacuateShardPrm) ([]pooledShard, []float64, error) { e.mtx.RLock() defer e.mtx.RUnlock() @@ -322,7 +364,7 @@ func (e *StorageEngine) getActualShards(shardIDs []string, handlerDefined bool) } } - if len(e.shards)-len(shardIDs) < 1 && !handlerDefined { + if len(e.shards)-len(shardIDs) < 1 && prm.ObjectsHandler == nil { return nil, nil, errMustHaveTwoShards } @@ -368,7 +410,7 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to getRes, err := sh.Get(ctx, getPrm) if err != nil { if prm.IgnoreErrors { - res.failed.Add(1) + res.objFailed.Add(1) continue } e.log.Error(logs.EngineShardsEvacuationFailedToReadObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, @@ -385,19 +427,19 @@ func (e *StorageEngine) evacuateObjects(ctx context.Context, sh *shard.Shard, to continue } - if prm.Handler == nil { + if prm.ObjectsHandler == nil { // Do not check ignoreErrors flag here because // ignoring errors on put make this command kinda useless. return fmt.Errorf("%w: %s", errPutShard, toEvacuate[i]) } - err = prm.Handler(ctx, addr, getRes.Object()) + err = prm.ObjectsHandler(ctx, addr, getRes.Object()) if err != nil { e.log.Error(logs.EngineShardsEvacuationFailedToMoveObject, zap.String("address", addr.EncodeToString()), zap.Error(err), evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } - res.evacuated.Add(1) + res.objEvacuated.Add(1) } return nil } @@ -418,7 +460,7 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add } switch e.putToShard(ctx, shards[j].hashedShard, j, shards[j].pool, addr, object).status { case putToShardSuccess: - res.evacuated.Add(1) + res.objEvacuated.Add(1) e.log.Debug(logs.EngineObjectIsMovedToAnotherShard, zap.Stringer("from", sh.ID()), zap.Stringer("to", shards[j].ID()), @@ -427,7 +469,7 @@ func (e *StorageEngine) tryEvacuateObjectLocal(ctx context.Context, addr oid.Add zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return true, nil case putToShardExists, putToShardRemoved: - res.skipped.Add(1) + res.objSkipped.Add(1) return true, nil default: continue diff --git a/pkg/local_object_storage/engine/evacuate_limiter.go b/pkg/local_object_storage/engine/evacuate_limiter.go index 83acd30d..63960e23 100644 --- a/pkg/local_object_storage/engine/evacuate_limiter.go +++ b/pkg/local_object_storage/engine/evacuate_limiter.go @@ -34,32 +34,53 @@ func (s *EvacuationState) ShardIDs() []string { return s.shardIDs } -func (s *EvacuationState) Evacuated() uint64 { +func (s *EvacuationState) ObjectsEvacuated() uint64 { if s == nil { return 0 } - return s.result.Evacuated() + return s.result.ObjectsEvacuated() } -func (s *EvacuationState) Total() uint64 { +func (s *EvacuationState) ObjectsTotal() uint64 { if s == nil { return 0 } - return s.result.Total() + return s.result.ObjectsTotal() } -func (s *EvacuationState) Failed() uint64 { +func (s *EvacuationState) ObjectsFailed() uint64 { if s == nil { return 0 } - return s.result.Failed() + return s.result.ObjectsFailed() } -func (s *EvacuationState) Skipped() uint64 { +func (s *EvacuationState) ObjectsSkipped() uint64 { if s == nil { return 0 } - return s.result.Skipped() + return s.result.ObjectsSkipped() +} + +func (s *EvacuationState) TreesEvacuated() uint64 { + if s == nil { + return 0 + } + return s.result.TreesEvacuated() +} + +func (s *EvacuationState) TreesTotal() uint64 { + if s == nil { + return 0 + } + return s.result.TreesTotal() +} + +func (s *EvacuationState) TreesFailed() uint64 { + if s == nil { + return 0 + } + return s.result.TreesFailed() } func (s *EvacuationState) ProcessingStatus() EvacuateProcessState { diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index 2dc4a177..cadcc293 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -107,14 +107,14 @@ func TestEvacuateShard(t *testing.T) { t.Run("must be read-only", func(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, ErrMustBeReadOnly) - require.Equal(t, uint64(0), res.Evacuated()) + require.Equal(t, uint64(0), res.ObjectsEvacuated()) }) require.NoError(t, e.shards[evacuateShardID].SetMode(mode.ReadOnly)) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, uint64(objPerShard), res.Evacuated()) + require.Equal(t, uint64(objPerShard), res.ObjectsEvacuated()) // We check that all objects are available both before and after shard removal. // First case is a real-world use-case. It ensures that an object can be put in presense @@ -125,7 +125,7 @@ func TestEvacuateShard(t *testing.T) { // Calling it again is OK, but all objects are already moved, so no new PUTs should be done. res, err = e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, uint64(0), res.Evacuated()) + require.Equal(t, uint64(0), res.ObjectsEvacuated()) checkHasObjects(t) @@ -177,13 +177,13 @@ func TestEvacuateNetwork(t *testing.T) { res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errMustHaveTwoShards) - require.Equal(t, uint64(0), res.Evacuated()) + require.Equal(t, uint64(0), res.ObjectsEvacuated()) - prm.Handler = acceptOneOf(objects, 2) + prm.ObjectsHandler = acceptOneOf(objects, 2) res, err = e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, uint64(2), res.Evacuated()) + require.Equal(t, uint64(2), res.ObjectsEvacuated()) }) t.Run("multiple shards, evacuate one", func(t *testing.T) { t.Parallel() @@ -197,18 +197,18 @@ func TestEvacuateNetwork(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] - prm.Handler = acceptOneOf(objects, 2) + prm.ObjectsHandler = acceptOneOf(objects, 2) res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, uint64(2), res.Evacuated()) + require.Equal(t, uint64(2), res.ObjectsEvacuated()) t.Run("no errors", func(t *testing.T) { - prm.Handler = acceptOneOf(objects, 3) + prm.ObjectsHandler = acceptOneOf(objects, 3) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, uint64(3), res.Evacuated()) + require.Equal(t, uint64(3), res.ObjectsEvacuated()) }) }) t.Run("multiple shards, evacuate many", func(t *testing.T) { @@ -234,18 +234,18 @@ func TestEvacuateNetwork(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = evacuateIDs - prm.Handler = acceptOneOf(objects, totalCount-1) + prm.ObjectsHandler = acceptOneOf(objects, totalCount-1) res, err := e.Evacuate(context.Background(), prm) require.ErrorIs(t, err, errReplication) - require.Equal(t, totalCount-1, res.Evacuated()) + require.Equal(t, totalCount-1, res.ObjectsEvacuated()) t.Run("no errors", func(t *testing.T) { - prm.Handler = acceptOneOf(objects, totalCount) + prm.ObjectsHandler = acceptOneOf(objects, totalCount) res, err := e.Evacuate(context.Background(), prm) require.NoError(t, err) - require.Equal(t, totalCount, res.Evacuated()) + require.Equal(t, totalCount, res.ObjectsEvacuated()) }) }) } @@ -262,7 +262,7 @@ func TestEvacuateCancellation(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] - prm.Handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { select { case <-ctx.Done(): return ctx.Err() @@ -276,7 +276,7 @@ func TestEvacuateCancellation(t *testing.T) { res, err := e.Evacuate(ctx, prm) require.ErrorContains(t, err, "context canceled") - require.Equal(t, uint64(0), res.Evacuated()) + require.Equal(t, uint64(0), res.ObjectsEvacuated()) } func TestEvacuateSingleProcess(t *testing.T) { @@ -293,7 +293,7 @@ func TestEvacuateSingleProcess(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] - prm.Handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { select { case <-running: default: @@ -307,14 +307,14 @@ func TestEvacuateSingleProcess(t *testing.T) { eg.Go(func() error { res, err := e.Evacuate(egCtx, prm) require.NoError(t, err, "first evacuation failed") - require.Equal(t, uint64(3), res.Evacuated()) + require.Equal(t, uint64(3), res.ObjectsEvacuated()) return nil }) eg.Go(func() error { <-running res, err := e.Evacuate(egCtx, prm) require.ErrorContains(t, err, "evacuate is already running for shard ids", "second evacuation not failed") - require.Equal(t, uint64(0), res.Evacuated()) + require.Equal(t, uint64(0), res.ObjectsEvacuated()) close(blocker) return nil }) @@ -335,7 +335,7 @@ func TestEvacuateAsync(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids[1:2] - prm.Handler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { + prm.ObjectsHandler = func(ctx context.Context, a oid.Address, o *objectSDK.Object) error { select { case <-running: default: @@ -348,7 +348,7 @@ func TestEvacuateAsync(t *testing.T) { st, err := e.GetEvacuationState(context.Background()) require.NoError(t, err, "get init state failed") require.Equal(t, EvacuateProcessStateUndefined, st.ProcessingStatus(), "invalid init state") - require.Equal(t, uint64(0), st.Evacuated(), "invalid init count") + require.Equal(t, uint64(0), st.ObjectsEvacuated(), "invalid init count") require.Nil(t, st.StartedAt(), "invalid init started at") require.Nil(t, st.FinishedAt(), "invalid init finished at") require.ElementsMatch(t, []string{}, st.ShardIDs(), "invalid init shard ids") @@ -358,7 +358,7 @@ func TestEvacuateAsync(t *testing.T) { eg.Go(func() error { res, err := e.Evacuate(egCtx, prm) require.NoError(t, err, "first evacuation failed") - require.Equal(t, uint64(3), res.Evacuated()) + require.Equal(t, uint64(3), res.ObjectsEvacuated()) return nil }) @@ -367,7 +367,7 @@ func TestEvacuateAsync(t *testing.T) { st, err = e.GetEvacuationState(context.Background()) require.NoError(t, err, "get running state failed") require.Equal(t, EvacuateProcessStateRunning, st.ProcessingStatus(), "invalid running state") - require.Equal(t, uint64(0), st.Evacuated(), "invalid running count") + require.Equal(t, uint64(0), st.ObjectsEvacuated(), "invalid running count") require.NotNil(t, st.StartedAt(), "invalid running started at") require.Nil(t, st.FinishedAt(), "invalid init finished at") expectedShardIDs := make([]string, 0, 2) @@ -385,7 +385,7 @@ func TestEvacuateAsync(t *testing.T) { }, 3*time.Second, 10*time.Millisecond, "invalid final state") require.NoError(t, err, "get final state failed") - require.Equal(t, uint64(3), st.Evacuated(), "invalid final count") + require.Equal(t, uint64(3), st.ObjectsEvacuated(), "invalid final count") require.NotNil(t, st.StartedAt(), "invalid final started at") require.NotNil(t, st.FinishedAt(), "invalid final finished at") require.ElementsMatch(t, expectedShardIDs, st.ShardIDs(), "invalid final shard ids") diff --git a/pkg/services/control/server/convert.go b/pkg/services/control/server/convert.go index f922d727..4f313235 100644 --- a/pkg/services/control/server/convert.go +++ b/pkg/services/control/server/convert.go @@ -47,15 +47,18 @@ func stateToResponse(state *engine.EvacuationState) (*control.GetShardEvacuation } return &control.GetShardEvacuationStatusResponse{ Body: &control.GetShardEvacuationStatusResponse_Body{ - Shard_ID: shardIDs, - Evacuated: state.Evacuated(), - Total: state.Total(), - Failed: state.Failed(), - Status: evacStatus, - StartedAt: startedAt, - Duration: duration, - ErrorMessage: state.ErrorMessage(), - Skipped: state.Skipped(), + Shard_ID: shardIDs, + EvacuatedObjects: state.ObjectsEvacuated(), + TotalObjects: state.ObjectsTotal(), + FailedObjects: state.ObjectsFailed(), + Status: evacStatus, + StartedAt: startedAt, + Duration: duration, + ErrorMessage: state.ErrorMessage(), + SkippedObjects: state.ObjectsSkipped(), + TotalTrees: state.TreesTotal(), + EvacuatedTrees: state.TreesEvacuated(), + FailedTrees: state.TreesFailed(), }, }, nil } diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index 6cba72d7..ac8b3d54 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -26,10 +26,10 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe } prm := engine.EvacuateShardPrm{ - ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), - IgnoreErrors: req.GetBody().GetIgnoreErrors(), - Handler: s.replicate, - Scope: engine.EvacuateScopeObjects, + ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), + IgnoreErrors: req.GetBody().GetIgnoreErrors(), + ObjectsHandler: s.replicate, + Scope: engine.EvacuateScopeObjects, } res, err := s.s.Evacuate(ctx, prm) @@ -39,7 +39,7 @@ func (s *Server) EvacuateShard(ctx context.Context, req *control.EvacuateShardRe resp := &control.EvacuateShardResponse{ Body: &control.EvacuateShardResponse_Body{ - Count: uint32(res.Evacuated()), + Count: uint32(res.ObjectsEvacuated()), }, } diff --git a/pkg/services/control/server/evacuate_async.go b/pkg/services/control/server/evacuate_async.go index 0b285127..91f0731c 100644 --- a/pkg/services/control/server/evacuate_async.go +++ b/pkg/services/control/server/evacuate_async.go @@ -22,11 +22,11 @@ func (s *Server) StartShardEvacuation(ctx context.Context, req *control.StartSha } prm := engine.EvacuateShardPrm{ - ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), - IgnoreErrors: req.GetBody().GetIgnoreErrors(), - Handler: s.replicate, - Async: true, - Scope: engine.EvacuateScope(req.GetBody().GetScope()), + ShardID: s.getShardIDList(req.GetBody().GetShard_ID()), + IgnoreErrors: req.GetBody().GetIgnoreErrors(), + ObjectsHandler: s.replicate, + Async: true, + Scope: engine.EvacuateScope(req.GetBody().GetScope()), } _, err = s.s.Evacuate(ctx, prm) diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 9c1a8741561d3a4412aca26b8a4b7548b4c31c19..4e95ac61734545f94c827b2f6b849d6e578094f3 100644 GIT binary patch delta 6594 zcmb_hYgAO{)m~@I9b^V11_9xKH$;S);Vw~zG-_Hkl3*el1qK)zuM7!TBo^_KimRrM zO_^;16^vSws1Yh=G(p8!W12`EYBe@lE7Gq1Xur?c_@_;3Bz@lZ%o(6DU8`OF!t*W!HbJT52Dh~-xPgj=+PpWm#+U?J&bv3GPlUi9*zrHRlBv_s=k;nC`*4Jfc zs^tz|-J}=`&8E~EM_uZA+d8}E{4&X9L^R~J`^G|2%19*GtGv>6iF0+Y%j+`eWHxhQ zqcSdcqzGNrs{4uY9cnj3WTGcXsXoi*$q)<+`HtB#Y>+nw^2W%B-A~pCTE3I4#Jsd` zulkZR#NR#Gq@?=!B6n|_GBr%PQ^)hfieX~juuTacMQijq;(qjz)7AO*I?V?C91>D} zEJiYSKeJt#GmIgjNqF%n)bL1hgXBFNc@`H!Mm&!smWumpZj_9DYHROC(i%r7?Yohx zU3OpHq1^o-C^s)vH8q%W+}EF=17m{hcKMjaTb`nHZZlD^zc{Yqvrkb{XjZj2&8|)j z3FcWwn#%7uX&euZp)l7{3X7@~r#YEE8u7qtEsf_jJ@f!C9!mz^7)TRBvLs=)VAH*T z5?zJl&o`G+02i#JKsKMC6u(??XpW>+CMu}oEJn;Y7}6EGmGSD zvWed&J3m%J(*wQgN{0NWWSSZf5-g3uX@amTJCkialpr+=)%ck61mi6ww1`jbpo#qG z5mNc{N63#2%kb;y1PbO)mgCRqL-Zh*ETddbBZ}m*%OGy4 zeES@k*%(Ygoc=b&ar1J@;F}6X5=32EPK!dJOOECg3tKPHB!2j5ki52qGI?_!9OUI` z;MwyKmcO1O6R$JUWWi}N?_L3QdTmf5W-hqDw*prOZ4?j=u2~h*AvR&u*p;-X^`{_P zY$qdcT1f@`+4E4gWdW@aJcHQrCN%CK3gylha9Z>#h49OzG?xQxpqn%S!H!)CiIdAH zHbgkON(;-r?@~N}zl`#9_WtZHqwjG`85yH2kd64{coh-Xs=2wG(tOd5;pQGP2AOju zNnYz7dQjj=u?w;(GH4?s#jCn0EXW(`3W3ZCVM0W<#9vl5HziYCcut1k=H+SAd8!g1 zz^J?+;p_#hS(;MGL#N?G%Vmlj>y@Bkp3~>ij8Q-h;H}9NJz7o$t5%ARlqrxUacPnX zHH=&h=k2Fx0_9}zGy_mmTtYFNy^;({4*&erzp|{l|;YmZHkHUS}Cm> zl>$}VR!TDjz_!ezs8$D(E7(S{oNyRmNwZO+I2ss(H*0JVpJPLow%aI5yzaJ97W=p1 z%{{!~$97VPlFeOC1X<*|xDm#;a`C6M0_TG_kRA0E8m7my=Mb`4R1Tr3S=Nf*RZy0q za%d$@7yPp-<>OlKqf2}2_dfnzdrYg+h)b#{OUHk#xV^s@d1#u8j0&|=INzF-k1wEeeH0R;OZyrab1AOqInc8)KP-T;#QB*s|wE|6gmxue6QeGd2@43@%FK-FOr-1{+X<%4-nTIS#lOmY?S~425!l{T0CW5phiIRo6d0e_1VOmf}soZ>53Ff9;5P!K5 z;zBpU&eJZMz(?<428GfQyu496h~QYpcf160l=u{0{yaS1zX{A8Gx7LYEut%3 zj7lEK&J8rv^&-4C^dX8ngcx{m3B?P~MjQgL)hAJSgLWXso;W-@XQBh6GvQeIN?GPw zk5KJ?g=VsKE5(wzQuOIUe#${pc)>y#`Y(Zbw+X@W{1$l;@&P1uY@m1uFBW#Gen`(u zHa<*C6bnc6&}{Y$p#1BEVH{pdPjLBfkt@0fOl-|3KlkOa$X|YXD-dIvgL0xHE0!;A zguh;Ig~dP?s)fi%&rBqv(FJ1wFKK9zi!6Q?lt^cEbwdnXJ^8-Q-=I$T12r@8v!kP&63>&Li}^e9p(dnqFSBQZ`i z>>`7Rb>c2C-@1bmLo>AoA?w9f@pj0gCE?>Ol+Ov<(Qb2A04jYw6sTlyS26SyQQ*oN zn&w(Ueo8eb5`tuS6K*UyIYPHm{#vLsI38)Rc}Pigm&MUig&%WLhL||K{Vj)^TPZ;( zb<7Fz-z*8qemMevvM9jfU~b!kKgwcc&f9zNr>GEEw|da;QMUq^n-O$9dsN=|Q;OzA z$AB!E4hXC&hNU>(w-;&tKP-E~g1lB(&{qKOytox0{Lczq0DtlZEJ5jz&GC#2yo%ys zDr$y&6VG^aY%iUV!QkV?Yjwp92 zeD(EZY; zY(BX<2m2@};hTy$Zy z+Xj>=-6_-fawm!(#%lwYrh$0haxmCUu(}deS_EqQAy|kZUJg7_{-WpRaKt<~5Pm@^ z5rYz{ak!wpT>^W8+E8$I{eohooJol6!Zs>-{Ti0RL z+t+9oZ@!LLqGH91=_1SxysX4@R7c-oXD-6@`62jSJ)-R?kDKTbZa9Lj1A|__hA8R6 zWIaye8Hwmz`pASK@hAXk6(0NSI&E~fM_@uSW=Op&1h$lXS+dMu7Dp(Z-nq= zJolXh=WVBfo^!k5X&_~oZnE&no^kNK;{q-Pp3oqYW+cmKy%+T$->?VIVrv(uhgD86 zVy;myVx$37$0|0y;{lR-pN3$=N!;mc2lU`jr-0;Gik$AS z!b}_OT@AQtG)6DJ{CL}9 zB-puyXn6B40DZ&oc=HTcNdvGe1oMln_*33X55QGneEn%mYFIN=(U8aUD`(*w=Q+Ry zJ4%TFWPra#`|44I^#Ui+oO1@hu$jkr@dX?R6n-jtj6PEOJ`M~oNBZXmFZAc1hi!Aj zrUMnM+<$&lP`+hXGVnuU6z&mAlp_WCIR^@gohpi7&V~B;!jZuqv7O*UM2e2ufnOm* zZ_dKoKVJ}1PKuQz8+>~6V7zb?puOS*JuKd;yeA9pLyo5Mf{XA_cOmfKbP?4lS@>EG zG{x_U0_IJ?0BsVHvF)FYcIGL>wI`MXF^F9tPx&=qyzP?aFY^l+be&?=0oXIxv;@Df zYDr%XpN(sW#?q^=eW2yGk=MTu|LSX-?BP>I0Fr)pW$9^5b->_ z55UV7V_Lr0Fe1bjAlH-KQtvTX8M)&_t&ZPW3$=?jYuiuCWr&nT6tj7_Z&GgddBmS{ z`ZdfYi}HXE?cO$Q{0(gGYe(cd_9Djr zZl?3rLZDEBeY&`DUgm#Dj`qz1yC;9oZp6WJdOxDsW&6PNs6nZPZj z$}=3bU2zkyA)F(g#F%Ebv?oVEKhY zDHi!==IBzI!QZ_?!#UMVzC6EzwA`6STE)VDeT9PAc{dGlvV^+3anKwZ$;ETYiQ`@+ z5B^sTjgT{)BE;8YuhNtVqe<*)w937c<*$6Us=r1tVinWzCq=$UH>_r!w1g%&3;(RV zbP0{n$SG#tUqFFeJdY-FTrj!2o5dQFLD*}VNAo!$k$ea*`gFJ+=S)64TthBgl}IMu zJDVo6?izV>Y!i*>AA%nri>DAi)JBt7|14r(Tt|brAd&ny&XY6*FN~=)n49CV+?+%K z!bn>(MjCTrWNi{z`QMFX>DxbmlSYy=e|&+WI4OnPxGRDDE0f5bzs{xSyiIAsakF$k zT!QJn4EAFekcO3I6y`6&Rl&0?6H^T;c*(+zNxUl!L6s~+j58}hiG3MO=gb7^m|)o|)A2QN2tRd^~uK#_=Co0OO=|3KDq5rDNpu z91v*B!0Oz18psXl6r$+4Bb_W_%Kl?CltVMfixV{fZB_T65y>qGT5I8*(akV)e_`j9myv))FV_{v~%<<4qg z*1Q?Khox|?k)Gz-w@JravnYVe11N}-g1}SEMYwt3C0c5K`7j+&c-<;8u+~O{L?C6E z6wlpOgc-4bW^>ayENH_w4veH2zB>n-6h;CG$4(J^B@ZGG}io5~;uqte(vW^5@$T`2BnuB$Q`NWhD>hw{M{r9s4@* zquDe{4QJyQt+?m(%{)$qK0ENShxU8zJN^Kza(7~9lu!}RVF zOfN1+X}h+U7O+hNB6ZI2rmBSWoLvP=pKhZGe5{71@ZAD@%@2iI0HT)5%@BS+hdqVS zlN+$P?J9^WucXMI0M*R~@E;>-ICeD1*t8L^w$T)&EaO}siWKhGi<%V1?K&V)djb}d z1Y_34K;7^TJ;M_|$Lg&IfLOO4{2SiH)4CvJSNSmHRO=LQVmOFcvP((2Qj`y0YfJI% z>>Lmdul0Pk4B&oO3Q9Nl#bc%>08ONIlt09&sn5n=$1mw6N(#yN%1W${f) z;PTC6K;8}Fiz(!8&wiN}DO@Vh6z@EF2<*MR7p@gofy`ZNUL9JXnK zzq&UO@X~y&ZLY$yCUJ<}e-pWvP(s`Ilz4J{^%A&U-45?&-2x<6iU3glPMYnIjSt7p zK*Bf}p33zz;M~HOXev*sR!?W;En397D=_pMh0%Von7&guVV_!1>JP%&{aPH94&EG9 z#YsRo&ciHw863pIwVzlFL{Jgc5M|LV+A4(la|X#|;G8nl^g$Vj;6g2!Y@R@)x$Gbf z6(PnRM9@&(5dQN))eWTst;&e$;2lSiUh)t7& z7)a>~KYRx{j7J^sJAue`!6^N)W=c@_=z9n++69Wx9tLqMzJXw4kEn%OMsxTGCOim) zuP_kGLod^dBAR|`?)(7k|FpgDQGDK02bIn|3gS^DoH-TViDSUbf4M_1h)4$UzNM7O zJL^b~!})j}YOg%J|5FEpBZ9FI9{TgjgUB{V0kt@z=KwzM9s|1fyO1)IXQ|4nJBFMW zBZS&BH61cPTn*LlvOuyXG>TFW=vDVuBPnI0(|lZw^5Svq^`suAdKzGK;bEkyIOI8H z1XVbr7lXa~^_YXtV+F9DBFS`oJP(Sk4JVDgcruMq$YfM0&6x+7(B(Pti)K-dPLj5A z8VIg2gKF9R@k!a-G@V4q&)cw7<6yNwjTh4JReOOw4Zsw=z%VXqgEQUrzz-hy+uL*L zhQjwxkrcz@Q}7IO^J1%=#)#ZWv}0tzX$m8wiO;qoGkWZ#7hm)}$_(ymgu$sFBMRC0 z;+u)n3C)M>MaGN|fmyzeZ0d@=xWAeoxpyyQc~9hXaf?h&$Smu1#47P%DLf+TF7 z2*4n|KozxhpOBlpQl;`$D;hi3&(IxpV31e(;E;Ub98Ina2kbE>$fGMByHtG&@AG{q z%-P)k#^;z9C`zMipADA*Ve+ORuK`9gd!K_D;4qIpGe}c84!ug&QV6wvJ2K|)aY8kB z!A-|`1dOg%+ow8jzVEgnx@vA8GQh=bnboz3YcO&8Mmei>4u`8?+9mm*4=jr_&%zIadWR5Q0NA&$Vj!6bEaMYb2 zUI26dlgiQ4<5K@8e)lVhNCef@Z?DR@oE?QRwS4}zbunc?Up>lWD51(Gp($hfg*V1 zGU7w0U=g(`gl|P+4B~fDa3c9oIG0?;fUXTWDcp0$tsYo*w815P9`u51$#4$c4%+iXZwTy4IflR+mC=-Tjl<17xF^pxm z zP#n-lYFPU!4Cj0WM`dx|cOBIX0m;(VujLF9$AmX8zAh7X4A)!-X#d_2++-CoMsSHg z1dIkx&y7`(c;FsDftyDR+rREjlqa{*2%z_r3bL*LNVV0}lbe$CuzlWCyB9zm`BYH@)|4kx%^SS5suy&|A1rlYRKc zH%OsfohW7~^j^I7G*~TviJY8%z%!f6t=+x4pvw0YM>QdewTvKiJZKv{v+Vq wyYjyVzFq3;+%AxH<1V_(ZC&co^nnqyKd*pWJzp;>kY5)KL diff --git a/pkg/services/control/service.proto b/pkg/services/control/service.proto index 73e8ca90..16e0f707 100644 --- a/pkg/services/control/service.proto +++ b/pkg/services/control/service.proto @@ -394,11 +394,11 @@ message GetShardEvacuationStatusResponse { } // Total objects to evacuate count. The value is approximate, so evacuated + failed + skipped == total is not guaranteed after completion. - uint64 total = 1; + uint64 total_objects = 1; // Evacuated objects count. - uint64 evacuated = 2; + uint64 evacuated_objects = 2; // Failed objects count. - uint64 failed = 3; + uint64 failed_objects = 3; // Shard IDs. repeated bytes shard_ID = 4; @@ -412,7 +412,14 @@ message GetShardEvacuationStatusResponse { string error_message = 8; // Skipped objects count. - uint64 skipped = 9; + uint64 skipped_objects = 9; + + // Total trees to evacuate count. + uint64 total_trees = 10; + // Evacuated trees count. + uint64 evacuated_trees = 11; + // Failed trees count. + uint64 failed_trees = 12; } Body body = 1; diff --git a/pkg/services/control/service_frostfs.pb.go b/pkg/services/control/service_frostfs.pb.go index b18d30dba167296cd71e7ba491163df66e94c18a..37527aa8c7d92f5fc33366940cb3e11cfa99738c 100644 GIT binary patch delta 327 zcmca{lI`{xwhe2ZviT=vr6!k5p1)6Q(sTC7C!X@5iU=F)C{*aVC1&QNra+VyYfeto zR+_Bv>@%uLenSJ0(vbX;#GH_#)KsXE!iI(*Vb`+6>i;UMwi&4KrJ zGEVOIo;-O2o51A!mul=#J0}YY2?P0jlXs>H3&71nmsHCY=0$NTNZV%lSEY<-=I}Eb qOs;z=&I+~^YBvAm#Gk@^aI;aRCT delta 93 zcmV-j0HXif%m&!W2C%HOGS|>LHV+>KT&}5)+e;mld<_>Rt6v(Cv%3hj=cH