From 3a797e46826e82575bb572192a35fe2afc439813 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Wed, 3 Jul 2024 11:47:50 +0300 Subject: [PATCH] [#1222] engine: Fix tree evacuation Do not fail if it is unable to evacuate tree to other node. Signed-off-by: Dmitrii Stepanov --- pkg/local_object_storage/engine/evacuate.go | 32 +++++++++++++------ .../engine/evacuate_test.go | 4 +-- pkg/services/control/server/evacuate.go | 10 +++--- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/pkg/local_object_storage/engine/evacuate.go b/pkg/local_object_storage/engine/evacuate.go index 0ce59a0be..04e427e49 100644 --- a/pkg/local_object_storage/engine/evacuate.go +++ b/pkg/local_object_storage/engine/evacuate.go @@ -75,7 +75,7 @@ func (s EvacuateScope) TreesOnly() bool { type EvacuateShardPrm struct { ShardID []*shard.ID ObjectsHandler func(context.Context, oid.Address, *objectSDK.Object) (bool, error) - TreeHandler func(context.Context, cid.ID, string, pilorama.Forest) (string, error) + TreeHandler func(context.Context, cid.ID, string, pilorama.Forest) (bool, string, error) IgnoreErrors bool Async bool Scope EvacuateScope @@ -450,7 +450,7 @@ func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, tree continue } - nodePK, err := e.evacuateTreeToOtherNode(ctx, sh, contTree, prm) + moved, nodePK, err := e.evacuateTreeToOtherNode(ctx, sh, contTree, prm) if err != nil { e.log.Error(logs.EngineShardsEvacuationFailedToMoveTree, zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), @@ -458,18 +458,32 @@ func (e *StorageEngine) evacuateTrees(ctx context.Context, sh *shard.Shard, tree zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) return err } - e.log.Debug(logs.EngineShardsEvacuationTreeEvacuatedRemote, - zap.String("cid", contTree.CID.EncodeToString()), zap.String("treeID", contTree.TreeID), - zap.String("from_shardID", sh.ID().String()), zap.String("to_node", nodePK), - evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) - res.trEvacuated.Add(1) + if moved { + e.log.Debug(logs.EngineShardsEvacuationTreeEvacuatedRemote, + zap.String("cid", contTree.CID.EncodeToString()), zap.String("treeID", contTree.TreeID), + zap.String("from_shardID", sh.ID().String()), zap.String("to_node", nodePK), + evacuationOperationLogField, zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + res.trEvacuated.Add(1) + } else if prm.IgnoreErrors { + res.trFailed.Add(1) + e.log.Warn(logs.EngineShardsEvacuationFailedToMoveTree, + zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), + zap.String("from_shard_id", sh.ID().String()), evacuationOperationLogField, + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + } else { + e.log.Error(logs.EngineShardsEvacuationFailedToMoveTree, + zap.String("cid", contTree.CID.EncodeToString()), zap.String("tree_id", contTree.TreeID), + zap.String("from_shard_id", sh.ID().String()), evacuationOperationLogField, + zap.Error(err), zap.String("trace_id", tracingPkg.GetTraceID(ctx))) + return fmt.Errorf("no remote nodes available to replicate tree '%s' of container %s", contTree.TreeID, contTree.CID) + } } return nil } -func (e *StorageEngine) evacuateTreeToOtherNode(ctx context.Context, sh *shard.Shard, tree pilorama.ContainerIDTreeID, prm EvacuateShardPrm) (string, error) { +func (e *StorageEngine) evacuateTreeToOtherNode(ctx context.Context, sh *shard.Shard, tree pilorama.ContainerIDTreeID, prm EvacuateShardPrm) (bool, string, error) { if prm.TreeHandler == nil { - return "", fmt.Errorf("failed to evacuate tree '%s' for container %s from shard %s: local evacuation failed, but no remote evacuation available", tree.TreeID, tree.CID, sh.ID()) + return false, "", fmt.Errorf("failed to evacuate tree '%s' for container %s from shard %s: local evacuation failed, but no remote evacuation available", tree.TreeID, tree.CID, sh.ID()) } return prm.TreeHandler(ctx, tree.CID, tree.TreeID, sh) diff --git a/pkg/local_object_storage/engine/evacuate_test.go b/pkg/local_object_storage/engine/evacuate_test.go index f097665bf..55268b549 100644 --- a/pkg/local_object_storage/engine/evacuate_test.go +++ b/pkg/local_object_storage/engine/evacuate_test.go @@ -507,7 +507,7 @@ func TestEvacuateTreesRemote(t *testing.T) { var prm EvacuateShardPrm prm.ShardID = ids prm.Scope = EvacuateScopeTrees - prm.TreeHandler = func(ctx context.Context, contID cid.ID, treeID string, f pilorama.Forest) (string, error) { + prm.TreeHandler = func(ctx context.Context, contID cid.ID, treeID string, f pilorama.Forest) (bool, string, error) { key := contID.String() + treeID var height uint64 for { @@ -515,7 +515,7 @@ func TestEvacuateTreesRemote(t *testing.T) { require.NoError(t, err) if op.Time == 0 { - return "", nil + return true, "", nil } evacuatedTreeOps[key] = append(evacuatedTreeOps[key], &op) height = op.Time + 1 diff --git a/pkg/services/control/server/evacuate.go b/pkg/services/control/server/evacuate.go index dd609caec..0ba8be765 100644 --- a/pkg/services/control/server/evacuate.go +++ b/pkg/services/control/server/evacuate.go @@ -89,22 +89,22 @@ func (s *Server) replicateObject(ctx context.Context, addr oid.Address, obj *obj return true, nil } -func (s *Server) replicateTree(ctx context.Context, contID cid.ID, treeID string, forest pilorama.Forest) (string, error) { +func (s *Server) replicateTree(ctx context.Context, contID cid.ID, treeID string, forest pilorama.Forest) (bool, string, error) { nodes, err := s.getContainerNodes(contID) if err != nil { - return "", err + return false, "", err } if len(nodes) == 0 { - return "", fmt.Errorf("no remote nodes available to replicate tree '%s' of container %s", treeID, contID) + return false, "", nil } for _, node := range nodes { err = s.replicateTreeToNode(ctx, forest, contID, treeID, node) if err == nil { - return hex.EncodeToString(node.PublicKey()), nil + return true, hex.EncodeToString(node.PublicKey()), nil } } - return "", err + return false, "", err } func (s *Server) replicateTreeToNode(ctx context.Context, forest pilorama.Forest, contID cid.ID, treeID string, node netmap.NodeInfo) error {