[#1731] services/control: Replicate object over network in EvacuateShard RPC

Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>

CHANGELOG.md

@@ -9,6 +9,8 @@ Changelog for NeoFS Node
 
 ### Changed
 
+- Allow to evacuate shard data with `EvacuateShard` control RPC (#1800)
+
 ### Fixed
 
 ### Removed

cmd/neofs-node/config.go

@@ -43,6 +43,7 @@ import (
 	getsvc "github.com/nspcc-dev/neofs-node/pkg/services/object/get"
 	"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/tombstone"
 	tsourse "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/tombstone/source"
+	"github.com/nspcc-dev/neofs-node/pkg/services/replicator"
 	trustcontroller "github.com/nspcc-dev/neofs-node/pkg/services/reputation/local/controller"
 	truststorage "github.com/nspcc-dev/neofs-node/pkg/services/reputation/local/storage"
 	"github.com/nspcc-dev/neofs-node/pkg/services/tree"
@@ -116,6 +117,8 @@ type cfg struct {
 
 	respSvc *response.Service
 
+	replicator *replicator.Replicator
+
 	cfgControlService cfgControlService
 
 	treeService *tree.Service

cmd/neofs-node/control.go

@@ -30,6 +30,8 @@ func initControlService(c *cfg) {
 		controlSvc.WithAuthorizedKeys(rawPubs),
 		controlSvc.WithHealthChecker(c),
 		controlSvc.WithNetMapSource(c.netMapSource),
+		controlSvc.WithContainerSource(c.cfgObject.cnrSource),
+		controlSvc.WithReplicator(c.replicator),
 		controlSvc.WithNodeState(c),
 		controlSvc.WithLocalStorage(c.cfgObject.cfgLocalStorage.localStorage),
 		controlSvc.WithTreeService(c.treeService),

cmd/neofs-node/object.go

@@ -207,7 +207,7 @@ func initObjectService(c *cfg) {
 		log:     c.log,
 	}
 
-	repl := replicator.New(
+	c.replicator = replicator.New(
 		replicator.WithLogger(c.log),
 		replicator.WithPutTimeout(
 			replicatorconfig.PutTimeout(c.appCfg),
@@ -232,7 +232,7 @@ func initObjectService(c *cfg) {
 		policer.WithHeadTimeout(
 			policerconfig.HeadTimeout(c.appCfg),
 		),
-		policer.WithReplicator(repl),
+		policer.WithReplicator(c.replicator),
 		policer.WithRedundantCopyCallback(func(addr oid.Address) {
 			var inhumePrm engine.InhumePrm
 			inhumePrm.MarkAsGarbage(addr)

pkg/services/control/server/evacuate.go

@@ -1,11 +1,20 @@
 package control
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha256"
+	"errors"
+	"fmt"
 
+	"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
 	"github.com/nspcc-dev/neofs-node/pkg/services/control"
+	"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/placement"
+	"github.com/nspcc-dev/neofs-node/pkg/services/replicator"
+	objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
+	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -21,6 +30,7 @@ func (s *Server) EvacuateShard(_ context.Context, req *control.EvacuateShardRequ
 	var prm engine.EvacuateShardPrm
 	prm.WithShardID(shardID)
 	prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors())
+	prm.WithFaultHandler(s.replicate)
 
 	res, err := s.s.Evacuate(prm)
 	if err != nil {
@@ -39,3 +49,61 @@ func (s *Server) EvacuateShard(_ context.Context, req *control.EvacuateShardRequ
 	}
 	return resp, nil
 }
+
+func (s *Server) replicate(addr oid.Address, obj *objectSDK.Object) error {
+	cid, ok := obj.ContainerID()
+	if !ok {
+		// Return nil to prevent situations where a shard can't be evacuated
+		// because of a single bad/corrupted object.
+		return nil
+	}
+
+	nm, err := s.netMapSrc.GetNetMap(0)
+	if err != nil {
+		return err
+	}
+
+	c, err := s.cnrSrc.Get(cid)
+	if err != nil {
+		return err
+	}
+
+	binCnr := make([]byte, sha256.Size)
+	cid.Encode(binCnr)
+
+	ns, err := nm.ContainerNodes(c.Value.PlacementPolicy(), binCnr)
+	if err != nil {
+		return fmt.Errorf("can't build a list of container nodes")
+	}
+
+	nodes := placement.FlattenNodes(ns)
+	bs := (*keys.PublicKey)(&s.key.PublicKey).Bytes()
+	for i := 0; i < len(nodes); i++ {
+		if bytes.Equal(nodes[i].PublicKey(), bs) {
+			copy(nodes[i:], nodes[i+1:])
+			nodes = nodes[:len(nodes)-1]
+		}
+	}
+
+	var res replicatorResult
+	task := new(replicator.Task).
+		WithObject(obj).
+		WithObjectAddress(addr).
+		WithCopiesNumber(1).
+		WithNodes(nodes)
+	s.replicator.HandleTask(context.TODO(), task, &res)
+
+	if res.count == 0 {
+		return errors.New("object was not replicated")
+	}
+	return nil
+}
+
+type replicatorResult struct {
+	count int
+}
+
+// SubmitSuccessfulReplication implements the replicator.TaskResult interface.
+func (r *replicatorResult) SubmitSuccessfulReplication(_ uint64) {
+	r.count++
+}

pkg/services/control/server/server.go

@@ -3,9 +3,11 @@ package control
 import (
 	"crypto/ecdsa"
 
+	"github.com/nspcc-dev/neofs-node/pkg/core/container"
 	"github.com/nspcc-dev/neofs-node/pkg/core/netmap"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
 	"github.com/nspcc-dev/neofs-node/pkg/services/control"
+	"github.com/nspcc-dev/neofs-node/pkg/services/replicator"
 )
 
 // Server is an entity that serves
@@ -47,6 +49,10 @@ type cfg struct {
 
 	netMapSrc netmap.Source
 
+	cnrSrc container.Source
+
+	replicator *replicator.Replicator
+
 	nodeState NodeState
 
 	treeService TreeService
@@ -102,6 +108,20 @@ func WithNetMapSource(netMapSrc netmap.Source) Option {
 	}
 }
 
+// WithContainerSource returns option to set container storage.
+func WithContainerSource(cnrSrc container.Source) Option {
+	return func(c *cfg) {
+		c.cnrSrc = cnrSrc
+	}
+}
+
+// WithReplicator returns option to set network map storage.
+func WithReplicator(r *replicator.Replicator) Option {
+	return func(c *cfg) {
+		c.replicator = r
+	}
+}
+
 // WithNodeState returns option to set node network state component.
 func WithNodeState(state NodeState) Option {
 	return func(c *cfg) {

pkg/services/replicator/process.go

@@ -26,17 +26,20 @@ func (p *Replicator) HandleTask(ctx context.Context, task *Task, res TaskResult)
 		)
 	}()
 
-	obj, err := engine.Get(p.localStorage, task.addr)
-	if err != nil {
-		p.log.Error("could not get object from local storage",
-			zap.Stringer("object", task.addr),
-			zap.Error(err))
+	if task.obj == nil {
+		var err error
+		task.obj, err = engine.Get(p.localStorage, task.addr)
+		if err != nil {
+			p.log.Error("could not get object from local storage",
+				zap.Stringer("object", task.addr),
+				zap.Error(err))
 
-		return
+			return
+		}
 	}
 
 	prm := new(putsvc.RemotePutPrm).
-		WithObject(obj)
+		WithObject(task.obj)
 
 	for i := 0; task.quantity > 0 && i < len(task.nodes); i++ {
 		select {
@@ -52,7 +55,7 @@ func (p *Replicator) HandleTask(ctx context.Context, task *Task, res TaskResult)
 
 		callCtx, cancel := context.WithTimeout(ctx, p.putTimeout)
 
-		err = p.remoteSender.PutObject(callCtx, prm.WithNodeInfo(task.nodes[i]))
+		err := p.remoteSender.PutObject(callCtx, prm.WithNodeInfo(task.nodes[i]))
 
 		cancel()
 

pkg/services/replicator/task.go

@@ -2,6 +2,7 @@ package replicator
 
 import (
 	"github.com/nspcc-dev/neofs-sdk-go/netmap"
+	objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 )
 
@@ -11,6 +12,8 @@ type Task struct {
 
 	addr oid.Address
 
+	obj *objectSDK.Object
+
 	nodes []netmap.NodeInfo
 }
 
@@ -32,6 +35,15 @@ func (t *Task) WithObjectAddress(v oid.Address) *Task {
 	return t
 }
 
+// WithObject sets object to avoid fetching it from the local storage.
+func (t *Task) WithObject(obj *objectSDK.Object) *Task {
+	if t != nil {
+		t.obj = obj
+	}
+
+	return t
+}
+
 // WithNodes sets a list of potential object holders.
 func (t *Task) WithNodes(v []netmap.NodeInfo) *Task {
 	if t != nil {