[#114] Add remove-node IR control command #310
10 changed files with 157 additions and 0 deletions
|
@ -10,6 +10,8 @@ var irCmd = &cobra.Command{
|
||||||
|
|
||||||
func initControlIRCmd() {
|
func initControlIRCmd() {
|
||||||
irCmd.AddCommand(tickEpochCmd)
|
irCmd.AddCommand(tickEpochCmd)
|
||||||
|
irCmd.AddCommand(removeNodeCmd)
|
||||||
|
|
||||||
initControlIRTickEpochCmd()
|
initControlIRTickEpochCmd()
|
||||||
|
initControlIRRemoveNodeCmd()
|
||||||
}
|
}
|
||||||
|
|
58
cmd/frostfs-cli/modules/control/ir_remove_node.go
Normal file
58
cmd/frostfs-cli/modules/control/ir_remove_node.go
Normal file
|
@ -0,0 +1,58 @@
|
||||||
|
package control
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client"
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key"
|
||||||
|
commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common"
|
||||||
|
ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
|
||||||
|
ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var removeNodeCmd = &cobra.Command{
|
||||||
|
Use: "remove-node",
|
||||||
|
Short: "Forces a node removal from netmap",
|
||||||
|
Long: "Forces a node removal from netmap via a notary request. It should be executed on other IR nodes as well.",
|
||||||
|
Run: removeNode,
|
||||||
|
}
|
||||||
|
|
||||||
|
func initControlIRRemoveNodeCmd() {
|
||||||
|
initControlFlags(removeNodeCmd)
|
||||||
|
|
||||||
|
flags := removeNodeCmd.Flags()
|
||||||
|
flags.String("node", "", "Node public key as a hex string")
|
||||||
|
_ = removeNodeCmd.MarkFlagRequired("node")
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeNode(cmd *cobra.Command, _ []string) {
|
||||||
|
pk := key.Get(cmd)
|
||||||
|
c := getClient(cmd, pk)
|
||||||
|
|
||||||
fyrchik marked this conversation as resolved
Outdated
|
|||||||
|
nodeKeyStr, _ := cmd.Flags().GetString("node")
|
||||||
|
if len(nodeKeyStr) == 0 {
|
||||||
|
commonCmd.ExitOnErr(cmd, "parsing node public key: ", errors.New("key cannot be empty"))
|
||||||
|
}
|
||||||
|
nodeKey, err := hex.DecodeString(nodeKeyStr)
|
||||||
|
commonCmd.ExitOnErr(cmd, "can't decode node public key: %w", err)
|
||||||
|
|
||||||
|
req := new(ircontrol.RemoveNodeRequest)
|
||||||
|
req.SetBody(&ircontrol.RemoveNodeRequest_Body{
|
||||||
|
Key: nodeKey,
|
||||||
|
})
|
||||||
|
|
||||||
|
commonCmd.ExitOnErr(cmd, "could not sign request: %w", ircontrolsrv.SignMessage(pk, req))
|
||||||
|
|
||||||
|
var resp *ircontrol.RemoveNodeResponse
|
||||||
|
err = c.ExecRaw(func(client *rawclient.Client) error {
|
||||||
|
resp, err = ircontrol.RemoveNode(client, req)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
commonCmd.ExitOnErr(cmd, "rpc error: %w", err)
|
||||||
|
|
||||||
|
verifyResponse(cmd, resp.GetSignature(), resp.GetBody())
|
||||||
|
|
||||||
|
cmd.Println("Node removed")
|
||||||
|
}
|
|
@ -41,3 +41,19 @@ func (c *Client) AddPeer(p AddPeerPrm) error {
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ForceRemovePeer marks the given peer as offline via a notary control transaction.
|
||||||
|
func (c *Client) ForceRemovePeer(nodeInfo netmap.NodeInfo) error {
|
||||||
|
if !c.client.WithNotary() {
|
||||||
|
return fmt.Errorf("peer can be forcefully removed only in notary environment")
|
||||||
|
}
|
||||||
|
|
||||||
|
prm := UpdatePeerPrm{}
|
||||||
dstepanov-yadro marked this conversation as resolved
Outdated
dstepanov-yadro
commented
NodeState = Offline ? NodeState = Offline ?
dstepanov-yadro
commented
Oh, my mistake:
Oh, my mistake:
```
if p.state == 0 {
p.state = netmap.NodeStateOffline
}
```
|
|||||||
|
prm.SetKey(nodeInfo.PublicKey())
|
||||||
|
prm.SetControlTX(true)
|
||||||
|
|
||||||
|
if err := c.UpdatePeerState(prm); err != nil {
|
||||||
|
return fmt.Errorf("updating peer state: %v", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -11,6 +11,7 @@ const serviceName = "ircontrol.ControlService"
|
||||||
const (
|
const (
|
||||||
rpcHealthCheck = "HealthCheck"
|
rpcHealthCheck = "HealthCheck"
|
||||||
rpcTickEpoch = "TickEpoch"
|
rpcTickEpoch = "TickEpoch"
|
||||||
|
rpcRemoveNode = "RemoveNode"
|
||||||
)
|
)
|
||||||
|
|
||||||
// HealthCheck executes ControlService.HealthCheck RPC.
|
// HealthCheck executes ControlService.HealthCheck RPC.
|
||||||
|
@ -31,6 +32,14 @@ func TickEpoch(
|
||||||
return sendUnary[TickEpochRequest, TickEpochResponse](cli, rpcTickEpoch, req, opts...)
|
return sendUnary[TickEpochRequest, TickEpochResponse](cli, rpcTickEpoch, req, opts...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func RemoveNode(
|
||||||
|
cli *client.Client,
|
||||||
|
req *RemoveNodeRequest,
|
||||||
|
opts ...client.CallOption,
|
||||||
|
) (*RemoveNodeResponse, error) {
|
||||||
|
return sendUnary[RemoveNodeRequest, RemoveNodeResponse](cli, rpcRemoveNode, req, opts...)
|
||||||
|
}
|
||||||
|
|
||||||
func sendUnary[I, O grpc.Message](cli *client.Client, rpcName string, req *I, opts ...client.CallOption) (*O, error) {
|
func sendUnary[I, O grpc.Message](cli *client.Client, rpcName string, req *I, opts ...client.CallOption) (*O, error) {
|
||||||
var resp O
|
var resp O
|
||||||
wResp := &responseWrapper[*O]{
|
wResp := &responseWrapper[*O]{
|
||||||
|
|
|
@ -1,9 +1,11 @@
|
||||||
package control
|
package control
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap"
|
||||||
control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
|
control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir"
|
||||||
"google.golang.org/grpc/codes"
|
"google.golang.org/grpc/codes"
|
||||||
"google.golang.org/grpc/status"
|
"google.golang.org/grpc/status"
|
||||||
|
@ -57,3 +59,43 @@ func (s *Server) TickEpoch(_ context.Context, req *control.TickEpochRequest) (*c
|
||||||
|
|
||||||
return resp, nil
|
return resp, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RemoveNode forces a node removal.
|
||||||
|
//
|
||||||
|
// If request is not signed with a key from white list, permission error returns.
|
||||||
|
func (s *Server) RemoveNode(_ context.Context, req *control.RemoveNodeRequest) (*control.RemoveNodeResponse, error) {
|
||||||
|
if err := s.isValidRequest(req); err != nil {
|
||||||
|
return nil, status.Error(codes.PermissionDenied, err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
resp := new(control.RemoveNodeResponse)
|
||||||
|
resp.SetBody(new(control.RemoveNodeResponse_Body))
|
||||||
|
|
||||||
|
nm, err := s.netmapClient.NetMap()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("getting netmap: %w", err)
|
||||||
|
}
|
||||||
|
var nodeInfo netmap.NodeInfo
|
||||||
|
for _, info := range nm.Nodes() {
|
||||||
|
if bytes.Equal(info.PublicKey(), req.GetBody().GetKey()) {
|
||||||
|
nodeInfo = info
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(nodeInfo.PublicKey()) == 0 {
|
||||||
dstepanov-yadro marked this conversation as resolved
Outdated
dstepanov-yadro
commented
Maybe validate node state too? What if state already Offline? Maybe validate node state too? What if state already Offline?
ale64bit
commented
would it appear in the list if it was offline? would it appear in the list if it was offline?
dstepanov-yadro
commented
I don't know) I don't know)
ale64bit
commented
Added the check just in case. Added the check just in case.
|
|||||||
|
return nil, status.Error(codes.NotFound, "no such node")
|
||||||
|
}
|
||||||
|
if nodeInfo.IsOffline() {
|
||||||
|
return nil, status.Error(codes.FailedPrecondition, "node is already offline")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.netmapClient.ForceRemovePeer(nodeInfo); err != nil {
|
||||||
|
return nil, fmt.Errorf("forcing node removal: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil {
|
||||||
|
return nil, status.Error(codes.Internal, err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
|
@ -32,3 +32,15 @@ func (x *TickEpochResponse) SetBody(v *TickEpochResponse_Body) {
|
||||||
x.Body = v
|
x.Body = v
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (x *RemoveNodeRequest) SetBody(v *RemoveNodeRequest_Body) {
|
||||||
|
if x != nil {
|
||||||
|
x.Body = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *RemoveNodeResponse) SetBody(v *RemoveNodeResponse_Body) {
|
||||||
|
if x != nil {
|
||||||
|
x.Body = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
BIN
pkg/services/control/ir/service.pb.go
generated
BIN
pkg/services/control/ir/service.pb.go
generated
Binary file not shown.
|
@ -12,6 +12,8 @@ service ControlService {
|
||||||
rpc HealthCheck (HealthCheckRequest) returns (HealthCheckResponse);
|
rpc HealthCheck (HealthCheckRequest) returns (HealthCheckResponse);
|
||||||
// Forces a new epoch to be signaled by the IR node with high probability.
|
// Forces a new epoch to be signaled by the IR node with high probability.
|
||||||
rpc TickEpoch (TickEpochRequest) returns (TickEpochResponse);
|
rpc TickEpoch (TickEpochRequest) returns (TickEpochResponse);
|
||||||
|
// Forces a node removal to be signaled by the IR node with high probability.
|
||||||
|
rpc RemoveNode (RemoveNodeRequest) returns (RemoveNodeResponse);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Health check request.
|
// Health check request.
|
||||||
|
@ -57,3 +59,19 @@ message TickEpochResponse {
|
||||||
Body body = 1;
|
Body body = 1;
|
||||||
Signature signature = 2;
|
Signature signature = 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
message RemoveNodeRequest {
|
||||||
|
message Body{
|
||||||
|
bytes key = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
Body body = 1;
|
||||||
|
Signature signature = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message RemoveNodeResponse {
|
||||||
|
message Body{}
|
||||||
|
|
||||||
|
Body body = 1;
|
||||||
|
Signature signature = 2;
|
||||||
|
}
|
||||||
|
|
BIN
pkg/services/control/ir/service_frostfs.pb.go
generated
BIN
pkg/services/control/ir/service_frostfs.pb.go
generated
Binary file not shown.
BIN
pkg/services/control/ir/service_grpc.pb.go
generated
BIN
pkg/services/control/ir/service_grpc.pb.go
generated
Binary file not shown.
Loading…
Reference in a new issue
Can we just make a flag required? I know, you still can provide empty value, but hey, control service should still return an error.
done