frostfs-node/cmd/neofs-cli/modules/control.go
Evgenii Stratonikov 6472a170eb [#1143] shard: Introduce explicit Degraded mode
`Degraded` mode is set automatically after error counter is over the
threshold. `ReadOnly` mode can still be set by an administrator.

Signed-off-by: Evgenii Stratonikov <evgeniy@nspcc.ru>
2022-03-31 15:33:22 +03:00

575 lines
14 KiB
Go

package cmd
import (
"crypto/ecdsa"
"crypto/tls"
"fmt"
"github.com/mr-tron/base58"
rawclient "github.com/nspcc-dev/neofs-api-go/v2/rpc/client"
"github.com/nspcc-dev/neofs-node/pkg/services/control"
ircontrol "github.com/nspcc-dev/neofs-node/pkg/services/control/ir"
ircontrolsrv "github.com/nspcc-dev/neofs-node/pkg/services/control/ir/server"
controlSvc "github.com/nspcc-dev/neofs-node/pkg/services/control/server"
"github.com/nspcc-dev/neofs-sdk-go/client"
addressSDK "github.com/nspcc-dev/neofs-sdk-go/object/address"
"github.com/nspcc-dev/neofs-sdk-go/util/signature"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
var controlCmd = &cobra.Command{
Use: "control",
Short: "Operations with storage node",
Long: `Operations with storage node`,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
ff := cmd.Flags()
_ = viper.BindPFlag(walletPath, ff.Lookup(walletPath))
_ = viper.BindPFlag(address, ff.Lookup(address))
_ = viper.BindPFlag(controlRPC, ff.Lookup(controlRPC))
},
}
var healthCheckCmd = &cobra.Command{
Use: "healthcheck",
Short: "Health check of the NeoFS node",
Long: "Health check of the NeoFS node. Checks storage node by default, use --ir flag to work with Inner Ring.",
Run: healthCheck,
}
var setNetmapStatusCmd = &cobra.Command{
Use: "set-status",
Short: "Set status of the storage node in NeoFS network map",
Long: "Set status of the storage node in NeoFS network map",
Run: setNetmapStatus,
}
var setShardModeCmd = &cobra.Command{
Use: "set-mode",
Short: "Set work mode of the shard",
Long: "Set work mode of the shard",
Run: setShardMode,
}
var listShardsCmd = &cobra.Command{
Use: "list",
Short: "List shards of the storage node",
Long: "List shards of the storage node",
Run: listShards,
}
var shardsCmd = &cobra.Command{
Use: "shards",
Short: "Operations with storage node's shards",
Long: "Operations with storage node's shards",
}
const (
netmapStatusFlag = "status"
netmapStatusOnline = "online"
netmapStatusOffline = "offline"
netmapStatusMaintenance = "maintenance"
shardModeFlag = "mode"
shardIDFlag = "id"
shardClearErrorsFlag = "clear-errors"
shardModeReadOnly = "read-only"
shardModeReadWrite = "read-write"
shardModeDegraded = "degraded"
)
const (
controlRPC = "endpoint"
controlRPCDefault = ""
controlRPCUsage = "remote node control address (as 'multiaddr' or '<host>:<port>')"
)
// control healthcheck flags
const (
healthcheckIRFlag = "ir"
)
// control healthcheck vars
var (
healthCheckIRVar bool
)
// control netmap vars
var (
netmapStatus string
)
// control shard vars
var (
shardMode string
shardID string
)
func initControlHealthCheckCmd() {
initCommonFlagsWithoutRPC(healthCheckCmd)
flags := healthCheckCmd.Flags()
flags.String(controlRPC, controlRPCDefault, controlRPCUsage)
flags.BoolVar(&healthCheckIRVar, healthcheckIRFlag, false, "Communicate with IR node")
}
func initControlSetShardModeCmd() {
initCommonFlagsWithoutRPC(setShardModeCmd)
flags := setShardModeCmd.Flags()
flags.String(controlRPC, controlRPCDefault, controlRPCUsage)
flags.StringVarP(&shardID, shardIDFlag, "", "", "ID of the shard in base58 encoding")
flags.StringVarP(&shardMode, shardModeFlag, "", "",
fmt.Sprintf("new shard mode keyword ('%s', '%s', '%s')",
shardModeReadWrite,
shardModeReadOnly,
shardModeDegraded,
),
)
flags.Bool(shardClearErrorsFlag, false, "Set shard error count to 0")
}
func initControlShardsListCmd() {
initCommonFlagsWithoutRPC(listShardsCmd)
flags := listShardsCmd.Flags()
flags.String(controlRPC, controlRPCDefault, controlRPCUsage)
}
func initControlSetNetmapStatusCmd() {
initCommonFlagsWithoutRPC(setNetmapStatusCmd)
flags := setNetmapStatusCmd.Flags()
flags.String(controlRPC, controlRPCDefault, controlRPCUsage)
flags.StringVarP(&netmapStatus, netmapStatusFlag, "", "",
fmt.Sprintf("new netmap status keyword ('%s', '%s', '%s')",
netmapStatusOnline,
netmapStatusOffline,
netmapStatusMaintenance,
),
)
_ = setNetmapStatusCmd.MarkFlagRequired(netmapStatusFlag)
}
func initControlDropObjectsCmd() {
initCommonFlagsWithoutRPC(dropObjectsCmd)
flags := dropObjectsCmd.Flags()
flags.String(controlRPC, controlRPCDefault, controlRPCUsage)
flags.StringSliceVarP(&dropObjectsList, dropObjectsFlag, "o", nil,
"List of object addresses to be removed in string format")
_ = dropObjectsCmd.MarkFlagRequired(dropObjectsFlag)
}
func initControlSnapshotCmd() {
initCommonFlagsWithoutRPC(snapshotCmd)
flags := snapshotCmd.Flags()
flags.String(controlRPC, controlRPCDefault, controlRPCUsage)
flags.BoolVar(&netmapSnapshotJSON, "json", false,
"print netmap structure in JSON format")
}
func init() {
rootCmd.AddCommand(controlCmd)
shardsCmd.AddCommand(listShardsCmd)
shardsCmd.AddCommand(setShardModeCmd)
shardsCmd.AddCommand(dumpShardCmd)
shardsCmd.AddCommand(restoreShardCmd)
controlCmd.AddCommand(
healthCheckCmd,
setNetmapStatusCmd,
dropObjectsCmd,
snapshotCmd,
shardsCmd,
)
initControlHealthCheckCmd()
initControlSetNetmapStatusCmd()
initControlDropObjectsCmd()
initControlSnapshotCmd()
initControlShardsListCmd()
initControlSetShardModeCmd()
initControlDumpShardCmd()
initControlRestoreShardCmd()
}
func healthCheck(cmd *cobra.Command, _ []string) {
key, err := getKeyNoGenerate()
exitOnErr(cmd, err)
cli, err := getControlSDKClient(key)
exitOnErr(cmd, err)
if healthCheckIRVar {
healthCheckIR(cmd, key, cli)
return
}
req := new(control.HealthCheckRequest)
req.SetBody(new(control.HealthCheckRequest_Body))
err = controlSvc.SignMessage(key, req)
exitOnErr(cmd, errf("could not sign message: %w", err))
var resp *control.HealthCheckResponse
err = cli.ExecRaw(func(client *rawclient.Client) error {
resp, err = control.HealthCheck(client, req)
return err
})
exitOnErr(cmd, errf("rpc error: %w", err))
sign := resp.GetSignature()
err = signature.VerifyDataWithSource(
resp,
func() ([]byte, []byte) {
return sign.GetKey(), sign.GetSign()
},
)
exitOnErr(cmd, errf("invalid response signature: %w", err))
cmd.Printf("Network status: %s\n", resp.GetBody().GetNetmapStatus())
cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus())
}
func healthCheckIR(cmd *cobra.Command, key *ecdsa.PrivateKey, c *client.Client) {
req := new(ircontrol.HealthCheckRequest)
req.SetBody(new(ircontrol.HealthCheckRequest_Body))
err := ircontrolsrv.SignMessage(key, req)
exitOnErr(cmd, errf("could not sign request: %w", err))
var resp *ircontrol.HealthCheckResponse
err = c.ExecRaw(func(client *rawclient.Client) error {
resp, err = ircontrol.HealthCheck(client, req)
return err
})
exitOnErr(cmd, errf("rpc error: %w", err))
sign := resp.GetSignature()
err = signature.VerifyDataWithSource(
resp,
func() ([]byte, []byte) {
return sign.GetKey(), sign.GetSign()
},
)
exitOnErr(cmd, errf("invalid response signature: %w", err))
cmd.Printf("Health status: %s\n", resp.GetBody().GetHealthStatus())
}
func setNetmapStatus(cmd *cobra.Command, _ []string) {
key, err := getKeyNoGenerate()
exitOnErr(cmd, err)
var status control.NetmapStatus
switch netmapStatus {
default:
exitOnErr(cmd, fmt.Errorf("unsupported status %s", netmapStatus))
case netmapStatusOnline:
status = control.NetmapStatus_ONLINE
case netmapStatusOffline:
status = control.NetmapStatus_OFFLINE
case netmapStatusMaintenance:
status = control.NetmapStatus_MAINTENANCE
}
req := new(control.SetNetmapStatusRequest)
body := new(control.SetNetmapStatusRequest_Body)
req.SetBody(body)
body.SetStatus(status)
err = controlSvc.SignMessage(key, req)
exitOnErr(cmd, errf("could not sign request: %w", err))
cli, err := getControlSDKClient(key)
exitOnErr(cmd, err)
var resp *control.SetNetmapStatusResponse
err = cli.ExecRaw(func(client *rawclient.Client) error {
resp, err = control.SetNetmapStatus(client, req)
return err
})
exitOnErr(cmd, errf("rpc error: %w", err))
sign := resp.GetSignature()
err = signature.VerifyDataWithSource(
resp,
func() ([]byte, []byte) {
return sign.GetKey(), sign.GetSign()
},
)
exitOnErr(cmd, errf("invalid response signature: %w", err))
cmd.Println("Network status update request successfully sent.")
}
const dropObjectsFlag = "objects"
var dropObjectsList []string
var dropObjectsCmd = &cobra.Command{
Use: "drop-objects",
Short: "Drop objects from the node's local storage",
Long: "Drop objects from the node's local storage",
Run: func(cmd *cobra.Command, args []string) {
key, err := getKeyNoGenerate()
exitOnErr(cmd, err)
binAddrList := make([][]byte, 0, len(dropObjectsList))
for i := range dropObjectsList {
a := addressSDK.NewAddress()
err := a.Parse(dropObjectsList[i])
if err != nil {
exitOnErr(cmd, fmt.Errorf("could not parse address #%d: %w", i, err))
}
binAddr, err := a.Marshal()
exitOnErr(cmd, errf("could not marshal the address: %w", err))
binAddrList = append(binAddrList, binAddr)
}
req := new(control.DropObjectsRequest)
body := new(control.DropObjectsRequest_Body)
req.SetBody(body)
body.SetAddressList(binAddrList)
err = controlSvc.SignMessage(key, req)
exitOnErr(cmd, errf("could not sign request: %w", err))
cli, err := getControlSDKClient(key)
exitOnErr(cmd, err)
var resp *control.DropObjectsResponse
err = cli.ExecRaw(func(client *rawclient.Client) error {
resp, err = control.DropObjects(client, req)
return err
})
exitOnErr(cmd, errf("rpc error: %w", err))
sign := resp.GetSignature()
err = signature.VerifyDataWithSource(
resp,
func() ([]byte, []byte) {
return sign.GetKey(), sign.GetSign()
},
)
exitOnErr(cmd, errf("invalid response signature: %w", err))
cmd.Println("Objects were successfully marked to be removed.")
},
}
var snapshotCmd = &cobra.Command{
Use: "netmap-snapshot",
Short: "Get network map snapshot",
Long: "Get network map snapshot",
Run: func(cmd *cobra.Command, args []string) {
key, err := getKeyNoGenerate()
exitOnErr(cmd, err)
req := new(control.NetmapSnapshotRequest)
req.SetBody(new(control.NetmapSnapshotRequest_Body))
err = controlSvc.SignMessage(key, req)
exitOnErr(cmd, errf("could not sign request: %w", err))
cli, err := getControlSDKClient(key)
exitOnErr(cmd, err)
var resp *control.NetmapSnapshotResponse
err = cli.ExecRaw(func(client *rawclient.Client) error {
resp, err = control.NetmapSnapshot(client, req)
return err
})
exitOnErr(cmd, errf("rpc error: %w", err))
sign := resp.GetSignature()
err = signature.VerifyDataWithSource(
resp,
func() ([]byte, []byte) {
return sign.GetKey(), sign.GetSign()
},
)
exitOnErr(cmd, errf("invalid response signature: %w", err))
prettyPrintNetmap(cmd, resp.GetBody().GetNetmap(), netmapSnapshotJSON)
},
}
func listShards(cmd *cobra.Command, _ []string) {
key, err := getKeyNoGenerate()
exitOnErr(cmd, err)
req := new(control.ListShardsRequest)
req.SetBody(new(control.ListShardsRequest_Body))
err = controlSvc.SignMessage(key, req)
exitOnErr(cmd, errf("could not sign request: %w", err))
cli, err := getControlSDKClient(key)
exitOnErr(cmd, err)
var resp *control.ListShardsResponse
err = cli.ExecRaw(func(client *rawclient.Client) error {
resp, err = control.ListShards(client, req)
return err
})
exitOnErr(cmd, errf("rpc error: %w", err))
sign := resp.GetSignature()
err = signature.VerifyDataWithSource(
resp,
func() ([]byte, []byte) {
return sign.GetKey(), sign.GetSign()
},
)
exitOnErr(cmd, errf("invalid response signature: %w", err))
prettyPrintShards(cmd, resp.GetBody().GetShards())
}
// getControlSDKClient is the same getSDKClient but with
// another RPC endpoint flag.
func getControlSDKClient(key *ecdsa.PrivateKey) (*client.Client, error) {
var (
c client.Client
prmInit client.PrmInit
prmDial client.PrmDial
)
netAddr, err := getEndpointAddress(controlRPC)
if err != nil {
return nil, err
}
prmInit.SetDefaultPrivateKey(*key)
prmDial.SetServerURI(netAddr.HostAddr())
if netAddr.TLSEnabled() {
prmDial.SetTLSConfig(&tls.Config{})
}
c.Init(prmInit)
return &c, c.Dial(prmDial)
}
func prettyPrintShards(cmd *cobra.Command, ii []*control.ShardInfo) {
for _, i := range ii {
var mode string
switch i.GetMode() {
case control.ShardMode_READ_WRITE:
mode = "read-write"
case control.ShardMode_READ_ONLY:
mode = "read-only"
case control.ShardMode_DEGRADED:
mode = "degraded"
default:
mode = "unknown"
}
pathPrinter := func(name, path string) string {
if path == "" {
return ""
}
return fmt.Sprintf("%s: %s\n", name, path)
}
cmd.Printf("Shard %s:\nMode: %s\n"+
pathPrinter("Metabase", i.GetMetabasePath())+
pathPrinter("Blobstor", i.GetBlobstorPath())+
pathPrinter("Write-cache", i.GetWritecachePath())+
fmt.Sprintf("Error count: %d\n", i.GetErrorCount()),
base58.Encode(i.Shard_ID),
mode,
)
}
}
func setShardMode(cmd *cobra.Command, _ []string) {
key, err := getKeyNoGenerate()
exitOnErr(cmd, err)
var mode control.ShardMode
switch shardMode {
default:
exitOnErr(cmd, fmt.Errorf("unsupported mode %s", mode))
case shardModeReadWrite:
mode = control.ShardMode_READ_WRITE
case shardModeReadOnly:
mode = control.ShardMode_READ_ONLY
case shardModeDegraded:
mode = control.ShardMode_DEGRADED
}
req := new(control.SetShardModeRequest)
body := new(control.SetShardModeRequest_Body)
req.SetBody(body)
rawID, err := base58.Decode(shardID)
exitOnErr(cmd, errf("incorrect shard ID encoding: %w", err))
body.SetMode(mode)
body.SetShardID(rawID)
reset, _ := cmd.Flags().GetBool(shardClearErrorsFlag)
body.ClearErrorCounter(reset)
err = controlSvc.SignMessage(key, req)
exitOnErr(cmd, errf("could not sign request: %w", err))
cli, err := getControlSDKClient(key)
exitOnErr(cmd, err)
var resp *control.SetShardModeResponse
err = cli.ExecRaw(func(client *rawclient.Client) error {
resp, err = control.SetShardMode(client, req)
return err
})
exitOnErr(cmd, errf("rpc error: %w", err))
sign := resp.GetSignature()
err = signature.VerifyDataWithSource(
resp,
func() ([]byte, []byte) {
return sign.GetKey(), sign.GetSign()
},
)
exitOnErr(cmd, errf("invalid response signature: %w", err))
cmd.Println("Shard mode update request successfully sent.")
}