forked from TrueCloudLab/frostfs-node
[#1916] control: Check maintenance allowance on Control server
In previous implementation turning to maintenance mode using NeoFS CLI required NeoFS API endpoint. This was not convenient from the user perspective. It's worth to move networks settings' check to the server side. Add `force_maintenance` field to `SetNetmapStatusRequest.Body` message of Control API. Add `force` flag to `neofs-cli control set-status` command which sets corresponding field in the requests body if status is `maintenance`. Force flag is ignored for any other status. Signed-off-by: Leonard Lyubich <ctulhurider@gmail.com>
This commit is contained in:
parent
c50603494b
commit
60e9de8d63
10 changed files with 94 additions and 39 deletions
|
@ -9,18 +9,24 @@ Changelog for NeoFS Node
|
||||||
- `tree list` CLI command (#1332)
|
- `tree list` CLI command (#1332)
|
||||||
- `TreeService.GetTrees` RPC (#1902)
|
- `TreeService.GetTrees` RPC (#1902)
|
||||||
- All trees synchronization on bootstrap (#1902)
|
- All trees synchronization on bootstrap (#1902)
|
||||||
|
- `--force` flag to `neofs-cli control set-status` command (#1916)
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
### Fixed
|
### Fixed
|
||||||
- `writecache.max_object_size` is now correctly handled (#1925)
|
- `writecache.max_object_size` is now correctly handled (#1925)
|
||||||
- Correctly handle setting ONLINE netmap status after maintenance (#1922)
|
- Correctly handle setting ONLINE netmap status after maintenance (#1922)
|
||||||
- Correctly reset shard errors in `ControlService.SetShardMode` RPC (#1931)
|
- Correctly reset shard errors in `ControlService.SetShardMode` RPC (#1931)
|
||||||
|
- Setting node's network state to `MAINTENANCE` while network settings forbid it (#1916)
|
||||||
|
|
||||||
### Removed
|
### Removed
|
||||||
### Updated
|
### Updated
|
||||||
- `neo-go` to `v0.99.4`
|
- `neo-go` to `v0.99.4`
|
||||||
|
|
||||||
### Updating from v0.33.0
|
### Updating from v0.33.0
|
||||||
|
Now storage node serves Control API `SetNemapStatus` request with `MAINTENANCE`
|
||||||
|
status only if the mode is allowed in the network settings. To force starting the local
|
||||||
|
maintenance on the node, provide `--force` flag to the `neofs-cli control set-status`
|
||||||
|
command.
|
||||||
|
|
||||||
## [0.33.0] - 2022-10-17 - Anmado (안마도, 鞍馬島)
|
## [0.33.0] - 2022-10-17 - Anmado (안마도, 鞍馬島)
|
||||||
|
|
||||||
|
|
|
@ -4,13 +4,11 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
rawclient "github.com/nspcc-dev/neofs-api-go/v2/rpc/client"
|
rawclient "github.com/nspcc-dev/neofs-api-go/v2/rpc/client"
|
||||||
internalclient "github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/client"
|
|
||||||
"github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/common"
|
"github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/common"
|
||||||
"github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/commonflags"
|
"github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/commonflags"
|
||||||
"github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/key"
|
"github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/key"
|
||||||
"github.com/nspcc-dev/neofs-node/pkg/services/control"
|
"github.com/nspcc-dev/neofs-node/pkg/services/control"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"github.com/spf13/viper"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -41,53 +39,40 @@ func initControlSetNetmapStatusCmd() {
|
||||||
)
|
)
|
||||||
|
|
||||||
_ = setNetmapStatusCmd.MarkFlagRequired(netmapStatusFlag)
|
_ = setNetmapStatusCmd.MarkFlagRequired(netmapStatusFlag)
|
||||||
|
|
||||||
|
flags.BoolP(commonflags.ForceFlag, commonflags.ForceFlagShorthand, false,
|
||||||
|
"Force turning to local maintenance")
|
||||||
}
|
}
|
||||||
|
|
||||||
func setNetmapStatus(cmd *cobra.Command, _ []string) {
|
func setNetmapStatus(cmd *cobra.Command, _ []string) {
|
||||||
pk := key.Get(cmd)
|
pk := key.Get(cmd)
|
||||||
|
body := new(control.SetNetmapStatusRequest_Body)
|
||||||
|
force, _ := cmd.Flags().GetBool(commonflags.ForceFlag)
|
||||||
|
|
||||||
var status control.NetmapStatus
|
printIgnoreForce := func(st control.NetmapStatus) {
|
||||||
|
if force {
|
||||||
|
common.PrintVerbose("Ignore --%s flag for %s state.", commonflags.ForceFlag, st)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
switch st, _ := cmd.Flags().GetString(netmapStatusFlag); st {
|
switch st, _ := cmd.Flags().GetString(netmapStatusFlag); st {
|
||||||
default:
|
default:
|
||||||
common.ExitOnErr(cmd, "", fmt.Errorf("unsupported status %s", st))
|
common.ExitOnErr(cmd, "", fmt.Errorf("unsupported status %s", st))
|
||||||
case netmapStatusOnline:
|
case netmapStatusOnline:
|
||||||
status = control.NetmapStatus_ONLINE
|
body.SetStatus(control.NetmapStatus_ONLINE)
|
||||||
|
printIgnoreForce(control.NetmapStatus_ONLINE)
|
||||||
case netmapStatusOffline:
|
case netmapStatusOffline:
|
||||||
status = control.NetmapStatus_OFFLINE
|
body.SetStatus(control.NetmapStatus_OFFLINE)
|
||||||
|
printIgnoreForce(control.NetmapStatus_OFFLINE)
|
||||||
case netmapStatusMaintenance:
|
case netmapStatusMaintenance:
|
||||||
status = control.NetmapStatus_MAINTENANCE
|
body.SetStatus(control.NetmapStatus_MAINTENANCE)
|
||||||
|
|
||||||
common.PrintVerbose("Reading network settings to check allowance of \"%s\" mode...", st)
|
if force {
|
||||||
|
body.SetForceMaintenance()
|
||||||
if !viper.IsSet(commonflags.RPC) {
|
common.PrintVerbose("Local maintenance will be forced.")
|
||||||
common.ExitOnErr(cmd, "",
|
|
||||||
fmt.Errorf("flag --%s (-%s) is not set, you must specify it for \"%s\" mode",
|
|
||||||
commonflags.RPC,
|
|
||||||
commonflags.RPCShorthand,
|
|
||||||
st,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cli := internalclient.GetSDKClientByFlag(cmd, pk, commonflags.RPC)
|
|
||||||
|
|
||||||
var prm internalclient.NetworkInfoPrm
|
|
||||||
prm.SetClient(cli)
|
|
||||||
|
|
||||||
res, err := internalclient.NetworkInfo(prm)
|
|
||||||
common.ExitOnErr(cmd, "receive network info: %v", err)
|
|
||||||
|
|
||||||
if !res.NetworkInfo().MaintenanceModeAllowed() {
|
|
||||||
common.ExitOnErr(cmd, "", fmt.Errorf("\"%s\" mode is not allowed by the network", st))
|
|
||||||
}
|
|
||||||
|
|
||||||
common.PrintVerbose("\"%s\" mode is allowed, continue processing...", st)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
body := new(control.SetNetmapStatusRequest_Body)
|
|
||||||
body.SetStatus(status)
|
|
||||||
|
|
||||||
req := new(control.SetNetmapStatusRequest)
|
req := new(control.SetNetmapStatusRequest)
|
||||||
req.SetBody(body)
|
req.SetBody(body)
|
||||||
|
|
||||||
|
|
|
@ -340,8 +340,7 @@ func (c *cfg) SetNetmapStatus(st control.NetmapStatus) error {
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unsupported status %v", st)
|
return fmt.Errorf("unsupported status %v", st)
|
||||||
case control.NetmapStatus_MAINTENANCE:
|
case control.NetmapStatus_MAINTENANCE:
|
||||||
c.startMaintenance()
|
return c.setMaintenanceStatus(false)
|
||||||
return c.updateNetMapState((*nmClient.UpdatePeerPrm).SetMaintenance)
|
|
||||||
case control.NetmapStatus_ONLINE, control.NetmapStatus_OFFLINE:
|
case control.NetmapStatus_ONLINE, control.NetmapStatus_OFFLINE:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -361,6 +360,33 @@ func (c *cfg) SetNetmapStatus(st control.NetmapStatus) error {
|
||||||
return c.updateNetMapState(func(*nmClient.UpdatePeerPrm) {})
|
return c.updateNetMapState(func(*nmClient.UpdatePeerPrm) {})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *cfg) ForceMaintenance() error {
|
||||||
|
return c.setMaintenanceStatus(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *cfg) setMaintenanceStatus(force bool) error {
|
||||||
|
netSettings, err := c.cfgNetmap.wrapper.ReadNetworkConfiguration()
|
||||||
|
if err != nil {
|
||||||
|
err = fmt.Errorf("read network settings to check maintenance allowance: %w", err)
|
||||||
|
} else if !netSettings.MaintenanceModeAllowed {
|
||||||
|
err = errors.New("maintenance mode is not allowed by the network")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err == nil || force {
|
||||||
|
c.startMaintenance()
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
err = c.updateNetMapState((*nmClient.UpdatePeerPrm).SetMaintenance)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("local maintenance is started, but state is not updated in the network: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// calls UpdatePeerState operation of Netmap contract's client for the local node.
|
// calls UpdatePeerState operation of Netmap contract's client for the local node.
|
||||||
// State setter is used to specify node state to switch to.
|
// State setter is used to specify node state to switch to.
|
||||||
func (c *cfg) updateNetMapState(stateSetter func(*nmClient.UpdatePeerPrm)) error {
|
func (c *cfg) updateNetMapState(stateSetter func(*nmClient.UpdatePeerPrm)) error {
|
||||||
|
|
|
@ -32,8 +32,9 @@ $ neofs-adm morph set-config MaintenanceModeAllowed=true|false
|
||||||
|
|
||||||
To switch the node to MM, exec:
|
To switch the node to MM, exec:
|
||||||
```shell
|
```shell
|
||||||
$ neofs-cli control set-status --status maintenance
|
$ neofs-cli control set-status --status maintenance [--force|-f]
|
||||||
```
|
```
|
||||||
|
`-f` flag allows to force local maintenance regardless of the network settings.
|
||||||
|
|
||||||
To stop the maintenance, use the same command but with any other supported state.
|
To stop the maintenance, use the same command but with any other supported state.
|
||||||
|
|
||||||
|
|
|
@ -34,7 +34,15 @@ type HealthChecker interface {
|
||||||
|
|
||||||
// NodeState is an interface of storage node network state.
|
// NodeState is an interface of storage node network state.
|
||||||
type NodeState interface {
|
type NodeState interface {
|
||||||
SetNetmapStatus(control.NetmapStatus) error
|
// SetNetmapStatus switches the storage node to the given network status.
|
||||||
|
//
|
||||||
|
// If status is control.NetmapStatus_MAINTENANCE and maintenance is allowed
|
||||||
|
// in the network settings, the node additionally starts local maintenance.
|
||||||
|
SetNetmapStatus(st control.NetmapStatus) error
|
||||||
|
|
||||||
|
// ForceMaintenance works like SetNetmapStatus(control.NetmapStatus_MAINTENANCE)
|
||||||
|
// but starts local maintenance regardless of the network settings.
|
||||||
|
ForceMaintenance() error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Option of the Server's constructor.
|
// Option of the Server's constructor.
|
||||||
|
|
|
@ -17,8 +17,23 @@ func (s *Server) SetNetmapStatus(ctx context.Context, req *control.SetNetmapStat
|
||||||
return nil, status.Error(codes.PermissionDenied, err.Error())
|
return nil, status.Error(codes.PermissionDenied, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
// set node status
|
var err error
|
||||||
if err := s.nodeState.SetNetmapStatus(req.GetBody().GetStatus()); err != nil {
|
bodyReq := req.GetBody()
|
||||||
|
st := bodyReq.GetStatus()
|
||||||
|
force := bodyReq.GetForceMaintenance()
|
||||||
|
|
||||||
|
if force {
|
||||||
|
if st != control.NetmapStatus_MAINTENANCE {
|
||||||
|
return nil, status.Errorf(codes.InvalidArgument,
|
||||||
|
"force_maintenance MUST be set for %s status only", control.NetmapStatus_MAINTENANCE)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = s.nodeState.ForceMaintenance()
|
||||||
|
} else {
|
||||||
|
err = s.nodeState.SetNetmapStatus(st)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
return nil, status.Error(codes.Aborted, err.Error())
|
return nil, status.Error(codes.Aborted, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -35,6 +35,11 @@ func (x *SetNetmapStatusRequest_Body) SetStatus(v NetmapStatus) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetForceMaintenance sets force_maintenance flag in the message.
|
||||||
|
func (x *SetNetmapStatusRequest_Body) SetForceMaintenance() {
|
||||||
|
x.ForceMaintenance = true
|
||||||
|
}
|
||||||
|
|
||||||
// SetBody sets body of the set netmap status request .
|
// SetBody sets body of the set netmap status request .
|
||||||
func (x *SetNetmapStatusRequest) SetBody(v *SetNetmapStatusRequest_Body) {
|
func (x *SetNetmapStatusRequest) SetBody(v *SetNetmapStatusRequest_Body) {
|
||||||
if x != nil {
|
if x != nil {
|
||||||
|
|
BIN
pkg/services/control/service.pb.go
generated
BIN
pkg/services/control/service.pb.go
generated
Binary file not shown.
|
@ -75,7 +75,16 @@ message SetNetmapStatusRequest {
|
||||||
// Set netmap status request body.
|
// Set netmap status request body.
|
||||||
message Body {
|
message Body {
|
||||||
// New storage node status in NeoFS network map.
|
// New storage node status in NeoFS network map.
|
||||||
|
// If status is MAINTENANCE, the node checks whether maintenance is
|
||||||
|
// allowed in the network settings. In case of prohibition, the request
|
||||||
|
// is denied. Otherwise, node switches to local maintenance state. To
|
||||||
|
// force local maintenance, use `force_maintenance` flag.
|
||||||
NetmapStatus status = 1;
|
NetmapStatus status = 1;
|
||||||
|
|
||||||
|
// MAINTENANCE status validation skip flag. If set, node starts local
|
||||||
|
// maintenance regardless of network settings. The flag MUST NOT be
|
||||||
|
// set for any other status.
|
||||||
|
bool force_maintenance = 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Body of set netmap status request message.
|
// Body of set netmap status request message.
|
||||||
|
|
BIN
pkg/services/control/service_neofs.pb.go
generated
BIN
pkg/services/control/service_neofs.pb.go
generated
Binary file not shown.
Loading…
Reference in a new issue