diff --git a/cmd/frostfs-cli/modules/control/ir.go b/cmd/frostfs-cli/modules/control/ir.go new file mode 100644 index 00000000..e89dda07 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir.go @@ -0,0 +1,15 @@ +package control + +import "github.com/spf13/cobra" + +var irCmd = &cobra.Command{ + Use: "ir", + Short: "Operations with inner ring nodes", + Long: "Operations with inner ring nodes", +} + +func initControlIRCmd() { + irCmd.AddCommand(tickEpochCmd) + + initControlIRTickEpochCmd() +} diff --git a/cmd/frostfs-cli/modules/control/ir_tick_epoch.go b/cmd/frostfs-cli/modules/control/ir_tick_epoch.go new file mode 100644 index 00000000..3e6af008 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir_tick_epoch.go @@ -0,0 +1,43 @@ +package control + +import ( + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + "github.com/spf13/cobra" +) + +var tickEpochCmd = &cobra.Command{ + Use: "tick-epoch", + Short: "Forces a new epoch", + Long: "Forces a new epoch via a notary request. It should be executed on other IR nodes as well.", + Run: tickEpoch, +} + +func initControlIRTickEpochCmd() { + initControlFlags(tickEpochCmd) +} + +func tickEpoch(cmd *cobra.Command, _ []string) { + pk := key.Get(cmd) + c := getClient(cmd, pk) + + req := new(ircontrol.TickEpochRequest) + req.SetBody(new(ircontrol.TickEpochRequest_Body)) + + err := ircontrolsrv.SignMessage(pk, req) + commonCmd.ExitOnErr(cmd, "could not sign request: %w", err) + + var resp *ircontrol.TickEpochResponse + err = c.ExecRaw(func(client *rawclient.Client) error { + resp, err = ircontrol.TickEpoch(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "rpc error: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + cmd.Println("Epoch tick requested") +} diff --git a/cmd/frostfs-cli/modules/control/root.go b/cmd/frostfs-cli/modules/control/root.go index d3b65639..01567618 100644 --- a/cmd/frostfs-cli/modules/control/root.go +++ b/cmd/frostfs-cli/modules/control/root.go @@ -33,6 +33,7 @@ func init() { dropObjectsCmd, shardsCmd, synchronizeTreeCmd, + irCmd, ) initControlHealthCheckCmd() @@ -40,4 +41,5 @@ func init() { initControlDropObjectsCmd() initControlShardsCmd() initControlSynchronizeTreeCmd() + initControlIRCmd() } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 7e0c310f..38bde141 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -459,7 +459,7 @@ func (s *Server) initGRPCServer(cfg *viper.Viper) error { p.SetPrivateKey(*s.key) p.SetHealthChecker(s) - controlSvc := controlsrv.New(p, + controlSvc := controlsrv.New(p, s.netmapClient, controlsrv.WithAllowedKeys(authKeys), ) diff --git a/pkg/innerring/processors/netmap/process_epoch.go b/pkg/innerring/processors/netmap/process_epoch.go index ebf128f8..17e445b1 100644 --- a/pkg/innerring/processors/netmap/process_epoch.go +++ b/pkg/innerring/processors/netmap/process_epoch.go @@ -79,7 +79,7 @@ func (np *Processor) processNewEpochTick() { nextEpoch := np.epochState.EpochCounter() + 1 np.log.Debug(logs.NetmapNextEpoch, zap.Uint64("value", nextEpoch)) - err := np.netmapClient.NewEpoch(nextEpoch) + err := np.netmapClient.NewEpoch(nextEpoch, false) if err != nil { np.log.Error(logs.NetmapCantInvokeNetmapNewEpoch, zap.Error(err)) } diff --git a/pkg/morph/client/netmap/new_epoch.go b/pkg/morph/client/netmap/new_epoch.go index 0b4d31b1..7a63f14d 100644 --- a/pkg/morph/client/netmap/new_epoch.go +++ b/pkg/morph/client/netmap/new_epoch.go @@ -8,10 +8,14 @@ import ( // NewEpoch updates FrostFS epoch number through // Netmap contract call. -func (c *Client) NewEpoch(epoch uint64) error { +// If `force` is true, this call is normally initiated by a control +// service command and uses a control notary transaction internally +// to ensure all nodes produce the same transaction with high probability. +func (c *Client) NewEpoch(epoch uint64, force bool) error { prm := client.InvokePrm{} prm.SetMethod(newEpochMethod) prm.SetArgs(epoch) + prm.SetControlTX(force) if err := c.client.Invoke(prm); err != nil { return fmt.Errorf("could not invoke method (%s): %w", newEpochMethod, err) diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 42755437..3e21911e 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -886,6 +886,16 @@ func CalculateNotaryDepositAmount(c *Client, gasMul, gasDiv int64) (fixedn.Fixed // CalculateNonceAndVUB calculates nonce and ValidUntilBlock values // based on transaction hash. func (c *Client) CalculateNonceAndVUB(hash util.Uint256) (nonce uint32, vub uint32, err error) { + return c.calculateNonceAndVUB(hash, false) +} + +// CalculateNonceAndVUBControl calculates nonce and rounded ValidUntilBlock values +// based on transaction hash for use in control transactions. +func (c *Client) CalculateNonceAndVUBControl(hash util.Uint256) (nonce uint32, vub uint32, err error) { + return c.calculateNonceAndVUB(hash, true) +} + +func (c *Client) calculateNonceAndVUB(hash util.Uint256, roundBlockHeight bool) (nonce uint32, vub uint32, err error) { c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -904,6 +914,14 @@ func (c *Client) CalculateNonceAndVUB(hash util.Uint256) (nonce uint32, vub uint return 0, 0, fmt.Errorf("could not get transaction height: %w", err) } + // For control transactions, we round down the block height to control the + // probability of all nodes producing the same transaction, since it depends + // on this value. + if roundBlockHeight { + inc := c.rpcActor.GetVersion().Protocol.MaxValidUntilBlockIncrement + height = height / inc * inc + } + return nonce, height + c.notary.txValidTime, nil } diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index afaf49f3..910f7853 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -94,6 +94,12 @@ type InvokePrmOptional struct { // `validUntilBlock` values by all notification // receivers. hash *util.Uint256 + // controlTX controls whether the invoke method will use a rounded + // block height value, which is useful for control transactions which + // are required to be produced by all nodes with very high probability. + // It's only used by notary transactions and it affects only the + // computation of `validUntilBlock` values. + controlTX bool } // SetHash sets optional hash of the transaction. @@ -104,6 +110,11 @@ func (i *InvokePrmOptional) SetHash(hash util.Uint256) { i.hash = &hash } +// SetControlTX sets whether a control transaction will be used. +func (i *InvokePrmOptional) SetControlTX(b bool) { + i.controlTX = b +} + // Invoke calls Invoke method of Client with static internal script hash and fee. // Supported args types are the same as in Client. // @@ -126,7 +137,11 @@ func (s StaticClient) Invoke(prm InvokePrm) error { ) if prm.hash != nil { - nonce, vub, err = s.client.CalculateNonceAndVUB(*prm.hash) + if prm.controlTX { + nonce, vub, err = s.client.CalculateNonceAndVUBControl(*prm.hash) + } else { + nonce, vub, err = s.client.CalculateNonceAndVUB(*prm.hash) + } if err != nil { return fmt.Errorf("could not calculate nonce and VUB for notary alphabet invoke: %w", err) } diff --git a/pkg/services/control/ir/convert.go b/pkg/services/control/ir/convert.go index 01bc4872..c892c5b6 100644 --- a/pkg/services/control/ir/convert.go +++ b/pkg/services/control/ir/convert.go @@ -14,18 +14,18 @@ func (w *requestWrapper) ToGRPCMessage() grpc.Message { return w.m } -type healthCheckResponseWrapper struct { - m *HealthCheckResponse +type responseWrapper[M grpc.Message] struct { + m M } -func (w *healthCheckResponseWrapper) ToGRPCMessage() grpc.Message { +func (w *responseWrapper[M]) ToGRPCMessage() grpc.Message { return w.m } -func (w *healthCheckResponseWrapper) FromGRPCMessage(m grpc.Message) error { +func (w *responseWrapper[M]) FromGRPCMessage(m grpc.Message) error { var ok bool - w.m, ok = m.(*HealthCheckResponse) + w.m, ok = m.(M) if !ok { return message.NewUnexpectedMessageType(m, w.m) } diff --git a/pkg/services/control/ir/rpc.go b/pkg/services/control/ir/rpc.go index a8b16b60..6b223495 100644 --- a/pkg/services/control/ir/rpc.go +++ b/pkg/services/control/ir/rpc.go @@ -3,12 +3,14 @@ package control import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/common" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/grpc" ) const serviceName = "ircontrol.ControlService" const ( rpcHealthCheck = "HealthCheck" + rpcTickEpoch = "TickEpoch" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -17,15 +19,29 @@ func HealthCheck( req *HealthCheckRequest, opts ...client.CallOption, ) (*HealthCheckResponse, error) { - wResp := &healthCheckResponseWrapper{ - m: new(HealthCheckResponse), + return sendUnary[HealthCheckRequest, HealthCheckResponse](cli, rpcHealthCheck, req, opts...) +} + +// TickEpoch executes ControlService.TickEpoch RPC. +func TickEpoch( + cli *client.Client, + req *TickEpochRequest, + opts ...client.CallOption, +) (*TickEpochResponse, error) { + return sendUnary[TickEpochRequest, TickEpochResponse](cli, rpcTickEpoch, req, opts...) +} + +func sendUnary[I, O grpc.Message](cli *client.Client, rpcName string, req *I, opts ...client.CallOption) (*O, error) { + var resp O + wResp := &responseWrapper[*O]{ + m: &resp, } wReq := &requestWrapper{ m: req, } - err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcHealthCheck), wReq, wResp, opts...) + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcName), wReq, wResp, opts...) if err != nil { return nil, err } diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 986da90f..56e2e3f7 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -2,6 +2,7 @@ package control import ( "context" + "fmt" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" "google.golang.org/grpc/codes" @@ -12,12 +13,10 @@ import ( // // If request is not signed with a key from white list, permission error returns. func (s *Server) HealthCheck(_ context.Context, req *control.HealthCheckRequest) (*control.HealthCheckResponse, error) { - // verify request if err := s.isValidRequest(req); err != nil { return nil, status.Error(codes.PermissionDenied, err.Error()) } - // create and fill response resp := new(control.HealthCheckResponse) body := new(control.HealthCheckResponse_Body) @@ -25,7 +24,33 @@ func (s *Server) HealthCheck(_ context.Context, req *control.HealthCheckRequest) body.SetHealthStatus(s.prm.healthChecker.HealthStatus()) - // sign the response + if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return resp, nil +} + +// TickEpoch forces a new epoch. +// +// If request is not signed with a key from white list, permission error returns. +func (s *Server) TickEpoch(_ context.Context, req *control.TickEpochRequest) (*control.TickEpochResponse, error) { + if err := s.isValidRequest(req); err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + resp := new(control.TickEpochResponse) + resp.SetBody(new(control.TickEpochResponse_Body)) + + epoch, err := s.netmapClient.Epoch() + if err != nil { + return nil, fmt.Errorf("getting current epoch: %w", err) + } + + if err := s.netmapClient.NewEpoch(epoch+1, true); err != nil { + return nil, fmt.Errorf("forcing new epoch: %w", err) + } + if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/pkg/services/control/ir/server/server.go b/pkg/services/control/ir/server/server.go index c75c1504..dc00809a 100644 --- a/pkg/services/control/ir/server/server.go +++ b/pkg/services/control/ir/server/server.go @@ -2,6 +2,8 @@ package control import ( "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" ) // Server is an entity that serves @@ -10,7 +12,8 @@ import ( // To gain access to the service, any request must be // signed with a key from the white list. type Server struct { - prm Prm + prm Prm + netmapClient *netmap.Client allowedKeys [][]byte } @@ -29,7 +32,7 @@ func panicOnPrmValue(n string, v any) { // Forms white list from all keys specified via // WithAllowedKeys option and a public key of // the parameterized private key. -func New(prm Prm, opts ...Option) *Server { +func New(prm Prm, netmapClient *netmap.Client, opts ...Option) *Server { // verify required parameters switch { case prm.healthChecker == nil: @@ -44,7 +47,8 @@ func New(prm Prm, opts ...Option) *Server { } return &Server{ - prm: prm, + prm: prm, + netmapClient: netmapClient, allowedKeys: append(o.allowedKeys, prm.key.PublicKey().Bytes()), } diff --git a/pkg/services/control/ir/service.go b/pkg/services/control/ir/service.go index dc04e490..1aaec2c8 100644 --- a/pkg/services/control/ir/service.go +++ b/pkg/services/control/ir/service.go @@ -20,3 +20,15 @@ func (x *HealthCheckResponse) SetBody(v *HealthCheckResponse_Body) { x.Body = v } } + +func (x *TickEpochRequest) SetBody(v *TickEpochRequest_Body) { + if x != nil { + x.Body = v + } +} + +func (x *TickEpochResponse) SetBody(v *TickEpochResponse_Body) { + if x != nil { + x.Body = v + } +} diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index 44ea2dd6..84acdfc8 100644 Binary files a/pkg/services/control/ir/service.pb.go and b/pkg/services/control/ir/service.pb.go differ diff --git a/pkg/services/control/ir/service.proto b/pkg/services/control/ir/service.proto index 5f99be16..5862e8fb 100644 --- a/pkg/services/control/ir/service.proto +++ b/pkg/services/control/ir/service.proto @@ -10,6 +10,8 @@ option go_package = "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/ir/ service ControlService { // Performs health check of the IR node. rpc HealthCheck (HealthCheckRequest) returns (HealthCheckResponse); + // Forces a new epoch to be signaled by the IR node with high probability. + rpc TickEpoch (TickEpochRequest) returns (TickEpochResponse); } // Health check request. @@ -41,3 +43,17 @@ message HealthCheckResponse { // Body signature. Signature signature = 2; } + +message TickEpochRequest { + message Body{} + + Body body = 1; + Signature signature = 2; +} + +message TickEpochResponse { + message Body{} + + Body body = 1; + Signature signature = 2; +} diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index f6dd94b3..d480f0b5 100644 Binary files a/pkg/services/control/ir/service_frostfs.pb.go and b/pkg/services/control/ir/service_frostfs.pb.go differ diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index bdcac73e..700d340c 100644 Binary files a/pkg/services/control/ir/service_grpc.pb.go and b/pkg/services/control/ir/service_grpc.pb.go differ diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index d713bb38..a126ce16 100644 Binary files a/pkg/services/control/service.pb.go and b/pkg/services/control/service.pb.go differ diff --git a/pkg/services/control/service_grpc.pb.go b/pkg/services/control/service_grpc.pb.go index 1e8dd9e3..3fa1e54d 100644 Binary files a/pkg/services/control/service_grpc.pb.go and b/pkg/services/control/service_grpc.pb.go differ diff --git a/pkg/services/tree/service_grpc.pb.go b/pkg/services/tree/service_grpc.pb.go index f981746d..fa259e80 100644 Binary files a/pkg/services/tree/service_grpc.pb.go and b/pkg/services/tree/service_grpc.pb.go differ