diff --git a/cmd/frostfs-cli/modules/control/ir.go b/cmd/frostfs-cli/modules/control/ir.go new file mode 100644 index 000000000..05e712275 --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir.go @@ -0,0 +1,15 @@ +package control + +import "github.com/spf13/cobra" + +var irCmd = &cobra.Command{ + Use: "ir", + Short: "Operations with inner ring nodes", + Long: "Operations with inner ring nodes", +} + +func initControlIRCmd() { + irCmd.AddCommand(removeContainerCmd) + + initControlIRRemoveContainerCmd() +} diff --git a/cmd/frostfs-cli/modules/control/ir_remove_container.go b/cmd/frostfs-cli/modules/control/ir_remove_container.go new file mode 100644 index 000000000..43173bcaa --- /dev/null +++ b/cmd/frostfs-cli/modules/control/ir_remove_container.go @@ -0,0 +1,94 @@ +package control + +import ( + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + rawclient "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/client" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/commonflags" + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-cli/internal/key" + commonCmd "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common" + ircontrol "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + ircontrolsrv "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir/server" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "github.com/spf13/cobra" +) + +const ( + ownerFlag = "owner" +) + +var removeContainerCmd = &cobra.Command{ + Use: "remove-container", + Short: "Schedules a container removal", + Long: `Schedules a container removal via a notary request. +Container data will be deleted asynchronously by policer. +To check removal status "frostfs-cli container list" command can be used.`, + Run: removeContainer, +} + +func initControlIRRemoveContainerCmd() { + initControlFlags(removeContainerCmd) + + flags := removeContainerCmd.Flags() + flags.String(commonflags.CIDFlag, "", commonflags.CIDFlagUsage) + flags.String(ownerFlag, "", "Container owner's wallet address.") + removeContainerCmd.MarkFlagsMutuallyExclusive(commonflags.CIDFlag, ownerFlag) +} + +func removeContainer(cmd *cobra.Command, _ []string) { + req := prepareRemoveContainerRequest(cmd) + + pk := key.Get(cmd) + c := getClient(cmd, pk) + + commonCmd.ExitOnErr(cmd, "could not sign request: %w", ircontrolsrv.SignMessage(pk, req)) + + var resp *ircontrol.RemoveContainerResponse + err := c.ExecRaw(func(client *rawclient.Client) error { + var err error + resp, err = ircontrol.RemoveContainer(client, req) + return err + }) + commonCmd.ExitOnErr(cmd, "failed to execute request: %w", err) + + verifyResponse(cmd, resp.GetSignature(), resp.GetBody()) + + if len(req.GetBody().GetContainerId()) > 0 { + cmd.Println("Container scheduled to removal") + } else { + cmd.Println("User containers sheduled to removal") + } +} + +func prepareRemoveContainerRequest(cmd *cobra.Command) *ircontrol.RemoveContainerRequest { + req := &ircontrol.RemoveContainerRequest{ + Body: &ircontrol.RemoveContainerRequest_Body{}, + } + + cidStr, err := cmd.Flags().GetString(commonflags.CIDFlag) + commonCmd.ExitOnErr(cmd, "failed to get cid: ", err) + + ownerStr, err := cmd.Flags().GetString(ownerFlag) + commonCmd.ExitOnErr(cmd, "failed to get owner: ", err) + + if len(ownerStr) == 0 && len(cidStr) == 0 { + commonCmd.ExitOnErr(cmd, "invalid usage: %w", errors.New("neither owner's wallet address nor container ID are specified")) + } + + if len(ownerStr) > 0 { + var owner user.ID + commonCmd.ExitOnErr(cmd, "invalid owner ID: %w", owner.DecodeString(ownerStr)) + var ownerID refs.OwnerID + owner.WriteToV2(&ownerID) + req.Body.Owner = ownerID.StableMarshal(nil) + } + + if len(cidStr) > 0 { + var containerID cid.ID + commonCmd.ExitOnErr(cmd, "invalid container ID: %w", containerID.DecodeString(cidStr)) + req.Body.ContainerId = containerID[:] + } + return req +} diff --git a/cmd/frostfs-cli/modules/control/root.go b/cmd/frostfs-cli/modules/control/root.go index d3b656392..015676185 100644 --- a/cmd/frostfs-cli/modules/control/root.go +++ b/cmd/frostfs-cli/modules/control/root.go @@ -33,6 +33,7 @@ func init() { dropObjectsCmd, shardsCmd, synchronizeTreeCmd, + irCmd, ) initControlHealthCheckCmd() @@ -40,4 +41,5 @@ func init() { initControlDropObjectsCmd() initControlShardsCmd() initControlSynchronizeTreeCmd() + initControlIRCmd() } diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 8db6328a2..b4f6f206b 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -520,7 +520,7 @@ func (s *Server) initGRPCServer(cfg *viper.Viper) error { p.SetPrivateKey(*s.key) p.SetHealthChecker(s) - controlSvc := controlsrv.New(p, + controlSvc := controlsrv.New(p, s.containerClient, controlsrv.WithAllowedKeys(authKeys), ) @@ -581,6 +581,7 @@ func (s *Server) initClientsFromMorph() (*serverMorphClients, error) { if err != nil { return nil, err } + s.containerClient = result.CnrClient s.netmapClient, err = nmClient.NewFromMorph(s.morphClient, s.contracts.netmap, fee, nmClient.TryNotary(), nmClient.AsAlphabet()) if err != nil { diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index a91d2fd0d..fbee056fb 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -14,6 +14,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" auditClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/audit" balanceClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/balance" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/subscriber" @@ -45,17 +46,18 @@ type ( epochTimer *timer.BlockTimer // global state - morphClient *client.Client - mainnetClient *client.Client - epochCounter atomic.Uint64 - epochDuration atomic.Uint64 - statusIndex *innerRingIndexer - precision precision.Fixed8Converter - auditClient *auditClient.Client - healthStatus atomic.Value - balanceClient *balanceClient.Client - netmapClient *nmClient.Client - persistate *state.PersistentStorage + morphClient *client.Client + mainnetClient *client.Client + epochCounter atomic.Uint64 + epochDuration atomic.Uint64 + statusIndex *innerRingIndexer + precision precision.Fixed8Converter + auditClient *auditClient.Client + healthStatus atomic.Value + balanceClient *balanceClient.Client + netmapClient *nmClient.Client + persistate *state.PersistentStorage + containerClient *container.Client // metrics metrics *metrics.InnerRingServiceMetrics diff --git a/pkg/morph/client/container/delete.go b/pkg/morph/client/container/delete.go index c9105a3ca..f04d86bde 100644 --- a/pkg/morph/client/container/delete.go +++ b/pkg/morph/client/container/delete.go @@ -60,7 +60,7 @@ func (d *DeletePrm) SetToken(token []byte) { // // If TryNotary is provided, calls notary contract. func (c *Client) Delete(p DeletePrm) error { - if len(p.signature) == 0 { + if len(p.signature) == 0 && !p.IsControl() { return errNilArgument } diff --git a/pkg/morph/client/static.go b/pkg/morph/client/static.go index afaf49f3d..b237a485b 100644 --- a/pkg/morph/client/static.go +++ b/pkg/morph/client/static.go @@ -94,6 +94,12 @@ type InvokePrmOptional struct { // `validUntilBlock` values by all notification // receivers. hash *util.Uint256 + // controlTX controls whether the invoke method will use a rounded + // block height value, which is useful for control transactions which + // are required to be produced by all nodes with very high probability. + // It's only used by notary transactions and it affects only the + // computation of `validUntilBlock` values. + controlTX bool } // SetHash sets optional hash of the transaction. @@ -104,6 +110,16 @@ func (i *InvokePrmOptional) SetHash(hash util.Uint256) { i.hash = &hash } +// SetControlTX sets whether a control transaction will be used. +func (i *InvokePrmOptional) SetControlTX(b bool) { + i.controlTX = b +} + +// IsControl gets whether a control transaction will be used. +func (i *InvokePrmOptional) IsControl() bool { + return i.controlTX +} + // Invoke calls Invoke method of Client with static internal script hash and fee. // Supported args types are the same as in Client. // diff --git a/pkg/services/control/ir/convert.go b/pkg/services/control/ir/convert.go index 01bc48724..4bf68fdbc 100644 --- a/pkg/services/control/ir/convert.go +++ b/pkg/services/control/ir/convert.go @@ -32,3 +32,22 @@ func (w *healthCheckResponseWrapper) FromGRPCMessage(m grpc.Message) error { return nil } + +type removeContainerResponseWrapper struct { + m *RemoveContainerResponse +} + +func (w *removeContainerResponseWrapper) ToGRPCMessage() grpc.Message { + return w.m +} + +func (w *removeContainerResponseWrapper) FromGRPCMessage(m grpc.Message) error { + var ok bool + + w.m, ok = m.(*RemoveContainerResponse) + if !ok { + return message.NewUnexpectedMessageType(m, w.m) + } + + return nil +} diff --git a/pkg/services/control/ir/rpc.go b/pkg/services/control/ir/rpc.go index a8b16b607..b4296dc36 100644 --- a/pkg/services/control/ir/rpc.go +++ b/pkg/services/control/ir/rpc.go @@ -8,7 +8,8 @@ import ( const serviceName = "ircontrol.ControlService" const ( - rpcHealthCheck = "HealthCheck" + rpcHealthCheck = "HealthCheck" + rpcRemoveContainer = "RemoveContainer" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -32,3 +33,24 @@ func HealthCheck( return wResp.m, nil } + +func RemoveContainer( + cli *client.Client, + req *RemoveContainerRequest, + opts ...client.CallOption, +) (*RemoveContainerResponse, error) { + wResp := &removeContainerResponseWrapper{ + m: new(RemoveContainerResponse), + } + + wReq := &requestWrapper{ + m: req, + } + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcRemoveContainer), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.m, nil +} diff --git a/pkg/services/control/ir/server/calls.go b/pkg/services/control/ir/server/calls.go index 986da90f1..3030436e6 100644 --- a/pkg/services/control/ir/server/calls.go +++ b/pkg/services/control/ir/server/calls.go @@ -2,8 +2,13 @@ package control import ( "context" + "fmt" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" control "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control/ir" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -32,3 +37,63 @@ func (s *Server) HealthCheck(_ context.Context, req *control.HealthCheckRequest) return resp, nil } + +// RemoveContainer forces a container removal. +func (s *Server) RemoveContainer(_ context.Context, req *control.RemoveContainerRequest) (*control.RemoveContainerResponse, error) { + if err := s.isValidRequest(req); err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + if len(req.Body.GetContainerId()) > 0 && len(req.Body.GetOwner()) > 0 { + return nil, status.Error(codes.InvalidArgument, "specify the owner and container at the same time is not allowed") + } + + if len(req.Body.GetContainerId()) > 0 { + var containerID cid.ID + if err := containerID.Decode(req.Body.GetContainerId()); err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed to parse container ID: %s", err.Error())) + } + if err := s.removeContainer(containerID); err != nil { + return nil, err + } + } else { + var ownerID refs.OwnerID + if err := ownerID.Unmarshal(req.GetBody().GetOwner()); err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed to parse ownerID: %s", err.Error())) + } + var owner user.ID + if err := owner.ReadFromV2(ownerID); err != nil { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("failed to read owner: %s", err.Error())) + } + + cids, err := s.containerClient.ContainersOf(&owner) + if err != nil { + return nil, fmt.Errorf("failed to get owner's containers: %w", err) + } + + for _, containerID := range cids { + if err := s.removeContainer(containerID); err != nil { + return nil, err + } + } + } + + resp := &control.RemoveContainerResponse{ + Body: &control.RemoveContainerResponse_Body{}, + } + if err := SignMessage(&s.prm.key.PrivateKey, resp); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil +} + +func (s *Server) removeContainer(containerID cid.ID) error { + var prm container.DeletePrm + prm.SetCID(containerID[:]) + prm.SetControlTX(true) + + if err := s.containerClient.Delete(prm); err != nil { + return fmt.Errorf("forcing container removal: %w", err) + } + return nil +} diff --git a/pkg/services/control/ir/server/server.go b/pkg/services/control/ir/server/server.go index c75c1504e..edd30303b 100644 --- a/pkg/services/control/ir/server/server.go +++ b/pkg/services/control/ir/server/server.go @@ -2,6 +2,8 @@ package control import ( "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/container" ) // Server is an entity that serves @@ -13,6 +15,8 @@ type Server struct { prm Prm allowedKeys [][]byte + + containerClient *container.Client } func panicOnPrmValue(n string, v any) { @@ -29,7 +33,7 @@ func panicOnPrmValue(n string, v any) { // Forms white list from all keys specified via // WithAllowedKeys option and a public key of // the parameterized private key. -func New(prm Prm, opts ...Option) *Server { +func New(prm Prm, containerClient *container.Client, opts ...Option) *Server { // verify required parameters switch { case prm.healthChecker == nil: @@ -46,6 +50,7 @@ func New(prm Prm, opts ...Option) *Server { return &Server{ prm: prm, - allowedKeys: append(o.allowedKeys, prm.key.PublicKey().Bytes()), + allowedKeys: append(o.allowedKeys, prm.key.PublicKey().Bytes()), + containerClient: containerClient, } } diff --git a/pkg/services/control/ir/service.pb.go b/pkg/services/control/ir/service.pb.go index 9f2834706..f01ba3e27 100644 Binary files a/pkg/services/control/ir/service.pb.go and b/pkg/services/control/ir/service.pb.go differ diff --git a/pkg/services/control/ir/service.proto b/pkg/services/control/ir/service.proto index 5f99be16b..c6c4d69a3 100644 --- a/pkg/services/control/ir/service.proto +++ b/pkg/services/control/ir/service.proto @@ -10,6 +10,8 @@ option go_package = "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/ir/ service ControlService { // Performs health check of the IR node. rpc HealthCheck (HealthCheckRequest) returns (HealthCheckResponse); + // Forces a container removal to be signaled by the IR node with high probability. + rpc RemoveContainer(RemoveContainerRequest) returns (RemoveContainerResponse); } // Health check request. @@ -41,3 +43,21 @@ message HealthCheckResponse { // Body signature. Signature signature = 2; } + + +message RemoveContainerRequest { + message Body{ + bytes container_id = 1; + bytes owner = 2; + } + + Body body = 1; + Signature signature = 2; +} + +message RemoveContainerResponse { + message Body{} + + Body body = 1; + Signature signature = 2; +} \ No newline at end of file diff --git a/pkg/services/control/ir/service_frostfs.pb.go b/pkg/services/control/ir/service_frostfs.pb.go index f6dd94b3a..438c3a8c5 100644 Binary files a/pkg/services/control/ir/service_frostfs.pb.go and b/pkg/services/control/ir/service_frostfs.pb.go differ diff --git a/pkg/services/control/ir/service_grpc.pb.go b/pkg/services/control/ir/service_grpc.pb.go index b6bc6fdba..b4e475a14 100644 Binary files a/pkg/services/control/ir/service_grpc.pb.go and b/pkg/services/control/ir/service_grpc.pb.go differ