[#191] control: Add Doctor RPC

Doctor RPC performs complex operations on the storage engine.
Currently only duplicate removal is supported.

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
pull/213/head
Evgenii Stratonikov 2023-04-07 14:21:05 +03:00 committed by Gitea
parent dbc3811ff4
commit 0b9622c418
7 changed files with 824 additions and 231 deletions

View File

@ -200,3 +200,21 @@ func (w *flushCacheResponseWrapper) FromGRPCMessage(m grpc.Message) error {
w.FlushCacheResponse = r
return nil
}
type doctorResponseWrapper struct {
*DoctorResponse
}
func (w *doctorResponseWrapper) ToGRPCMessage() grpc.Message {
return w.DoctorResponse
}
func (w *doctorResponseWrapper) FromGRPCMessage(m grpc.Message) error {
r, ok := m.(*DoctorResponse)
if !ok {
return message.NewUnexpectedMessageType(m, (*DoctorResponse)(nil))
}
w.DoctorResponse = r
return nil
}

View File

@ -18,6 +18,7 @@ const (
rpcSynchronizeTree = "SynchronizeTree"
rpcEvacuateShard = "EvacuateShard"
rpcFlushCache = "FlushCache"
rpcDoctor = "Doctor"
)
// HealthCheck executes ControlService.HealthCheck RPC.
@ -191,3 +192,16 @@ func FlushCache(cli *client.Client, req *FlushCacheRequest, opts ...client.CallO
return wResp.FlushCacheResponse, nil
}
// Doctor executes ControlService.Doctor RPC.
func Doctor(cli *client.Client, req *DoctorRequest, opts ...client.CallOption) (*DoctorResponse, error) {
wResp := &doctorResponseWrapper{new(DoctorResponse)}
wReq := &requestWrapper{m: req}
err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcDoctor), wReq, wResp, opts...)
if err != nil {
return nil, err
}
return wResp.DoctorResponse, nil
}

View File

@ -0,0 +1,37 @@
package control
import (
"context"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
func (s *Server) Doctor(ctx context.Context, req *control.DoctorRequest) (*control.DoctorResponse, error) {
err := s.isValidRequest(req)
if err != nil {
return nil, status.Error(codes.PermissionDenied, err.Error())
}
if !req.Body.RemoveDuplicates {
return nil, status.Error(codes.InvalidArgument, "operation not specified")
}
var prm engine.RemoveDuplicatesPrm
prm.Concurrency = int(req.Body.Concurrency)
err = s.s.RemoveDuplicates(ctx, prm)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
resp := &control.DoctorResponse{Body: &control.DoctorResponse_Body{}}
err = SignMessage(s.key, resp)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
return resp, nil
}

File diff suppressed because it is too large Load Diff

View File

@ -37,6 +37,9 @@ service ControlService {
// FlushCache moves all data from one shard to the others.
rpc FlushCache (FlushCacheRequest) returns (FlushCacheResponse);
// Doctor performs storage restructuring operations on engine.
rpc Doctor (DoctorRequest) returns (DoctorResponse);
}
// Health check request.
@ -345,3 +348,28 @@ message FlushCacheResponse {
Body body = 1;
Signature signature = 2;
}
// Doctor request.
message DoctorRequest {
// Request body structure.
message Body {
// Number of threads to use for the operation.
uint32 concurrency = 1;
// Flag to search engine for duplicate objects and leave only one copy.
bool remove_duplicates = 2;
}
Body body = 1;
Signature signature = 2;
}
// Doctor response.
message DoctorResponse {
// Response body structure.
message Body {
}
Body body = 1;
Signature signature = 2;
}

View File

@ -1548,3 +1548,156 @@ func (x *FlushCacheResponse) ReadSignedData(buf []byte) ([]byte, error) {
func (x *FlushCacheResponse) SetSignature(sig *Signature) {
x.Signature = sig
}
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *DoctorRequest_Body) StableSize() (size int) {
size += proto.UInt32Size(1, x.Concurrency)
size += proto.BoolSize(2, x.RemoveDuplicates)
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *DoctorRequest_Body) StableMarshal(buf []byte) []byte {
if x == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, x.StableSize())
}
var offset int
offset += proto.UInt32Marshal(1, buf[offset:], x.Concurrency)
offset += proto.BoolMarshal(2, buf[offset:], x.RemoveDuplicates)
return buf
}
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *DoctorRequest) StableSize() (size int) {
size += proto.NestedStructureSize(1, x.Body)
size += proto.NestedStructureSize(2, x.Signature)
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *DoctorRequest) StableMarshal(buf []byte) []byte {
if x == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, x.StableSize())
}
var offset int
offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body)
offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature)
return buf
}
// ReadSignedData fills buf with signed data of x.
// If buffer length is less than x.SignedDataSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same signed data.
func (x *DoctorRequest) SignedDataSize() int {
return x.GetBody().StableSize()
}
// SignedDataSize returns size of the request signed data in bytes.
//
// Structures with the same field values have the same signed data size.
func (x *DoctorRequest) ReadSignedData(buf []byte) ([]byte, error) {
return x.GetBody().StableMarshal(buf), nil
}
func (x *DoctorRequest) SetSignature(sig *Signature) {
x.Signature = sig
}
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *DoctorResponse_Body) StableSize() (size int) {
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *DoctorResponse_Body) StableMarshal(buf []byte) []byte {
return buf
}
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *DoctorResponse) StableSize() (size int) {
size += proto.NestedStructureSize(1, x.Body)
size += proto.NestedStructureSize(2, x.Signature)
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *DoctorResponse) StableMarshal(buf []byte) []byte {
if x == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, x.StableSize())
}
var offset int
offset += proto.NestedStructureMarshal(1, buf[offset:], x.Body)
offset += proto.NestedStructureMarshal(2, buf[offset:], x.Signature)
return buf
}
// ReadSignedData fills buf with signed data of x.
// If buffer length is less than x.SignedDataSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same signed data.
func (x *DoctorResponse) SignedDataSize() int {
return x.GetBody().StableSize()
}
// SignedDataSize returns size of the request signed data in bytes.
//
// Structures with the same field values have the same signed data size.
func (x *DoctorResponse) ReadSignedData(buf []byte) ([]byte, error) {
return x.GetBody().StableMarshal(buf), nil
}
func (x *DoctorResponse) SetSignature(sig *Signature) {
x.Signature = sig
}

View File

@ -1,6 +1,6 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.2.0
// - protoc-gen-go-grpc v1.3.0
// - protoc v3.21.12
// source: pkg/services/control/service.proto
@ -18,6 +18,20 @@ import (
// Requires gRPC-Go v1.32.0 or later.
const _ = grpc.SupportPackageIsVersion7
const (
ControlService_HealthCheck_FullMethodName = "/control.ControlService/HealthCheck"
ControlService_SetNetmapStatus_FullMethodName = "/control.ControlService/SetNetmapStatus"
ControlService_DropObjects_FullMethodName = "/control.ControlService/DropObjects"
ControlService_ListShards_FullMethodName = "/control.ControlService/ListShards"
ControlService_SetShardMode_FullMethodName = "/control.ControlService/SetShardMode"
ControlService_DumpShard_FullMethodName = "/control.ControlService/DumpShard"
ControlService_RestoreShard_FullMethodName = "/control.ControlService/RestoreShard"
ControlService_SynchronizeTree_FullMethodName = "/control.ControlService/SynchronizeTree"
ControlService_EvacuateShard_FullMethodName = "/control.ControlService/EvacuateShard"
ControlService_FlushCache_FullMethodName = "/control.ControlService/FlushCache"
ControlService_Doctor_FullMethodName = "/control.ControlService/Doctor"
)
// ControlServiceClient is the client API for ControlService service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
@ -42,6 +56,8 @@ type ControlServiceClient interface {
EvacuateShard(ctx context.Context, in *EvacuateShardRequest, opts ...grpc.CallOption) (*EvacuateShardResponse, error)
// FlushCache moves all data from one shard to the others.
FlushCache(ctx context.Context, in *FlushCacheRequest, opts ...grpc.CallOption) (*FlushCacheResponse, error)
// Doctor performs storage restructuring operations on engine.
Doctor(ctx context.Context, in *DoctorRequest, opts ...grpc.CallOption) (*DoctorResponse, error)
}
type controlServiceClient struct {
@ -54,7 +70,7 @@ func NewControlServiceClient(cc grpc.ClientConnInterface) ControlServiceClient {
func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) {
out := new(HealthCheckResponse)
err := c.cc.Invoke(ctx, "/control.ControlService/HealthCheck", in, out, opts...)
err := c.cc.Invoke(ctx, ControlService_HealthCheck_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -63,7 +79,7 @@ func (c *controlServiceClient) HealthCheck(ctx context.Context, in *HealthCheckR
func (c *controlServiceClient) SetNetmapStatus(ctx context.Context, in *SetNetmapStatusRequest, opts ...grpc.CallOption) (*SetNetmapStatusResponse, error) {
out := new(SetNetmapStatusResponse)
err := c.cc.Invoke(ctx, "/control.ControlService/SetNetmapStatus", in, out, opts...)
err := c.cc.Invoke(ctx, ControlService_SetNetmapStatus_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -72,7 +88,7 @@ func (c *controlServiceClient) SetNetmapStatus(ctx context.Context, in *SetNetma
func (c *controlServiceClient) DropObjects(ctx context.Context, in *DropObjectsRequest, opts ...grpc.CallOption) (*DropObjectsResponse, error) {
out := new(DropObjectsResponse)
err := c.cc.Invoke(ctx, "/control.ControlService/DropObjects", in, out, opts...)
err := c.cc.Invoke(ctx, ControlService_DropObjects_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -81,7 +97,7 @@ func (c *controlServiceClient) DropObjects(ctx context.Context, in *DropObjectsR
func (c *controlServiceClient) ListShards(ctx context.Context, in *ListShardsRequest, opts ...grpc.CallOption) (*ListShardsResponse, error) {
out := new(ListShardsResponse)
err := c.cc.Invoke(ctx, "/control.ControlService/ListShards", in, out, opts...)
err := c.cc.Invoke(ctx, ControlService_ListShards_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -90,7 +106,7 @@ func (c *controlServiceClient) ListShards(ctx context.Context, in *ListShardsReq
func (c *controlServiceClient) SetShardMode(ctx context.Context, in *SetShardModeRequest, opts ...grpc.CallOption) (*SetShardModeResponse, error) {
out := new(SetShardModeResponse)
err := c.cc.Invoke(ctx, "/control.ControlService/SetShardMode", in, out, opts...)
err := c.cc.Invoke(ctx, ControlService_SetShardMode_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -99,7 +115,7 @@ func (c *controlServiceClient) SetShardMode(ctx context.Context, in *SetShardMod
func (c *controlServiceClient) DumpShard(ctx context.Context, in *DumpShardRequest, opts ...grpc.CallOption) (*DumpShardResponse, error) {
out := new(DumpShardResponse)
err := c.cc.Invoke(ctx, "/control.ControlService/DumpShard", in, out, opts...)
err := c.cc.Invoke(ctx, ControlService_DumpShard_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -108,7 +124,7 @@ func (c *controlServiceClient) DumpShard(ctx context.Context, in *DumpShardReque
func (c *controlServiceClient) RestoreShard(ctx context.Context, in *RestoreShardRequest, opts ...grpc.CallOption) (*RestoreShardResponse, error) {
out := new(RestoreShardResponse)
err := c.cc.Invoke(ctx, "/control.ControlService/RestoreShard", in, out, opts...)
err := c.cc.Invoke(ctx, ControlService_RestoreShard_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -117,7 +133,7 @@ func (c *controlServiceClient) RestoreShard(ctx context.Context, in *RestoreShar
func (c *controlServiceClient) SynchronizeTree(ctx context.Context, in *SynchronizeTreeRequest, opts ...grpc.CallOption) (*SynchronizeTreeResponse, error) {
out := new(SynchronizeTreeResponse)
err := c.cc.Invoke(ctx, "/control.ControlService/SynchronizeTree", in, out, opts...)
err := c.cc.Invoke(ctx, ControlService_SynchronizeTree_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -126,7 +142,7 @@ func (c *controlServiceClient) SynchronizeTree(ctx context.Context, in *Synchron
func (c *controlServiceClient) EvacuateShard(ctx context.Context, in *EvacuateShardRequest, opts ...grpc.CallOption) (*EvacuateShardResponse, error) {
out := new(EvacuateShardResponse)
err := c.cc.Invoke(ctx, "/control.ControlService/EvacuateShard", in, out, opts...)
err := c.cc.Invoke(ctx, ControlService_EvacuateShard_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -135,7 +151,16 @@ func (c *controlServiceClient) EvacuateShard(ctx context.Context, in *EvacuateSh
func (c *controlServiceClient) FlushCache(ctx context.Context, in *FlushCacheRequest, opts ...grpc.CallOption) (*FlushCacheResponse, error) {
out := new(FlushCacheResponse)
err := c.cc.Invoke(ctx, "/control.ControlService/FlushCache", in, out, opts...)
err := c.cc.Invoke(ctx, ControlService_FlushCache_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *controlServiceClient) Doctor(ctx context.Context, in *DoctorRequest, opts ...grpc.CallOption) (*DoctorResponse, error) {
out := new(DoctorResponse)
err := c.cc.Invoke(ctx, ControlService_Doctor_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
@ -166,6 +191,8 @@ type ControlServiceServer interface {
EvacuateShard(context.Context, *EvacuateShardRequest) (*EvacuateShardResponse, error)
// FlushCache moves all data from one shard to the others.
FlushCache(context.Context, *FlushCacheRequest) (*FlushCacheResponse, error)
// Doctor performs storage restructuring operations on engine.
Doctor(context.Context, *DoctorRequest) (*DoctorResponse, error)
}
// UnimplementedControlServiceServer should be embedded to have forward compatible implementations.
@ -202,6 +229,9 @@ func (UnimplementedControlServiceServer) EvacuateShard(context.Context, *Evacuat
func (UnimplementedControlServiceServer) FlushCache(context.Context, *FlushCacheRequest) (*FlushCacheResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method FlushCache not implemented")
}
func (UnimplementedControlServiceServer) Doctor(context.Context, *DoctorRequest) (*DoctorResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Doctor not implemented")
}
// UnsafeControlServiceServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to ControlServiceServer will
@ -224,7 +254,7 @@ func _ControlService_HealthCheck_Handler(srv interface{}, ctx context.Context, d
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/control.ControlService/HealthCheck",
FullMethod: ControlService_HealthCheck_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ControlServiceServer).HealthCheck(ctx, req.(*HealthCheckRequest))
@ -242,7 +272,7 @@ func _ControlService_SetNetmapStatus_Handler(srv interface{}, ctx context.Contex
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/control.ControlService/SetNetmapStatus",
FullMethod: ControlService_SetNetmapStatus_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ControlServiceServer).SetNetmapStatus(ctx, req.(*SetNetmapStatusRequest))
@ -260,7 +290,7 @@ func _ControlService_DropObjects_Handler(srv interface{}, ctx context.Context, d
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/control.ControlService/DropObjects",
FullMethod: ControlService_DropObjects_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ControlServiceServer).DropObjects(ctx, req.(*DropObjectsRequest))
@ -278,7 +308,7 @@ func _ControlService_ListShards_Handler(srv interface{}, ctx context.Context, de
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/control.ControlService/ListShards",
FullMethod: ControlService_ListShards_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ControlServiceServer).ListShards(ctx, req.(*ListShardsRequest))
@ -296,7 +326,7 @@ func _ControlService_SetShardMode_Handler(srv interface{}, ctx context.Context,
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/control.ControlService/SetShardMode",
FullMethod: ControlService_SetShardMode_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ControlServiceServer).SetShardMode(ctx, req.(*SetShardModeRequest))
@ -314,7 +344,7 @@ func _ControlService_DumpShard_Handler(srv interface{}, ctx context.Context, dec
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/control.ControlService/DumpShard",
FullMethod: ControlService_DumpShard_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ControlServiceServer).DumpShard(ctx, req.(*DumpShardRequest))
@ -332,7 +362,7 @@ func _ControlService_RestoreShard_Handler(srv interface{}, ctx context.Context,
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/control.ControlService/RestoreShard",
FullMethod: ControlService_RestoreShard_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ControlServiceServer).RestoreShard(ctx, req.(*RestoreShardRequest))
@ -350,7 +380,7 @@ func _ControlService_SynchronizeTree_Handler(srv interface{}, ctx context.Contex
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/control.ControlService/SynchronizeTree",
FullMethod: ControlService_SynchronizeTree_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ControlServiceServer).SynchronizeTree(ctx, req.(*SynchronizeTreeRequest))
@ -368,7 +398,7 @@ func _ControlService_EvacuateShard_Handler(srv interface{}, ctx context.Context,
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/control.ControlService/EvacuateShard",
FullMethod: ControlService_EvacuateShard_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ControlServiceServer).EvacuateShard(ctx, req.(*EvacuateShardRequest))
@ -386,7 +416,7 @@ func _ControlService_FlushCache_Handler(srv interface{}, ctx context.Context, de
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/control.ControlService/FlushCache",
FullMethod: ControlService_FlushCache_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ControlServiceServer).FlushCache(ctx, req.(*FlushCacheRequest))
@ -394,6 +424,24 @@ func _ControlService_FlushCache_Handler(srv interface{}, ctx context.Context, de
return interceptor(ctx, in, info, handler)
}
func _ControlService_Doctor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(DoctorRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(ControlServiceServer).Doctor(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: ControlService_Doctor_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(ControlServiceServer).Doctor(ctx, req.(*DoctorRequest))
}
return interceptor(ctx, in, info, handler)
}
// ControlService_ServiceDesc is the grpc.ServiceDesc for ControlService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@ -441,6 +489,10 @@ var ControlService_ServiceDesc = grpc.ServiceDesc{
MethodName: "FlushCache",
Handler: _ControlService_FlushCache_Handler,
},
{
MethodName: "Doctor",
Handler: _ControlService_Doctor_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "pkg/services/control/service.proto",