From 0e60b1d6c9514ef148f8c6d4508efaa1b1e41706 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Mon, 24 Jan 2022 13:13:37 +0300 Subject: [PATCH] [#1086] services/control: implement `DumpShard` RPC Signed-off-by: Evgenii Stratonikov --- pkg/local_object_storage/engine/dump.go | 19 +++ pkg/services/control/convert.go | 18 +++ pkg/services/control/rpc.go | 14 ++ pkg/services/control/server/dump.go | 37 ++++++ pkg/services/control/service.go | 169 ++++++++++++++++++++++++ pkg/services/control/service.pb.go | Bin 67146 -> 77800 bytes pkg/services/control/service.proto | 37 ++++++ pkg/services/control/service_grpc.pb.go | Bin 11811 -> 13424 bytes pkg/services/control/types.pb.go | Bin 25708 -> 25708 bytes 9 files changed, 294 insertions(+) create mode 100644 pkg/local_object_storage/engine/dump.go create mode 100644 pkg/services/control/server/dump.go diff --git a/pkg/local_object_storage/engine/dump.go b/pkg/local_object_storage/engine/dump.go new file mode 100644 index 000000000..f9a1fffa1 --- /dev/null +++ b/pkg/local_object_storage/engine/dump.go @@ -0,0 +1,19 @@ +package engine + +import "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + +// DumpShard dumps objects from the shard with provided identifier. +// +// Returns an error if shard is not read-only. +func (e *StorageEngine) DumpShard(id *shard.ID, prm *shard.DumpPrm) error { + e.mtx.RLock() + defer e.mtx.RUnlock() + + sh, ok := e.shards[id.String()] + if !ok { + return errShardNotFound + } + + _, err := sh.Dump(prm) + return err +} diff --git a/pkg/services/control/convert.go b/pkg/services/control/convert.go index 02c74d136..c64b80b81 100644 --- a/pkg/services/control/convert.go +++ b/pkg/services/control/convert.go @@ -130,3 +130,21 @@ func (w *setShardModeResponseWrapper) FromGRPCMessage(m grpc.Message) error { return nil } + +type dumpShardResponseWrapper struct { + *DumpShardResponse +} + +func (w *dumpShardResponseWrapper) ToGRPCMessage() grpc.Message { + return w.DumpShardResponse +} + +func (w *dumpShardResponseWrapper) FromGRPCMessage(m grpc.Message) error { + r, ok := m.(*DumpShardResponse) + if !ok { + return message.NewUnexpectedMessageType(m, (*DumpShardResponse)(nil)) + } + + w.DumpShardResponse = r + return nil +} diff --git a/pkg/services/control/rpc.go b/pkg/services/control/rpc.go index c8edde492..46fba470e 100644 --- a/pkg/services/control/rpc.go +++ b/pkg/services/control/rpc.go @@ -14,6 +14,7 @@ const ( rpcDropObjects = "DropObjects" rpcListShards = "ListShards" rpcSetShardMode = "SetShardMode" + rpcDumpShard = "DumpShard" ) // HealthCheck executes ControlService.HealthCheck RPC. @@ -144,3 +145,16 @@ func SetShardMode( return wResp.m, nil } + +// DumpShard executes ControlService.DumpShard RPC. +func DumpShard(cli *client.Client, req *DumpShardRequest, opts ...client.CallOption) (*DumpShardResponse, error) { + wResp := new(dumpShardResponseWrapper) + wReq := &requestWrapper{m: req} + + err := client.SendUnary(cli, common.CallMethodInfoUnary(serviceName, rpcDumpShard), wReq, wResp, opts...) + if err != nil { + return nil, err + } + + return wResp.DumpShardResponse, nil +} diff --git a/pkg/services/control/server/dump.go b/pkg/services/control/server/dump.go new file mode 100644 index 000000000..024af053f --- /dev/null +++ b/pkg/services/control/server/dump.go @@ -0,0 +1,37 @@ +package control + +import ( + "context" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard" + "github.com/nspcc-dev/neofs-node/pkg/services/control" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func (s *Server) DumpShard(_ context.Context, req *control.DumpShardRequest) (*control.DumpShardResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + + shardID := shard.NewIDFromBytes(req.GetBody().GetShard_ID()) + + prm := new(shard.DumpPrm) + prm.WithPath(req.GetBody().GetFilepath()) + prm.WithIgnoreErrors(req.GetBody().GetIgnoreErrors()) + + err = s.s.DumpShard(shardID, prm) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := new(control.DumpShardResponse) + resp.SetBody(new(control.DumpShardResponse_Body)) + + err = SignMessage(s.key, resp) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return resp, nil +} diff --git a/pkg/services/control/service.go b/pkg/services/control/service.go index cdd6cede6..b57d70c80 100644 --- a/pkg/services/control/service.go +++ b/pkg/services/control/service.go @@ -910,3 +910,172 @@ func (x *SetShardModeResponse) ReadSignedData(buf []byte) ([]byte, error) { func (x *SetShardModeResponse) SignedDataSize() int { return x.GetBody().StableSize() } + +// SetShardID sets shard ID for the dump shard request. +func (x *DumpShardRequest_Body) SetShardID(id []byte) { + x.Shard_ID = id +} + +// SetFilepath sets filepath for the dump shard request. +func (x *DumpShardRequest_Body) SetFilepath(p string) { + x.Filepath = p +} + +// SetIgnoreErrors sets ignore errors flag for the dump shard request. +func (x *DumpShardRequest_Body) SetIgnoreErrors(ignore bool) { + x.IgnoreErrors = ignore +} + +const ( + _ = iota + dumpShardReqBodyShardIDFNum + dumpShardReqBodyFilepathFNum + dumpShardReqBodyIgnoreErrorsFNum +) + +// StableMarshal reads binary representation of request body binary format. +// +// If buffer length is less than StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *DumpShardRequest_Body) StableMarshal(buf []byte) ([]byte, error) { + if x == nil { + return []byte{}, nil + } + + if sz := x.StableSize(); len(buf) < sz { + buf = make([]byte, sz) + } + + var ( + offset, n int + err error + ) + + n, err = proto.BytesMarshal(dumpShardReqBodyShardIDFNum, buf, x.Shard_ID) + if err != nil { + return nil, err + } + + offset += n + + n, err = proto.StringMarshal(dumpShardReqBodyFilepathFNum, buf[offset:], x.Filepath) + if err != nil { + return nil, err + } + + offset += n + + _, err = proto.BoolMarshal(dumpShardReqBodyIgnoreErrorsFNum, buf[offset:], x.IgnoreErrors) + if err != nil { + return nil, err + } + + return buf, nil +} + +// StableSize returns binary size of the request body in protobuf binary format. +// +// Structures with the same field values have the same binary size. +func (x *DumpShardRequest_Body) StableSize() int { + if x == nil { + return 0 + } + + size := 0 + + size += proto.BytesSize(dumpShardReqBodyShardIDFNum, x.Shard_ID) + size += proto.StringSize(dumpShardReqBodyFilepathFNum, x.Filepath) + size += proto.BoolSize(dumpShardReqBodyIgnoreErrorsFNum, x.IgnoreErrors) + + return size +} + +// SetBody sets request body. +func (x *DumpShardRequest) SetBody(v *DumpShardRequest_Body) { + if x != nil { + x.Body = v + } +} + +// SetSignature sets body signature of the request. +func (x *DumpShardRequest) SetSignature(v *Signature) { + if x != nil { + x.Signature = v + } +} + +// ReadSignedData reads signed data from request to buf. +// +// If buffer length is less than x.SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *DumpShardRequest) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf) +} + +// SignedDataSize returns size of the request signed data in bytes. +// +// Structures with the same field values have the same signed data size. +func (x *DumpShardRequest) SignedDataSize() int { + return x.GetBody().StableSize() +} + +// StableMarshal reads binary representation of the response body in protobuf binary format. +// +// If buffer length is less than x.StableSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same binary format. +func (x *DumpShardResponse_Body) StableMarshal(buf []byte) ([]byte, error) { + return buf, nil +} + +// StableSize returns binary size of the response body +// in protobuf binary format. +// +// Structures with the same field values have the same binary size. +func (x *DumpShardResponse_Body) StableSize() int { + return 0 +} + +// SetBody sets response body. +func (x *DumpShardResponse) SetBody(v *DumpShardResponse_Body) { + if x != nil { + x.Body = v + } +} + +// SetSignature sets response body signature. +func (x *DumpShardResponse) SetSignature(v *Signature) { + if x != nil { + x.Signature = v + } +} + +// ReadSignedData reads signed data from response to buf. +// +// If buffer length is less than SignedDataSize(), new buffer is allocated. +// +// Returns any error encountered which did not allow writing the data completely. +// Otherwise, returns the buffer in which the data is written. +// +// Structures with the same field values have the same signed data. +func (x *DumpShardResponse) ReadSignedData(buf []byte) ([]byte, error) { + return x.GetBody().StableMarshal(buf) +} + +// SignedDataSize returns binary size of the signed data. +// +// Structures with the same field values have the same signed data size. +func (x *DumpShardResponse) SignedDataSize() int { + return x.GetBody().StableSize() +} diff --git a/pkg/services/control/service.pb.go b/pkg/services/control/service.pb.go index 49d32bf8434a35fb91c2554acdd3cd44bc5dac1e..5ab9bd96308139183d74f2e9a331998132b7f26a 100644 GIT binary patch delta 4786 zcmbVQdr(tX8lPN&kPse11&IbP*eZbtxi=v^>H{m*t+o_uTPr$9AmO1%LI_ssd)i&c zZWYVV)>^Ftb*$FWLQT8wR@=h#p`}%PYejjexVv>*v`*J?yXu~EZbGgWcW0JAZtnLu z-}%1Z_c-^^{#wA_x&ziftFxpS)v2kx%IR{r9VNW?Z;>fQ$V@SRSbgrv_?+5>&Y9&_ zmyLJXt848w?i97V&S~cd_*lxVQ`?9ZUqjXKk3?;-Zw(^!NBqR{h**_>nbT5?ZMmf~ z&gAqbWz@XDM`ToeU^yc5uE;pfsYV&q7_~4~5SOW!snw;mRV94lVm`^g*NSn&{BCR= zH{DOoQ|p*`)^_thSkesJUmS++dDh8!*8%~yj;{ax&$9TD-Uw0kn2~D zpFT!rr*lFD#y&5Pu{BnZ$)65&SWFKH@fok@dVZSS{g}g6m#CKmutz&xW~gKAJkN}$ zW-G9$z>clTNOULz!9RTiKh5bw=&Vpo%vF2B^LGqE{X9FmW@*tse+rg|M1h|bi1p9C z1+}%$6FRG62(s;VtSyQ_M1dNc>ZT%V&PRk7fX_>N(QjpZ&yRE8A z)`q}Qv>87v==1bj^M~Mo!;URB9X8iZ!OFO3blZqerJIpl-REg6`;No=ZaXR$#1RX> zV#K08MDRmBJ(ZmtbakcZcEz%lg+eGkcMip7XFb-ubj1_ldW%Ega;gxD4{Af%{)Ivm zR@MX}msqPGcvqsewgB!GSN6Ub$Kl4yb`)l7uzs8_-vCR{+>O6;5T{%Xr zj$y7Ry&8u4DS=2`twhHwZz{}KnRAm+f%tK*9*t9@n54pG^QcxAa*8yx>B>AEYeE&| zb!E=a4QuE=2CVx-2#lvRc)Hlm2?APQO!Q~ob6{1F;^922yPS7Bcz3y-chtI_wWOz2 zk5^RMom5ccc2!iBL3m8JqSVVXu<>9@;OEyks%~yDp(fts-OO@k(nM zUr}YND6zVU4gSS)JK4j~PGk4zb;$mID2+W<@5=U{8{K8nl)9CotlH$=|*=UPDtLIcG`3 z_U0$RZ_>~zKtRic|CRdfdyN?X-Y}Z_YaTsX8v}5)Weh0+mh1eF^a$T&BxMkUJ0I)4 zWe`NQdq12>@~_dd%~`HY%y}yd)<`mmeV#1N_`|{O|E~46GUt| zC}91@;gnO|s>jvOEkUfDtk`oX8+jOkqK3k-G-Lv=7& zIB3D7U5UtU3CB`T5-JZ4#nJtG>%IuVI|oc?-5-P(>h&o8 zCWE-F#_dm)GFc@4>`5?4*qEi_lp2X;`~g)NHJlX%Q@1ccy?M&Ke<+2qVjJHl%8?3<--FO?%DIwX^7^wx=q1ENV}sp}yCi8c38acEst-)3T?WZ8A`Of;1B0PC6kdG6|K>iAGc(jX`6l27f)F=R|>!I^>;K!g4Z+9CFr^ zPby4k>xoClNdw*{pI{+fQP7?;D$*LdhNI(mL6Ir+G`}ZYv>l5@%;};2ONjTB9z?bN zq&Aj&&-t*0k) zCKI|&&r*_FBQ}j*_<}I|j9HO^b!T!3h!@VxR+4c|#G zBpK7;@P%YjLANdxc-f-R&=rTeuH<0SL=s_MKlkVo9fQd zqVG<8Z+=AyJuz+2WnHEb5}2Z8F!`tFMw?~LWT!=CsZbK?o*0j z6bGSK#IcK7EbE;}iYl)!7N>f1uz>AJkWG044S z2$n3HjZ`F?FIz7Tf9O6`)hgmKruU5^dug!al8G~m%-}DVrgCPI_`Z>N;c^biHT7rT zJGh9hal|s*KdqbP`YPho3&271nv4c{A_L!e9sEg)w78@X|5!=FN;Z78@m!Qxm)-O+tzRPw5zyrqf+Nd$SsZ+w=>LPHw>PAJ=Q>fkO!iCL=*uOEG z|8GIX_o5iy`9ZI4C!%e!i>T(3*N^r$&*O2<@;%5NqZ64W%-7?QkDjfV!o^ z-iT#CjvD)#f0cl;gApu8T1@q`N1a$dV#81ir0KH*y4R!aj`+o@1-z;PwYMClr&1(tdZXzm zXxE{=H_PnTMexDxxzNt$;G-|oP~9$zr1|fyz_t%n+`BEKtj~$n@8&{l-;IVlvLh6) z8;(`SkIsFowbP7-o7pDb&5!W4CuXu`{Q941@O(In2NhAY`D_L1zLKr}da+fHpypkT zSN)&7%f|E2Zkph|WtCNhOQ^?@tI4Rkm5-|5rI-ce$C>LZ(D0W;jw*QboV&I237qdt z!SRj-2!FL4>)%%~HDIG4ww`_lLtP3=KTi}SYVV{Y6!pUSd2AYcqvdd2P%w1eg28U9 z$hgj->fa`t!eTbd`pR*?JCUc$P(E923x;B{W3lPJaWk-I+b9F02{@A{B^N)!0% zQ~c`n%|YE@Hg}CT2Wurcn5*sz??8Lyzs49$iH` zR*k)64SGHNZRr=n=3!tePx13p;YduikZM86bQ)V{;!Rt|EF5t1+fzAS#nNPORtl>51pKTACDgc=9=GVZ6G=WoO~WWbxPS{4t1xvO@$`#dmn{6x$4{ERB=k#cZrc;jKbeUfZowi6amnz?wio zNl<4GRC2RxDmm?1Ox&7)R>IqGIPbp^O(4l^nT?YFKdiFXY&3^odXaao-A0wO*m5`Z zvapLP*m4Is@YX{+Ys{ca?BP5Pb$Q5zMwgkrw2&^dc^R})K(|==d>*`$$*rB9xFtSu zc9x$<7uktSp0@Q#UaV@bjh$JxAt zWTYw-XCxM-=y7p^WP?HM$r}{KCJXSfAc=fnRYCAJuVYK$n0%gB9w9_X%Va@h)suK^ zIpCHx@cM5S;um3wCc%Xo$@zIDsTC!9&JbEhK@0BspwzMjttdljw zH7()FNHqfJ1Chx#BIZbbb`TTYd`==(aPoP1Q6yo1+385USm_!h-a+LIgwGGCRmdT^ nr=&thp(wRb56Rt=Kgif@*4A3YJh?zu7|ED^T_q&mUfnbRf8oU~ delta 39 xcmV+?0NDTVXro+^HnFgY2D8NnQwXz65flWo%oV5wvp*bN5wjsMq63o}G*(L%4y^zH diff --git a/pkg/services/control/types.pb.go b/pkg/services/control/types.pb.go index 746d0cda9533a904111d658656b915e29cfe7a7a..a67cecd97ca0d6349eae50d9183b17c5541ad536 100644 GIT binary patch delta 42 scmaEJg7M7>#t9*^W_ku(`uYl-1x5KK`N;}UP-d)WXsKtivG8{?04*pDO8@`> delta 42 scmaEJg7M7>#t9*^=6Z%)`uYl-1x5KK`N;}UP-d)WXsKtovG8{?04-S!N&o-=