From f0642d7f139e97378c5d8c0fe21b200c5bbbe90c Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Tue, 13 Jun 2023 01:33:53 +0300 Subject: [PATCH] [#40] types: Generate StableMarshaler methods for refs * Add plugin option for protogen in Makefile * Fix the generator for the plugin in util/protogen * Fix marshaler.go for refs Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- Makefile | 4 + refs/grpc/types.pb.go | Bin 27235 -> 27235 bytes refs/grpc/types_frostfs.pb.go | Bin 0 -> 7474 bytes refs/marshal.go | 154 ++++------------------------------ rpc/message/encoding.go | 11 +++ util/proto/marshal.go | 6 +- util/proto/test/test.pb.go | Bin 13851 -> 13851 bytes util/protogen/main.go | 110 ++++++++++++++---------- 8 files changed, 98 insertions(+), 187 deletions(-) create mode 100644 refs/grpc/types_frostfs.pb.go diff --git a/Makefile b/Makefile index 6ceea25..807744e 100755 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ #!/usr/bin/make -f SHELL = bash +BIN = bin + VERSION ?= $(shell git describe --tags --match "v*" --abbrev=8 --dirty --always) .PHONY: dep fmts fmt imports protoc test lint version help @@ -47,6 +49,8 @@ protoc: echo "⇒ Processing $$f "; \ protoc \ --proto_path=.:./vendor:/usr/local/include \ + --plugin=protoc-gen-go-frostfs=$(BIN)/protogen \ + --go-frostfs_out=. --go-frostfs_opt=paths=source_relative \ --go_out=. --go_opt=paths=source_relative \ --go-grpc_opt=require_unimplemented_servers=false \ --go-grpc_out=. --go-grpc_opt=paths=source_relative $$f; \ diff --git a/refs/grpc/types.pb.go b/refs/grpc/types.pb.go index f16583be011e0011556868b8ec61a9fbff9f637f..7db06b0535a875a52372c3b31ce1063a3892451c 100644 GIT binary patch delta 43 tcmaESh4Jwf#tFf4#s+!@T>AP7oCQVsCHct;P*7&9XK18nva#?}CIC1$4Rini delta 43 tcmaESh4Jwf#tFf4MizR8T>AP7oCQVsCHct;P*7&9XJn{nxv}t5CIC5B4S@gv diff --git a/refs/grpc/types_frostfs.pb.go b/refs/grpc/types_frostfs.pb.go new file mode 100644 index 0000000000000000000000000000000000000000..4a5ebd150126a662611cde55a8ae3cf438917dd8 GIT binary patch literal 7474 zcmeI1-*4J55XYa%UvcUaY(O+BZR(^xY_yZm)*%(vm#PmYxq#bXBin(ps{i}WPQs5s zwIXd*)b$#`mEsLxeF3RLo!qqb1CgG%{$>XRWbpMM`QD={B78;d_4w z=V!fP+w*2DxMO1kg^~6=KAlNr;AqTEJ9pqBlKxO7*o|cpeq$p)`=-TaoFe=4j-MDF z`|f^6p6B~8Fl-d#fd90CO`?PbGeOYy0~x_RaFM1rN+O7)nljTSztmMF0lA=t1vit_ zk4+IGj&TTc7AN%8gw1jN)rbqGmM(nTixLq){T|+Y3PXyb8ynfw8&G#iaA6wo;MLs4 z+tW0e_IK1&yMf6vu=&^DHQ~P9mBKJiwe5wC=Qwq&Q-5@efpH#b5nO`a<3@!pn9>s# z!!&)>h00|AS6ZO79NJYjD~41ds=RQ$2&kG7Du_{xsd8M?J8dh^1f{d?Nxjhofs5Rn z>{uMjz;crZyUe=A#1c>`sQ^VF6JbyhT#E@0CJ^!vgfz~10SmzmQiU?Y3A zkZ?=v=_v>v+sj(k&D8MW(W}uP*;wZdD|bnqy~BEHX_?q;0+B?HbEZfVkw!zJFkUTt z{}hvXri3Nk;5xPbc(V)XGvg4c#W=g%lEoIbRv24dbTWD|VP^@R9iZ&b)Kb0H^Ob#y zQ2i@imQ0t$*5a=80UWUggNf4=eY}ZE7i`y^X{5ae|KZ%WSqEtyoi6AbQjGCe?|tEH9(oBWA1 wT+ZA=>yCI=r_#lRn{HE`rk3tkjv@jj7U<^cxHDE4Su8HCzg>|x@l0j<3ji15W&i*H literal 0 HcmV?d00001 diff --git a/refs/marshal.go b/refs/marshal.go index 0382190..8ca50cc 100644 --- a/refs/marshal.go +++ b/refs/marshal.go @@ -30,25 +30,11 @@ const ( ) func (o *OwnerID) StableMarshal(buf []byte) []byte { - if o == nil { - return []byte{} - } - - if buf == nil { - buf = make([]byte, o.StableSize()) - } - - proto.BytesMarshal(ownerIDValField, buf, o.val) - - return buf + return message.StableMarshal[*refs.OwnerID](o, buf) } func (o *OwnerID) StableSize() int { - if o == nil { - return 0 - } - - return proto.BytesSize(ownerIDValField, o.val) + return message.StableSize[*refs.OwnerID](o) } func (o *OwnerID) Unmarshal(data []byte) error { @@ -56,25 +42,11 @@ func (o *OwnerID) Unmarshal(data []byte) error { } func (c *ContainerID) StableMarshal(buf []byte) []byte { - if c == nil { - return []byte{} - } - - if buf == nil { - buf = make([]byte, c.StableSize()) - } - - proto.BytesMarshal(containerIDValField, buf, c.val) - - return buf + return message.StableMarshal[*refs.ContainerID](c, buf) } func (c *ContainerID) StableSize() int { - if c == nil { - return 0 - } - - return proto.BytesSize(containerIDValField, c.val) + return message.StableSize[*refs.ContainerID](c) } func (c *ContainerID) Unmarshal(data []byte) error { @@ -82,22 +54,13 @@ func (c *ContainerID) Unmarshal(data []byte) error { } func (o *ObjectID) StableMarshal(buf []byte) []byte { - if o == nil { - return []byte{} - } - - if buf == nil { - buf = make([]byte, o.StableSize()) - } - - proto.BytesMarshal(objectIDValField, buf, o.val) - - return buf + return message.StableMarshal[*refs.ObjectID](o, buf) } // ObjectIDNestedListSize returns byte length of nested // repeated ObjectID field with fNum number. func ObjectIDNestedListSize(fNum int64, ids []ObjectID) (sz int) { + // TODO (aarifullin): remove this method when all marshalers are refactored for i := range ids { sz += proto.NestedStructureSize(fNum, &ids[i]) } @@ -106,16 +69,13 @@ func ObjectIDNestedListSize(fNum int64, ids []ObjectID) (sz int) { } func (o *ObjectID) StableSize() int { - if o == nil { - return 0 - } - - return proto.BytesSize(objectIDValField, o.val) + return message.StableSize[*refs.ObjectID](o) } // ObjectIDNestedListMarshal writes protobuf repeated ObjectID field // with fNum number to buf. func ObjectIDNestedListMarshal(fNum int64, buf []byte, ids []ObjectID) (off int) { + // TODO (aarifullin): remove this method when all marshalers are refactored prefix, _ := proto.NestedStructurePrefix(fNum) for i := range ids { off += binary.PutUvarint(buf[off:], prefix) @@ -133,31 +93,11 @@ func (o *ObjectID) Unmarshal(data []byte) error { } func (a *Address) StableMarshal(buf []byte) []byte { - if a == nil { - return []byte{} - } - - if buf == nil { - buf = make([]byte, a.StableSize()) - } - - var offset int - - offset += proto.NestedStructureMarshal(addressContainerField, buf[offset:], a.cid) - proto.NestedStructureMarshal(addressObjectField, buf[offset:], a.oid) - - return buf + return message.StableMarshal[*refs.Address](a, buf) } func (a *Address) StableSize() (size int) { - if a == nil { - return 0 - } - - size += proto.NestedStructureSize(addressContainerField, a.cid) - size += proto.NestedStructureSize(addressObjectField, a.oid) - - return size + return message.StableSize[*refs.Address](a) } func (a *Address) Unmarshal(data []byte) error { @@ -165,31 +105,11 @@ func (a *Address) Unmarshal(data []byte) error { } func (c *Checksum) StableMarshal(buf []byte) []byte { - if c == nil { - return []byte{} - } - - if buf == nil { - buf = make([]byte, c.StableSize()) - } - - var offset int - - offset += proto.EnumMarshal(checksumTypeField, buf[offset:], int32(c.typ)) - proto.BytesMarshal(checksumValueField, buf[offset:], c.sum) - - return buf + return message.StableMarshal[*refs.Checksum](c, buf) } func (c *Checksum) StableSize() (size int) { - if c == nil { - return 0 - } - - size += proto.EnumSize(checksumTypeField, int32(c.typ)) - size += proto.BytesSize(checksumValueField, c.sum) - - return size + return message.StableSize[*refs.Checksum](c) } func (c *Checksum) Unmarshal(data []byte) error { @@ -197,33 +117,11 @@ func (c *Checksum) Unmarshal(data []byte) error { } func (s *Signature) StableMarshal(buf []byte) []byte { - if s == nil { - return []byte{} - } - - if buf == nil { - buf = make([]byte, s.StableSize()) - } - - var offset int - - offset += proto.BytesMarshal(signatureKeyField, buf[offset:], s.key) - offset += proto.BytesMarshal(signatureValueField, buf[offset:], s.sign) - proto.EnumMarshal(signatureSchemeField, buf[offset:], int32(s.scheme)) - - return buf + return message.StableMarshal[*refs.Signature](s, buf) } func (s *Signature) StableSize() (size int) { - if s == nil { - return 0 - } - - size += proto.BytesSize(signatureKeyField, s.key) - size += proto.BytesSize(signatureValueField, s.sign) - size += proto.EnumSize(signatureSchemeField, int32(s.scheme)) - - return size + return message.StableSize[*refs.Signature](s) } func (s *Signature) Unmarshal(data []byte) error { @@ -231,31 +129,11 @@ func (s *Signature) Unmarshal(data []byte) error { } func (v *Version) StableMarshal(buf []byte) []byte { - if v == nil { - return []byte{} - } - - if buf == nil { - buf = make([]byte, v.StableSize()) - } - - var offset int - - offset += proto.UInt32Marshal(versionMajorField, buf[offset:], v.major) - proto.UInt32Marshal(versionMinorField, buf[offset:], v.minor) - - return buf + return message.StableMarshal[*refs.Version](v, buf) } func (v *Version) StableSize() (size int) { - if v == nil { - return 0 - } - - size += proto.UInt32Size(versionMajorField, v.major) - size += proto.UInt32Size(versionMinorField, v.minor) - - return size + return message.StableSize[*refs.Version](v) } func (v *Version) Unmarshal(data []byte) error { diff --git a/rpc/message/encoding.go b/rpc/message/encoding.go index 715f015..649b10a 100644 --- a/rpc/message/encoding.go +++ b/rpc/message/encoding.go @@ -2,6 +2,7 @@ package message import ( "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/grpc" + protoutil "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" ) @@ -46,3 +47,13 @@ func UnmarshalJSON(m Message, data []byte, gm GRPCConvertedMessage) error { return m.FromGRPCMessage(gm) } + +// StableMarshal encodes message 'm' using the invocation from related gRPC message +func StableMarshal[M protoutil.StableMarshaller](m Message, buf []byte) []byte { + return m.ToGRPCMessage().(M).StableMarshal(buf) +} + +// StableSize encodes message 'm' using the invocation from related gRPC message +func StableSize[M protoutil.StableMarshaller](m Message) int { + return m.ToGRPCMessage().(M).StableSize() +} diff --git a/util/proto/marshal.go b/util/proto/marshal.go index a82478b..2ac14f2 100644 --- a/util/proto/marshal.go +++ b/util/proto/marshal.go @@ -14,7 +14,7 @@ import ( ) type ( - stableMarshaller interface { + StableMarshaller interface { StableMarshal([]byte) []byte StableSize() int } @@ -302,7 +302,7 @@ func NestedStructurePrefix(field int64) (prefix uint64, ln int) { return prefix, VarUIntSize(prefix) } -func NestedStructureMarshal(field int64, buf []byte, v stableMarshaller) int { +func NestedStructureMarshal(field int64, buf []byte, v StableMarshaller) int { if v == nil || reflect.ValueOf(v).IsNil() { return 0 } @@ -317,7 +317,7 @@ func NestedStructureMarshal(field int64, buf []byte, v stableMarshaller) int { return offset + n } -func NestedStructureSize(field int64, v stableMarshaller) (size int) { +func NestedStructureSize(field int64, v StableMarshaller) (size int) { if v == nil || reflect.ValueOf(v).IsNil() { return 0 } diff --git a/util/proto/test/test.pb.go b/util/proto/test/test.pb.go index 0dfa4dc28064e4b471d35a3d873fabaf5bacb9ce..1d17e190d4fb238bab2ea88bac2fcbae0428b691 100644 GIT binary patch delta 41 rcmbQ8GdpKOu$-}ho&lG>z5-`KQGQ8&vH}#88S5Du>6vUSTxALX`@Rd7 delta 41 rcmbQ8GdpKOu$+;Fo*|dMz5-`KQGQ8&vH}#88S5Dt>RE0qTxALX{M`$q diff --git a/util/protogen/main.go b/util/protogen/main.go index 99f5b7b..aea5c6c 100644 --- a/util/protogen/main.go +++ b/util/protogen/main.go @@ -8,14 +8,18 @@ import ( "google.golang.org/protobuf/reflect/protoreflect" ) +type MethodMask int + +const ( + Signature MethodMask = 1 << iota +) + func main() { protogen.Options{}.Run(func(gen *protogen.Plugin) error { for _, f := range gen.Files { - //if !f.Generate { - // continue - //} - imp := string(f.GoImportPath) - if strings.HasSuffix(imp, "/tree") || strings.HasSuffix(imp, "/control") { + impPath := f.GoImportPath.String() + if strings.HasSuffix(impPath, "/tree") || strings.HasSuffix(impPath, "/control") || + strings.Contains(impPath, "/refs/") { generateFile(gen, f) } } @@ -25,6 +29,11 @@ func main() { // generateFile generates a *.pb.go file enforcing field-order serialization. func generateFile(gen *protogen.Plugin, file *protogen.File) *protogen.GeneratedFile { + emitMask := MethodMask(0) + if imp := string(file.GoImportPath); strings.HasSuffix(imp, "/tree") || strings.HasSuffix(imp, "/control") { + emitMask |= Signature + } + filename := file.GeneratedFilenamePrefix + "_frostfs.pb.go" g := gen.NewGeneratedFile(filename, file.GoImportPath) g.P("// Code generated by protoc-gen-go-frostfs. DO NOT EDIT.") @@ -33,28 +42,14 @@ func generateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated g.P() g.P(`import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto"`) - //for _, e := range file.Enums { - // g.P("type " + e.GoIdent.GoName + " int32") - // g.P("const (") - // for _, ev := range e.Values { - // g.P(ev.GoIdent.GoName, " = ", ev.Desc.Number()) - // } - // g.P(")") - //} for _, msg := range file.Messages { - emitMessage(g, msg) + emitMessage(g, msg, emitMask) } return g } -func emitMessage(g *protogen.GeneratedFile, msg *protogen.Message) { - for _, inner := range msg.Messages { - emitMessage(g, inner) - } - +func genStableSizeFunc(g *protogen.GeneratedFile, msg *protogen.Message) { fs := sortFields(msg.Fields) - - // StableSize implementation. g.P("// StableSize returns the size of x in protobuf format.") g.P("//") g.P("// Structures with the same field values have the same binary size.") @@ -72,8 +67,10 @@ func emitMessage(g *protogen.GeneratedFile, msg *protogen.Message) { } g.P("return size") g.P("}\n") +} - // StableMarshal implementation. +func genStableMarshalFunc(g *protogen.GeneratedFile, msg *protogen.Message) { + fs := sortFields(msg.Fields) g.P("// StableMarshal marshals x in protobuf binary format with stable field order.") g.P("//") g.P("// If buffer length is less than x.StableSize(), new buffer is allocated.") @@ -93,32 +90,53 @@ func emitMessage(g *protogen.GeneratedFile, msg *protogen.Message) { } g.P("return buf") g.P("}\n") +} + +func genReadSignedDataFunc(g *protogen.GeneratedFile, msg *protogen.Message) { + g.P("// ReadSignedData fills buf with signed data of x.") + g.P("// If buffer length is less than x.SignedDataSize(), new buffer is allocated.") + g.P("//") + g.P("// Returns any error encountered which did not allow writing the data completely.") + g.P("// Otherwise, returns the buffer in which the data is written.") + g.P("//") + + g.P("func (x *", msg.GoIdent.GoName, ") ReadSignedData(buf []byte) ([]byte, error) {") + g.P("return x.GetBody().StableMarshal(buf), nil") + g.P("}\n") +} + +func genSignedDataSizeFunc(g *protogen.GeneratedFile, msg *protogen.Message) { + g.P("// SignedDataSize returns size of the request signed data in bytes.") + g.P("//") + g.P("// Structures with the same field values have the same signed data size.") + + g.P("// Structures with the same field values have the same signed data.") + g.P("func (x *", msg.GoIdent.GoName, ") SignedDataSize() int {") + g.P("return x.GetBody().StableSize()") + g.P("}\n") +} + +func genSetSignatureFunc(g *protogen.GeneratedFile, msg *protogen.Message) { + g.P("func (x *", msg.GoIdent.GoName, ") SetSignature(sig *Signature) {") + g.P("x.Signature = sig") + g.P("}\n") +} + +func emitMessage(g *protogen.GeneratedFile, msg *protogen.Message, mask MethodMask) { + for _, inner := range msg.Messages { + emitMessage(g, inner, mask) + } + + genStableSizeFunc(g, msg) + + genStableMarshalFunc(g, msg) if strings.HasSuffix(msg.GoIdent.GoName, "Request") || strings.HasSuffix(msg.GoIdent.GoName, "Response") { - // SignedDataSize implementation (only for requests and responses). - g.P("// ReadSignedData fills buf with signed data of x.") - g.P("// If buffer length is less than x.SignedDataSize(), new buffer is allocated.") - g.P("//") - g.P("// Returns any error encountered which did not allow writing the data completely.") - g.P("// Otherwise, returns the buffer in which the data is written.") - g.P("//") - g.P("// Structures with the same field values have the same signed data.") - g.P("func (x *", msg.GoIdent.GoName, ") SignedDataSize() int {") - g.P("return x.GetBody().StableSize()") - g.P("}\n") - - // ReadSignedData implementation (only for requests and responses). - g.P("// SignedDataSize returns size of the request signed data in bytes.") - g.P("//") - g.P("// Structures with the same field values have the same signed data size.") - g.P("func (x *", msg.GoIdent.GoName, ") ReadSignedData(buf []byte) ([]byte, error) {") - g.P("return x.GetBody().StableMarshal(buf), nil") - g.P("}\n") - - // Signature setters and getters. - g.P("func (x *", msg.GoIdent.GoName, ") SetSignature(sig *Signature) {") - g.P("x.Signature = sig") - g.P("}\n") + if mask&Signature != 0 { + genReadSignedDataFunc(g, msg) + genSignedDataSizeFunc(g, msg) + genSetSignatureFunc(g, msg) + } } }