aarifullin
/
frostfs-api-go
forked from TrueCloudLab/frostfs-api-go
1
0
Fork 0
Code Pull Requests Activity

[#40] types: Generate StableMarshaler methods for refs 

* Add plugin option for protogen in Makefile
* Fix the generator for the plugin in util/protogen
* Fix marshaler.go for refs

Signed-off-by: Airat Arifullin a.arifullin@yadro.com
feature/40-autogen_marsh/refs_grpc
Airat Arifullin 2023-06-13 01:33:53 +03:00
parent 8266b31092
commit f0642d7f13
8 changed files with 343 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -1,6 +1,8 @@
#!/usr/bin/make -f
SHELL = bash
BIN = bin
VERSION ?= $(shell git describe --tags --match "v*" --abbrev=8 --dirty --always)
.PHONY: dep fmts fmt imports protoc test lint version help
@ -47,6 +49,8 @@ protoc:
echo "⇒ Processing $$f "; \
protoc \
--proto_path=.:./vendor:/usr/local/include \
--plugin=protoc-gen-go-frostfs=$(BIN)/protogen \
--go-frostfs_out=. --go-frostfs_opt=paths=source_relative \
--go_out=. --go_opt=paths=source_relative \
--go-grpc_opt=require_unimplemented_servers=false \
--go-grpc_out=. --go-grpc_opt=paths=source_relative $$f; \

4
refs/grpc/types.pb.go generated
View File

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.1
// protoc v3.21.9
// protoc-gen-go v1.30.0
// protoc v3.12.4
// source: refs/grpc/types.proto
package refs

241
refs/grpc/types_frostfs.pb.go generated 100644
View File

@ -0,0 +1,241 @@
// Code generated by protoc-gen-go-frostfs. DO NOT EDIT.
package refs
import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto"
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *Address) StableSize() (size int) {
size += proto.NestedStructureSize(1, x.ContainerId)
size += proto.NestedStructureSize(2, x.ObjectId)
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *Address) StableMarshal(buf []byte) []byte {
if x == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, x.StableSize())
}
var offset int
offset += proto.NestedStructureMarshal(1, buf[offset:], x.ContainerId)
offset += proto.NestedStructureMarshal(2, buf[offset:], x.ObjectId)
return buf
}
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *ObjectID) StableSize() (size int) {
size += proto.BytesSize(1, x.Value)
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *ObjectID) StableMarshal(buf []byte) []byte {
if x == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, x.StableSize())
}
var offset int
offset += proto.BytesMarshal(1, buf[offset:], x.Value)
return buf
}
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *ContainerID) StableSize() (size int) {
size += proto.BytesSize(1, x.Value)
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *ContainerID) StableMarshal(buf []byte) []byte {
if x == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, x.StableSize())
}
var offset int
offset += proto.BytesMarshal(1, buf[offset:], x.Value)
return buf
}
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *OwnerID) StableSize() (size int) {
size += proto.BytesSize(1, x.Value)
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *OwnerID) StableMarshal(buf []byte) []byte {
if x == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, x.StableSize())
}
var offset int
offset += proto.BytesMarshal(1, buf[offset:], x.Value)
return buf
}
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *Version) StableSize() (size int) {
size += proto.UInt32Size(1, x.Major)
size += proto.UInt32Size(2, x.Minor)
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *Version) StableMarshal(buf []byte) []byte {
if x == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, x.StableSize())
}
var offset int
offset += proto.UInt32Marshal(1, buf[offset:], x.Major)
offset += proto.UInt32Marshal(2, buf[offset:], x.Minor)
return buf
}
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *Signature) StableSize() (size int) {
size += proto.BytesSize(1, x.Key)
size += proto.BytesSize(2, x.Sign)
size += proto.EnumSize(3, int32(x.Scheme))
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *Signature) StableMarshal(buf []byte) []byte {
if x == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, x.StableSize())
}
var offset int
offset += proto.BytesMarshal(1, buf[offset:], x.Key)
offset += proto.BytesMarshal(2, buf[offset:], x.Sign)
offset += proto.EnumMarshal(3, buf[offset:], int32(x.Scheme))
return buf
}
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *SignatureRFC6979) StableSize() (size int) {
size += proto.BytesSize(1, x.Key)
size += proto.BytesSize(2, x.Sign)
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *SignatureRFC6979) StableMarshal(buf []byte) []byte {
if x == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, x.StableSize())
}
var offset int
offset += proto.BytesMarshal(1, buf[offset:], x.Key)
offset += proto.BytesMarshal(2, buf[offset:], x.Sign)
return buf
}
// StableSize returns the size of x in protobuf format.
//
// Structures with the same field values have the same binary size.
func (x *Checksum) StableSize() (size int) {
size += proto.EnumSize(1, int32(x.Type))
size += proto.BytesSize(2, x.Sum)
return size
}
// StableMarshal marshals x in protobuf binary format with stable field order.
//
// If buffer length is less than x.StableSize(), new buffer is allocated.
//
// Returns any error encountered which did not allow writing the data completely.
// Otherwise, returns the buffer in which the data is written.
//
// Structures with the same field values have the same binary format.
func (x *Checksum) StableMarshal(buf []byte) []byte {
if x == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, x.StableSize())
}
var offset int
offset += proto.EnumMarshal(1, buf[offset:], int32(x.Type))
offset += proto.BytesMarshal(2, buf[offset:], x.Sum)
return buf
}

154
refs/marshal.go
View File

@ -30,25 +30,11 @@ const (
)
func (o *OwnerID) StableMarshal(buf []byte) []byte {
if o == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, o.StableSize())
}
proto.BytesMarshal(ownerIDValField, buf, o.val)
return buf
return message.StableMarshal[*refs.OwnerID](o, buf)
}
func (o *OwnerID) StableSize() int {
if o == nil {
return 0
}
return proto.BytesSize(ownerIDValField, o.val)
return message.StableSize[*refs.OwnerID](o)
}
func (o *OwnerID) Unmarshal(data []byte) error {
@ -56,25 +42,11 @@ func (o *OwnerID) Unmarshal(data []byte) error {
}
func (c *ContainerID) StableMarshal(buf []byte) []byte {
if c == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, c.StableSize())
}
proto.BytesMarshal(containerIDValField, buf, c.val)
return buf
return message.StableMarshal[*refs.ContainerID](c, buf)
}
func (c *ContainerID) StableSize() int {
if c == nil {
return 0
}
return proto.BytesSize(containerIDValField, c.val)
return message.StableSize[*refs.ContainerID](c)
}
func (c *ContainerID) Unmarshal(data []byte) error {
@ -82,22 +54,13 @@ func (c *ContainerID) Unmarshal(data []byte) error {
}
func (o *ObjectID) StableMarshal(buf []byte) []byte {
if o == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, o.StableSize())
}
proto.BytesMarshal(objectIDValField, buf, o.val)
return buf
return message.StableMarshal[*refs.ObjectID](o, buf)
}
// ObjectIDNestedListSize returns byte length of nested
// repeated ObjectID field with fNum number.
func ObjectIDNestedListSize(fNum int64, ids []ObjectID) (sz int) {
// TODO (aarifullin): remove this method when all marshalers are refactored
for i := range ids {
sz += proto.NestedStructureSize(fNum, &ids[i])
}
@ -106,16 +69,13 @@ func ObjectIDNestedListSize(fNum int64, ids []ObjectID) (sz int) {
}
func (o *ObjectID) StableSize() int {
if o == nil {
return 0
}
return proto.BytesSize(objectIDValField, o.val)
return message.StableSize[*refs.ObjectID](o)
}
// ObjectIDNestedListMarshal writes protobuf repeated ObjectID field
// with fNum number to buf.
func ObjectIDNestedListMarshal(fNum int64, buf []byte, ids []ObjectID) (off int) {
// TODO (aarifullin): remove this method when all marshalers are refactored
prefix, _ := proto.NestedStructurePrefix(fNum)
for i := range ids {
off += binary.PutUvarint(buf[off:], prefix)
@ -133,31 +93,11 @@ func (o *ObjectID) Unmarshal(data []byte) error {
}
func (a *Address) StableMarshal(buf []byte) []byte {
if a == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, a.StableSize())
}
var offset int
offset += proto.NestedStructureMarshal(addressContainerField, buf[offset:], a.cid)
proto.NestedStructureMarshal(addressObjectField, buf[offset:], a.oid)
return buf
return message.StableMarshal[*refs.Address](a, buf)
}
func (a *Address) StableSize() (size int) {
if a == nil {
return 0
}
size += proto.NestedStructureSize(addressContainerField, a.cid)
size += proto.NestedStructureSize(addressObjectField, a.oid)
return size
return message.StableSize[*refs.Address](a)
}
func (a *Address) Unmarshal(data []byte) error {
@ -165,31 +105,11 @@ func (a *Address) Unmarshal(data []byte) error {
}
func (c *Checksum) StableMarshal(buf []byte) []byte {
if c == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, c.StableSize())
}
var offset int
offset += proto.EnumMarshal(checksumTypeField, buf[offset:], int32(c.typ))
proto.BytesMarshal(checksumValueField, buf[offset:], c.sum)
return buf
return message.StableMarshal[*refs.Checksum](c, buf)
}
func (c *Checksum) StableSize() (size int) {
if c == nil {
return 0
}
size += proto.EnumSize(checksumTypeField, int32(c.typ))
size += proto.BytesSize(checksumValueField, c.sum)
return size
return message.StableSize[*refs.Checksum](c)
}
func (c *Checksum) Unmarshal(data []byte) error {
@ -197,33 +117,11 @@ func (c *Checksum) Unmarshal(data []byte) error {
}
func (s *Signature) StableMarshal(buf []byte) []byte {
if s == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, s.StableSize())
}
var offset int
offset += proto.BytesMarshal(signatureKeyField, buf[offset:], s.key)
offset += proto.BytesMarshal(signatureValueField, buf[offset:], s.sign)
proto.EnumMarshal(signatureSchemeField, buf[offset:], int32(s.scheme))
return buf
return message.StableMarshal[*refs.Signature](s, buf)
}
func (s *Signature) StableSize() (size int) {
if s == nil {
return 0
}
size += proto.BytesSize(signatureKeyField, s.key)
size += proto.BytesSize(signatureValueField, s.sign)
size += proto.EnumSize(signatureSchemeField, int32(s.scheme))
return size
return message.StableSize[*refs.Signature](s)
}
func (s *Signature) Unmarshal(data []byte) error {
@ -231,31 +129,11 @@ func (s *Signature) Unmarshal(data []byte) error {
}
func (v *Version) StableMarshal(buf []byte) []byte {
if v == nil {
return []byte{}
}
if buf == nil {
buf = make([]byte, v.StableSize())
}
var offset int
offset += proto.UInt32Marshal(versionMajorField, buf[offset:], v.major)
proto.UInt32Marshal(versionMinorField, buf[offset:], v.minor)
return buf
return message.StableMarshal[*refs.Version](v, buf)
}
func (v *Version) StableSize() (size int) {
if v == nil {
return 0
}
size += proto.UInt32Size(versionMajorField, v.major)
size += proto.UInt32Size(versionMinorField, v.minor)
return size
return message.StableSize[*refs.Version](v)
}
func (v *Version) Unmarshal(data []byte) error {

11
rpc/message/encoding.go
View File

@ -2,6 +2,7 @@ package message
import (
"git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/rpc/grpc"
protoutil "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/proto"
)
@ -46,3 +47,13 @@ func UnmarshalJSON(m Message, data []byte, gm GRPCConvertedMessage) error {
return m.FromGRPCMessage(gm)
}
// StableMarshal encodes message 'm' using the invocation from related gRPC message
func StableMarshal[M protoutil.StableMarshaller](m Message, buf []byte) []byte {
return m.ToGRPCMessage().(M).StableMarshal(buf)
}
// StableSize encodes message 'm' using the invocation from related gRPC message
func StableSize[M protoutil.StableMarshaller](m Message) int {
return m.ToGRPCMessage().(M).StableSize()
}

6
util/proto/marshal.go
View File

@ -14,7 +14,7 @@ import (
)
type (
stableMarshaller interface {
StableMarshaller interface {
StableMarshal([]byte) []byte
StableSize() int
}
@ -302,7 +302,7 @@ func NestedStructurePrefix(field int64) (prefix uint64, ln int) {
return prefix, VarUIntSize(prefix)
}
func NestedStructureMarshal(field int64, buf []byte, v stableMarshaller) int {
func NestedStructureMarshal(field int64, buf []byte, v StableMarshaller) int {
if v == nil || reflect.ValueOf(v).IsNil() {
return 0
}
@ -317,7 +317,7 @@ func NestedStructureMarshal(field int64, buf []byte, v stableMarshaller) int {
return offset + n
}
func NestedStructureSize(field int64, v stableMarshaller) (size int) {
func NestedStructureSize(field int64, v StableMarshaller) (size int) {
if v == nil || reflect.ValueOf(v).IsNil() {
return 0
}

4
util/proto/test/test.pb.go generated
View File

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.28.1
// protoc v3.21.9
// protoc-gen-go v1.30.0
// protoc v3.12.4
// source: util/proto/test/test.proto
package test

110
util/protogen/main.go
View File

@ -8,14 +8,18 @@ import (
"google.golang.org/protobuf/reflect/protoreflect"
)
type MethodMask int
const (
Signature MethodMask = 1 << iota
)
func main() {
protogen.Options{}.Run(func(gen *protogen.Plugin) error {
for _, f := range gen.Files {
//if !f.Generate {
// continue
//}
imp := string(f.GoImportPath)
if strings.HasSuffix(imp, "/tree") || strings.HasSuffix(imp, "/control") {
impPath := f.GoImportPath.String()
if strings.HasSuffix(impPath, "/tree") || strings.HasSuffix(impPath, "/control") ||
strings.Contains(impPath, "/refs/") {
generateFile(gen, f)
}
}
@ -25,6 +29,11 @@ func main() {
// generateFile generates a *.pb.go file enforcing field-order serialization.
func generateFile(gen *protogen.Plugin, file *protogen.File) *protogen.GeneratedFile {
emitMask := MethodMask(0)
if imp := string(file.GoImportPath); strings.HasSuffix(imp, "/tree") || strings.HasSuffix(imp, "/control") {
emitMask |= Signature
}
filename := file.GeneratedFilenamePrefix + "_frostfs.pb.go"
g := gen.NewGeneratedFile(filename, file.GoImportPath)
g.P("// Code generated by protoc-gen-go-frostfs. DO NOT EDIT.")
@ -33,28 +42,14 @@ func generateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated
g.P()
g.P(`import "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto"`)
//for _, e := range file.Enums {
// g.P("type " + e.GoIdent.GoName + " int32")
// g.P("const (")
// for _, ev := range e.Values {
// g.P(ev.GoIdent.GoName, " = ", ev.Desc.Number())
// }
// g.P(")")
//}
for _, msg := range file.Messages {
emitMessage(g, msg)
emitMessage(g, msg, emitMask)
}
return g
}
func emitMessage(g *protogen.GeneratedFile, msg *protogen.Message) {
for _, inner := range msg.Messages {
emitMessage(g, inner)
}
func genStableSizeFunc(g *protogen.GeneratedFile, msg *protogen.Message) {
fs := sortFields(msg.Fields)
// StableSize implementation.
g.P("// StableSize returns the size of x in protobuf format.")
g.P("//")
g.P("// Structures with the same field values have the same binary size.")
@ -72,8 +67,10 @@ func emitMessage(g *protogen.GeneratedFile, msg *protogen.Message) {
}
g.P("return size")
g.P("}\n")
}
// StableMarshal implementation.
func genStableMarshalFunc(g *protogen.GeneratedFile, msg *protogen.Message) {
fs := sortFields(msg.Fields)
g.P("// StableMarshal marshals x in protobuf binary format with stable field order.")
g.P("//")
g.P("// If buffer length is less than x.StableSize(), new buffer is allocated.")
@ -93,32 +90,53 @@ func emitMessage(g *protogen.GeneratedFile, msg *protogen.Message) {
}
g.P("return buf")
g.P("}\n")
}
func genReadSignedDataFunc(g *protogen.GeneratedFile, msg *protogen.Message) {
g.P("// ReadSignedData fills buf with signed data of x.")
g.P("// If buffer length is less than x.SignedDataSize(), new buffer is allocated.")
g.P("//")
g.P("// Returns any error encountered which did not allow writing the data completely.")
g.P("// Otherwise, returns the buffer in which the data is written.")
g.P("//")
g.P("func (x *", msg.GoIdent.GoName, ") ReadSignedData(buf []byte) ([]byte, error) {")
g.P("return x.GetBody().StableMarshal(buf), nil")
g.P("}\n")
}
func genSignedDataSizeFunc(g *protogen.GeneratedFile, msg *protogen.Message) {
g.P("// SignedDataSize returns size of the request signed data in bytes.")
g.P("//")
g.P("// Structures with the same field values have the same signed data size.")
g.P("// Structures with the same field values have the same signed data.")
g.P("func (x *", msg.GoIdent.GoName, ") SignedDataSize() int {")
g.P("return x.GetBody().StableSize()")
g.P("}\n")
}
func genSetSignatureFunc(g *protogen.GeneratedFile, msg *protogen.Message) {
g.P("func (x *", msg.GoIdent.GoName, ") SetSignature(sig *Signature) {")
g.P("x.Signature = sig")
g.P("}\n")
}
func emitMessage(g *protogen.GeneratedFile, msg *protogen.Message, mask MethodMask) {
for _, inner := range msg.Messages {
emitMessage(g, inner, mask)
}
genStableSizeFunc(g, msg)
genStableMarshalFunc(g, msg)
if strings.HasSuffix(msg.GoIdent.GoName, "Request") || strings.HasSuffix(msg.GoIdent.GoName, "Response") {
// SignedDataSize implementation (only for requests and responses).
g.P("// ReadSignedData fills buf with signed data of x.")
g.P("// If buffer length is less than x.SignedDataSize(), new buffer is allocated.")
g.P("//")
g.P("// Returns any error encountered which did not allow writing the data completely.")
g.P("// Otherwise, returns the buffer in which the data is written.")
g.P("//")
g.P("// Structures with the same field values have the same signed data.")
g.P("func (x *", msg.GoIdent.GoName, ") SignedDataSize() int {")
g.P("return x.GetBody().StableSize()")
g.P("}\n")
// ReadSignedData implementation (only for requests and responses).
g.P("// SignedDataSize returns size of the request signed data in bytes.")
g.P("//")
g.P("// Structures with the same field values have the same signed data size.")
g.P("func (x *", msg.GoIdent.GoName, ") ReadSignedData(buf []byte) ([]byte, error) {")
g.P("return x.GetBody().StableMarshal(buf), nil")
g.P("}\n")
// Signature setters and getters.
g.P("func (x *", msg.GoIdent.GoName, ") SetSignature(sig *Signature) {")
g.P("x.Signature = sig")
g.P("}\n")
if mask&Signature != 0 {
genReadSignedDataFunc(g, msg)
genSignedDataSizeFunc(g, msg)
genSetSignatureFunc(g, msg)
}
}
}