diff --git a/.forgejo/workflows/dco.yml b/.forgejo/workflows/dco.yml index 6746408..9aa0d31 100644 --- a/.forgejo/workflows/dco.yml +++ b/.forgejo/workflows/dco.yml @@ -16,6 +16,6 @@ jobs: go-version: '1.21' - name: Run commit format checker - uses: https://git.frostfs.info/TrueCloudLab/dco-go@v2 + uses: https://git.frostfs.info/TrueCloudLab/dco-go@v3 with: from: 'origin/${{ github.event.pull_request.base.ref }}' diff --git a/Makefile b/Makefile index 6ceea25..e421213 100755 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ protoc: # Run Unit Test with go test test: @echo "⇒ Running go test" - @GO111MODULE=on go test ./... + @GO111MODULE=on go test ./... -count=1 # Run linters lint: diff --git a/go.mod b/go.mod index 78c1bdb..5b3e91e 100644 --- a/go.mod +++ b/go.mod @@ -5,22 +5,22 @@ go 1.20 require ( git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 github.com/stretchr/testify v1.8.3 - golang.org/x/sync v0.2.0 - google.golang.org/grpc v1.55.0 - google.golang.org/protobuf v1.30.0 + golang.org/x/sync v0.7.0 + google.golang.org/grpc v1.61.2 + google.golang.org/protobuf v1.34.2 ) require ( git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index bb23dce..b9bddda 100644 --- a/go.sum +++ b/go.sum @@ -5,11 +5,9 @@ git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0/go.mod h1:okpbKfVYf/BpejtfFTfhZqFP+ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -23,23 +21,20 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d h1:JU0iKnSg02Gmb5ZdV8nYsKEKsP6o/FGVWTrw4i1DA9A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/grpc v1.61.2 h1:TzJay21lXCf7BiNFKl7mSskt5DlkKAumAYTs52SpJeo= +google.golang.org/grpc v1.61.2/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/object/marshal.go b/object/marshal.go index fe0e232..ddd5746 100644 --- a/object/marshal.go +++ b/object/marshal.go @@ -322,6 +322,14 @@ func (o *Object) StableMarshal(buf []byte) []byte { return []byte{} } + if o.marshalData != nil { + if buf == nil { + return o.marshalData + } + copy(buf, o.marshalData) + return buf + } + if buf == nil { buf = make([]byte, o.StableSize()) } @@ -336,6 +344,16 @@ func (o *Object) StableMarshal(buf []byte) []byte { return buf } +// SetMarshalData sets marshal data to reduce memory allocations. +// +// It is unsafe to modify/copy object data after setting marshal data. +func (o *Object) SetMarshalData(data []byte) { + if o == nil { + return + } + o.marshalData = data +} + func (o *Object) StableSize() (size int) { if o == nil { return 0 @@ -1071,6 +1089,15 @@ func (r *PutSingleRequestBody) StableMarshal(buf []byte) []byte { if r == nil { return []byte{} } + + if r.marshalData != nil { + if buf == nil { + return r.marshalData + } + copy(buf, r.marshalData) + return buf + } + if buf == nil { buf = make([]byte, r.StableSize()) } @@ -1082,6 +1109,19 @@ func (r *PutSingleRequestBody) StableMarshal(buf []byte) []byte { return buf } +// SetMarshalData sets marshal data to reduce memory allocations. +// +// It is unsafe to modify/copy request data after setting marshal data. +func (r *PutSingleRequestBody) SetMarshalData(data []byte) { + if r == nil { + return + } + + r.marshalData = data + + proto.NestedStructureSetMarshalData(putSingleReqObjectField, r.marshalData, r.object) +} + func (r *PutSingleRequestBody) StableSize() int { if r == nil { return 0 diff --git a/object/types.go b/object/types.go index c28bf35..895afb0 100644 --- a/object/types.go +++ b/object/types.go @@ -75,6 +75,9 @@ type Object struct { header *Header payload []byte + + // marshalData holds marshaled data, must not be marshaled by StableMarshal + marshalData []byte } type SplitInfo struct { @@ -304,6 +307,9 @@ type GetRangeHashResponse struct { type PutSingleRequestBody struct { object *Object copyNum []uint32 + + // marshalData holds marshaled data, must not be marshaled by StableMarshal + marshalData []byte } type PutSingleRequest struct { diff --git a/session/marshal.go b/session/marshal.go index cda9579..8632398 100644 --- a/session/marshal.go +++ b/session/marshal.go @@ -211,7 +211,7 @@ func (c *ObjectSessionContext) StableMarshal(buf []byte) []byte { } offset := proto.EnumMarshal(objectCtxVerbField, buf, int32(c.verb)) - proto.NestedStructureMarshal(objectCtxTargetField, buf[offset:], objectSessionContextTarget{ + proto.NestedStructureMarshalUnchecked(objectCtxTargetField, buf[offset:], objectSessionContextTarget{ cnr: c.cnr, objs: c.objs, }) @@ -225,7 +225,7 @@ func (c *ObjectSessionContext) StableSize() (size int) { } size += proto.EnumSize(objectCtxVerbField, int32(c.verb)) - size += proto.NestedStructureSize(objectCtxTargetField, objectSessionContextTarget{ + size += proto.NestedStructureSizeUnchecked(objectCtxTargetField, objectSessionContextTarget{ cnr: c.cnr, objs: c.objs, }) diff --git a/util/pool/buffer.go b/util/pool/buffer.go new file mode 100644 index 0000000..e0a7185 --- /dev/null +++ b/util/pool/buffer.go @@ -0,0 +1,54 @@ +package pool + +import ( + "sync" +) + +// Buffer contains a byte slice. +type Buffer struct { + Data []byte +} + +// BufferPool manages a pool of Buffers. +type BufferPool struct { + poolSliceSize uint32 // Size for the buffer slices in the pool. + buffersPool *sync.Pool +} + +// NewBufferPool creates a BufferPool with a specified size. +func NewBufferPool(poolSliceSize uint32) BufferPool { + pool := sync.Pool{ + New: func() any { + return new(Buffer) + }, + } + return BufferPool{poolSliceSize: poolSliceSize, buffersPool: &pool} +} + +// Get retrieves a Buffer from the pool or creates a new one if necessary. +// It ensures the buffer's capacity is at least the specified size. +func (pool BufferPool) Get(size uint32) *Buffer { + result := pool.buffersPool.Get().(*Buffer) + + if cap(result.Data) < int(size) { + result.Data = make([]byte, size) + } else { + result.Data = result.Data[:size] + } + return result +} + +// Put returns a Buffer to the pool if its capacity does not exceed poolSliceSize. +func (pool BufferPool) Put(buf *Buffer) { + if cap(buf.Data) > int(pool.poolSliceSize) { + return + } + + buf.Data = buf.Data[:0] + pool.buffersPool.Put(buf) +} + +// PoolSliceSize returns the size for buffer slices in the pool. +func (pool BufferPool) PoolSliceSize() uint32 { + return uint32(pool.poolSliceSize) +} diff --git a/util/proto/marshal.go b/util/proto/marshal.go index 606086a..26b3eb0 100644 --- a/util/proto/marshal.go +++ b/util/proto/marshal.go @@ -15,10 +15,16 @@ import ( ) type ( - stableMarshaller interface { + stableMarshaler interface { StableMarshal([]byte) []byte StableSize() int } + + setMarshalData[T any] interface { + SetMarshalData([]byte) + StableSize() int + ~*T + } ) func BytesMarshal(field int, buf, v []byte) int { @@ -249,12 +255,21 @@ func VarUIntSize(x uint64) int { return (bits.Len64(x|1) + 6) / 7 } -func NestedStructureMarshal[T stableMarshaller](field int64, buf []byte, v T) int { - n := v.StableSize() - if n == 0 { +type ptrStableMarshaler[T any] interface { + stableMarshaler + ~*T +} + +func NestedStructureMarshal[T any, M ptrStableMarshaler[T]](field int64, buf []byte, v M) int { + if v == nil { return 0 } + return NestedStructureMarshalUnchecked(field, buf, v) +} + +func NestedStructureMarshalUnchecked[T stableMarshaler](field int64, buf []byte, v T) int { + n := v.StableSize() prefix := protowire.EncodeTag(protowire.Number(field), protowire.BytesType) offset := binary.PutUvarint(buf, prefix) offset += binary.PutUvarint(buf[offset:], uint64(n)) @@ -263,13 +278,42 @@ func NestedStructureMarshal[T stableMarshaller](field int64, buf []byte, v T) in return offset + n } -func NestedStructureSize[T stableMarshaller](field int64, v T) (size int) { - n := v.StableSize() - if n == 0 { +// NestedStructureSetMarshalData calculates offset for field in parentData +// and calls SetMarshalData for nested structure. +// +// Returns marshalled data length of nested structure. +func NestedStructureSetMarshalData[T any, M setMarshalData[T]](field int64, parentData []byte, v M) int { + if v == nil { return 0 } - size = protowire.SizeGroup(protowire.Number(field), protowire.SizeBytes(n)) - return + + if parentData == nil { + v.SetMarshalData(nil) + return 0 + } + + n := v.StableSize() + buf := make([]byte, binary.MaxVarintLen64) + prefix := protowire.EncodeTag(protowire.Number(field), protowire.BytesType) + offset := binary.PutUvarint(buf, prefix) + offset += binary.PutUvarint(buf, uint64(n)) + + v.SetMarshalData(parentData[offset : offset+n]) + + return offset + n +} + +func NestedStructureSize[T any, M ptrStableMarshaler[T]](field int64, v M) (size int) { + if v == nil { + return 0 + } + + return NestedStructureSizeUnchecked(field, v) +} + +func NestedStructureSizeUnchecked[T stableMarshaler](field int64, v T) int { + n := v.StableSize() + return protowire.SizeGroup(protowire.Number(field), protowire.SizeBytes(n)) } func Fixed64Marshal(field int, buf []byte, v uint64) int { diff --git a/util/proto/marshal_test.go b/util/proto/marshal_test.go index 11a4d8f..73129c2 100644 --- a/util/proto/marshal_test.go +++ b/util/proto/marshal_test.go @@ -1,12 +1,14 @@ package proto_test import ( + "encoding/binary" "math" "testing" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/proto/test" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/encoding/protowire" goproto "google.golang.org/protobuf/proto" ) @@ -33,6 +35,8 @@ type stableRepPrimitives struct { FieldD []uint32 FieldE []int64 FieldF []uint64 + + FieldFu []uint64 } const ( @@ -187,6 +191,20 @@ func (s *stableRepPrimitives) stableMarshal(buf []byte, wrongField bool) ([]byte } i += proto.RepeatedUInt64Marshal(fieldNum, buf, s.FieldF) + fieldNum = 7 + if wrongField { + fieldNum++ + } + for j := range s.FieldFu { + { + prefix := protowire.EncodeTag( + protowire.Number(fieldNum), + protowire.VarintType) + i += binary.PutUvarint(buf[i:], uint64(prefix)) + i += binary.PutUvarint(buf[i:], s.FieldFu[j]) + } + } + return buf, nil } @@ -198,7 +216,12 @@ func (s *stableRepPrimitives) stableSize() int { f5, _ := proto.RepeatedInt64Size(5, s.FieldE) f6, _ := proto.RepeatedUInt64Size(6, s.FieldF) - return f1 + f2 + f3 + f4 + f5 + f6 + var f7 int + for i := range s.FieldFu { + f7 += protowire.SizeGroup(protowire.Number(7), protowire.SizeVarint(s.FieldFu[i])) + } + + return f1 + f2 + f3 + f4 + f5 + f6 + f7 } func TestBytesMarshal(t *testing.T) { @@ -406,6 +429,22 @@ func TestRepeatedUInt64Marshal(t *testing.T) { }) } +func TestRepeatedUInt64MarshalUnpacked(t *testing.T) { + t.Run("not empty", func(t *testing.T) { + data := []uint64{0, 1, 2, 3, 4, 5} + testRepeatedUInt64MarshalUnpacked(t, data, false) + testRepeatedUInt64MarshalUnpacked(t, data, true) + }) + + t.Run("empty", func(t *testing.T) { + testRepeatedUInt64MarshalUnpacked(t, []uint64{}, false) + }) + + t.Run("nil", func(t *testing.T) { + testRepeatedUInt64MarshalUnpacked(t, nil, false) + }) +} + func TestFixed64Marshal(t *testing.T) { t.Run("zero", func(t *testing.T) { testFixed64Marshal(t, 0, false) @@ -441,7 +480,7 @@ func TestFixed32Marshal(t *testing.T) { }) } -func testMarshal(t *testing.T, c stablePrimitives, tr test.Primitives, wrongField bool) *test.Primitives { +func testMarshal(t *testing.T, c stablePrimitives, tr *test.Primitives, wrongField bool) *test.Primitives { var ( wire []byte err error @@ -449,7 +488,7 @@ func testMarshal(t *testing.T, c stablePrimitives, tr test.Primitives, wrongFiel wire, err = c.stableMarshal(nil, wrongField) require.NoError(t, err) - wireGen, err := goproto.Marshal(&tr) + wireGen, err := goproto.Marshal(tr) require.NoError(t, err) if !wrongField { @@ -472,7 +511,7 @@ func testBytesMarshal(t *testing.T, data []byte, wrongField bool) { transport = test.Primitives{FieldA: data} ) - result := testMarshal(t, custom, transport, wrongField) + result := testMarshal(t, custom, &transport, wrongField) if !wrongField { require.Len(t, result.FieldA, len(data)) @@ -490,7 +529,7 @@ func testStringMarshal(t *testing.T, s string, wrongField bool) { transport = test.Primitives{FieldB: s} ) - result := testMarshal(t, custom, transport, wrongField) + result := testMarshal(t, custom, &transport, wrongField) if !wrongField { require.Len(t, result.FieldB, len(s)) @@ -508,7 +547,7 @@ func testBoolMarshal(t *testing.T, b bool, wrongField bool) { transport = test.Primitives{FieldC: b} ) - result := testMarshal(t, custom, transport, wrongField) + result := testMarshal(t, custom, &transport, wrongField) if !wrongField { require.Equal(t, b, result.FieldC) @@ -523,7 +562,7 @@ func testInt32Marshal(t *testing.T, n int32, wrongField bool) { transport = test.Primitives{FieldD: n} ) - result := testMarshal(t, custom, transport, wrongField) + result := testMarshal(t, custom, &transport, wrongField) if !wrongField { require.Equal(t, n, result.FieldD) @@ -538,7 +577,7 @@ func testUInt32Marshal(t *testing.T, n uint32, wrongField bool) { transport = test.Primitives{FieldE: n} ) - result := testMarshal(t, custom, transport, wrongField) + result := testMarshal(t, custom, &transport, wrongField) if !wrongField { require.Equal(t, n, result.FieldE) @@ -553,7 +592,7 @@ func testInt64Marshal(t *testing.T, n int64, wrongField bool) { transport = test.Primitives{FieldF: n} ) - result := testMarshal(t, custom, transport, wrongField) + result := testMarshal(t, custom, &transport, wrongField) if !wrongField { require.Equal(t, n, result.FieldF) @@ -568,7 +607,7 @@ func testUInt64Marshal(t *testing.T, n uint64, wrongField bool) { transport = test.Primitives{FieldG: n} ) - result := testMarshal(t, custom, transport, wrongField) + result := testMarshal(t, custom, &transport, wrongField) if !wrongField { require.Equal(t, n, result.FieldG) @@ -583,7 +622,7 @@ func testFloat64Marshal(t *testing.T, n float64, wrongField bool) { transport = test.Primitives{FieldJ: n} ) - result := testMarshal(t, custom, transport, wrongField) + result := testMarshal(t, custom, &transport, wrongField) if !wrongField { require.Equal(t, n, result.FieldJ) @@ -598,7 +637,7 @@ func testEnumMarshal(t *testing.T, e SomeEnum, wrongField bool) { transport = test.Primitives{FieldH: test.Primitives_SomeEnum(e)} ) - result := testMarshal(t, custom, transport, wrongField) + result := testMarshal(t, custom, &transport, wrongField) if !wrongField { require.EqualValues(t, custom.FieldH, result.FieldH) @@ -607,7 +646,7 @@ func testEnumMarshal(t *testing.T, e SomeEnum, wrongField bool) { } } -func testRepMarshal(t *testing.T, c stableRepPrimitives, tr test.RepPrimitives, wrongField bool) *test.RepPrimitives { +func testRepMarshal(t *testing.T, c stableRepPrimitives, tr *test.RepPrimitives, wrongField bool) *test.RepPrimitives { var ( wire []byte err error @@ -615,7 +654,7 @@ func testRepMarshal(t *testing.T, c stableRepPrimitives, tr test.RepPrimitives, wire, err = c.stableMarshal(nil, wrongField) require.NoError(t, err) - wireGen, err := goproto.Marshal(&tr) + wireGen, err := goproto.Marshal(tr) require.NoError(t, err) if !wrongField { @@ -638,7 +677,7 @@ func testRepeatedBytesMarshal(t *testing.T, data [][]byte, wrongField bool) { transport = test.RepPrimitives{FieldA: data} ) - result := testRepMarshal(t, custom, transport, wrongField) + result := testRepMarshal(t, custom, &transport, wrongField) if !wrongField { require.Len(t, result.FieldA, len(data)) @@ -656,7 +695,7 @@ func testRepeatedStringMarshal(t *testing.T, s []string, wrongField bool) { transport = test.RepPrimitives{FieldB: s} ) - result := testRepMarshal(t, custom, transport, wrongField) + result := testRepMarshal(t, custom, &transport, wrongField) if !wrongField { require.Len(t, result.FieldB, len(s)) @@ -674,7 +713,7 @@ func testRepeatedInt32Marshal(t *testing.T, n []int32, wrongField bool) { transport = test.RepPrimitives{FieldC: n} ) - result := testRepMarshal(t, custom, transport, wrongField) + result := testRepMarshal(t, custom, &transport, wrongField) if !wrongField { require.Len(t, result.FieldC, len(n)) @@ -692,7 +731,7 @@ func testRepeatedUInt32Marshal(t *testing.T, n []uint32, wrongField bool) { transport = test.RepPrimitives{FieldD: n} ) - result := testRepMarshal(t, custom, transport, wrongField) + result := testRepMarshal(t, custom, &transport, wrongField) if !wrongField { require.Len(t, result.FieldD, len(n)) @@ -710,7 +749,7 @@ func testRepeatedInt64Marshal(t *testing.T, n []int64, wrongField bool) { transport = test.RepPrimitives{FieldE: n} ) - result := testRepMarshal(t, custom, transport, wrongField) + result := testRepMarshal(t, custom, &transport, wrongField) if !wrongField { require.Len(t, result.FieldE, len(n)) @@ -728,7 +767,7 @@ func testRepeatedUInt64Marshal(t *testing.T, n []uint64, wrongField bool) { transport = test.RepPrimitives{FieldF: n} ) - result := testRepMarshal(t, custom, transport, wrongField) + result := testRepMarshal(t, custom, &transport, wrongField) if !wrongField { require.Len(t, result.FieldF, len(n)) @@ -740,13 +779,31 @@ func testRepeatedUInt64Marshal(t *testing.T, n []uint64, wrongField bool) { } } +func testRepeatedUInt64MarshalUnpacked(t *testing.T, n []uint64, wrongField bool) { + var ( + custom = stableRepPrimitives{FieldFu: n} + transport = test.RepPrimitives{FieldFu: n} + ) + + result := testRepMarshal(t, custom, &transport, wrongField) + + if !wrongField { + require.Len(t, result.FieldFu, len(n)) + if len(n) > 0 { + require.Equal(t, n, result.FieldFu) + } + } else { + require.Len(t, result.FieldFu, 0) + } +} + func testFixed64Marshal(t *testing.T, n uint64, wrongField bool) { var ( custom = stablePrimitives{FieldI: n} transport = test.Primitives{FieldI: n} ) - result := testMarshal(t, custom, transport, wrongField) + result := testMarshal(t, custom, &transport, wrongField) if !wrongField { require.Equal(t, n, result.FieldI) @@ -761,7 +818,7 @@ func testFixed32Marshal(t *testing.T, n uint32, wrongField bool) { transport = test.Primitives{FieldK: n} ) - result := testMarshal(t, custom, transport, wrongField) + result := testMarshal(t, custom, &transport, wrongField) if !wrongField { require.Equal(t, n, result.FieldK) diff --git a/util/proto/test/test.pb.go b/util/proto/test/test.pb.go index 2fa2d4e..e8ec719 100644 Binary files a/util/proto/test/test.pb.go and b/util/proto/test/test.pb.go differ diff --git a/util/proto/test/test.proto b/util/proto/test/test.proto index 65e350f..14f6ac2 100644 --- a/util/proto/test/test.proto +++ b/util/proto/test/test.proto @@ -5,30 +5,31 @@ package test; option go_package = "util/proto/test"; message Primitives { - bytes field_a = 1; - string field_b = 2; - bool field_c = 200; - int32 field_d = 201; - uint32 field_e = 202; - int64 field_f = 203; - uint64 field_g = 204; - fixed64 field_i = 205; - double field_j = 206; - fixed32 field_k = 207; + bytes field_a = 1; + string field_b = 2; + bool field_c = 200; + int32 field_d = 201; + uint32 field_e = 202; + int64 field_f = 203; + uint64 field_g = 204; + fixed64 field_i = 205; + double field_j = 206; + fixed32 field_k = 207; - enum SomeEnum { - UNKNOWN = 0; - POSITIVE = 1; - NEGATIVE = -1; - } - SomeEnum field_h = 300; + enum SomeEnum { + UNKNOWN = 0; + POSITIVE = 1; + NEGATIVE = -1; + } + SomeEnum field_h = 300; } message RepPrimitives { - repeated bytes field_a = 1; - repeated string field_b = 2; - repeated int32 field_c = 3; - repeated uint32 field_d = 4; - repeated int64 field_e = 5; - repeated uint64 field_f = 6; + repeated bytes field_a = 1; + repeated string field_b = 2; + repeated int32 field_c = 3; + repeated uint32 field_d = 4; + repeated int64 field_e = 5; + repeated uint64 field_f = 6; + repeated uint64 field_fu = 7 [ packed = false ]; } diff --git a/util/protogen/main.go b/util/protogen/main.go index fe89fbf..f1a276d 100644 --- a/util/protogen/main.go +++ b/util/protogen/main.go @@ -8,6 +8,11 @@ import ( "google.golang.org/protobuf/reflect/protoreflect" ) +var ( + protowirePackage = protogen.GoImportPath("google.golang.org/protobuf/encoding/protowire") + binaryPackage = protogen.GoImportPath("encoding/binary") +) + func main() { protogen.Options{}.Run(func(gen *protogen.Plugin) error { for _, f := range gen.Files { @@ -62,7 +67,7 @@ func emitMessage(g *protogen.GeneratedFile, msg *protogen.Message) { g.P("if x == nil { return 0 }") if len(fs) != 0 { for _, f := range fs { - if f.Desc.IsList() && marshalers[f.Desc.Kind()].RepeatedDouble { + if f.Desc.IsList() && marshalers[f.Desc.Kind()].RepeatedDouble && !(f.Desc.Kind() == protoreflect.Uint64Kind && !f.Desc.IsPacked()) { g.P("var n int") break } @@ -140,9 +145,19 @@ func emitFieldSize(g *protogen.GeneratedFile, f *protogen.Field) { } switch { - case f.Desc.IsList() && f.Desc.Kind() == protoreflect.MessageKind: + case f.Desc.IsList() && (f.Desc.Kind() == protoreflect.MessageKind || f.Desc.Kind() == protoreflect.Uint64Kind && !f.Desc.IsPacked()): g.P("for i := range ", name, "{") - g.P("size += proto.NestedStructureSize(", f.Desc.Number(), ", ", name, "[i])") + if f.Desc.Kind() == protoreflect.MessageKind { + g.P("size += proto.NestedStructureSize(", f.Desc.Number(), ", ", name, "[i])") + } else { + if f.Desc.Kind() != protoreflect.Uint64Kind { + panic("only uint64 unpacked primitive is supported") + } + + g.P("size += ", protowirePackage.Ident("SizeGroup"), "(", + protowirePackage.Ident("Number"), "(", f.Desc.Number(), "), ", + protowirePackage.Ident("SizeVarint"), "(", name, "[i]))") + } g.P("}") case f.Desc.IsList(): if m.RepeatedDouble { @@ -177,9 +192,22 @@ func emitFieldMarshal(g *protogen.GeneratedFile, f *protogen.Field) { prefix = "Repeated" + m.Prefix } switch { - case f.Desc.IsList() && f.Desc.Kind() == protoreflect.MessageKind: + case f.Desc.IsList() && (f.Desc.Kind() == protoreflect.MessageKind || f.Desc.Kind() == protoreflect.Uint64Kind && !f.Desc.IsPacked()): g.P("for i := range ", name, "{") - g.P("offset += proto.NestedStructureMarshal(", f.Desc.Number(), ", buf[offset:], ", name, "[i])") + if f.Desc.Kind() == protoreflect.MessageKind { + g.P("offset += proto.NestedStructureMarshal(", f.Desc.Number(), ", buf[offset:], ", name, "[i])") + } else { + if f.Desc.Kind() != protoreflect.Uint64Kind { + panic("only uint64 unpacked primitive is supported") + } + g.P("{") + g.P("prefix := ", protowirePackage.Ident("EncodeTag"), "(", + protowirePackage.Ident("Number"), "(", f.Desc.Number(), "), ", + protowirePackage.Ident("VarintType"), ")") + g.P("offset += ", binaryPackage.Ident("PutUvarint"), "(buf[offset:], uint64(prefix))") + g.P("offset += ", binaryPackage.Ident("PutUvarint"), "(buf[offset:], ", name, "[i])") + g.P("}") + } g.P("}") case f.Desc.IsList(): g.P("offset += proto.Repeated", m.Prefix, "Marshal(", f.Desc.Number(), ", buf[offset:], ", name, ")") diff --git a/util/signature/buffer.go b/util/signature/buffer.go deleted file mode 100644 index f9b003d..0000000 --- a/util/signature/buffer.go +++ /dev/null @@ -1,33 +0,0 @@ -package signature - -import "sync" - -const poolSliceMaxSize = 128 * 1024 - -type buffer struct { - data []byte -} - -var buffersPool = sync.Pool{ - New: func() any { - return new(buffer) - }, -} - -func newBufferFromPool(size int) *buffer { - result := buffersPool.Get().(*buffer) - if cap(result.data) < size { - result.data = make([]byte, size) - } else { - result.data = result.data[:size] - } - return result -} - -func returnBufferToPool(buf *buffer) { - if cap(buf.data) > poolSliceMaxSize { - return - } - buf.data = buf.data[:0] - buffersPool.Put(buf) -} diff --git a/util/signature/data.go b/util/signature/data.go index 1db46dc..5e7c310 100644 --- a/util/signature/data.go +++ b/util/signature/data.go @@ -4,9 +4,14 @@ import ( "crypto/ecdsa" "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/util/pool" crypto "git.frostfs.info/TrueCloudLab/frostfs-crypto" ) +const poolSliceMaxSize = 128 * 1024 + +var buffersPool = pool.NewBufferPool(poolSliceMaxSize) + type DataSource interface { ReadSignedData([]byte) ([]byte, error) SignedDataSize() int @@ -35,10 +40,10 @@ func SignDataWithHandler(key *ecdsa.PrivateKey, src DataSource, handler KeySigna opts[i](cfg) } - buffer := newBufferFromPool(src.SignedDataSize()) - defer returnBufferToPool(buffer) + buffer := buffersPool.Get(uint32(src.SignedDataSize())) + defer buffersPool.Put(buffer) - data, err := src.ReadSignedData(buffer.data) + data, err := src.ReadSignedData(buffer.Data) if err != nil { return err } @@ -64,10 +69,10 @@ func VerifyDataWithSource(dataSrc DataSource, sigSrc KeySignatureSource, opts .. opts[i](cfg) } - buffer := newBufferFromPool(dataSrc.SignedDataSize()) - defer returnBufferToPool(buffer) + buffer := buffersPool.Get(uint32(dataSrc.SignedDataSize())) + defer buffersPool.Put(buffer) - data, err := dataSrc.ReadSignedData(buffer.data) + data, err := dataSrc.ReadSignedData(buffer.Data) if err != nil { return err } diff --git a/util/signature/options.go b/util/signature/options.go index b9bf68d..ca2a5e9 100644 --- a/util/signature/options.go +++ b/util/signature/options.go @@ -35,10 +35,10 @@ func verify(cfg *cfg, data []byte, sig *refs.Signature) error { case refs.ECDSA_RFC6979_SHA256: return crypto.VerifyRFC6979(pub, data, sig.GetSign()) case refs.ECDSA_RFC6979_SHA256_WALLET_CONNECT: - buffer := newBufferFromPool(base64.StdEncoding.EncodedLen(len(data))) - defer returnBufferToPool(buffer) - base64.StdEncoding.Encode(buffer.data, data) - if !walletconnect.Verify(pub, buffer.data, sig.GetSign()) { + buffer := buffersPool.Get(uint32(base64.StdEncoding.EncodedLen(len(data)))) + defer buffersPool.Put(buffer) + base64.StdEncoding.Encode(buffer.Data, data) + if !walletconnect.Verify(pub, buffer.Data, sig.GetSign()) { return crypto.ErrInvalidSignature } return nil @@ -54,10 +54,10 @@ func sign(cfg *cfg, key *ecdsa.PrivateKey, data []byte) ([]byte, error) { case refs.ECDSA_RFC6979_SHA256: return crypto.SignRFC6979(key, data) case refs.ECDSA_RFC6979_SHA256_WALLET_CONNECT: - buffer := newBufferFromPool(base64.StdEncoding.EncodedLen(len(data))) - defer returnBufferToPool(buffer) - base64.StdEncoding.Encode(buffer.data, data) - return walletconnect.Sign(key, buffer.data) + buffer := buffersPool.Get(uint32(base64.StdEncoding.EncodedLen(len(data)))) + defer buffersPool.Put(buffer) + base64.StdEncoding.Encode(buffer.Data, data) + return walletconnect.Sign(key, buffer.Data) default: panic(fmt.Sprintf("unsupported scheme %s", cfg.scheme)) }