From 29f215756383cf58cd1de326dd1c2961b06ea201 Mon Sep 17 00:00:00 2001
From: Evgenii Stratonikov <e.stratonikov@yadro.com>
Date: Fri, 11 Oct 2024 10:17:42 +0300
Subject: [PATCH] [#123] protogen: Treat bytes field as non-nullable

In protobuf 3.12 they have added an support for `optional` keyword,
which has made it into the main branch in 3.15.
https://github.com/protocolbuffers/protobuf/blob/main/docs/implementing_proto3_presence.md
https://github.com/protocolbuffers/protobuf/blob/v3.12.0/docs/field_presence.md#presence-in-proto3-apis

This means that without an explicit `optional` keyword field presence
for scalars is not tracked, thus empty string in JSON should be
unmarshaled to a nil byte slice. Relevant decoding code and tests from
protojson:
https://github.com/protocolbuffers/protobuf-go/blob/fb995f184a1719ec42b247a3771d1036d92adf67/internal/impl/message_reflect_field.go#L327
https://github.com/protocolbuffers/protobuf-go/blob/fb995f184a1719ec42b247a3771d1036d92adf67/encoding/protojson/decode_test.go#L134
https://github.com/protocolbuffers/protobuf-go/blob/fb995f184a1719ec42b247a3771d1036d92adf67/encoding/protojson/decode_test.go#L156

We do not support `optional` keyword and the generator will fail if it sees on.
So only implement the default behaviour.

Refs #122

Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
---
 acl/grpc/types_frostfs.pb.go              | Bin 48047 -> 48129 bytes
 ape/grpc/types_frostfs.pb.go              | Bin 8592 -> 8668 bytes
 apemanager/grpc/service_frostfs.pb.go     | Bin 57148 -> 57300 bytes
 container/grpc/types_frostfs.pb.go        | Bin 12208 -> 12284 bytes
 netmap/grpc/types_frostfs.pb.go           | Bin 57422 -> 57650 bytes
 object/grpc/service_frostfs.pb.go         | Bin 223720 -> 224182 bytes
 object/grpc/types_frostfs.pb.go           | Bin 65148 -> 65528 bytes
 refs/grpc/types_frostfs.pb.go             | Bin 30568 -> 31176 bytes
 session/grpc/service_frostfs.pb.go        | Bin 20196 -> 20348 bytes
 session/grpc/types_frostfs.pb.go          | Bin 70734 -> 70886 bytes
 status/grpc/types_frostfs.pb.go           | Bin 14144 -> 14220 bytes
 tombstone/grpc/types_frostfs.pb.go        | Bin 5766 -> 5842 bytes
 util/proto/marshal_test.go                |  10 +---------
 util/proto/test/custom/test_frostfs.pb.go | Bin 34803 -> 35037 bytes
 util/protogen/internalgengo/json.go       |  12 +++++++++++-
 15 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/acl/grpc/types_frostfs.pb.go b/acl/grpc/types_frostfs.pb.go
index e5a6f32c25a89a77a238be5194e7f155542e87de..bc90cb77608035d512ef50ab7773ac6c17306b99 100644
GIT binary patch
delta 111
zcmZ4govHB$(}p6;glaBMFeu3_P_VL9$jsAosw_z@*3g8CXQnCSq~>V=r8O07Z50d@
gU>Z0<I&2m4GIO9RYoT-+P!uQ!;ni+#u+(n?07ixx@Bjb+

delta 32
ocmZqt!L<H6(}p5T-ZTYUh0HuXr^=GlVvWfLS$3QGtv)pZ0MeceeE<Le

diff --git a/ape/grpc/types_frostfs.pb.go b/ape/grpc/types_frostfs.pb.go
index cf9499fe275a585696ccba9d903e6fdb66f1941d..1f2a1a57376ec53415ae2069ea5dcdc0c678ecf2 100644
GIT binary patch
delta 102
zcmbQ>e8+jiVuh$`E=~|A$t_T@vQ@~;({rjUNiEjUga~J*DdeQ)X#iz46>M!43>2Ws
d!Rl=l@-lNE>S`fWnu4tYP%W5KJ9(qReE>~#7{LGl

delta 26
icmccPJi&RxVg<f51zUy8JUyq%lGI`i&B=ci?gIda6AD8B

diff --git a/apemanager/grpc/service_frostfs.pb.go b/apemanager/grpc/service_frostfs.pb.go
index 7f3ed10eca70e1d6a8f4355bca8cd9943eab9c6a..b2633e145e13c9ace7905f4117fa45b7090de340 100644
GIT binary patch
delta 209
zcmdn9kNL`e<_!j>QPo_WAW)K9pkQUIkeR3FR9TW*tf2`J&P-FtNzKy$%4#au+A0_*
wK$U~l+bZN`=0McdLZ~zaTLqw6FsF92ovEQZ4b7iyXPP~ELJ9BYuQBdT0FNaxxc~qF

delta 53
wcmcbzpLx$d<_!j>d}#`{3YmF&PL(C8#TuHMyG#w$(ZnWqnPzWphz)510Qu(<vH$=8

diff --git a/container/grpc/types_frostfs.pb.go b/container/grpc/types_frostfs.pb.go
index 4ca23fa4cdaef715406511d0e09dba21145c6189..a4f0882c2260a1dcd27464c8bca2d32ba8c7b47e 100644
GIT binary patch
delta 103
zcmdlG|0jM!i%wKE7bggm<Q6Dc*(zk_={Z%Fq!w#vLWDEZ6mnAYG=Q?23bwWi1`1H+
eVD+{Nd6_v7b+r&GO~F<Hs20qr-8@N0ju`-rL>P4d

delta 27
icmewpzaf4@iw<9!f~`Vko}N=>Noui%=H`1ka?AjlX$kiL

diff --git a/netmap/grpc/types_frostfs.pb.go b/netmap/grpc/types_frostfs.pb.go
index 032853121cfcd121428f38beab724a28bcaf8ef6..24003c6e6eefb18cf886df8e12d44977e00d99da 100644
GIT binary patch
delta 307
zcmX?ifO*p)<_+A_qpG<$L7*hJK*7pZAu~_Usj?)sSVI#coSCMOlbWXil+{$QwN)@s
zfGP*8w^hi?%z>z@g-~e<whBPCU{393$?1(sG&Fy5cZ2by>D;um;lF~!WZl^zn-g}u
GQw0F+r%49@

delta 70
zcmdmVi22+B<_+A_`O*|@6*BYmoGMFFi#0Sir%rEFLKB<pu*;4IS=r>j3R0Ut>~c~A
E0Dij}o&W#<

diff --git a/object/grpc/service_frostfs.pb.go b/object/grpc/service_frostfs.pb.go
index 9ee5a3daeb4e63982e93f5f36e52224ac772978a..a55a41b0d7faaacf642369cef942b9c726f3c47f 100644
GIT binary patch
delta 620
zcmaEHnRnZD-VLqcQPo_WAW)K9pkQUIkeR3FR9TW*tf2`J&P-FtNzKy$%4#au+A0_*
zK$U~l+bZN`=0McdLZ~zaTLqw6FsF9<G&d%j&F90Hx+T!g60lVj)6a@Br!>ErxBbmL
z#yQJr=%?*KKbacQ(t?{6Oh>h00fsADIB~`cdc<%-V}=tsYB+1BJ5(^)ZhyIi>0$vb
LJ##{oSs?@fRdSGI

delta 173
zcmdmXo%h9M-VLqcd}#`{3YmF&PL(C8#TuHM?}tBhLlc{PKU{M9I&tQ}=9c-}Tjn#)
zS&pf|W_p1KlhE{dQzowMm)x1|8loxL&RNBDRGSyYvgwKX%y!$emoZ%|5Jr*E<l+Q^
P$qMsRw*MDr-X079u+lu>

diff --git a/object/grpc/types_frostfs.pb.go b/object/grpc/types_frostfs.pb.go
index e4a916fd1789ec371552e5fa40c0fd87ef518a4b..a6d4f016d1d4929c2d66d4899396de81a78a765b 100644
GIT binary patch
delta 498
zcmezKhxx~U<_(RJQPo_WAW)K9pkQUIkeR3FR9TW*tf2`J&P-FtNzKy$%4#au+A0_*
zK$U~l+bZN`=0McdLZ~zaTLqw6FsF8Nf25!a4b7jtuGTY#cFyPGoXnpgx>=^~kRJ^_
WvH8szML`;xKe<0rYP0D*cLe}J?sC5X

delta 119
zcmezIpZU)p<_(RJd}#`{3YmF&PL(C8#TuHMZ$}EM@S=!Kt~;wVIlWE;UFFNVn||nG
YGUpTp(ZnX-j+B}_zn*RL_WO<s0CKx4>Hq)$

diff --git a/refs/grpc/types_frostfs.pb.go b/refs/grpc/types_frostfs.pb.go
index 5a16f8e46fb89b57c4f311094b9d162b09c1f51d..f2a2663ad151c4c1ec537e82d17f22e24f96dc80 100644
GIT binary patch
delta 788
zcmaFyj`752#ti{VQPo_WAW)K9pkQUIkeR3FR9TW*tf2`J&P-FtNzKy$%4#au+A0_*
zK$U~l+bZN`=0McdLZ~zaTLqw6FsF8NoRShZEzR%KeZ)me^Xn`Y%F)pL$-3cAG&Fv5
nZTLQZ8k$dI=TEK;7vH=g-IG6sb}1B`PAl|+OLHfGbPERn<>|08

delta 178
zcmX@{neoLt#ti{Vd}#`{3YmF&PL(C8#TuHMS1BoRqlrykrIfk3U$2r2oA_x<3psRg
f!AvJ~p|zRfn*}15^P`JnD&4#`GXX<cdSxO2?ov9Q

diff --git a/session/grpc/service_frostfs.pb.go b/session/grpc/service_frostfs.pb.go
index 6d3ea9fe0e64de1700672326eac9629d158f9abe..26213b9938989721d9625fd234a3ce785fba7d4b 100644
GIT binary patch
delta 200
zcmaDdm+{X$#tq8mQPo_WAW)K9pkQUIkeR3FR9TW*tf2`J&P-FtNzKy$%4#au+A0_*
nK$U~l+bZN`=0McdLZ~zaTLqw6FsF91p}7+cjo;j8o-7Uk-SI7Y

delta 48
rcmew}kMYS|#tq8md}#`{3YmF&PL(C8#TuHEYt5a|gf>fBB#Q$8pi&RZ

diff --git a/session/grpc/types_frostfs.pb.go b/session/grpc/types_frostfs.pb.go
index e9f3b424f06ee9769ba81ac145f695d6ebcdda41..542a02b0c622031964a5bd42c5c8f8cd6a03c5f0 100644
GIT binary patch
delta 215
zcmX@Ng5}vtmJKT_lB>BmL7*hJK*7pZAu~_Usj?)sSVI#coSCMOlbWXil+{$QwN)@s
zfGP*8w^hi?%z>z@g-~e<whBPCU=Bp6LeDd0`b8l|;mwyT+!&*1XWM4=D)m_ag;O$t

delta 49
rcmaF1lI7eAmJKT__|g<?6*BYmoGMFFi#0SiGgl@tqKoaXQlA9?%XksG

diff --git a/status/grpc/types_frostfs.pb.go b/status/grpc/types_frostfs.pb.go
index 1d8b4772c2a424133b457182eb1b0b3cf4596d6c..6c62a0f24f54ab8f16b527a8f624d5965de6945a 100644
GIT binary patch
delta 103
zcmX?**OR}2Q6;LHixUJ&atjo!Y!x!|^qeY7Qj0Y-A;Oty3OT8H8bDc11zTGM0|lsZ
euzFjCyv!Vkx>^X8reLc8R14<RZsu1p6b1l1q8C8`

delta 27
icmeClKajV9QH3u}!B!zNPtU2cB(+#Wb91zcp)df2SqQ@b

diff --git a/tombstone/grpc/types_frostfs.pb.go b/tombstone/grpc/types_frostfs.pb.go
index acb1836c7ee54ba3caa721aec30567ed6b5d34db..e56832a0e878c5eb811f26f064eff76ae879295f 100644
GIT binary patch
delta 103
zcmZqEy`;N=Ni?dOixUJ&atjo!Y!x!|^qeY7Qj0Y-A;Oty3OT8H8bDc11zTGM0|lsZ
euzFjCyv!Vkx>^X8reLc8R14<RZWa(NV*~&QJ{I8s

delta 27
icmcbl+orpLNt7>5!B!zNPtU2cB(+#Wb90Pn86yCBZwO-m

diff --git a/util/proto/marshal_test.go b/util/proto/marshal_test.go
index 5498cf96..29af711a 100644
--- a/util/proto/marshal_test.go
+++ b/util/proto/marshal_test.go
@@ -50,9 +50,6 @@ func TestStableMarshalSingle(t *testing.T) {
 
 			var actualFrostfs generated.Primitives
 			require.NoError(t, actualFrostfs.UnmarshalJSON(r))
-			if len(actualFrostfs.FieldA) == 0 {
-				actualFrostfs.FieldA = nil
-			}
 			require.Equal(t, input, &actualFrostfs)
 
 			primitivesEqual(t, input, &actual)
@@ -110,9 +107,6 @@ func TestStableMarshalSingle(t *testing.T) {
 
 				var actualFrostfs generated.Primitives
 				require.NoError(t, actualFrostfs.UnmarshalJSON(r))
-				if len(actualFrostfs.FieldA) == 0 {
-					actualFrostfs.FieldA = nil
-				}
 				require.Equal(t, tc.input, &actualFrostfs)
 
 				primitivesEqual(t, tc.input, &actual)
@@ -124,9 +118,7 @@ func TestStableMarshalSingle(t *testing.T) {
 func primitivesEqual(t *testing.T, a *generated.Primitives, b *test.Primitives) {
 	// Compare each field directly, because proto-generated code has private fields.
 	require.Equal(t, len(a.FieldA), len(b.FieldA))
-	if len(a.FieldA) != 0 {
-		require.Equal(t, a.FieldA, b.FieldA)
-	}
+	require.Equal(t, a.FieldA, b.FieldA)
 	require.Equal(t, a.FieldB, b.FieldB)
 	require.Equal(t, a.FieldC, b.FieldC)
 	require.Equal(t, a.FieldD, b.FieldD)
diff --git a/util/proto/test/custom/test_frostfs.pb.go b/util/proto/test/custom/test_frostfs.pb.go
index f979038b001687ad4d8f19cff3409197253f7038..5ef95a6c1d4ed711000a03af021d64afa0f513ae 100644
GIT binary patch
delta 324
zcmey|&vbVp(}t;z>D64EAW)K9pkQUIkeR3FR9TW*tf2`J&P-FtNzKy$%4#au+A0_*
zK$U~l+bZN`=0McdLZ~zaTLqw6Fb5)3q34#Fnv>!<nbApn^Kr))98t8haPwZTyG9Af
gF2v#^PMlss_Y)`7Q=G`Y;;fxKr^RXWt6EMj09hhShyVZp

delta 71
zcmcaRk?C_k(}t;zd}#`{3YmF&PL(C8#TuHM-#hYfqKk!k-!<YvRyVnz-evQZI!-PC
Dl@S?A

diff --git a/util/protogen/internalgengo/json.go b/util/protogen/internalgengo/json.go
index 15facfd9..55c9cd31 100644
--- a/util/protogen/internalgengo/json.go
+++ b/util/protogen/internalgengo/json.go
@@ -129,7 +129,17 @@ func emitJSONFieldRead(g *protogen.GeneratedFile, f *protogen.Field, name string
 	case protoreflect.StringKind:
 		template = "%s = in.String()"
 	case protoreflect.BytesKind:
-		template = "%s = in.Bytes()"
+		// Since some time ago proto3 support optional keyword, thus the presence is not tracked by default:
+		// https://github.com/protocolbuffers/protobuf-go/blob/fb995f184a1719ec42b247a3771d1036d92adf67/internal/impl/message_reflect_field.go#L327
+		// We do not explicitly support `optional` keyword, protoc will fail on such fileds.
+		// Thus, treat empty string as `[]byte(nil)`.
+		template = `{
+			tmp := in.Bytes()
+			if len(tmp) == 0 {
+				tmp = nil
+			}
+			%s = tmp
+		}`
 	case protoreflect.MessageKind:
 		if f.Desc.IsList() {
 			g.P("f = ", fieldType(g, f)[2:], "{}")