From 51c16cf546c7bf0c97b854b1cbbfaec724bdc24b Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 20 Mar 2024 12:40:43 +0300 Subject: [PATCH 1/4] [#205] go.mod: Update api-go Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- go.sum | Bin 72928 -> 72928 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 48c9453..84be6b5 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.frostfs.info/TrueCloudLab/frostfs-sdk-go go 1.20 require ( - git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240306101814-c1c7b344b9c0 + git.frostfs.info/TrueCloudLab/frostfs-api-go/v2 v2.16.1-0.20240319122301-1772b921826b git.frostfs.info/TrueCloudLab/frostfs-contract v0.0.0-20230307110621-19a8ef2d02fb git.frostfs.info/TrueCloudLab/frostfs-crypto v0.6.0 git.frostfs.info/TrueCloudLab/hrw v1.2.1 diff --git a/go.sum b/go.sum index 33959db353cd453de7a01861eae8b15932f1853d..e11d2e888868b8b398a7358430d26ee2b46f0f90 100644 GIT binary patch delta 117 zcmaE`ljXrqmJL(6T?{P^jf{*940R37&5e>QjSMY}%#su`46Oq4-P5a*(zHuMwG#up zk}Oh73LOg*Q}aD@eIkQ0le42L)AaKK&9%#tBPSQ~YD})?mJ-EkShE4ob^{*94FUj) C^dk%a delta 117 zcmaE`ljXrqmJL(6T@1_&4Gb*|O>~nDlg*QiO-zz3lMNIy46RHuD;={ev(3YeEF%jl zt3pG{eM^0vl6;KajeRZ6vVD^x%#8fY-TmCtTqhUuYD})?mJ-EkShE4ob^{*94FUjm C<|4HK -- 2.45.2 From 31ba724610efe2bd47dad37608800a68ee2dcf8a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 22 Feb 2024 22:29:25 +0300 Subject: [PATCH 2/4] [#205] netmap: Add EC statement to placement policy Signed-off-by: Evgenii Stratonikov --- netmap/netmap.go | 26 +- netmap/parser/Query.g4 | 6 +- netmap/parser/Query.interp | Bin 5170 -> 5562 bytes netmap/parser/Query.tokens | Bin 372 -> 400 bytes netmap/parser/QueryLexer.g4 | 2 + netmap/parser/QueryLexer.interp | Bin 7459 -> 7764 bytes netmap/parser/QueryLexer.tokens | Bin 372 -> 400 bytes netmap/parser/query_base_visitor.go | 6 +- netmap/parser/query_lexer.go | 242 +++++----- netmap/parser/query_parser.go | 711 ++++++++++++++++++++-------- netmap/parser/query_visitor.go | 5 +- netmap/policy.go | 72 ++- netmap/policy_decode_test.go | 2 + 13 files changed, 728 insertions(+), 344 deletions(-) diff --git a/netmap/netmap.go b/netmap/netmap.go index 530b06d..f0ece7d 100644 --- a/netmap/netmap.go +++ b/netmap/netmap.go @@ -209,6 +209,25 @@ func (m NetMap) SelectFilterNodes(expr *SelectFilterExpr) ([][]NodeInfo, error) return ret, nil } +func countNodes(r netmap.Replica) uint32 { + if r.GetCount() != 0 { + return r.GetCount() + } + return r.GetECDataCount() + r.GetECParityCount() +} + +func (p PlacementPolicy) isUnique() bool { + if p.unique { + return true + } + for _, r := range p.replicas { + if r.GetECDataCount() != 0 || r.GetECParityCount() != 0 { + return true + } + } + return false +} + // ContainerNodes returns two-dimensional list of nodes as a result of applying // given PlacementPolicy to the NetMap. Each line of the list corresponds to a // replica descriptor. Line order corresponds to order of ReplicaDescriptor list @@ -230,6 +249,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e return nil, err } + unique := p.isUnique() result := make([][]NodeInfo, len(p.replicas)) // Note that the cached selectors are not used when the policy contains the UNIQUE flag. @@ -240,7 +260,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e sName := p.replicas[i].GetSelector() if sName == "" && !(len(p.replicas) == 1 && len(p.selectors) == 1) { var s netmap.Selector - s.SetCount(p.replicas[i].GetCount()) + s.SetCount(countNodes(p.replicas[i])) s.SetFilter(mainFilterName) nodes, err := c.getSelection(s) @@ -250,14 +270,14 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e result[i] = append(result[i], flattenNodes(nodes)...) - if p.unique { + if unique { c.addUsedNodes(result[i]...) } continue } - if p.unique { + if unique { if c.processedSelectors[sName] == nil { return nil, fmt.Errorf("selector not found: '%s'", sName) } diff --git a/netmap/parser/Query.g4 b/netmap/parser/Query.g4 index 72fa880..0a5b314 100644 --- a/netmap/parser/Query.g4 +++ b/netmap/parser/Query.g4 @@ -4,10 +4,14 @@ options { tokenVocab = QueryLexer; } -policy: UNIQUE? repStmt+ cbfStmt? selectStmt* filterStmt* EOF; +policy: UNIQUE? (repStmt | ecStmt)+ cbfStmt? selectStmt* filterStmt* EOF; selectFilterExpr: cbfStmt? selectStmt? filterStmt* EOF; +ecStmt: + EC Data = NUMBER1 DOT Parity = NUMBER1 // erasure code configuration + (IN Selector = ident)?; // optional selector name + repStmt: REP Count = NUMBER1 // number of object replicas (IN Selector = ident)?; // optional selector name diff --git a/netmap/parser/Query.interp b/netmap/parser/Query.interp index a8fb219e6ebe10b7437f9f3f946010b562b52f2d..3f7a9ad50456f472f2f5103076fb7f55c6cfe1e3 100644 GIT binary patch literal 5562 zcmbW4TW=dT5QX3QEBxlRa0^j)<*ACS6hdvumE;C!Q3Q^&0gT8_V9RLz@B5t@?rKT3 zK~Z2$at`NmX2@NQ_n$vi4=203ef4;=J9)Uduby6ohv(fcoGw?}({MUno-?l2rwP4Y zE`GV51Fh#*jPn`8V#zSwFw9=RVcg6w<}(?;S+6cNEiSfmJe+>V-w*t4rk8SYzSwLR zOVj)Y@mv0WOmUn-r!9rOaeaEZfB$)Rcbk!g>$}xefUKs~TH|JMd37<@$~gxZhxsfl zmSMUHFb&Q#h-Gk&;qBt$d^TO5hx64o%r2(aoB2Djt=Bc>aPjVHx}Gn?I?dBgCVRj~~L_ zhw5SPW4m4@c!)ub{1T;x<3Ysv*7fKr0B_u0H zeFW4V8)+|X;zu~V+CKWYYWpArT75E1C=!oI3dfp9fQ_y_VDq1}2W+>s6A?2eEaDJP zShDgbq0Xs6J?)`$66*2A32Eq@8trD`+g}+8!|n)sIJlk@C`Lq{F*MNgJ&H`6_UI&w z5oM5xpHs7!P|hBI4T`^F?Rv4p+$d566n)g?@hkMdM%>Cn#5d1>bOuFfcKtB49rG{ zCO=rceF7v62ruKb<0qvTQVNoWETnMQNw-uMR11lS2wpj3D+*~_r_hxSSQktM?t=Be zc$^mtxr$!hTAMu{B(~hSN>yhw!l4{A9uQ>7ZS#t{%t%xzW7t4)htP)LG!}=JmU0;v z&aAE-CD&Gzd|>ZkEF-0vX{wZckT$6L#J(5UZ5dUtI4I0j!KlwtW-i);f|6<+2O3&D zjm@wy?;-1pOMD)Wuc`{qjD;0ZB=?$AOSX8?R~mJQF#`9ratsR`3&XBgByCu63T9WF zg1+`gal!idZF82a*DS*j^y|O_Bj_ZZ@>FCez04N5U*y}zgv5N)6_<0jFNt6 zG7M+s$tys=I629Kl8h2A$mqi8en^#YSP39vd6T51-?26s1(5C$RwSakHYiR>zDk19 z89>lW0uk|W!dVDd>IakPtne+$l;W-Lrvrh%%Q zArFh94&kIR?lGV=SY-B3!ER3)+G;&dF{UVG#WnHC22Vh(l zM?jICemW+YC`=Puzyz;=hssXao)o!EV5|lUSid=53@K@&aVKxWP|PsoNs*r93Di+< z9hJB!d}b}IsD$JRIc9pwIUOOSygCA62a!1TdqY@uV#fx|MWa@Z9fejSn6okmZc?k7 zxl)&Qn}q}Nlc}mA9b&=TiJ1d{)C;vh(v2PTm>Iu!KZeAPpL65dfY?b%v*~UOiMk>p zM`({~NYvpg?jdVyV$>Nmenrk&3$c^(!L{*~&%(75)Eis4mU!%1DGHSjCa2;pOGkTk zBE|#qPUaXr(p3^&8DzTnkJi=T51x#2Y)IFu(cG6P!20e(e#*i6pRO=WnQ9+G)CQr` zLRAd4mfB?vAw#SfB2(KNjmCY3Dz)yBu2QpqxHDZtm3oE>knaq%Qj`U<7o&%w)DLM0 ziPn!Xdm18aotRoOv~iazVV$ovVA%^Dtw%cwXn{zPS;tWOP||92aNhbTMC+|m!&aZ8 zF;R+YzHkG-+KX~jt0gabMZE$BnfI(B)@PNCpZCONOXtS&UdIv%Ysyl`)h2EImT={G zU}x=B*~odjU3O?u)^;DWcq*p)qM}IFqt&=wH?xY?<5|y@@vsi8?AN@*FT1^jmWj)! zy%BA8_TPXyulA8g0Y}%?{zp)whU(${6L9nfTYIO5OMY}om{ogg->f+UZ%)Pk(I{KO zgtmE^3%nk^XV%xHrmcOZ*0KqQq}~V|$cMFjWiOaeST)2<7i)X5YHL8$A2F7}>f-WHl{nAM7_)WzY|se>SbURyVuQcFXLDt!*84|M(l=8}GFM literal 5170 zcmbW5TTfd@6oudCSMWw#QOV=G^Hc|%$|@LYLy=ll2*`_xVXbP<{{8wKo|~_$&EqEA-+i*#2@b7q zUtgN=X-D>@Z8j*o{9E&uR)5^CUmKm$<6ZL{J~wawe0lyD9zHft8y~NaLkPE#?mh zYCW;FMy;beWdnP0gfJtO&3ue@ooM`2!wm}tJLLJ5=xZ=#U`@(W< zsR`-JbV4QRN!opBD+vG_6I;-?*aNxD*(nfHmIH|cF!@uIQqZch%A25vvgvU_qqqZ{We^d0 z<+dy_Mq4+IDv}XVXNqJn*~aEH%cxU%u&`t0!*cFPt1yX+9$997$TRjTb;=a`q7_Uu z`2rOn(e(WmdXO5aY#dbTM8h{xsrz{$i&IBw82+;$(W?XRDH5)xCIuz}_Ot>obm3Gr zL*BCoRwaU5*QTfBxa)F*hWl0hzze^^;N=){zd8WQOjY$@+3fX|PF;RDeCG>$qQl+8 zlyiYe9@MeO)?O^iw@)romuX1^OjuQ6An{7(PH+IX#_EfJJivolmy#ZA)d9kxB~a{6 zLQk@$LY3S&r|dj<3kOvV46D*WZ=h5PdrE=#TlTi+0cTLBCJbJ#i#+rYJ!E7N_>I)F ztLaervQa3LL3O|wgIn;xs^pM|RdU$c3r3uw5)s`4Bsj?g*Fwas+(w$Q+QP^q@r3cs z@Mk2g5ogz47D$r$As3g=EZ_8`3Ojd$IhX||S=I^&>$0RtaV5u+N1R>}jex2Md{#%C zo=mzXu?QFoDU`?B7+0x`6$YJ!9<;?zBsTI8QG{4vlE+nc$YX`Bt*kc0g8M=njPkV^ z<5}{!N)UPMC(4hlSP%tFB;iI&?j5Moo@mB$%S5L8DFZ?m_SPUQ2`~#xa!r$^iTo)n zDfxq(I)NRgC??9spP$_-z(ktb+qyxXqy)8{ERvu+%Nak4jswJw2&ixU8)7Zk)}$!0 zBPn*I3wdkTkl2yg!m=T;qckehBt~~vnuY|54Vm_0M}D{`Y_&;P7y><(eRPkDBl~49 zxH2C*mb$*b?T#bq(ke?YTLiGqa+e-n>~Orhu&7(M6eI7iI2v0NT^gmG4Le_}1;MmT z^JQDqx7uTR&&)Aa5xhs7fW;PJ5yQyrDU^iVC(RGv-2pRJBN=*h^+X74RGUwSB3io$v-(`g~IOaOITW>)vw zR%_H!ca_)K8nYHBCJ#MY(*<6|Vb$2n(9pdU4W+Dk%-FQ6P=KOD`mNlC0rvkF1(jPx zdnM|wJ^);d(iH%W6E&t(&X3mYfwQk5Q;n?^>+cyMq`EyctK6*89|_ZNRcC7bmo;MA z*Ix2x?Hm1GVZ_k7V>7E<&Ya3EwOg;_pqHiwC|W-znHGgczU5N-&6j{|Cx$s!jj^ diff --git a/netmap/parser/Query.tokens b/netmap/parser/Query.tokens index 6376ea2554427a870ced03dce5ce8ad0847fdf62..b873682b36d215fd3270670b8b92f86b7f882279 100644 GIT binary patch literal 400 zcmXw!%WA_g5JmU*N2cwfSc>yn3{N%SPVeIwYl~^uoH1=nD3;V+P~Lb&XzDw}(GtNSt@F)_ zQ}6L-C9CJpTZ!pceUE2Ve}tsmgo(mNVWH$A43t-uFiY4a3=lHT+t-vPUWh&3f~?Sye5<i>KFbAO85fdHVD5vANvsPnXT*a<^8w zKU_xiZnydRt`l_VZdGgPZriOG{PM8BL2S0CP8Kdd(es&}&@}B*_q}?YJRNei#U%{80D?z*|`4k*V`Z8J-+#=ASadUIK?Wj5D zMvR+o)ogan^4N%J<2)N!HqNp6X0u(dmWOq--k+M)c6oQ~?qQqiiE^{Ozg-@>U2}+X zd1^N6Zg*;ScQ-G(LukJ14*TYKI&5}dHs2hZ=J!7z9}?&P5#9BhU*0@7yLV4Nd*was zyW@%wuJ-G$dDVT}9G5RU8n^q+?sSxY+pfgc`@0vy@cO4;Uq3xP^gCb=&6EX06aQ&D(cB_aRD}H_Y0+et!DA`TqLqVsv%UPV}_x)y4Gc;?tnI zp)>Ehx)7qeOx)SyWwiRaeD-s?V*iQuT$^RjM9DM@k4< zacZAC2%%4cKA#>z%LyQ8IRykQP0u)I5u5{pJ|}{pA?b3{hSMe zmUBVSaxNBQwVVrrKIej<l93;BKzMA} zxD4>~!aF1FN9x=AR`q=lS5FG=jf;G_UY)BO4xA(~( zjkmf4a}2_R!=VB603a|-Yo;CyQ^zn8!`3Gy3{#izuqEMv;*8NJp`W$p(IP!qL}C#+ zSAj+99}wy`m4%cbhk9*l5xpKviKdTIpwlQpzFe6=PhdNWM5BT)4;KT}J|tuul_=uU zx!|@?UlOW{CYMIb@@s@$AZX|zEZ*m)40A|k^czb7J=J8vLad4mx$n5HKKMJ#?!}Ze zput zv}AqaS`*irNK48SDM_%a5mJd;LM@MlD%Yx^?*^taun@XH^1v$YYm{XXWj8z~>O5-l zK#pf2g7(Ptf{EAzUJ$9^g$4$@9qg!6IE6$ADl@1+0Sn$Ogb=1V(_Cd|Bvy8cq0KB+ z7KUZ#X;~R)*-w@Jhho+gE=m;wxl)H|N+jO0y=LPo|jz9_HOB;ZQ}|$j}5C<1&4iJL+1n8c$F4gzk=A}IixzfD2Jpmt!znS zRt*$^d4EFOa}DBP+nt%;wy-006pD}-vEg$~yc|`o+HGRHG}H8}r5f({Of`|6 z?|2FK3uen)FPU9O^`hBQ+eMVkZX>Tb!8wRW^$OTXZlXeyXmc;+pe;HVNdYplUN7#w zFs(Id*UM;IWah7#-gl$8s6i&Oiz)L6itVT(55;y=?-#Y(`b=}lw)LsDD>r5`JtaJx zijl-jd(3eFyabf_2nv`=Mc`bkA3*`tVh{;#N(}39J%E`=VX>bEfLsL40TfVeF3!VD z7=wE{05c)_k$uEzFr{#&HnXZdb6;Q<^#qVYGq_iV81@Bb90f$pVfb8D7>8y*9N=72 z)aIgJY6?iR_mL!~q)KRtvhxHu*HrgBuFjx2c$D$PBdPQp;SHvz#_;J36Z>>xGKmPPM)t?>es)U-g#hY_nyPWcB!)oPnNxp1 zRCVr3jvb>lWgIPF;d>w(Eg+1ffU4`MG=y|6HyZ4+B|j+Uk2!~X#mRCj!A^J;P`{c- z8m}K)Kk2F&M()`VEko)_QoZ#mI+?G z(Q%OYIW)Lpukf)fpgWKQiJwQAF8+<(zibq*lI3D;@}$^r@!cz>fkgdZe+ zNDUKO*1!i0x|$_g1_{^hUlwPO!24dmjGkoxxCShhWJE~dYeR5u1Rw1aLU1-EKHArd zm`EiV!N-XjrEL(RU95_w5))rRf)GuB5baVI4=3&+kx9BjEZPYX?zz##{hJQ{fmo~y zUA&tzqdLWzLt{2@j}!!44=2iS{7ZPIKkWzQ)*WR?fvOcCR-&3Y*nmRUP@FGNSkY5} zSj3eQ70JLY*RK&w0c>{^1}qkG9GN--+l_ zq?!~1=}%!9qrnCgx`x&vBY$FNmIknBRtf`n5OXMoB*x;;-YhL(lJg=?5DwCu+HYJ-xa#Sv-#+T$R6zjYcTmX5Y0VTKY; z4dAV5kj4jw9C zyc7Zbk}AGgmX;uOm|9zBj|Q}LS{l)MnMlssI!iR5!O4!tr0ij*Yo( Z4%s#4H->`lN5YQ*ez4;b?HG+8{svh1Zi@f_ literal 7459 zcmd6rTW=dT5QX3OSNP3s;R2#=mhx1^S~o&v$(3ZJXi*qxVzfpb2S}WU{`;P9hTJ8o zG)U2x0$N@UXNG5nTyj=V@BX}dJbAc(x_kfX;pFkv+q)0Xn#Ye156$UjdpK=Q7n`NZ z?d~+9H=EUuH?5#ud#!S{L1wSw;`yb@eY61y^}40yR2va(R-0zA zZ$z*mF3 zshBm*+jnoe*d)m-W^Gw*m3l+xLPgKHP|?$a1)BOf7mK5M&V`CG=R!r#xlqw_ zE>!fK3l%-*LZvCi;ghAnPyS>;cwUO4kpEapUB8)p5lOctFJ<>jN@+>*xKu2J=cVEy zBri3ckmJ&$^Ccx)N+v#AF|Oa2R~}$?Bsw0D&sH8MwkZn#y7J($uTl4qrOLy_zGlor zl~w2fPDGZPhmcUi-^L_FENScKVT~29fxuKTuAUz0P=zTwBeo@Lm<;=vsLGnPRe8Kp zzmXo{`oY;k ztcE6(QK>*xWAJd8Eym1}z#wo ziIszrwwWc$LX0q}XGlEtlTnX|cbVr15v587a;DyCF-?iYL$Z@lBOZJObZk5xP=(a7 z`FK*~QvEu2Xne)Gz8dlEDWGHXwn0mWY_C?(F?-v=r4-w@3JyEDhK_^E=FO$lp6baV znP$u6kh;@KQKo0?yo{G5W`3%4kzmC*sB$ru%=TiYYPZWkYK5R}*jgJNgUT^KBPu$! zcJU(V2-)hzv#$p+7Z1H2WTJY3Yz@b}g&BDh8dtO1@iVT%VJkWAjtrlRw=Ya8Nsg&7 zsU&UHr_#!2w{@RZ0paurvl?Wg8AV2(lP=6F@|=6adO0`+hjs$x>Y8IXsMR&cuwEm! z*;J8}m2>&!z5{TF14=1^0;V!4kS!KLwW_$YOQ(t(m7XXkfM^KFrV%jJq)7I2t5p$a zlXFy}+|dCz7kPo)=spcVwPfcSBzs_zUReNlJ<3vv%9{4v!JZ>Pl+4sM$9eRoR~K+B zDR1B`D{zYw2}rYdk(_A^E=bC}O9C8A%A;9aonEr{sOQN#;eR(XbAahN!m7BLoY>cL z@I1omIS5)MCiCq47|!S(A(J3JtBRmnybgeJDuM#IxKZzBx0_tO!sc867o)&F5CC>L zCVMi)0mvJMwuXHk01|^zTfMpXBk5wO6R*l|&QCG|3HHl} ze{}+gWeCjQMZ^=1YSF?#kI31}vQfs~ZOH8Shhqu_~&XHv?rYkZ(#S zXMxB=20<`^d)5|S?j@#_Pf1!x{PyVSd8#Er{W4eV z1|#{xM7PSpb280@bjAZ8@fssKfLY~66r6cz zcO;(>g0nE$M+H|U(!Dbv@q43>hh3&*KpNy`jI2TGd)Dt*z8SEaamGWw76fP8(Y_zJ zaRp~VvOAcB6X`yrmV&ZJ5(U}w`bsaNthIb~YtGj-!wLih~3$`_eW@{CslCCMKT)Ie=u*^PeF&UpQ{*2&nu9CO3!<5o|CMb)p zQlL0T_E9i_L@+ajtf4G#feK)yN)U@?r4YqqKFzu{GGgl4QWc^W3puL6&T=5bt~117 zNgJ`1sJxL$tXk$x`ok{^8d;GKCQ*~dHO|y-E<^Fv6be~G>yYto)p@F0gj#79pJK{H z!kQSXStqrDmoRYC%RHMa%5pf`-znnFN^^1;q=^zvph>(9$7@qcCVCW=0woq4ZzYNs zbsA#K0@+n53&5#TqL{3c78i@@L+&8!AM+!YPjrnj;o#J){I+~1;+9Kma^y)f0(V2f z#LzkgGP8j^t&}AirjSUOT|)^;fdU+TrkP1xDP@U9^1FA?zAJ3e3&X^c1Ftl}hB*Rp zHl|KgF>qvA_P3TSiAq9c&rm_U?p#79$aXQ6B-*2>6ezk37a);{DOlhif;1uFCFw#* z54V_^CK@DOHIC!tUNw%}q+c~Ih(APgGWM}GB>$2z@p38#OsW;MPp~2Bm+X5iByB+b z5osT0L$Wc6x)&xYp#HQRlAPJUd?KJyn3{N%SPVeIwYl~^uoH1=nD3;V+P~Lb&XzDw}(GtNSt@F)_ zQ}6L-C9CJpTZ!pceUE2Ve}tsmgo(mNVWH$A43t-uFiY4a3=lHT+t-vPUWh&3f~?Sye5< Date: Thu, 21 Mar 2024 09:57:34 +0300 Subject: [PATCH 3/4] [#205] netmap: Add well-known EC parameters to network config Signed-off-by: Evgenii Stratonikov --- netmap/network_info.go | 32 ++++++++++++++++++++++++++++++++ netmap/network_info_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/netmap/network_info.go b/netmap/network_info.go index 186d433..a149bf4 100644 --- a/netmap/network_info.go +++ b/netmap/network_info.go @@ -62,6 +62,8 @@ func (x *NetworkInfo) readFromV2(m netmap.NetworkInfo, checkFieldPresence bool) configEpochDuration, configIRCandidateFee, configMaxObjSize, + configMaxECDataCount, + configMaxECParityCount, configWithdrawalFee: _, err = decodeConfigValueUint64(prm.GetValue()) case configHomomorphicHashingDisabled, @@ -234,6 +236,8 @@ func (x *NetworkInfo) IterateRawNetworkParameters(f func(name string, value []by configEpochDuration, configIRCandidateFee, configMaxObjSize, + configMaxECDataCount, + configMaxECParityCount, configWithdrawalFee, configHomomorphicHashingDisabled, configMaintenanceModeAllowed: @@ -432,6 +436,34 @@ func (x NetworkInfo) MaxObjectSize() uint64 { return x.configUint64(configMaxObjSize) } +const configMaxECDataCount = "MaxECDataCount" + +// SetMaxECDataCount sets maximum number of data shards for erasure codes. +// +// Zero means no restrictions. +func (x *NetworkInfo) SetMaxECDataCount(dataCount uint64) { + x.setConfigUint64(configMaxECDataCount, dataCount) +} + +// MaxECDataCount returns maximum number of data shards for erasure codes. +func (x NetworkInfo) MaxECDataCount() uint64 { + return x.configUint64(configMaxECDataCount) +} + +const configMaxECParityCount = "MaxECParityCount" + +// SetMaxECParityCount sets maximum number of parity shards for erasure codes. +// +// Zero means no restrictions. +func (x *NetworkInfo) SetMaxECParityCount(parityCount uint64) { + x.setConfigUint64(configMaxECParityCount, parityCount) +} + +// MaxECParityCount returns maximum number of parity shards for erasure codes. +func (x NetworkInfo) MaxECParityCount() uint64 { + return x.configUint64(configMaxECParityCount) +} + const configWithdrawalFee = "WithdrawFee" // SetWithdrawalFee sets fee for withdrawals from the FrostFS accounts that diff --git a/netmap/network_info_test.go b/netmap/network_info_test.go index 54b1b30..161d152 100644 --- a/netmap/network_info_test.go +++ b/netmap/network_info_test.go @@ -173,6 +173,32 @@ func TestNetworkInfo_MaxObjectSize(t *testing.T) { ) } +func TestNetworkInfo_MaxECDataCount(t *testing.T) { + testConfigValue(t, + func(x NetworkInfo) any { return x.MaxECDataCount() }, + func(info *NetworkInfo, val any) { info.SetMaxECDataCount(val.(uint64)) }, + uint64(1), uint64(2), + "MaxECDataCount", func(val any) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, val.(uint64)) + return data + }, + ) +} + +func TestNetworkInfo_MaxECParityCount(t *testing.T) { + testConfigValue(t, + func(x NetworkInfo) any { return x.MaxECParityCount() }, + func(info *NetworkInfo, val any) { info.SetMaxECParityCount(val.(uint64)) }, + uint64(1), uint64(2), + "MaxECParityCount", func(val any) []byte { + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, val.(uint64)) + return data + }, + ) +} + func TestNetworkInfo_WithdrawalFee(t *testing.T) { testConfigValue(t, func(x NetworkInfo) any { return x.WithdrawalFee() }, -- 2.45.2 From 50e538f27a921eb3e1a825d37d9d6c839945f20a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 28 Feb 2024 14:51:48 +0300 Subject: [PATCH 4/4] [#205] object: Initial EC implementation Signed-off-by: Evgenii Stratonikov --- go.mod | 2 + go.sum | Bin 72928 -> 73369 bytes object/erasure_code.go | 121 +++++++++++ object/erasurecode/constructor.go | 87 ++++++++ object/erasurecode/constructor_test.go | 31 +++ object/erasurecode/reconstruct.go | 142 +++++++++++++ object/erasurecode/reconstruct_test.go | 284 +++++++++++++++++++++++++ object/erasurecode/split.go | 69 ++++++ object/erasurecode/split_test.go | 36 ++++ object/erasurecode/target.go | 44 ++++ object/erasurecode/verify.go | 104 +++++++++ object/object.go | 8 + 12 files changed, 928 insertions(+) create mode 100644 object/erasure_code.go create mode 100644 object/erasurecode/constructor.go create mode 100644 object/erasurecode/constructor_test.go create mode 100644 object/erasurecode/reconstruct.go create mode 100644 object/erasurecode/reconstruct_test.go create mode 100644 object/erasurecode/split.go create mode 100644 object/erasurecode/split_test.go create mode 100644 object/erasurecode/target.go create mode 100644 object/erasurecode/verify.go diff --git a/go.mod b/go.mod index 84be6b5..97bcd88 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/antlr4-go/antlr/v4 v4.13.0 github.com/google/uuid v1.3.0 github.com/hashicorp/golang-lru/v2 v2.0.2 + github.com/klauspost/reedsolomon v1.12.1 github.com/mr-tron/base58 v1.2.0 github.com/nspcc-dev/neo-go v0.101.2-0.20230601131642-a0117042e8fc github.com/stretchr/testify v1.8.3 @@ -28,6 +29,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/golang-lru v0.6.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 // indirect github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20230615193820-9185820289ce // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect diff --git a/go.sum b/go.sum index e11d2e888868b8b398a7358430d26ee2b46f0f90..66eeb94990dfe5816264d1ba10804c942932f02d 100644 GIT binary patch delta 368 zcmZ|Jy>5a)007{jPOg0cc9MfbCDg=0s$LOAO(RG*xI+&Gj>>^QN#jGf)Uj{S#;HTM zzCq*O_vq5Nb*L}!<@!1tW6)T#HFNteJCP9X)cNO${{@v5%xV mBp!tLMm2O-zfWw$b3NPtIQ8SFb9%a}0eS!O)jJ+e2>k-!n1S>F delta 23 fcmbQam*v4umJKt)Cl^QRY}SjsTDv*uy1P37h-(WN diff --git a/object/erasure_code.go b/object/erasure_code.go new file mode 100644 index 0000000..43abe03 --- /dev/null +++ b/object/erasure_code.go @@ -0,0 +1,121 @@ +package object + +import ( + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/object" + refs "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" +) + +// ECHeader represents erasure coding header. +type ECHeader struct { + parent oid.ID + index uint32 + total uint32 + header []byte + headerLength uint32 +} + +// NewECHeader constructs new erasure coding header. +func NewECHeader(parent oid.ID, index, total uint32, header []byte, headerLength uint32) *ECHeader { + return &ECHeader{ + parent: parent, + index: index, + total: total, + header: header, + headerLength: headerLength, + } +} + +// WriteToV2 converts SDK structure to v2-api one. +func (e *ECHeader) WriteToV2(h *object.ECHeader) { + var parent refs.ObjectID + e.parent.WriteToV2(&parent) + h.Parent = &parent + h.Index = e.index + h.Total = e.total + h.Header = e.header + h.HeaderLength = e.headerLength +} + +// ReadFromV2 converts v2-api structure to SDK one. +func (e *ECHeader) ReadFromV2(h *object.ECHeader) error { + if h == nil { + return nil + } + if h.Parent == nil { + return errors.New("empty parent") + } + + _ = e.parent.ReadFromV2(*h.Parent) + e.index = h.Index + e.total = h.Total + e.header = h.Header + e.headerLength = h.HeaderLength + return nil +} + +func (o *Object) ECHeader() *ECHeader { + ec := (*object.Object)(o).GetHeader().GetEC() + if ec == nil { + return nil + } + + h := new(ECHeader) + _ = h.ReadFromV2(ec) + return h +} + +func (o *Object) SetECHeader(ec *ECHeader) { + o.setHeaderField(func(h *object.Header) { + if ec == nil { + h.SetEC(nil) + return + } + + v2 := new(object.ECHeader) + ec.WriteToV2(v2) + h.SetEC(v2) + }) +} + +func (e *ECHeader) Parent() oid.ID { + return e.parent +} + +func (e *ECHeader) SetParent(id oid.ID) { + e.parent = id +} + +func (e *ECHeader) Index() uint32 { + return e.index +} + +func (e *ECHeader) SetIndex(i uint32) { + e.index = i +} + +func (e *ECHeader) Total() uint32 { + return e.total +} + +func (e *ECHeader) SetTotal(i uint32) { + e.total = i +} + +func (e *ECHeader) Header() []byte { + return e.header +} + +func (e *ECHeader) SetHeader(header []byte) { + e.header = header +} + +func (e *ECHeader) HeaderLength() uint32 { + return e.headerLength +} + +func (e *ECHeader) SetHeaderLength(l uint32) { + e.headerLength = l +} diff --git a/object/erasurecode/constructor.go b/object/erasurecode/constructor.go new file mode 100644 index 0000000..447372d --- /dev/null +++ b/object/erasurecode/constructor.go @@ -0,0 +1,87 @@ +package erasurecode + +import ( + "errors" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "github.com/klauspost/reedsolomon" +) + +var ( + // ErrMalformedSlice is returned when a slice of EC chunks is inconsistent. + ErrMalformedSlice = errors.New("inconsistent EC headers") + // ErrInvShardNum is returned from NewConstructor when the number of shards is invalid. + ErrInvShardNum = reedsolomon.ErrInvShardNum + // ErrMaxShardNum is returned from NewConstructor when the number of shards is too big. + ErrMaxShardNum = reedsolomon.ErrMaxShardNum +) + +// MaxShardCount is the maximum number of shards. +const MaxShardCount = 256 + +// Constructor is a wrapper around encoder allowing to reconstruct objects. +// It's methods are not thread-safe. +type Constructor struct { + enc reedsolomon.Encoder + headerLength uint32 + payloadShards [][]byte + headerShards [][]byte +} + +// NewConstructor returns new constructor instance. +func NewConstructor(dataCount int, parityCount int) (*Constructor, error) { + // The library supports up to 65536 shards with some restrictions. + // This can easily result in OOM or panic, thus SDK declares it's own restriction. + if dataCount+parityCount > MaxShardCount { + return nil, ErrMaxShardNum + } + + enc, err := reedsolomon.New(dataCount, parityCount) + if err != nil { + return nil, err + } + return &Constructor{enc: enc}, nil +} + +// clear clears internal state of the constructor, so it can be reused. +func (c *Constructor) clear() { + c.headerLength = 0 + c.payloadShards = nil + c.headerShards = nil +} + +func (c *Constructor) fillHeader(parts []*objectSDK.Object) error { + shards := make([][]byte, len(parts)) + headerLength := 0 + for i := range parts { + if parts[i] == nil { + continue + } + + var err error + headerLength, err = validatePart(parts, i, headerLength) + if err != nil { + return err + } + + shards[i] = parts[i].GetECHeader().Header() + } + + c.headerLength = uint32(headerLength) + c.headerShards = shards + return nil +} + +// fillPayload fills the payload shards. +// Currently there is no case when it can be called without reconstructing header, +// thus fillHeader() must be called before and this function performs no validation. +func (c *Constructor) fillPayload(parts []*objectSDK.Object) { + shards := make([][]byte, len(parts)) + for i := range parts { + if parts[i] == nil { + continue + } + shards[i] = parts[i].Payload() + } + c.payloadShards = shards +} diff --git a/object/erasurecode/constructor_test.go b/object/erasurecode/constructor_test.go new file mode 100644 index 0000000..3268d35 --- /dev/null +++ b/object/erasurecode/constructor_test.go @@ -0,0 +1,31 @@ +package erasurecode_test + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" + "github.com/stretchr/testify/require" +) + +func TestErasureConstruct(t *testing.T) { + t.Run("negative, no panic", func(t *testing.T) { + _, err := erasurecode.NewConstructor(-1, 2) + require.ErrorIs(t, err, erasurecode.ErrInvShardNum) + }) + t.Run("negative, no panic", func(t *testing.T) { + _, err := erasurecode.NewConstructor(2, -1) + require.ErrorIs(t, err, erasurecode.ErrInvShardNum) + }) + t.Run("zero parity", func(t *testing.T) { + _, err := erasurecode.NewConstructor(1, 0) + require.NoError(t, err) + }) + t.Run("max shard num", func(t *testing.T) { + _, err := erasurecode.NewConstructor(erasurecode.MaxShardCount, 0) + require.NoError(t, err) + }) + t.Run("max+1 shard num", func(t *testing.T) { + _, err := erasurecode.NewConstructor(erasurecode.MaxShardCount+1, 0) + require.ErrorIs(t, err, erasurecode.ErrMaxShardNum) + }) +} diff --git a/object/erasurecode/reconstruct.go b/object/erasurecode/reconstruct.go new file mode 100644 index 0000000..68ade85 --- /dev/null +++ b/object/erasurecode/reconstruct.go @@ -0,0 +1,142 @@ +package erasurecode + +import ( + "bytes" + "crypto/ecdsa" + "fmt" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "github.com/klauspost/reedsolomon" +) + +// Reconstruct returns full object reconstructed from parts. +// All non-nil objects in parts must have EC header with the same `total` field equal to len(parts). +// The slice must contain at least one non nil object. +// Index of the objects in parts must be equal to it's index field in the EC header. +// The parts slice isn't changed and can be used concurrently for reading. +func (c *Constructor) Reconstruct(parts []*objectSDK.Object) (*objectSDK.Object, error) { + res, err := c.ReconstructHeader(parts) + if err != nil { + return nil, err + } + + c.fillPayload(parts) + + payload, err := reconstructExact(c.enc, int(res.PayloadSize()), c.payloadShards) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedSlice, err) + } + + res.SetPayload(payload) + return res, nil +} + +// ReconstructHeader returns object header reconstructed from parts. +// All non-nil objects in parts must have EC header with the same `total` field equal to len(parts). +// The slice must contain at least one non nil object. +// Index of the objects in parts must be equal to it's index field in the EC header. +// The parts slice isn't changed and can be used concurrently for reading. +func (c *Constructor) ReconstructHeader(parts []*objectSDK.Object) (*objectSDK.Object, error) { + c.clear() + + if err := c.fillHeader(parts); err != nil { + return nil, err + } + + obj, err := c.reconstructHeader() + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedSlice, err) + } + return obj, nil +} + +// ReconstructParts reconstructs specific EC parts without reconstructing full object. +// All non-nil objects in parts must have EC header with the same `total` field equal to len(parts). +// The slice must contain at least one non nil object. +// Index of the objects in parts must be equal to it's index field in the EC header. +// Those parts for which corresponding element in required is true must be nil and will be overwritten. +// Because partial reconstruction only makes sense for full objects, all parts must have non-empty payload. +// If key is not nil, all reconstructed parts are signed with this key. +func (c *Constructor) ReconstructParts(parts []*objectSDK.Object, required []bool, key *ecdsa.PrivateKey) error { + if len(required) != len(parts) { + return fmt.Errorf("len(parts) != len(required): %d != %d", len(parts), len(required)) + } + + c.clear() + + if err := c.fillHeader(parts); err != nil { + return err + } + c.fillPayload(parts) + + if err := c.enc.ReconstructSome(c.payloadShards, required); err != nil { + return fmt.Errorf("%w: %w", ErrMalformedSlice, err) + } + if err := c.enc.ReconstructSome(c.headerShards, required); err != nil { + return fmt.Errorf("%w: %w", ErrMalformedSlice, err) + } + + nonNilPart := 0 + for i := range parts { + if parts[i] != nil { + nonNilPart = i + break + } + } + + ec := parts[nonNilPart].GetECHeader() + parent := ec.Parent() + total := ec.Total() + + for i := range required { + if parts[i] != nil || !required[i] { + continue + } + + part := objectSDK.New() + copyRequiredFields(part, parts[nonNilPart]) + part.SetPayload(c.payloadShards[i]) + part.SetPayloadSize(uint64(len(c.payloadShards[i]))) + part.SetECHeader(objectSDK.NewECHeader(parent, uint32(i), total, + c.headerShards[i], c.headerLength)) + + if err := setIDWithSignature(part, key); err != nil { + return err + } + parts[i] = part + } + return nil +} + +func (c *Constructor) reconstructHeader() (*objectSDK.Object, error) { + data, err := reconstructExact(c.enc, int(c.headerLength), c.headerShards) + if err != nil { + return nil, err + } + + var obj objectSDK.Object + return &obj, obj.Unmarshal(data) +} + +func reconstructExact(enc reedsolomon.Encoder, size int, shards [][]byte) ([]byte, error) { + if err := enc.ReconstructData(shards); err != nil { + return nil, err + } + + // Technically, this error will be returned from enc.Join(). + // However, allocating based on unvalidated user data is an easy attack vector. + // Preallocating seems to have enough benefits to justify a slight increase in code complexity. + maxSize := 0 + for i := range shards { + maxSize += len(shards[i]) + } + if size > maxSize { + return nil, reedsolomon.ErrShortData + } + + buf := bytes.NewBuffer(make([]byte, 0, size)) + if err := enc.Join(buf, shards, size); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/object/erasurecode/reconstruct_test.go b/object/erasurecode/reconstruct_test.go new file mode 100644 index 0000000..b638a4f --- /dev/null +++ b/object/erasurecode/reconstruct_test.go @@ -0,0 +1,284 @@ +package erasurecode_test + +import ( + "context" + "crypto/rand" + "math" + "testing" + + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/user" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/version" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestErasureCodeReconstruct(t *testing.T) { + const payloadSize = 99 + const dataCount = 3 + const parityCount = 2 + + // We would also like to test padding behaviour, + // so ensure padding is done. + require.NotZero(t, payloadSize%(dataCount+parityCount)) + + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + original := newObject(t, payloadSize, pk) + + c, err := erasurecode.NewConstructor(dataCount, parityCount) + require.NoError(t, err) + + parts, err := c.Split(original, &pk.PrivateKey) + require.NoError(t, err) + + t.Run("reconstruct header", func(t *testing.T) { + original := original.CutPayload() + parts := cloneSlice(parts) + for i := range parts { + parts[i] = parts[i].CutPayload() + } + t.Run("from data", func(t *testing.T) { + parts := cloneSlice(parts) + for i := dataCount; i < dataCount+parityCount; i++ { + parts[i] = nil + } + reconstructed, err := c.ReconstructHeader(parts) + require.NoError(t, err) + verifyReconstruction(t, original, reconstructed) + }) + t.Run("from parity", func(t *testing.T) { + parts := cloneSlice(parts) + for i := 0; i < parityCount; i++ { + parts[i] = nil + } + reconstructed, err := c.ReconstructHeader(parts) + require.NoError(t, err) + verifyReconstruction(t, original, reconstructed) + + t.Run("not enough shards", func(t *testing.T) { + parts[parityCount] = nil + _, err := c.ReconstructHeader(parts) + require.ErrorIs(t, err, erasurecode.ErrMalformedSlice) + }) + }) + t.Run("only nil parts", func(t *testing.T) { + parts := make([]*objectSDK.Object, len(parts)) + _, err := c.ReconstructHeader(parts) + require.ErrorIs(t, err, erasurecode.ErrMalformedSlice) + }) + t.Run("missing EC header", func(t *testing.T) { + parts := cloneSlice(parts) + parts[0] = deepCopy(t, parts[0]) + parts[0].SetECHeader(nil) + + _, err := c.ReconstructHeader(parts) + require.ErrorIs(t, err, erasurecode.ErrMalformedSlice) + }) + t.Run("invalid index", func(t *testing.T) { + parts := cloneSlice(parts) + parts[0] = deepCopy(t, parts[0]) + + ec := parts[0].GetECHeader() + ec.SetIndex(1) + parts[0].SetECHeader(ec) + + _, err := c.ReconstructHeader(parts) + require.ErrorIs(t, err, erasurecode.ErrMalformedSlice) + }) + t.Run("invalid total", func(t *testing.T) { + parts := cloneSlice(parts) + parts[0] = deepCopy(t, parts[0]) + + ec := parts[0].GetECHeader() + ec.SetTotal(uint32(len(parts) + 1)) + parts[0].SetECHeader(ec) + + _, err := c.ReconstructHeader(parts) + require.ErrorIs(t, err, erasurecode.ErrMalformedSlice) + }) + t.Run("inconsistent header length", func(t *testing.T) { + parts := cloneSlice(parts) + parts[0] = deepCopy(t, parts[0]) + + ec := parts[0].GetECHeader() + ec.SetHeaderLength(ec.HeaderLength() - 1) + parts[0].SetECHeader(ec) + + _, err := c.ReconstructHeader(parts) + require.ErrorIs(t, err, erasurecode.ErrMalformedSlice) + }) + t.Run("invalid header length", func(t *testing.T) { + parts := cloneSlice(parts) + for i := range parts { + parts[i] = deepCopy(t, parts[i]) + + ec := parts[0].GetECHeader() + ec.SetHeaderLength(math.MaxUint32) + parts[0].SetECHeader(ec) + } + + _, err := c.ReconstructHeader(parts) + require.ErrorIs(t, err, erasurecode.ErrMalformedSlice) + }) + }) + t.Run("reconstruct data", func(t *testing.T) { + t.Run("from data", func(t *testing.T) { + parts := cloneSlice(parts) + for i := dataCount; i < dataCount+parityCount; i++ { + parts[i] = nil + } + reconstructed, err := c.Reconstruct(parts) + require.NoError(t, err) + verifyReconstruction(t, original, reconstructed) + }) + t.Run("from parity", func(t *testing.T) { + parts := cloneSlice(parts) + for i := 0; i < parityCount; i++ { + parts[i] = nil + } + reconstructed, err := c.Reconstruct(parts) + require.NoError(t, err) + verifyReconstruction(t, original, reconstructed) + + t.Run("not enough shards", func(t *testing.T) { + parts[parityCount] = nil + _, err := c.Reconstruct(parts) + require.ErrorIs(t, err, erasurecode.ErrMalformedSlice) + }) + }) + }) + t.Run("reconstruct parts", func(t *testing.T) { + // We would like to also test that ReconstructParts doesn't perform + // excessive work, so ensure this test makes sense. + require.GreaterOrEqual(t, parityCount, 2) + + t.Run("from data", func(t *testing.T) { + oldParts := parts + parts := cloneSlice(parts) + for i := dataCount; i < dataCount+parityCount; i++ { + parts[i] = nil + } + + required := make([]bool, len(parts)) + required[dataCount] = true + + require.NoError(t, c.ReconstructParts(parts, required, nil)) + + old := deepCopy(t, oldParts[dataCount]) + old.SetSignature(nil) + require.Equal(t, old, parts[dataCount]) + + for i := dataCount + 1; i < dataCount+parityCount; i++ { + require.Nil(t, parts[i]) + } + }) + t.Run("from parity", func(t *testing.T) { + oldParts := parts + parts := cloneSlice(parts) + for i := 0; i < parityCount; i++ { + parts[i] = nil + } + + required := make([]bool, len(parts)) + required[0] = true + + require.NoError(t, c.ReconstructParts(parts, required, nil)) + + old := deepCopy(t, oldParts[0]) + old.SetSignature(nil) + require.Equal(t, old, parts[0]) + + for i := 1; i < parityCount; i++ { + require.Nil(t, parts[i]) + } + }) + }) +} + +func newObject(t *testing.T, size uint64, pk *keys.PrivateKey) *objectSDK.Object { + // Use transformer to form object to avoid potential bugs with yet another helper object creation in tests. + tt := &testTarget{} + p := transformer.NewPayloadSizeLimiter(transformer.Params{ + Key: &pk.PrivateKey, + NextTargetInit: func() transformer.ObjectWriter { return tt }, + NetworkState: dummyEpochSource(123), + MaxSize: size + 1, + WithoutHomomorphicHash: true, + }) + cnr := cidtest.ID() + ver := version.Current() + hdr := objectSDK.New() + hdr.SetContainerID(cnr) + hdr.SetType(objectSDK.TypeRegular) + hdr.SetVersion(&ver) + + var owner user.ID + user.IDFromKey(&owner, pk.PrivateKey.PublicKey) + hdr.SetOwnerID(owner) + + var attr objectSDK.Attribute + attr.SetKey("somekey") + attr.SetValue("somevalue") + hdr.SetAttributes(attr) + + expectedPayload := make([]byte, size) + _, _ = rand.Read(expectedPayload) + writeObject(t, context.Background(), p, hdr, expectedPayload) + require.Len(t, tt.objects, 1) + return tt.objects[0] +} + +func writeObject(t *testing.T, ctx context.Context, target transformer.ChunkedObjectWriter, header *objectSDK.Object, payload []byte) *transformer.AccessIdentifiers { + require.NoError(t, target.WriteHeader(ctx, header)) + + _, err := target.Write(ctx, payload) + require.NoError(t, err) + + ids, err := target.Close(ctx) + require.NoError(t, err) + + return ids +} + +func verifyReconstruction(t *testing.T, original, reconstructed *objectSDK.Object) { + require.True(t, reconstructed.VerifyIDSignature()) + reconstructed.ToV2().SetMarshalData(nil) + original.ToV2().SetMarshalData(nil) + + require.Equal(t, original, reconstructed) +} + +func deepCopy(t *testing.T, obj *objectSDK.Object) *objectSDK.Object { + data, err := obj.Marshal() + require.NoError(t, err) + + res := objectSDK.New() + require.NoError(t, res.Unmarshal(data)) + return res +} + +func cloneSlice[T any](src []T) []T { + dst := make([]T, len(src)) + copy(dst, src) + return dst +} + +type dummyEpochSource uint64 + +func (s dummyEpochSource) CurrentEpoch() uint64 { + return uint64(s) +} + +type testTarget struct { + objects []*objectSDK.Object +} + +func (tt *testTarget) WriteObject(_ context.Context, o *objectSDK.Object) error { + tt.objects = append(tt.objects, o) + return nil // AccessIdentifiers should not be used. +} diff --git a/object/erasurecode/split.go b/object/erasurecode/split.go new file mode 100644 index 0000000..b449b27 --- /dev/null +++ b/object/erasurecode/split.go @@ -0,0 +1,69 @@ +package erasurecode + +import ( + "crypto/ecdsa" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" +) + +// Split splits fully formed object into multiple chunks. +func (c *Constructor) Split(obj *objectSDK.Object, key *ecdsa.PrivateKey) ([]*objectSDK.Object, error) { + c.clear() + + header, err := obj.CutPayload().Marshal() + if err != nil { + return nil, err + } + + headerShards, err := c.encodeRaw(header) + if err != nil { + return nil, err + } + payloadShards, err := c.encodeRaw(obj.Payload()) + if err != nil { + return nil, err + } + + parts := make([]*objectSDK.Object, len(payloadShards)) + parent, _ := obj.ID() + for i := range parts { + chunk := objectSDK.New() + copyRequiredFields(chunk, obj) + chunk.SetPayload(payloadShards[i]) + chunk.SetPayloadSize(uint64(len(payloadShards[i]))) + + ec := objectSDK.NewECHeader(parent, uint32(i), uint32(len(payloadShards)), headerShards[i], uint32(len(header))) + chunk.SetECHeader(ec) + if err := setIDWithSignature(chunk, key); err != nil { + return nil, err + } + + parts[i] = chunk + } + return parts, nil +} + +func setIDWithSignature(obj *objectSDK.Object, key *ecdsa.PrivateKey) error { + if err := objectSDK.CalculateAndSetID(obj); err != nil { + return err + } + + objectSDK.CalculateAndSetPayloadChecksum(obj) + + if key == nil { + return nil + } + + return objectSDK.CalculateAndSetSignature(*key, obj) +} + +func (c *Constructor) encodeRaw(data []byte) ([][]byte, error) { + shards, err := c.enc.Split(data) + if err != nil { + return nil, err + } + if err := c.enc.Encode(shards); err != nil { + return nil, err + } + return shards, nil +} diff --git a/object/erasurecode/split_test.go b/object/erasurecode/split_test.go new file mode 100644 index 0000000..9fcba76 --- /dev/null +++ b/object/erasurecode/split_test.go @@ -0,0 +1,36 @@ +package erasurecode_test + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/erasurecode" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +// The library can behave differently for big shard counts. +// This test checks we support the maximum number of chunks we promise. +func TestSplitMaxShardCount(t *testing.T) { + pk, err := keys.NewPrivateKey() + require.NoError(t, err) + + original := newObject(t, 1024, pk) + + t.Run("only data", func(t *testing.T) { + c, err := erasurecode.NewConstructor(erasurecode.MaxShardCount, 0) + require.NoError(t, err) + + parts, err := c.Split(original, &pk.PrivateKey) + require.NoError(t, err) + require.Len(t, parts, erasurecode.MaxShardCount) + }) + t.Run("data + parity", func(t *testing.T) { + c, err := erasurecode.NewConstructor(1, erasurecode.MaxShardCount-1) + require.NoError(t, err) + + parts, err := c.Split(original, &pk.PrivateKey) + require.NoError(t, err) + require.Len(t, parts, erasurecode.MaxShardCount) + }) + +} diff --git a/object/erasurecode/target.go b/object/erasurecode/target.go new file mode 100644 index 0000000..5cd672b --- /dev/null +++ b/object/erasurecode/target.go @@ -0,0 +1,44 @@ +package erasurecode + +import ( + "context" + "crypto/ecdsa" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/transformer" +) + +// Target accepts regular objects and splits them into erasure-coded chunks. +type Target struct { + c *Constructor + key *ecdsa.PrivateKey + next transformer.ObjectWriter +} + +// ObjectWriter is an interface of the object writer that writes prepared object. +type ObjectWriter interface { + WriteObject(context.Context, *objectSDK.Object) error +} + +// NewTarget returns new target instance. +func NewTarget(c *Constructor, key *ecdsa.PrivateKey, next ObjectWriter) *Target { + return &Target{ + c: c, + key: key, + next: next, + } +} + +// WriteObject implements the transformer.ObjectWriter interface. +func (t *Target) WriteObject(ctx context.Context, obj *objectSDK.Object) error { + parts, err := t.c.Split(obj, t.key) + if err != nil { + return err + } + for i := range parts { + if err := t.next.WriteObject(ctx, parts[i]); err != nil { + return err + } + } + return nil +} diff --git a/object/erasurecode/verify.go b/object/erasurecode/verify.go new file mode 100644 index 0000000..8f1acd4 --- /dev/null +++ b/object/erasurecode/verify.go @@ -0,0 +1,104 @@ +package erasurecode + +import ( + "fmt" + + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" +) + +// Verify verifies that parts are well formed. +// All parts are expected to be non-nil. +// The number of parts must be equal to `total` field of the EC header +// and parts must be sorted by index. +func (c *Constructor) Verify(parts []*objectSDK.Object) error { + c.clear() + + var headerLength int + for i := range parts { + if parts[i] == nil { + return ErrMalformedSlice + } + + var err error + headerLength, err = validatePart(parts, i, headerLength) + if err != nil { + return err + } + } + + p0 := parts[0] + for i := 1; i < len(parts); i++ { + // This part must be kept in sync with copyRequiredFields(). + pi := parts[i] + if p0.OwnerID().Equals(pi.OwnerID()) { + return fmt.Errorf("%w: owner id mismatch: %s != %s", ErrMalformedSlice, p0.OwnerID(), pi.OwnerID()) + } + if p0.Version() == nil && pi.Version() != nil || !p0.Version().Equal(*pi.Version()) { + return fmt.Errorf("%w: version mismatch: %s != %s", ErrMalformedSlice, p0.Version(), pi.Version()) + } + + cnr0, _ := p0.ContainerID() + cnri, _ := pi.ContainerID() + if !cnr0.Equals(cnri) { + return fmt.Errorf("%w: container id mismatch: %s != %s", ErrMalformedSlice, cnr0, cnri) + } + } + + if err := c.fillHeader(parts); err != nil { + return err + } + c.fillPayload(parts) + + ok, err := c.enc.Verify(c.headerShards) + if err != nil { + return err + } + if !ok { + return ErrMalformedSlice + } + + ok, err = c.enc.Verify(c.payloadShards) + if err != nil { + return err + } + if !ok { + return ErrMalformedSlice + } + return nil +} + +// copyRequiredFields sets all fields in dst which are copied from src and shared among all chunks. +// src can be either another chunk of full object. +// dst must be a chunk. +func copyRequiredFields(dst *objectSDK.Object, src *objectSDK.Object) { + dst.SetVersion(src.Version()) + dst.SetOwnerID(src.OwnerID()) + dst.SetCreationEpoch(src.CreationEpoch()) + dst.SetSessionToken(src.SessionToken()) + + cnr, _ := src.ContainerID() + dst.SetContainerID(cnr) +} + +// validatePart makes i-th part is consistent with the rest. +// If headerLength is not zero it is asserted to be equal in the ec header. +// Otherwise, new headerLength is returned. +func validatePart(parts []*objectSDK.Object, i int, headerLength int) (int, error) { + ec := parts[i].GetECHeader() + if ec == nil { + return headerLength, fmt.Errorf("%w: missing EC header", ErrMalformedSlice) + } + if ec.Index() != uint32(i) { + return headerLength, fmt.Errorf("%w: index=%d, ec.index=%d", ErrMalformedSlice, i, ec.Index()) + } + if ec.Total() != uint32(len(parts)) { + return headerLength, fmt.Errorf("%w: len(parts)=%d, total=%d", ErrMalformedSlice, len(parts), ec.Total()) + } + if headerLength == 0 { + return int(ec.HeaderLength()), nil + } + if ec.HeaderLength() != uint32(headerLength) { + return headerLength, fmt.Errorf("%w: header length mismatch %d != %d", ErrMalformedSlice, headerLength, ec.HeaderLength()) + } + return headerLength, nil +} diff --git a/object/object.go b/object/object.go index fde26ae..af16128 100644 --- a/object/object.go +++ b/object/object.go @@ -366,6 +366,14 @@ func (o *Object) Children() []oid.ID { return res } +func (o *Object) GetECHeader() *ECHeader { + v2 := (*object.Object)(o).GetHeader().GetEC() + + var ec ECHeader + _ = ec.ReadFromV2(v2) // Errors is checked on unmarshal. + return &ec +} + // SetChildren sets list of the identifiers of the child objects. func (o *Object) SetChildren(v ...oid.ID) { var ( -- 2.45.2