From 31ba724610efe2bd47dad37608800a68ee2dcf8a Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Thu, 22 Feb 2024 22:29:25 +0300 Subject: [PATCH] [#205] netmap: Add EC statement to placement policy Signed-off-by: Evgenii Stratonikov --- netmap/netmap.go | 26 +- netmap/parser/Query.g4 | 6 +- netmap/parser/Query.interp | Bin 5170 -> 5562 bytes netmap/parser/Query.tokens | Bin 372 -> 400 bytes netmap/parser/QueryLexer.g4 | 2 + netmap/parser/QueryLexer.interp | Bin 7459 -> 7764 bytes netmap/parser/QueryLexer.tokens | Bin 372 -> 400 bytes netmap/parser/query_base_visitor.go | 6 +- netmap/parser/query_lexer.go | 242 +++++----- netmap/parser/query_parser.go | 711 ++++++++++++++++++++-------- netmap/parser/query_visitor.go | 5 +- netmap/policy.go | 72 ++- netmap/policy_decode_test.go | 2 + 13 files changed, 728 insertions(+), 344 deletions(-) diff --git a/netmap/netmap.go b/netmap/netmap.go index 530b06d..f0ece7d 100644 --- a/netmap/netmap.go +++ b/netmap/netmap.go @@ -209,6 +209,25 @@ func (m NetMap) SelectFilterNodes(expr *SelectFilterExpr) ([][]NodeInfo, error) return ret, nil } +func countNodes(r netmap.Replica) uint32 { + if r.GetCount() != 0 { + return r.GetCount() + } + return r.GetECDataCount() + r.GetECParityCount() +} + +func (p PlacementPolicy) isUnique() bool { + if p.unique { + return true + } + for _, r := range p.replicas { + if r.GetECDataCount() != 0 || r.GetECParityCount() != 0 { + return true + } + } + return false +} + // ContainerNodes returns two-dimensional list of nodes as a result of applying // given PlacementPolicy to the NetMap. Each line of the list corresponds to a // replica descriptor. Line order corresponds to order of ReplicaDescriptor list @@ -230,6 +249,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e return nil, err } + unique := p.isUnique() result := make([][]NodeInfo, len(p.replicas)) // Note that the cached selectors are not used when the policy contains the UNIQUE flag. @@ -240,7 +260,7 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e sName := p.replicas[i].GetSelector() if sName == "" && !(len(p.replicas) == 1 && len(p.selectors) == 1) { var s netmap.Selector - s.SetCount(p.replicas[i].GetCount()) + s.SetCount(countNodes(p.replicas[i])) s.SetFilter(mainFilterName) nodes, err := c.getSelection(s) @@ -250,14 +270,14 @@ func (m NetMap) ContainerNodes(p PlacementPolicy, pivot []byte) ([][]NodeInfo, e result[i] = append(result[i], flattenNodes(nodes)...) - if p.unique { + if unique { c.addUsedNodes(result[i]...) } continue } - if p.unique { + if unique { if c.processedSelectors[sName] == nil { return nil, fmt.Errorf("selector not found: '%s'", sName) } diff --git a/netmap/parser/Query.g4 b/netmap/parser/Query.g4 index 72fa880..0a5b314 100644 --- a/netmap/parser/Query.g4 +++ b/netmap/parser/Query.g4 @@ -4,10 +4,14 @@ options { tokenVocab = QueryLexer; } -policy: UNIQUE? repStmt+ cbfStmt? selectStmt* filterStmt* EOF; +policy: UNIQUE? (repStmt | ecStmt)+ cbfStmt? selectStmt* filterStmt* EOF; selectFilterExpr: cbfStmt? selectStmt? filterStmt* EOF; +ecStmt: + EC Data = NUMBER1 DOT Parity = NUMBER1 // erasure code configuration + (IN Selector = ident)?; // optional selector name + repStmt: REP Count = NUMBER1 // number of object replicas (IN Selector = ident)?; // optional selector name diff --git a/netmap/parser/Query.interp b/netmap/parser/Query.interp index a8fb219e6ebe10b7437f9f3f946010b562b52f2d..3f7a9ad50456f472f2f5103076fb7f55c6cfe1e3 100644 GIT binary patch literal 5562 zcmbW4TW=dT5QX3QEBxlRa0^j)<*ACS6hdvumE;C!Q3Q^&0gT8_V9RLz@B5t@?rKT3 zK~Z2$at`NmX2@NQ_n$vi4=203ef4;=J9)Uduby6ohv(fcoGw?}({MUno-?l2rwP4Y zE`GV51Fh#*jPn`8V#zSwFw9=RVcg6w<}(?;S+6cNEiSfmJe+>V-w*t4rk8SYzSwLR zOVj)Y@mv0WOmUn-r!9rOaeaEZfB$)Rcbk!g>$}xefUKs~TH|JMd37<@$~gxZhxsfl zmSMUHFb&Q#h-Gk&;qBt$d^TO5hx64o%r2(aoB2Djt=Bc>aPjVHx}Gn?I?dBgCVRj~~L_ zhw5SPW4m4@c!)ub{1T;x<3Ysv*7fKr0B_u0H zeFW4V8)+|X;zu~V+CKWYYWpArT75E1C=!oI3dfp9fQ_y_VDq1}2W+>s6A?2eEaDJP zShDgbq0Xs6J?)`$66*2A32Eq@8trD`+g}+8!|n)sIJlk@C`Lq{F*MNgJ&H`6_UI&w z5oM5xpHs7!P|hBI4T`^F?Rv4p+$d566n)g?@hkMdM%>Cn#5d1>bOuFfcKtB49rG{ zCO=rceF7v62ruKb<0qvTQVNoWETnMQNw-uMR11lS2wpj3D+*~_r_hxSSQktM?t=Be zc$^mtxr$!hTAMu{B(~hSN>yhw!l4{A9uQ>7ZS#t{%t%xzW7t4)htP)LG!}=JmU0;v z&aAE-CD&Gzd|>ZkEF-0vX{wZckT$6L#J(5UZ5dUtI4I0j!KlwtW-i);f|6<+2O3&D zjm@wy?;-1pOMD)Wuc`{qjD;0ZB=?$AOSX8?R~mJQF#`9ratsR`3&XBgByCu63T9WF zg1+`gal!idZF82a*DS*j^y|O_Bj_ZZ@>FCez04N5U*y}zgv5N)6_<0jFNt6 zG7M+s$tys=I629Kl8h2A$mqi8en^#YSP39vd6T51-?26s1(5C$RwSakHYiR>zDk19 z89>lW0uk|W!dVDd>IakPtne+$l;W-Lrvrh%%Q zArFh94&kIR?lGV=SY-B3!ER3)+G;&dF{UVG#WnHC22Vh(l zM?jICemW+YC`=Puzyz;=hssXao)o!EV5|lUSid=53@K@&aVKxWP|PsoNs*r93Di+< z9hJB!d}b}IsD$JRIc9pwIUOOSygCA62a!1TdqY@uV#fx|MWa@Z9fejSn6okmZc?k7 zxl)&Qn}q}Nlc}mA9b&=TiJ1d{)C;vh(v2PTm>Iu!KZeAPpL65dfY?b%v*~UOiMk>p zM`({~NYvpg?jdVyV$>Nmenrk&3$c^(!L{*~&%(75)Eis4mU!%1DGHSjCa2;pOGkTk zBE|#qPUaXr(p3^&8DzTnkJi=T51x#2Y)IFu(cG6P!20e(e#*i6pRO=WnQ9+G)CQr` zLRAd4mfB?vAw#SfB2(KNjmCY3Dz)yBu2QpqxHDZtm3oE>knaq%Qj`U<7o&%w)DLM0 ziPn!Xdm18aotRoOv~iazVV$ovVA%^Dtw%cwXn{zPS;tWOP||92aNhbTMC+|m!&aZ8 zF;R+YzHkG-+KX~jt0gabMZE$BnfI(B)@PNCpZCONOXtS&UdIv%Ysyl`)h2EImT={G zU}x=B*~odjU3O?u)^;DWcq*p)qM}IFqt&=wH?xY?<5|y@@vsi8?AN@*FT1^jmWj)! zy%BA8_TPXyulA8g0Y}%?{zp)whU(${6L9nfTYIO5OMY}om{ogg->f+UZ%)Pk(I{KO zgtmE^3%nk^XV%xHrmcOZ*0KqQq}~V|$cMFjWiOaeST)2<7i)X5YHL8$A2F7}>f-WHl{nAM7_)WzY|se>SbURyVuQcFXLDt!*84|M(l=8}GFM literal 5170 zcmbW5TTfd@6oudCSMWw#QOV=G^Hc|%$|@LYLy=ll2*`_xVXbP<{{8wKo|~_$&EqEA-+i*#2@b7q zUtgN=X-D>@Z8j*o{9E&uR)5^CUmKm$<6ZL{J~wawe0lyD9zHft8y~NaLkPE#?mh zYCW;FMy;beWdnP0gfJtO&3ue@ooM`2!wm}tJLLJ5=xZ=#U`@(W< zsR`-JbV4QRN!opBD+vG_6I;-?*aNxD*(nfHmIH|cF!@uIQqZch%A25vvgvU_qqqZ{We^d0 z<+dy_Mq4+IDv}XVXNqJn*~aEH%cxU%u&`t0!*cFPt1yX+9$997$TRjTb;=a`q7_Uu z`2rOn(e(WmdXO5aY#dbTM8h{xsrz{$i&IBw82+;$(W?XRDH5)xCIuz}_Ot>obm3Gr zL*BCoRwaU5*QTfBxa)F*hWl0hzze^^;N=){zd8WQOjY$@+3fX|PF;RDeCG>$qQl+8 zlyiYe9@MeO)?O^iw@)romuX1^OjuQ6An{7(PH+IX#_EfJJivolmy#ZA)d9kxB~a{6 zLQk@$LY3S&r|dj<3kOvV46D*WZ=h5PdrE=#TlTi+0cTLBCJbJ#i#+rYJ!E7N_>I)F ztLaervQa3LL3O|wgIn;xs^pM|RdU$c3r3uw5)s`4Bsj?g*Fwas+(w$Q+QP^q@r3cs z@Mk2g5ogz47D$r$As3g=EZ_8`3Ojd$IhX||S=I^&>$0RtaV5u+N1R>}jex2Md{#%C zo=mzXu?QFoDU`?B7+0x`6$YJ!9<;?zBsTI8QG{4vlE+nc$YX`Bt*kc0g8M=njPkV^ z<5}{!N)UPMC(4hlSP%tFB;iI&?j5Moo@mB$%S5L8DFZ?m_SPUQ2`~#xa!r$^iTo)n zDfxq(I)NRgC??9spP$_-z(ktb+qyxXqy)8{ERvu+%Nak4jswJw2&ixU8)7Zk)}$!0 zBPn*I3wdkTkl2yg!m=T;qckehBt~~vnuY|54Vm_0M}D{`Y_&;P7y><(eRPkDBl~49 zxH2C*mb$*b?T#bq(ke?YTLiGqa+e-n>~Orhu&7(M6eI7iI2v0NT^gmG4Le_}1;MmT z^JQDqx7uTR&&)Aa5xhs7fW;PJ5yQyrDU^iVC(RGv-2pRJBN=*h^+X74RGUwSB3io$v-(`g~IOaOITW>)vw zR%_H!ca_)K8nYHBCJ#MY(*<6|Vb$2n(9pdU4W+Dk%-FQ6P=KOD`mNlC0rvkF1(jPx zdnM|wJ^);d(iH%W6E&t(&X3mYfwQk5Q;n?^>+cyMq`EyctK6*89|_ZNRcC7bmo;MA z*Ix2x?Hm1GVZ_k7V>7E<&Ya3EwOg;_pqHiwC|W-znHGgczU5N-&6j{|Cx$s!jj^ diff --git a/netmap/parser/Query.tokens b/netmap/parser/Query.tokens index 6376ea2554427a870ced03dce5ce8ad0847fdf62..b873682b36d215fd3270670b8b92f86b7f882279 100644 GIT binary patch literal 400 zcmXw!%WA_g5JmU*N2cwfSc>yn3{N%SPVeIwYl~^uoH1=nD3;V+P~Lb&XzDw}(GtNSt@F)_ zQ}6L-C9CJpTZ!pceUE2Ve}tsmgo(mNVWH$A43t-uFiY4a3=lHT+t-vPUWh&3f~?Sye5<i>KFbAO85fdHVD5vANvsPnXT*a<^8w zKU_xiZnydRt`l_VZdGgPZriOG{PM8BL2S0CP8Kdd(es&}&@}B*_q}?YJRNei#U%{80D?z*|`4k*V`Z8J-+#=ASadUIK?Wj5D zMvR+o)ogan^4N%J<2)N!HqNp6X0u(dmWOq--k+M)c6oQ~?qQqiiE^{Ozg-@>U2}+X zd1^N6Zg*;ScQ-G(LukJ14*TYKI&5}dHs2hZ=J!7z9}?&P5#9BhU*0@7yLV4Nd*was zyW@%wuJ-G$dDVT}9G5RU8n^q+?sSxY+pfgc`@0vy@cO4;Uq3xP^gCb=&6EX06aQ&D(cB_aRD}H_Y0+et!DA`TqLqVsv%UPV}_x)y4Gc;?tnI zp)>Ehx)7qeOx)SyWwiRaeD-s?V*iQuT$^RjM9DM@k4< zacZAC2%%4cKA#>z%LyQ8IRykQP0u)I5u5{pJ|}{pA?b3{hSMe zmUBVSaxNBQwVVrrKIej<l93;BKzMA} zxD4>~!aF1FN9x=AR`q=lS5FG=jf;G_UY)BO4xA(~( zjkmf4a}2_R!=VB603a|-Yo;CyQ^zn8!`3Gy3{#izuqEMv;*8NJp`W$p(IP!qL}C#+ zSAj+99}wy`m4%cbhk9*l5xpKviKdTIpwlQpzFe6=PhdNWM5BT)4;KT}J|tuul_=uU zx!|@?UlOW{CYMIb@@s@$AZX|zEZ*m)40A|k^czb7J=J8vLad4mx$n5HKKMJ#?!}Ze zput zv}AqaS`*irNK48SDM_%a5mJd;LM@MlD%Yx^?*^taun@XH^1v$YYm{XXWj8z~>O5-l zK#pf2g7(Ptf{EAzUJ$9^g$4$@9qg!6IE6$ADl@1+0Sn$Ogb=1V(_Cd|Bvy8cq0KB+ z7KUZ#X;~R)*-w@Jhho+gE=m;wxl)H|N+jO0y=LPo|jz9_HOB;ZQ}|$j}5C<1&4iJL+1n8c$F4gzk=A}IixzfD2Jpmt!znS zRt*$^d4EFOa}DBP+nt%;wy-006pD}-vEg$~yc|`o+HGRHG}H8}r5f({Of`|6 z?|2FK3uen)FPU9O^`hBQ+eMVkZX>Tb!8wRW^$OTXZlXeyXmc;+pe;HVNdYplUN7#w zFs(Id*UM;IWah7#-gl$8s6i&Oiz)L6itVT(55;y=?-#Y(`b=}lw)LsDD>r5`JtaJx zijl-jd(3eFyabf_2nv`=Mc`bkA3*`tVh{;#N(}39J%E`=VX>bEfLsL40TfVeF3!VD z7=wE{05c)_k$uEzFr{#&HnXZdb6;Q<^#qVYGq_iV81@Bb90f$pVfb8D7>8y*9N=72 z)aIgJY6?iR_mL!~q)KRtvhxHu*HrgBuFjx2c$D$PBdPQp;SHvz#_;J36Z>>xGKmPPM)t?>es)U-g#hY_nyPWcB!)oPnNxp1 zRCVr3jvb>lWgIPF;d>w(Eg+1ffU4`MG=y|6HyZ4+B|j+Uk2!~X#mRCj!A^J;P`{c- z8m}K)Kk2F&M()`VEko)_QoZ#mI+?G z(Q%OYIW)Lpukf)fpgWKQiJwQAF8+<(zibq*lI3D;@}$^r@!cz>fkgdZe+ zNDUKO*1!i0x|$_g1_{^hUlwPO!24dmjGkoxxCShhWJE~dYeR5u1Rw1aLU1-EKHArd zm`EiV!N-XjrEL(RU95_w5))rRf)GuB5baVI4=3&+kx9BjEZPYX?zz##{hJQ{fmo~y zUA&tzqdLWzLt{2@j}!!44=2iS{7ZPIKkWzQ)*WR?fvOcCR-&3Y*nmRUP@FGNSkY5} zSj3eQ70JLY*RK&w0c>{^1}qkG9GN--+l_ zq?!~1=}%!9qrnCgx`x&vBY$FNmIknBRtf`n5OXMoB*x;;-YhL(lJg=?5DwCu+HYJ-xa#Sv-#+T$R6zjYcTmX5Y0VTKY; z4dAV5kj4jw9C zyc7Zbk}AGgmX;uOm|9zBj|Q}LS{l)MnMlssI!iR5!O4!tr0ij*Yo( Z4%s#4H->`lN5YQ*ez4;b?HG+8{svh1Zi@f_ literal 7459 zcmd6rTW=dT5QX3OSNP3s;R2#=mhx1^S~o&v$(3ZJXi*qxVzfpb2S}WU{`;P9hTJ8o zG)U2x0$N@UXNG5nTyj=V@BX}dJbAc(x_kfX;pFkv+q)0Xn#Ye156$UjdpK=Q7n`NZ z?d~+9H=EUuH?5#ud#!S{L1wSw;`yb@eY61y^}40yR2va(R-0zA zZ$z*mF3 zshBm*+jnoe*d)m-W^Gw*m3l+xLPgKHP|?$a1)BOf7mK5M&V`CG=R!r#xlqw_ zE>!fK3l%-*LZvCi;ghAnPyS>;cwUO4kpEapUB8)p5lOctFJ<>jN@+>*xKu2J=cVEy zBri3ckmJ&$^Ccx)N+v#AF|Oa2R~}$?Bsw0D&sH8MwkZn#y7J($uTl4qrOLy_zGlor zl~w2fPDGZPhmcUi-^L_FENScKVT~29fxuKTuAUz0P=zTwBeo@Lm<;=vsLGnPRe8Kp zzmXo{`oY;k ztcE6(QK>*xWAJd8Eym1}z#wo ziIszrwwWc$LX0q}XGlEtlTnX|cbVr15v587a;DyCF-?iYL$Z@lBOZJObZk5xP=(a7 z`FK*~QvEu2Xne)Gz8dlEDWGHXwn0mWY_C?(F?-v=r4-w@3JyEDhK_^E=FO$lp6baV znP$u6kh;@KQKo0?yo{G5W`3%4kzmC*sB$ru%=TiYYPZWkYK5R}*jgJNgUT^KBPu$! zcJU(V2-)hzv#$p+7Z1H2WTJY3Yz@b}g&BDh8dtO1@iVT%VJkWAjtrlRw=Ya8Nsg&7 zsU&UHr_#!2w{@RZ0paurvl?Wg8AV2(lP=6F@|=6adO0`+hjs$x>Y8IXsMR&cuwEm! z*;J8}m2>&!z5{TF14=1^0;V!4kS!KLwW_$YOQ(t(m7XXkfM^KFrV%jJq)7I2t5p$a zlXFy}+|dCz7kPo)=spcVwPfcSBzs_zUReNlJ<3vv%9{4v!JZ>Pl+4sM$9eRoR~K+B zDR1B`D{zYw2}rYdk(_A^E=bC}O9C8A%A;9aonEr{sOQN#;eR(XbAahN!m7BLoY>cL z@I1omIS5)MCiCq47|!S(A(J3JtBRmnybgeJDuM#IxKZzBx0_tO!sc867o)&F5CC>L zCVMi)0mvJMwuXHk01|^zTfMpXBk5wO6R*l|&QCG|3HHl} ze{}+gWeCjQMZ^=1YSF?#kI31}vQfs~ZOH8Shhqu_~&XHv?rYkZ(#S zXMxB=20<`^d)5|S?j@#_Pf1!x{PyVSd8#Er{W4eV z1|#{xM7PSpb280@bjAZ8@fssKfLY~66r6cz zcO;(>g0nE$M+H|U(!Dbv@q43>hh3&*KpNy`jI2TGd)Dt*z8SEaamGWw76fP8(Y_zJ zaRp~VvOAcB6X`yrmV&ZJ5(U}w`bsaNthIb~YtGj-!wLih~3$`_eW@{CslCCMKT)Ie=u*^PeF&UpQ{*2&nu9CO3!<5o|CMb)p zQlL0T_E9i_L@+ajtf4G#feK)yN)U@?r4YqqKFzu{GGgl4QWc^W3puL6&T=5bt~117 zNgJ`1sJxL$tXk$x`ok{^8d;GKCQ*~dHO|y-E<^Fv6be~G>yYto)p@F0gj#79pJK{H z!kQSXStqrDmoRYC%RHMa%5pf`-znnFN^^1;q=^zvph>(9$7@qcCVCW=0woq4ZzYNs zbsA#K0@+n53&5#TqL{3c78i@@L+&8!AM+!YPjrnj;o#J){I+~1;+9Kma^y)f0(V2f z#LzkgGP8j^t&}AirjSUOT|)^;fdU+TrkP1xDP@U9^1FA?zAJ3e3&X^c1Ftl}hB*Rp zHl|KgF>qvA_P3TSiAq9c&rm_U?p#79$aXQ6B-*2>6ezk37a);{DOlhif;1uFCFw#* z54V_^CK@DOHIC!tUNw%}q+c~Ih(APgGWM}GB>$2z@p38#OsW;MPp~2Bm+X5iByB+b z5osT0L$Wc6x)&xYp#HQRlAPJUd?KJyn3{N%SPVeIwYl~^uoH1=nD3;V+P~Lb&XzDw}(GtNSt@F)_ zQ}6L-C9CJpTZ!pceUE2Ve}tsmgo(mNVWH$A43t-uFiY4a3=lHT+t-vPUWh&3f~?Sye5<