From bb6bef9072f9c7f0f841cdd8938c71edc888d8b1 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 7 Mar 2024 10:40:37 +0300 Subject: [PATCH 1/3] [#9999] services: Generate BillingService proto Signed-off-by: Dmitrii Stepanov --- pkg/services/billing/service.pb.go | Bin 0 -> 35700 bytes pkg/services/billing/service.proto | 87 +++++++++++++++++++++ pkg/services/billing/service_frostfs.pb.go | Bin 0 -> 11631 bytes pkg/services/billing/service_grpc.pb.go | Bin 0 -> 4050 bytes 4 files changed, 87 insertions(+) create mode 100644 pkg/services/billing/service.pb.go create mode 100644 pkg/services/billing/service.proto create mode 100644 pkg/services/billing/service_frostfs.pb.go create mode 100644 pkg/services/billing/service_grpc.pb.go diff --git a/pkg/services/billing/service.pb.go b/pkg/services/billing/service.pb.go new file mode 100644 index 0000000000000000000000000000000000000000..e13e7fb36c36a1d63fe6c6497e88a1d6909b6840 GIT binary patch literal 35700 zcmeHQTW{OQ6@Hd}#nca5YvCxm_>!ndflcfzjAj#Kle9&Hz@;V1VIz?$NjY`{|M#97 z$-}Ei$+ClN7Q0@1ICJiEzB9ui`S`JU;Z1CFX}h*>h4#c;+?a*$h2HoPv^~1?y5{(e zdHv>{IXOOk*KI$3Y_4oSa6C5{;UBG3XLHr>4u{>Ix?<#?tNrfafUgAJ(jVI+v-o`Z zII#UIXKV+LFPz!TapQ(A(QUUE*7&n^X&brRZaedZ=Z9vu&76IEI2*}3$*%bDH1 z^k$ZO+4cO($3)>`IejdBxIzBdk?}xoRy}y>hR)pP!Ppx{#!M4p!HqjMJNR>_-D$VS zU>_0^O!Rl#ccwQc{A7jj!wHnJ$7p+K36|5TGj?n@oZXn^;!)^5nt(UD?bf+@#{NLo z-4l1}f$4xEdSG|ZGmq5O-yB!2blR;9V?V3L25iZ&B+qZwwQvMM2E7id9^Nc$^U4Xr z7oHnh4#ZXPyB#dRAolrlZ*p@Ux1WMG=WjzRTn5H*!{H!pxA*SEp2GNv38>GWfxoTg zEHqcvY>CFH&*lB=A1Y8p-e);<^r>}l_{_12M zDf)RSCicvRIon8v@7A-!0No35b>c4P>;p5m7J-47Y^+eXeP#JLi|hN4Bjwz+ z<~B?U;QWDt=3@~0kT|}!Tdm&6?8Khl*@Koow8U1YmI1Vg4e;;VH6)<$6A_sCN0J0Z zQ2a-xhaxEcBh#mri2n$3R@7)Fc^=7CCGymn+2??7=Mlc03xGup=afXxZBXYO zB+TGL?_)}tP2@#_U`OJht%2=I#-G*^GLdVYNELq+$i`QcbUEo`U!n+*g^uloZ;!EN z8JYJ%bI4KV2J{I++k!;OldyS+U|1%B0LI__@bTg%wD-&h%nco-SY508))%Ds>%UIV zFt&-#*TEhn75ev9%<$H^bOAwq+jvtGc#s?Cb`U_x)MX@2uEV@qLZt-r3h~sA3Nu|m z@JLLwHuQ4-XTPwhB&e(jmp}+|M z(-AZRkR4Jt;Md=vnJs4iO`M=^8||P4gT4Nqw+Qzz?LULjdvtlYWBwC(?q~;FytxzF z^F?^G^KWAH+;J`chNzQefPP|yGA@IJ3yMVpGrW>^%~2)2pJO&112vBwLLh*-f4+u1 z8DIJekmF2K#tN*ded1y{F~R<=xr14`_WYa&mr)ME!2u3V-1X1yZ_2HZHIqwc9>k{P zM;zFF3yIX0?pU4e&K=0}6lxJXX&vjO?3g{mG-hp;j5W10O!Lwi4Oe1V>h?RA{n;!h zqtS?)*s#JqKmBB`&7S}=of*4oU26jmF!(FanrM4>%(H}cxks>UiO_K#P!%IsId>v= z$CfV0m2oh3wKXd;qW05$8PIkSxIQGLga6V9JzF#1z+E=%{T2KGX1}bpoof zP3qB(yBBZ>i=OnVd*XIvuK1n;1C8n)sJo zmcNtyn+0ytE1!!CyKONq_dni4UM^IsW9}eFmr$do0jv{~dmN|7feAghRTY|XbRP$H zmSB%|9CXcVFEqaW-=*Wj-3J4Aft)$q5*T}MlL_vQ&?Zd32mct{mzA6Ei(`CVRNm^DJ&Ton+P^@fGZwT8n^6ARJ2(iRZ#iI2)vAA7^zr*0-4O zP7mrqok%Ndv4CAH+^I2;Su|t1pSG~a&eYL1;d=!_r$={cV7JOF>4t&EKDZqMZF>s* zW)AessLE|w9benmp;_Q<($M>CyS%{$-b48ucWIj$+>XPIwv6LWr^a2*W(Kz6X#Wgu zMS#VmGv0OcL%6%+dvh|`g?;I`vR{WB&w$}O;#cO+VX~*@!hmxW@tj0tmttc~C8KF} zfYw>>3>8*$lo3{geKPO12>8Q?0Q^L=?X3XC$W4G>i=}s-Ah#}X_b<9zaR>W&X!buG z_$T&Vy8uW?+|#NMZf8{20a_xc1ztpObUJ~siNT}_{?(9zWm@&ZP@)_dzruVBEG2hW zF{G}VRWG&iSe3JYQ7XBTj(8)aav7Q$q8ZDcXT?&KtEUI7JK#uXWG<|5{HbdIDS1~J?$E@LF4#$+$@qyl z>Bay$_TbEp9MDKPDB8QO+5|=JcLtcK>hE-y9^_|JV7;;cfOo7}*Sy1HNl?DcCgu~A zIJP^%@`mmv2A?b+?(JD{gVAwG)ro!STN5BO_u%vr)DE#_EZ5k+@A**XoH1#0;dwLd zEh%$UWgtRi{1vrohgHG83YFEkuR3oL`1c57aDE`y64LF1+G!$D}MF6Q_@s4vv;nGeUb zZr~trR9F2HbIso`DmS(or3}uMq$*7kX*1P{x{x;rZi2yCEZk-CF8%|D)A4k3q67sj z_Nkw~AXIQbPSpKme}(qmGE#C|RzN(ywL(7X(m*`wpOfw&kuCt98R4i&%T}|xOHZ3d zwO)GY7AwDl2d@0mif+3O%yL_^Dl88bud5vbMDI>UMBrv35@uIPh#d+9Y_?E<`G%Ft11=ZN zKH;H9+_OF0*A?`|h%LHAX}$6;;S75zF@5t6duB?W8MUPd!h;Kbg>9Cc=1xkh zNC6_Yy#fop_M^bU=U?eLSMuQ9msycB@8!G>KY6r%y%h-y^$w(Ez$?k5RWJk{H40VV zNGq*96lcoe^>|a)yXB4vzaDe!nd99MeF@|r5PVnQr2=2Z#swY%h@b7^w#@+!t*{Bc z;rK1W+tIfLZ)~l5MK@-7H{hmdZ4cYfWk0Yj#x}J17NCt?d9TQ};u(c7yJ*61t%i)ijcSe)fx`FXBbhFX0YCpD{hG@7lw>zbNtAQ0wjkVGGuJRI>h$~0Xgn8rmp;xTi{J3aCI#<0~!nBB7-B6=#VAdC!+LwpW1uG z0AxXCbwr__L$U$bhD=1m4p>bl!9-buK9W8n#z)FThx9uO(I>;Qse4R^jeNiw5Yhe< zB-A5v10KM?b32*tfOSffK=w?5uJ);?Fnh`j){~tX6VVER5)&P*#fD?gW{Z=?EPI~} zxIZQh*mcZ+ZN1OTIMg^E*!e_4T+JR*USSGsM)uGVz5oj>NS&mKNHZID;E)_HHf1(s z7LK|@c7C}Rk->niNoo$+ro@J0xj}%EPGFWepg||nfTXxXvtV3BRv`w;EG+eYi4i1B zs|`hChtbSH20gOidQl{^hYg#bJr`smwmZ9YJ-b(Afy4*-G$01qL_f|&oG*1Cmia)m zR1>PlF;hNoP=m6GbHEdgG~jqI$X6WD`FzF6j>A-@Ozj!FuUV3N`xIAV0}jw=HjoAr zH8rT~irAq3;-NG-oMLHe$UrPh5GYcpEF>v}jKC>`O()>W*@^Lkn8{V1D^pH(5jY@! zGL&e*_RU8(sV^F^ygKn|Q5Z50^HN0ok!6lOr|2i7B?Bt!-6tDtKNoT0A-y4d$jKZ( z9Lr=!Nz@#CRMhlXr;;315WZv;$kWK6rxP_1S}$Q_jDtCKpBUU(qD~u!Lk2aFJW9s- zk**=E(?uJZh#b4cbp3?GQKt%GS)H`^Iehsijg=age93+Vp=ln_)R$6)nQ;_zlHi2F z?JA(9{@^6R^MkW2=h0L)%@~$r9u`jHF1KRTm%$omRzD zDF8T3)|+Tj`KS~kUesBcu0l|tiF!PtIJIeI$@-kshGala0F4$^$e<9c(FkjqE|b}i zf#l1(NNqU(RXTzBv^t8)f|bq1Ya5-k`FFubcCySp=De0!D%d0@;!r!FnJKfBW^XD& zI8HQv(eENJUDb#ZMG;9|u%V#=k{JJpA`G%LWG5*hm+}O4X<|y`6nTi8%#}>eMG{?` zxf&U8{OpsA5cjF1_>fo?0fG#MghXpZ2pW)*bH$bxGGM#w*;uwT&*T|k zurkdc192BmAD++bE|o#oOMH-K5)BSj2_-C5+Q|DrLSU7*kbyXv=B+rHHDCs$#}UU8 zkrVgQ?-lL^1Hyh^VxmScQpdf}2fT|6w&Y%50H#I;WebhGC+f#rj3US&?_Rw{rmZiI zt@aRki=z%aiVS+>E**&ZnBhRk@3KqT$#pG-gMO-<&CgB3L>a2gGenBk=$MN28; za+b(IAduZfKT~nVF6D4kI;{w6nsgu{GT_N2DIrIQ_ZWv1MZ%bnqoQ0y-$b*480d15 zIg>TH4A>`|?sE%~=0qk}xt;CKD-8O{siikz4af+5n_{0_cff?`cOge!;j5f|8Sa^} zDHL{i@oV@ch_OxDtj z-rSRCI9+Ag4@i%g8EUYyw!!(SO0J6CA?;rHz0Zio$o1`EGMPf4i492Rl37K8)91bG zZ&d@9w{mR?u|!TLWe-THVGf~KWzmx2N476HJ1Mhp@xXIf`j?iDG?-_AgeF;;F;2<@ z4r^w^S9!*8I|&VegBr*Yh5FJ_YLU6Z8WU+|&(x$;klhxVP;x;{oa$*MreFp%tEK2q zPg~SL&s&}iJ=TUCn6fxaN<<;QM(0o9OIHm~j=XWla7qzADA&c`8V^6VLJJO`G-Qfj z_@alC8jj38TzQ(daj=V6M7s&9|%nWxR@ zUV88F%T;v3iM}H+`;FIR{cmRuGISZy# z#6!JUaizI{$9_2vqC~DK;-gN@uT%*59F+56N_7~ek2n^v1xAOp7^Q=4g??$a^3<6& zF0egXgDnU_&x_|{Q5iSGoE}Ngym5j(S%*1NYNoh4QYsPv@2BhVPAEY(#gm+M)852h zoKCLe^fVkmVghVlk!3iNFCfW`@L4Jj)4dx+-xn)pp;8Z& zRY%kQErwnNrfSP}L!ReA zyM;R&K>q_U==VS3yIk>gW5M#`(F|VoPVZ5Jm-T=0CPoo%wL6HgU>qG>XPgY=9V<#~*s55?1hXeRxN0!X|qB%5mw{%wKDxgABNBw%Uo< z(9<+8Yd%t-z81(!33@^S9>@7pQry3Hv)_QM3v*F*j~|ddxyEm!!y83l-G>XQEBvgs z?Q>neXB_;qquXeq)e62i@RDP5h2M?5>gp=`FE9qzVy&?SUs&yp@YiTP^wU)r6DD{~ zF86bnVXo2#MB^@lY?ssrM6sKE;ycw3A0&ZGx9|i1YiybRomnOeNxTsFKCCp@nw2`iky)$e48NMHNW`&rh{WsV95>yA5$_`7(SueC8fDX*+VKt=HJ(I^AZW&;F4QPXN$N?%Z}G~nym2mM ziFn9F1L%_;ov+6E?dsmRtCQi<*=kcg~jT`KY23Z$cMBt)d@(ur0u uK<2hfKHr7AE-!1YuAbovm!8yae2Wa0Nwl=6D`;}VqtUQrBNB(*w*L>XF{Ysa literal 0 HcmV?d00001 diff --git a/pkg/services/billing/service.proto b/pkg/services/billing/service.proto new file mode 100644 index 000000000..6c06cc0f6 --- /dev/null +++ b/pkg/services/billing/service.proto @@ -0,0 +1,87 @@ +syntax = "proto3"; + +package billing; + +option go_package = "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/billing"; + +// `BillingService` provides an interface to get usage statistics. +service BillingService { + // Returns list that contains container usage statistic. + rpc ListContainers(ListContainersRequest) returns (ListContainersResponse); +} + +// Signature of some message. +message Signature { + // Public key used for signing. + bytes key = 1; + + // Binary signature. + bytes signature = 2; +} + +message ListContainersRequest { + message Body { + // Container IDs to get container metrics. Not required. Maximum count is + // 10000 items. + repeated bytes container_id = 1; + // Max items count in the response. Not required. Default value is 1000. Not + // applicable if container_id is specified. Maximum value is 10000. + uint32 limit = 2; + // Next page token used to continue listing. Not required. If null or empty, + // listing starts from the beginning. + bytes next_page_token = 3; + } + + Body body = 1; + Signature signature = 2; +} + +message ListContainersResponse { + message Body { + message ContainerInfo { + message Count { + uint64 phy = 1; + uint64 logic = 2; + uint64 user = 3; + } + message Size { uint64 logic = 1; } + message Attributes { + // Container owner's wallet bytes. + bytes owner_wallet = 1; + // Container zone. + string zone = 2; + } + enum Status { + // Undefined status, default value. + UNDEFINED = 0; + // Container is available. + AVAILABLE = 1; + // Container is deleted. + DELETED = 2; + } + + // Container ID. + bytes container_id = 1; + // Container status. + Status container_status = 2; + // Container attributes. May be null if container attributes are + // unavailable for current moment or container already deleted. + Attributes attributes = 3; + // Count of the objects in container. + Count count = 4; + // Size of the objects in container. + Size size = 5; + } + + repeated ContainerInfo result = 1; + // Next page token used to get next batch. If returned value is null or + // empty, then listing completed. + bytes next_page_token = 2; + // Returned result is partial. This could happend if some shards are in + // degraded mode or returned an error. + bool partial = 3; + } + + Body body = 1; + Signature signature = 2; +} \ No newline at end of file diff --git a/pkg/services/billing/service_frostfs.pb.go b/pkg/services/billing/service_frostfs.pb.go new file mode 100644 index 0000000000000000000000000000000000000000..c5ada111fa3219bf5e1ff870715eb5737b6b2e1e GIT binary patch literal 11631 zcmeHN(Qex|6n&1qg6NaICH9tddnn+CH7SN*SrXVO1}q2)+M;YCGU<_2Y`5sY?~)W{ zk&+!ZOH;Ws5Fl~PdqrKIb964LZWo>Mn4mFXL|{c?G+d&Y;EIoq;D1MBev}9V=3Jc}>zu$zB# zgl80zcOQEb-QSw@GvC<`jH?(R3?~2YC@3I|KJH)Aj<3vhgpIWE~Z#@ zAkK!0Yy=@hqB&I)8;7R^C6uHwdcbK0Urq1>S$#F642z{n--(ipjZkZWei+a(!#Y#D zG@n)*wahdqQ*HDqI-q(wy&hsT)gKS+e9=$Q!RP3}jQp`@=h}I@R7C2VYW;kO7M(xH zvb}!Du^zITLgW(bl+ou>mHxy+PH>8*_O}eGli!;yk}Zie)y)gSMNEV%-g$zcq6rZw zC2S1UqZ0m-y84*u`4sEYY9As-=59EQ!)eM#8a^&rKVLb_mWT+!1tM(3Gp2|DjLj!> zG(jA ziZC~)fqF7J9L(0^BVUP@2IcMD5LUaL|K0!;JMa^L1%3cbI=BQU^AXnep0w;p1B=o! z0E8g$0Sh2TB8dWOsOTX7+lQW;AD|D?-r6y5?p9db7zKa~qiWy@Lp*e9xO2_k_XVTxa%j&=fg%9BPh#n$r!_2M@(=o;5n*!#dOp3v3r&BNz_p zVW7dfivzfaLbC?^yC62tr8AcxTqG`@Uq(HQjWA*s`iQ#Gdq>f9M^P&w4nU3`4OEW+ zZEmJD`jLY*BWonhSn z$Mu$~RqBb>AvCCxHlf<{<|bv)`JE`e^sTnnLO7@BMHg;STN~Z&X*(~hbAuL5uWs({ z88oWUQT-QhU(Tnm_xP>)81uY-;m}RDW4gn&1#lbzHfv1LRM(6yEMicaz3V`iXktJGLHPcBw}yGo7a~=5ojewHqD(|6hr zR`W2u$Z@_ou*Q%3&~La6UoSLT6~30vxZ3Xf^oDKQZ7&aHcsTMk(0%(fuTh5piaSf! z-;@$`n1SVcF4C^{*o-&0n!_&PU$Bu)s@rkUdBfJU@e#%&{LLBJKmB+4>0gCYgGuh~ zkkxS9hi>>lx8QZRSK|bHH1DcHd?i^}btl!m{C6yyI#Gb8_vE;>=oI7OxJ+fTUFjr( z?G6)fT8a`Rx7pEqtq%rlx3tahU(DH&_B=&rF5#2cB<2{|pC-QaY2wzl#DUzgxq*)4 zF|jHLVh+bbvC@=sH&>CW4IB&e1T1qYYuXu)c3Y+a?m^|fLsZAaEi(We!wgypR^;3r z8FrX7O7=x>NWiH&qUcQz3C!S4l0~OaWR{xE!{F_SU^&gDjBB$IxAYDwx8a=&dhGtM3z3;(-gUhBcB|A;(ULl>>(%`4{PwFBEIFXnz>{-MrpaZ^#^i95oss zFClXSaJre+NXY&)c30j(ES*zsIHI&W-2hSVKowqD{SJ>~-MxO3^H_KB!4nQ-{{`mp Bgy8@H literal 0 HcmV?d00001 diff --git a/pkg/services/billing/service_grpc.pb.go b/pkg/services/billing/service_grpc.pb.go new file mode 100644 index 0000000000000000000000000000000000000000..be81d72e0ffc95e7039570fffc397e6ab427cd0b GIT binary patch literal 4050 zcmcgvZExE)5dLiaikk;?a^cWjw_$*XVCd498Q9t)P4{IO0$MsdLS<1UDW^t}|Gwu) z$+2vw$$(--li1>syyv;+UNo7|naP=!T(Q-jbDD3cw#J+61kX;E=45H>Owsc<^!m*O zy?8#mP|0LMYqqX1m7B_!lOtPb{k{4@J=tBR-GA#J)zhEix-*TtCa z$zN%YUogI9e}Nrx3?9mU?TdNIbY1u>SRzb6)LoX7Uh1N9KE&vpKQ@I0 z!sYqfvy)dQgygAuLdH^wU|W@dD(C5vPQ%gaT~iC2ZzIVw_g9Sa&q-?tmVb+Jc^Bh! z`A6aW8HmTNu)TcQl;xl7uT1`01NJEmyFjY`#z}u;M_=FQtSq<+C1wBzIM~e1R8~qRL<+kaam1%^?H7qo9x@@J;MZz)V$|=2NUwO3!O22 zP6mK*&c3mgqY@NHv52rGya;#8IN(S&hz4O*WGWf-iA@Ewm?JY3e38FnDk^&1w-c?f z!;-1JBc)VG`!ijZZ)(x)h|+@{u7-MeOl+;OqvR%u;O0GrwBW#RlX*dwWl?Ll^vpWo za>SykG6G-y@J8eG3wuY$NyUXW@JNr2`Sm%=Znn3k89hE`G@?FN{~xEqmuWmrPpRVT zwC~0_#{i6y0U{tanaVOXtJdZNi;v?yO1_+|q6&n+X*B=|;NW5*(WP&2e^H7z9h>+X z0-p?SwhV{8SP(*?-+lz8U?7!Hu_TuzTk-AwT&8DbNvQE9uTTo0a!8Kcm1)YHcr{1E zdS7jqIX`x%_#y=ZKiAqK^V`B&l!f0M%g4^uPL^LcANO|XdCTXT+t z<={*TnqKn(CFnI9DV zK-dwU7h5qRgTyJz&@plX;T&VUAe~EKBM9V}Ii7yksLhfOm>kI-Yxq=CFp;Lmx9e#Q2v83q=aIVBA-3Pw|_?U7^VQ+R6hjn=Y3tJt=^b&Rl7$(gLnX23bDx{Qz9eIJ`;=`mI-aNvwd&xDHDk5{V)FMKz7=eWrZ9qSd{c2#`um@NF@A3^9EdnullIoW&gHg2cVE!&!^((k hyTLE{iEjn)pmQNU2rkGvJkq)L8j$aQJBC9y$-kKNm~#LC literal 0 HcmV?d00001 -- 2.45.2 From 020281a0e9349f6723754b9594d741b3060cd748 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 7 Mar 2024 12:03:58 +0300 Subject: [PATCH 2/3] [#9999] node: Add billing service Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/billing.go | 61 ++++++++++++ cmd/frostfs-node/config.go | 5 + cmd/frostfs-node/config/billing/config.go | 58 ++++++++++++ .../config/billing/config_test.go | 37 ++++++++ cmd/frostfs-node/main.go | 1 + config/example/node.env | 4 + config/example/node.json | 9 ++ config/example/node.yaml | 7 ++ dev/.vscode-example/launch.json | 2 + docs/storage-node-configuration.md | 15 +++ internal/logs/logs.go | 2 + .../billing/server/list_containers.go | 12 +++ pkg/services/billing/server/server.go | 34 +++++++ pkg/services/billing/server/sign.go | 92 +++++++++++++++++++ 14 files changed, 339 insertions(+) create mode 100644 cmd/frostfs-node/billing.go create mode 100644 cmd/frostfs-node/config/billing/config.go create mode 100644 cmd/frostfs-node/config/billing/config_test.go create mode 100644 pkg/services/billing/server/list_containers.go create mode 100644 pkg/services/billing/server/server.go create mode 100644 pkg/services/billing/server/sign.go diff --git a/cmd/frostfs-node/billing.go b/cmd/frostfs-node/billing.go new file mode 100644 index 000000000..e09999257 --- /dev/null +++ b/cmd/frostfs-node/billing.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "net" + + billingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/billing" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/billing" + billingSvc "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/billing/server" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +const serviceNameBilling = "billing" + +func initBillingService(c *cfg) { + endpoint := billingconfig.GRPC(c.appCfg).Endpoint() + if endpoint == billingconfig.GRPCEndpointDefault { + c.log.Info(logs.FrostFSNodeBillingServiceDisabled) + return + } + + pubs := billingconfig.AuthorizedKeys(c.appCfg) + rawPubs := make([][]byte, 0, len(pubs)+1) + rawPubs = append(rawPubs, c.key.PublicKey().Bytes()) + + for i := range pubs { + rawPubs = append(rawPubs, pubs[i].Bytes()) + } + + billingSvc := billingSvc.New( + &c.key.PrivateKey, + rawPubs, + c.cfgObject.cnrSource, + c.cfgObject.cfgLocalStorage.localStorage, + ) + + lis, err := net.Listen("tcp", endpoint) + if err != nil { + c.log.Error(logs.FrostFSNodeCantListenGRPCEndpointBilling, zap.Error(err)) + return + } + + c.cfgBillingService.server = grpc.NewServer() + + c.onShutdown(func() { + stopGRPC("FrostFS Billing API", c.cfgBillingService.server, c.log) + }) + + billing.RegisterBillingServiceServer(c.cfgBillingService.server, billingSvc) + + c.workers = append(c.workers, newWorkerFromFunc(func(ctx context.Context) { + runAndLog(ctx, c, serviceNameBilling, false, func(context.Context, *cfg) { + c.log.Info(logs.FrostFSNodeStartListeningEndpoint, + zap.String("service", serviceNameBilling), + zap.String("endpoint", endpoint)) + fatalOnErr(c.cfgBillingService.server.Serve(lis)) + }) + })) +} diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 2b185cfc8..f8dc0d4ab 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -476,6 +476,7 @@ type cfg struct { cfgNetmap cfgNetmap cfgControlService cfgControlService cfgObject cfgObject + cfgBillingService cfgBillingService } // ReadCurrentNetMap reads network map which has been cached at the @@ -656,6 +657,10 @@ type cfgControlService struct { server *grpc.Server } +type cfgBillingService struct { + server *grpc.Server +} + var persistateSideChainLastBlockKey = []byte("side_chain_last_processed_block") func initCfg(appCfg *config.Config) *cfg { diff --git a/cmd/frostfs-node/config/billing/config.go b/cmd/frostfs-node/config/billing/config.go new file mode 100644 index 000000000..15d4149d0 --- /dev/null +++ b/cmd/frostfs-node/config/billing/config.go @@ -0,0 +1,58 @@ +package billing + +import ( + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" +) + +type GRPCConfig struct { + cfg *config.Config +} + +const ( + subsection = "billing" + grpcSubsection = "grpc" + + // GRPCEndpointDefault is a default endpoint of gRPC Billing service. + GRPCEndpointDefault = "" +) + +// AuthorizedKeys parses and returns an array of "authorized_keys" config +// parameter from "control" section. +// +// Returns an empty list if not set. +func AuthorizedKeys(c *config.Config) keys.PublicKeys { + strKeys := config.StringSliceSafe(c.Sub(subsection), "authorized_keys") + pubs := make(keys.PublicKeys, 0, len(strKeys)) + + for i := range strKeys { + pub, err := keys.NewPublicKeyFromString(strKeys[i]) + if err != nil { + panic(fmt.Errorf("invalid permitted key for Billing service %s: %w", strKeys[i], err)) + } + + pubs = append(pubs, pub) + } + + return pubs +} + +func GRPC(c *config.Config) GRPCConfig { + return GRPCConfig{ + c.Sub(subsection).Sub(grpcSubsection), + } +} + +// Endpoint returns the value of "endpoint" config parameter. +// +// Returns GRPCEndpointDefault if the value is not a non-empty string. +func (g GRPCConfig) Endpoint() string { + v := config.String(g.cfg, "endpoint") + if v != "" { + return v + } + + return GRPCEndpointDefault +} diff --git a/cmd/frostfs-node/config/billing/config_test.go b/cmd/frostfs-node/config/billing/config_test.go new file mode 100644 index 000000000..85251e9e5 --- /dev/null +++ b/cmd/frostfs-node/config/billing/config_test.go @@ -0,0 +1,37 @@ +package billing_test + +import ( + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" + billingconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/billing" + configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" + "github.com/nspcc-dev/neo-go/pkg/crypto/keys" + "github.com/stretchr/testify/require" +) + +func TestBillingSection(t *testing.T) { + t.Run("defaults", func(t *testing.T) { + empty := configtest.EmptyConfig() + + require.Empty(t, billingconfig.AuthorizedKeys(empty)) + require.Equal(t, billingconfig.GRPCEndpointDefault, billingconfig.GRPC(empty).Endpoint()) + }) + + const path = "../../../../config/example/node" + + pubs := make(keys.PublicKeys, 2) + pubs[0], _ = keys.NewPublicKeyFromString("035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11") + pubs[1], _ = keys.NewPublicKeyFromString("028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6") + + fileConfigTest := func(c *config.Config) { + require.Equal(t, pubs, billingconfig.AuthorizedKeys(c)) + require.Equal(t, "localhost:8092", billingconfig.GRPC(c).Endpoint()) + } + + configtest.ForEachFileType(path, fileConfigTest) + + t.Run("ENV", func(t *testing.T) { + configtest.ForEnvFileType(t, path, fileConfigTest) + }) +} diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index e4f0a434c..3eb7fcd95 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -115,6 +115,7 @@ func initApp(ctx context.Context, c *cfg) { initAndLog(c, "tree", initTreeService) initAndLog(c, "apemanager", initAPEManagerService) initAndLog(c, "control", initControlService) + initAndLog(c, "billing", initBillingService) initAndLog(c, "morph notifications", func(c *cfg) { listenMorphNotifications(ctx, c) }) } diff --git a/config/example/node.env b/config/example/node.env index 72f56e96c..60cdfcf8f 100644 --- a/config/example/node.env +++ b/config/example/node.env @@ -49,6 +49,10 @@ FROSTFS_GRPC_1_TLS_ENABLED=false FROSTFS_CONTROL_AUTHORIZED_KEYS="035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6" FROSTFS_CONTROL_GRPC_ENDPOINT=localhost:8090 +# Billing service section +FROSTFS_BILLING_AUTHORIZED_KEYS="035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6" +FROSTFS_BILLING_GRPC_ENDPOINT=localhost:8092 + # Contracts section FROSTFS_CONTRACTS_BALANCE=5263abba1abedbf79bb57f3e40b50b4425d2d6cd FROSTFS_CONTRACTS_CONTAINER=5d084790d7aa36cea7b53fe897380dab11d2cd3c diff --git a/config/example/node.json b/config/example/node.json index b9dc6014c..1ef5979b4 100644 --- a/config/example/node.json +++ b/config/example/node.json @@ -82,6 +82,15 @@ "endpoint": "localhost:8090" } }, + "billing": { + "authorized_keys": [ + "035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11", + "028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6" + ], + "grpc": { + "endpoint": "localhost:8092" + } + }, "contracts": { "balance": "5263abba1abedbf79bb57f3e40b50b4425d2d6cd", "container": "5d084790d7aa36cea7b53fe897380dab11d2cd3c", diff --git a/config/example/node.yaml b/config/example/node.yaml index bad67816a..df5fb1c7e 100644 --- a/config/example/node.yaml +++ b/config/example/node.yaml @@ -69,6 +69,13 @@ control: grpc: endpoint: localhost:8090 # endpoint that is listened by the Control Service +billing: + authorized_keys: # list of hex-encoded public keys that have rights to use the Billing Service + - 035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 + - 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6 + grpc: + endpoint: localhost:8092 # endpoint that is listened by the Billing Service + contracts: # side chain NEOFS contract script hashes; optional, override values retrieved from NNS contract balance: 5263abba1abedbf79bb57f3e40b50b4425d2d6cd container: 5d084790d7aa36cea7b53fe897380dab11d2cd3c diff --git a/dev/.vscode-example/launch.json b/dev/.vscode-example/launch.json index 990fd42a8..e09a952ba 100644 --- a/dev/.vscode-example/launch.json +++ b/dev/.vscode-example/launch.json @@ -49,6 +49,8 @@ "FROSTFS_GRPC_0_ENDPOINT":"127.0.0.1:8080", "FROSTFS_CONTROL_GRPC_ENDPOINT":"127.0.0.1:8081", "FROSTFS_CONTROL_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", + "FROSTFS_BILLING_GRPC_ENDPOINT":"127.0.0.1:8082", + "FROSTFS_BILLING_AUTHORIZED_KEYS":"031a6c6fbbdf02ca351745fa86b9ba5a9452d785ac4f7fc2b7548ca2a46c4fcf4a", "FROSTFS_NODE_ATTRIBUTE_0":"User-Agent:FrostFS/dev", "FROSTFS_NODE_ATTRIBUTE_1":"UN-LOCODE:RU MOW", "FROSTFS_NODE_PERSISTENT_STATE_PATH":"${workspaceFolder}/.cache/state/.frostfs-node-s1-state", diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md index 5389bfbb5..0db423b33 100644 --- a/docs/storage-node-configuration.md +++ b/docs/storage-node-configuration.md @@ -18,6 +18,7 @@ There are some custom types used for brevity: | `pprof` | [PProf configuration](#pprof-section) | | `prometheus` | [Prometheus metrics configuration](#prometheus-section) | | `control` | [Control service configuration](#control-section) | +| `billing` | [Billing service configuration](#billing-section) | | `contracts` | [Override FrostFS contracts hashes](#contracts-section) | | `morph` | [N3 blockchain client configuration](#morph-section) | | `apiclient` | [FrostFS API client configuration](#apiclient-section) | @@ -42,6 +43,20 @@ control: | `authorized_keys` | `[]public key` | empty | List of public keys which are used to authorize requests to the control service. | | `grpc.endpoint` | `string` | empty | Address that control service listener binds to. | +# `billing` section +```yaml +billing: + authorized_keys: + - 035839e45d472a3b7769a2a1bd7d54c4ccd4943c3b40f547870e83a8fcbfb3ce11 + - 028f42cfcb74499d7b15b35d9bff260a1c8d27de4f446a627406a382d8961486d6 + grpc: + endpoint: 127.0.0.1:8092 +``` +| Parameter | Type | Default value | Description | +|-------------------|----------------|---------------|----------------------------------------------------------------------------------| +| `authorized_keys` | `[]public key` | empty | List of public keys which are used to authorize requests to the billing service. | +| `grpc.endpoint` | `string` | empty | Address that billing service listener binds to. | + # `grpc` section ```yaml grpc: diff --git a/internal/logs/logs.go b/internal/logs/logs.go index 78f00c4ee..e53432e8d 100644 --- a/internal/logs/logs.go +++ b/internal/logs/logs.go @@ -443,6 +443,8 @@ const ( FrostFSNodeRemovingAllTreesForContainer = "removing all trees for container" FrostFSNodeContainerRemovalEventReceivedButTreesWerentRemoved = "container removal event received, but trees weren't removed" FrostFSNodeCantListenGRPCEndpointControl = "can't listen gRPC endpoint (control)" + FrostFSNodeCantListenGRPCEndpointBilling = "can't listen gRPC endpoint (billing)" + FrostFSNodeBillingServiceDisabled = "billing service is disabled" FrostFSNodePolicerIsDisabled = "policer is disabled" CommonApplicationStarted = "application started" ShardGCCollectingExpiredObjectsStarted = "collecting expired objects started" diff --git a/pkg/services/billing/server/list_containers.go b/pkg/services/billing/server/list_containers.go new file mode 100644 index 000000000..e121e37f7 --- /dev/null +++ b/pkg/services/billing/server/list_containers.go @@ -0,0 +1,12 @@ +package server + +import ( + "context" + "errors" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/billing" +) + +func (s *Server) ListContainers(context.Context, *billing.ListContainersRequest) (*billing.ListContainersResponse, error) { + return nil, errors.New("not implemented") +} diff --git a/pkg/services/billing/server/server.go b/pkg/services/billing/server/server.go new file mode 100644 index 000000000..0e01cb72d --- /dev/null +++ b/pkg/services/billing/server/server.go @@ -0,0 +1,34 @@ +package server + +import ( + "crypto/ecdsa" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" +) + +type cfg struct { + key *ecdsa.PrivateKey + allowedKeys [][]byte + cnrSrc container.Source + se *engine.StorageEngine +} + +type Server struct { + *cfg +} + +func New(key *ecdsa.PrivateKey, + allowedKeys [][]byte, + cnrSrc container.Source, + se *engine.StorageEngine, +) *Server { + return &Server{ + cfg: &cfg{ + key: key, + allowedKeys: allowedKeys, + cnrSrc: cnrSrc, + se: se, + }, + } +} diff --git a/pkg/services/billing/server/sign.go b/pkg/services/billing/server/sign.go new file mode 100644 index 000000000..579d43baf --- /dev/null +++ b/pkg/services/billing/server/sign.go @@ -0,0 +1,92 @@ +package server + +import ( + "bytes" + "crypto/ecdsa" + "errors" + "fmt" + + "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/refs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/billing" + frostfscrypto "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto" + frostfsecdsa "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/crypto/ecdsa" +) + +// SignedMessage is an interface of Control service message. +type SignedMessage interface { + ReadSignedData([]byte) ([]byte, error) + GetSignature() *billing.Signature + SetSignature(*billing.Signature) +} + +var ( + errDisallowedKey = errors.New("key is not in the allowed list") + errMissingSignature = errors.New("missing signature") + errInvalidSignature = errors.New("invalid signature") +) + +func (s *Server) isValidRequest(req SignedMessage) error { + sign := req.GetSignature() + if sign == nil { + return errMissingSignature + } + + var ( + key = sign.GetKey() + allowed = false + ) + + for i := range s.allowedKeys { + if allowed = bytes.Equal(s.allowedKeys[i], key); allowed { + break + } + } + if !allowed { + return errDisallowedKey + } + + binBody, err := req.ReadSignedData(nil) + if err != nil { + return fmt.Errorf("marshal request body: %w", err) + } + + var sigV2 refs.Signature + sigV2.SetKey(sign.GetKey()) + sigV2.SetSign(sign.GetSignature()) + sigV2.SetScheme(refs.ECDSA_SHA512) + + var sig frostfscrypto.Signature + if err := sig.ReadFromV2(sigV2); err != nil { + return fmt.Errorf("can't read signature: %w", err) + } + + if !sig.Verify(binBody) { + return errInvalidSignature + } + + return nil +} + +func SignMessage(key *ecdsa.PrivateKey, msg SignedMessage) error { + binBody, err := msg.ReadSignedData(nil) + if err != nil { + return fmt.Errorf("marshal request body: %w", err) + } + + var sig frostfscrypto.Signature + + err = sig.Calculate(frostfsecdsa.Signer(*key), binBody) + if err != nil { + return fmt.Errorf("calculate signature: %w", err) + } + + var sigV2 refs.Signature + sig.WriteToV2(&sigV2) + + var sigBilling billing.Signature + sigBilling.Key = sigV2.GetKey() + sigBilling.Signature = sigV2.GetSign() + msg.SetSignature(&sigBilling) + + return nil +} -- 2.45.2 From 11ccf9fec977e35b2adbd9120249a0b2688014cb Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Thu, 7 Mar 2024 17:03:15 +0300 Subject: [PATCH 3/3] [#9999] billing: Implement list containers. Signed-off-by: Dmitrii Stepanov --- .../engine/container_stat.go | 167 ++++++++++++ .../engine/container_stat_test.go | 221 ++++++++++++++++ .../metabase/container_stat.go | 237 ++++++++++++++++++ .../metabase/containers.go | 9 +- pkg/local_object_storage/metabase/counter.go | 23 +- .../shard/container_stat.go | 63 +++++ .../billing/server/list_containers.go | 129 +++++++++- 7 files changed, 834 insertions(+), 15 deletions(-) create mode 100644 pkg/local_object_storage/engine/container_stat.go create mode 100644 pkg/local_object_storage/engine/container_stat_test.go create mode 100644 pkg/local_object_storage/metabase/container_stat.go create mode 100644 pkg/local_object_storage/shard/container_stat.go diff --git a/pkg/local_object_storage/engine/container_stat.go b/pkg/local_object_storage/engine/container_stat.go new file mode 100644 index 000000000..f1e419e09 --- /dev/null +++ b/pkg/local_object_storage/engine/container_stat.go @@ -0,0 +1,167 @@ +package engine + +import ( + "bytes" + "context" + "errors" + "sort" + "sync" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/errgroup" +) + +type ContainerStatPrm struct { + ContainerID []cid.ID + + Limit uint32 + StartFromContainerID *cid.ID +} + +type ContainerStatRes struct { + ContainerStats []ContainerStat + Partial bool +} + +type ContainerStat struct { + ContainerID cid.ID + SizeLogic uint64 + CountPhy, CountLogic, CountUser uint64 +} + +var errInvalidLimit = errors.New("limit must be greater than zero") + +func (e *StorageEngine) ContainerStat(ctx context.Context, prm ContainerStatPrm) (*ContainerStatRes, error) { + if e.metrics != nil { + defer elapsed("ContainerStat", e.metrics.AddMethodDuration)() + } + + ctx, span := tracing.StartSpanFromContext(ctx, "StorageEngine.ContainerStat", + trace.WithAttributes( + attribute.Int("container_ids", len(prm.ContainerID)), + attribute.Int64("limit", int64(prm.Limit)), + attribute.Bool("start_from_container_id", prm.StartFromContainerID != nil), + )) + defer span.End() + + if len(prm.ContainerID) == 0 && prm.Limit == 0 { + return nil, errInvalidLimit + } + + var result *ContainerStatRes + err := e.execIfNotBlocked(func() error { + var sErr error + result, sErr = e.containerStat(ctx, prm) + return sErr + }) + return result, err +} + +func (e *StorageEngine) containerStat(ctx context.Context, prm ContainerStatPrm) (*ContainerStatRes, error) { + e.mtx.RLock() + defer e.mtx.RUnlock() + + if len(prm.ContainerID) > 0 { + sort.Slice(prm.ContainerID, func(i, j int) bool { + return bytes.Compare(prm.ContainerID[i][:], prm.ContainerID[j][:]) < 0 + }) + } + + shardResults, partial, err := e.collectShardContainerStats(ctx, prm) + if err != nil { + return nil, err + } + + return &ContainerStatRes{ + ContainerStats: e.mergeShardContainerStats(shardResults, int(prm.Limit)), + Partial: partial, + }, nil +} + +func (e *StorageEngine) collectShardContainerStats(ctx context.Context, prm ContainerStatPrm) ([][]shard.ContainerStat, bool, error) { + if len(prm.ContainerID) > 0 { + sort.Slice(prm.ContainerID, func(i, j int) bool { + return bytes.Compare(prm.ContainerID[i][:], prm.ContainerID[j][:]) < 0 + }) + } + + var shardResults [][]shard.ContainerStat + var shardErrors []error + var resultsGuard sync.Mutex + + shPrm := shard.ContainerStatPrm{ + ContainerID: prm.ContainerID, + Limit: prm.Limit, + StartFromContainerID: prm.StartFromContainerID, + } + + eg, egCtx := errgroup.WithContext(ctx) + var shardsCount int + e.iterateOverUnsortedShards(func(hs hashedShard) (stop bool) { + shardsCount++ + eg.Go(func() error { + s, err := hs.ContainerStat(egCtx, shPrm) + resultsGuard.Lock() + defer resultsGuard.Unlock() + + if err != nil { + shardErrors = append(shardErrors, err) + return nil + } + + if len(s) > 0 { + shardResults = append(shardResults, s) + } + + return nil + }) + return false + }) + if err := eg.Wait(); err != nil { + return nil, false, err + } + if shardsCount == len(shardErrors) { + return nil, false, errors.Join(shardErrors...) + } + + return shardResults, len(shardErrors) > 0, nil +} + +func (e *StorageEngine) mergeShardContainerStats(shardResults [][]shard.ContainerStat, limit int) []ContainerStat { + var stats []ContainerStat + for len(stats) <= limit && len(shardResults) > 0 { + // shard results are sorted by container ID + sort.Slice(shardResults, func(i, j int) bool { + return bytes.Compare(shardResults[i][0].ContainerID[:], shardResults[j][0].ContainerID[:]) < 0 + }) + + if len(stats) > 0 && stats[len(stats)-1].ContainerID == shardResults[0][0].ContainerID { + stats[len(stats)-1].SizeLogic += shardResults[0][0].SizeLogic + stats[len(stats)-1].CountPhy += shardResults[0][0].CountPhy + stats[len(stats)-1].CountLogic += shardResults[0][0].CountLogic + stats[len(stats)-1].CountUser += shardResults[0][0].CountUser + } else { + stats = append(stats, ContainerStat{ + ContainerID: shardResults[0][0].ContainerID, + SizeLogic: shardResults[0][0].SizeLogic, + CountPhy: shardResults[0][0].CountPhy, + CountLogic: shardResults[0][0].CountLogic, + CountUser: shardResults[0][0].CountUser, + }) + } + + if len(shardResults[0]) == 1 { // last item for shard + shardResults = shardResults[1:] + } else { + shardResults[0] = shardResults[0][1:] + } + } + if len(stats) > limit { + stats = stats[:limit] + } + return stats +} diff --git a/pkg/local_object_storage/engine/container_stat_test.go b/pkg/local_object_storage/engine/container_stat_test.go new file mode 100644 index 000000000..264f6d4b6 --- /dev/null +++ b/pkg/local_object_storage/engine/container_stat_test.go @@ -0,0 +1,221 @@ +package engine + +import ( + "context" + "testing" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/testutil" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/shard/mode" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger/test" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + cidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id/test" + objectSDK "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object" + oid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id" + oidtest "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +func TestContainerList(t *testing.T) { + t.Parallel() + + s1 := testNewShard(t) + s2 := testNewShard(t) + s3 := testNewShard(t) + + e := testNewEngine(t).setInitializedShards(t, s1, s2, s3).engine + e.log = test.NewLogger(t) + defer e.Close(context.Background()) + + const containerCount = 10 + expStat := testPutComplexObject(t, []*shard.Shard{s1, s2, s3}, containerCount, nil) + expStat = testPutSimpleObject(t, []*shard.Shard{s1, s2, s3}, containerCount, expStat) + expStat = testPutSimpleObject(t, []*shard.Shard{s1, s2}, containerCount, expStat) + expStat = testPutSimpleObject(t, []*shard.Shard{s2, s3}, containerCount, expStat) + expStat = testPutSimpleObject(t, []*shard.Shard{s1}, containerCount, expStat) + expStat = testPutSimpleObject(t, []*shard.Shard{s2}, containerCount, expStat) + expStat = testPutSimpleObject(t, []*shard.Shard{s3}, containerCount, expStat) + + t.Run("with default limit", func(t *testing.T) { + var prm ContainerStatPrm + prm.Limit = 10_000 + res, err := e.ContainerStat(context.Background(), prm) + require.NoError(t, err) + require.NotNil(t, res) + require.ElementsMatch(t, expStat, res.ContainerStats) + require.False(t, res.Partial) + }) + + t.Run("with limit, batched", func(t *testing.T) { + var prm ContainerStatPrm + prm.Limit = 1 + var stats []ContainerStat + + for { + res, err := e.ContainerStat(context.Background(), prm) + require.NoError(t, err) + require.NotNil(t, res) + require.False(t, res.Partial) + if len(res.ContainerStats) == 0 { + break + } + stats = append(stats, res.ContainerStats...) + last := res.ContainerStats[len(res.ContainerStats)-1].ContainerID + prm.StartFromContainerID = &last + prm.Limit += 1 + } + + require.ElementsMatch(t, expStat, stats) + }) + + t.Run("by container id", func(t *testing.T) { + for _, cc := range []int{1, 2, 3, 4, 5} { + var prm ContainerStatPrm + for idx := 0; idx+cc < len(expStat); idx += cc { + prm.ContainerID = nil + for i := 0; i < cc; i++ { + prm.ContainerID = append(prm.ContainerID, expStat[idx+i].ContainerID) + } + prm.Limit = uint32(len(prm.ContainerID)) + res, err := e.ContainerStat(context.Background(), prm) + require.NoError(t, err) + require.NotNil(t, res) + require.False(t, res.Partial) + require.ElementsMatch(t, expStat[idx:idx+cc], res.ContainerStats) + } + } + }) + + t.Run("unknown container id", func(t *testing.T) { + var prm ContainerStatPrm + prm.ContainerID = append(prm.ContainerID, cidtest.ID()) + prm.Limit = uint32(len(prm.ContainerID)) + res, err := e.ContainerStat(context.Background(), prm) + require.NoError(t, err) + require.NotNil(t, res) + require.False(t, res.Partial) + require.ElementsMatch(t, []ContainerStat{{ContainerID: prm.ContainerID[0]}}, res.ContainerStats) + }) + + t.Run("degraded shard", func(t *testing.T) { + s1.SetMode(mode.Degraded) + var prm ContainerStatPrm + prm.Limit = 10_000 + res, err := e.ContainerStat(context.Background(), prm) + require.NoError(t, err) + require.NotNil(t, res) + require.True(t, res.Partial) + }) +} + +func testPutComplexObject(t *testing.T, shards []*shard.Shard, count int, stats []ContainerStat) []ContainerStat { + const payloadSize = 10 * 1024 + for count > 0 { + var cnr cid.ID + var stat *ContainerStat + var newStat bool + if len(stats) == 0 || count%2 == 0 { + cnr = cidtest.ID() + stat = &ContainerStat{ContainerID: cnr} + newStat = true + } else { + cnr = stats[count%len(stats)].ContainerID + stat = &stats[count%len(stats)] + } + + parentID := oidtest.ID() + splitID := objectSDK.NewSplitID() + + parent := testutil.GenerateObjectWithCID(cnr) + parent.SetID(parentID) + parent.SetPayload(nil) + + const childCount = 10 + children := make([]*objectSDK.Object, childCount) + childIDs := make([]oid.ID, childCount) + for i := range children { + children[i] = testutil.GenerateObjectWithCID(cnr) + if i != 0 { + children[i].SetPreviousID(childIDs[i-1]) + } + if i == len(children)-1 { + children[i].SetParent(parent) + } + children[i].SetSplitID(splitID) + children[i].SetPayload(make([]byte, payloadSize)) + children[i].SetPayloadSize(payloadSize) + childIDs[i], _ = children[i].ID() + + stat.SizeLogic += payloadSize + stat.CountLogic += 1 + stat.CountPhy += 1 + } + stat.CountUser += 1 + + link := testutil.GenerateObjectWithCID(cnr) + link.SetParent(parent) + link.SetParentID(parentID) + link.SetSplitID(splitID) + link.SetChildren(childIDs...) + + stat.CountLogic += 1 + stat.CountPhy += 1 + stat.SizeLogic += link.PayloadSize() + + for i := range children { + sh := shards[i%len(shards)] + var putPrm shard.PutPrm + putPrm.SetObject(children[i]) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + } + sh := shards[count%len(shards)] + var putPrm shard.PutPrm + putPrm.SetObject(link) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + if newStat { + stats = append(stats, *stat) + } + count-- + } + return stats +} + +func testPutSimpleObject(t *testing.T, shards []*shard.Shard, count int, stats []ContainerStat) []ContainerStat { + const payloadSize = 7 * 1024 + for count > 0 { + var cnr cid.ID + var stat *ContainerStat + var newStat bool + if len(stats) == 0 || count%2 == 0 { + cnr = cidtest.ID() + stat = &ContainerStat{ContainerID: cnr} + newStat = true + } else { + cnr = stats[count%len(stats)].ContainerID + stat = &stats[count%len(stats)] + } + + obj := testutil.GenerateObjectWithCID(cnr) + obj.SetPayload(make([]byte, payloadSize)) + obj.SetPayloadSize(payloadSize) + stat.SizeLogic += payloadSize + stat.CountLogic += 1 + stat.CountPhy += 1 + stat.CountUser += 1 + + sh := shards[count%len(shards)] + var putPrm shard.PutPrm + putPrm.SetObject(obj) + _, err := sh.Put(context.Background(), putPrm) + require.NoError(t, err) + + if newStat { + stats = append(stats, *stat) + } + count-- + } + return stats +} diff --git a/pkg/local_object_storage/metabase/container_stat.go b/pkg/local_object_storage/metabase/container_stat.go new file mode 100644 index 000000000..db4c1ee8d --- /dev/null +++ b/pkg/local_object_storage/metabase/container_stat.go @@ -0,0 +1,237 @@ +package meta + +import ( + "bytes" + "context" + "crypto/sha256" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/internal/metaerr" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "go.etcd.io/bbolt" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +type ContainerStatPrm struct { + ContainerID []cid.ID + + Limit uint32 + StartFromContainerID *cid.ID +} + +type ContainerStat struct { + ContainerID cid.ID + SizeLogic uint64 + CountPhy, CountLogic, CountUser uint64 +} + +// ContainerStat returns object count and size for containers. +// If len(prm.ContainerID) > 0, then result slice contains records in the same order as prm.ContainerID. +// Otherwise result slice sorted by ContainerID. +func (db *DB) ContainerStat(ctx context.Context, prm ContainerStatPrm) ([]ContainerStat, error) { + var ( + startedAt = time.Now() + success = false + ) + defer func() { + db.metrics.AddMethodDuration("ContainerStat", time.Since(startedAt), success) + }() + + _, span := tracing.StartSpanFromContext(ctx, "metabase.ContainerStat", + trace.WithAttributes( + attribute.Int("container_ids", len(prm.ContainerID)), + attribute.Int64("limit", int64(prm.Limit)), + attribute.Bool("start_from_container_id", prm.StartFromContainerID != nil), + )) + defer span.End() + + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return nil, ErrDegradedMode + } + + if len(prm.ContainerID) > 0 { + return db.containerStatByContainerID(prm.ContainerID) + } + return db.containerStatByLimit(prm.StartFromContainerID, prm.Limit) +} + +func (db *DB) containerStatByContainerID(containerID []cid.ID) ([]ContainerStat, error) { + var result []ContainerStat + err := db.boltDB.View(func(tx *bbolt.Tx) error { + for _, contID := range containerID { + var stat ContainerStat + stat.ContainerID = contID + stat.SizeLogic = db.containerSize(tx, contID) + + counters, err := db.containerCounters(tx, contID) + if err != nil { + return err + } + stat.CountPhy = counters.Phy + stat.CountLogic = counters.Logic + stat.CountUser = counters.User + result = append(result, stat) + } + return nil + }) + if err != nil { + return nil, metaerr.Wrap(err) + } + return result, nil +} + +func (db *DB) containerStatByLimit(startFrom *cid.ID, limit uint32) ([]ContainerStat, error) { + var result []ContainerStat + + var lastKey []byte + if startFrom != nil { + lastKey = make([]byte, sha256.Size) + startFrom.Encode(lastKey) + } + + var counts []containerIDObjectCounters + var sizes []containerIDSize + + err := db.boltDB.View(func(tx *bbolt.Tx) error { + var e error + counts, e = getContainerCountersBatch(tx, lastKey, limit) + if e != nil { + return e + } + + sizes, e = getContainerSizesBatch(tx, lastKey, limit) + return e + }) + if err != nil { + return nil, metaerr.Wrap(err) + } + + result = mergeSizeAndCounts(counts, sizes) + if len(result) > int(limit) { + result = result[:limit] + } + return result, nil +} + +type containerIDObjectCounters struct { + ContainerID cid.ID + ObjectCounters +} + +func getContainerCountersBatch(tx *bbolt.Tx, lastKey []byte, limit uint32) ([]containerIDObjectCounters, error) { + var result []containerIDObjectCounters + b := tx.Bucket(containerCounterBucketName) + if b == nil { + return result, nil + } + c := b.Cursor() + var key, value []byte + for key, value = c.Seek(lastKey); key != nil && uint32(len(result)) < limit; key, value = c.Next() { + if bytes.Equal(lastKey, key) { + continue + } + + cnrID, err := parseContainerCounterKey(key) + if err != nil { + return nil, err + } + ent, err := parseContainerCounterValue(value) + if err != nil { + return nil, err + } + result = append(result, containerIDObjectCounters{ + ContainerID: cnrID, + ObjectCounters: ent, + }) + } + return result, nil +} + +type containerIDSize struct { + ContainerID cid.ID + Size uint64 +} + +func getContainerSizesBatch(tx *bbolt.Tx, lastKey []byte, limit uint32) ([]containerIDSize, error) { + var result []containerIDSize + b := tx.Bucket(containerVolumeBucketName) + c := b.Cursor() + var key, value []byte + for key, value = c.Seek(lastKey); key != nil && uint32(len(result)) < limit; key, value = c.Next() { + if bytes.Equal(lastKey, key) { + continue + } + + var r containerIDSize + r.Size = parseContainerSize(value) + if err := r.ContainerID.Decode(key); err != nil { + return nil, err + } + result = append(result, r) + } + return result, nil +} + +// mergeSizeAndCounts merges sizes and counts. +// As records are deleted in background, it can happen that metabase contains size record for container, +// but doesn't contain record for count. +func mergeSizeAndCounts(counts []containerIDObjectCounters, sizes []containerIDSize) []ContainerStat { + var result []ContainerStat + + for len(counts) > 0 || len(sizes) > 0 { + if len(counts) == 0 { + result = append(result, ContainerStat{ + ContainerID: sizes[0].ContainerID, + SizeLogic: sizes[0].Size, + }) + sizes = sizes[1:] + continue + } + + if len(sizes) == 0 { + result = append(result, ContainerStat{ + ContainerID: counts[0].ContainerID, + CountPhy: counts[0].Phy, + CountLogic: counts[0].Logic, + CountUser: counts[0].User, + }) + counts = counts[1:] + continue + } + + v := bytes.Compare(sizes[0].ContainerID[:], counts[0].ContainerID[:]) + + if v == 0 { // equal + result = append(result, ContainerStat{ + ContainerID: counts[0].ContainerID, + CountPhy: counts[0].Phy, + CountLogic: counts[0].Logic, + CountUser: counts[0].User, + SizeLogic: sizes[0].Size, + }) + counts = counts[1:] + sizes = sizes[1:] + } else if v < 0 { // from sizes + result = append(result, ContainerStat{ + ContainerID: sizes[0].ContainerID, + SizeLogic: sizes[0].Size, + }) + sizes = sizes[1:] + } else { // from counts + result = append(result, ContainerStat{ + ContainerID: counts[0].ContainerID, + CountPhy: counts[0].Phy, + CountLogic: counts[0].Logic, + CountUser: counts[0].User, + }) + counts = counts[1:] + } + } + + return result +} diff --git a/pkg/local_object_storage/metabase/containers.go b/pkg/local_object_storage/metabase/containers.go index 472b2affc..24a515486 100644 --- a/pkg/local_object_storage/metabase/containers.go +++ b/pkg/local_object_storage/metabase/containers.go @@ -65,20 +65,19 @@ func (db *DB) ContainerSize(id cid.ID) (size uint64, err error) { } err = db.boltDB.View(func(tx *bbolt.Tx) error { - size, err = db.containerSize(tx, id) - - return err + size = db.containerSize(tx, id) + return nil }) return size, metaerr.Wrap(err) } -func (db *DB) containerSize(tx *bbolt.Tx, id cid.ID) (uint64, error) { +func (db *DB) containerSize(tx *bbolt.Tx, id cid.ID) uint64 { containerVolume := tx.Bucket(containerVolumeBucketName) key := make([]byte, cidSize) id.Encode(key) - return parseContainerSize(containerVolume.Get(key)), nil + return parseContainerSize(containerVolume.Get(key)) } func parseContainerID(dst *cid.ID, name []byte, ignore map[string]struct{}) bool { diff --git a/pkg/local_object_storage/metabase/counter.go b/pkg/local_object_storage/metabase/counter.go index 275099ff2..d11710fd9 100644 --- a/pkg/local_object_storage/metabase/counter.go +++ b/pkg/local_object_storage/metabase/counter.go @@ -216,21 +216,28 @@ func (db *DB) ContainerCount(ctx context.Context, id cid.ID) (ObjectCounters, er var result ObjectCounters err := db.boltDB.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(containerCounterBucketName) - key := make([]byte, cidSize) - id.Encode(key) - v := b.Get(key) - if v == nil { - return nil - } var err error - result, err = parseContainerCounterValue(v) + result, err = db.containerCounters(tx, id) return err }) return result, metaerr.Wrap(err) } +func (*DB) containerCounters(tx *bbolt.Tx, id cid.ID) (ObjectCounters, error) { + b := tx.Bucket(containerCounterBucketName) + if b == nil { + return ObjectCounters{}, nil + } + key := make([]byte, cidSize) + id.Encode(key) + v := b.Get(key) + if v == nil { + return ObjectCounters{}, nil + } + return parseContainerCounterValue(v) +} + func (db *DB) incCounters(tx *bbolt.Tx, cnrID cid.ID, isUserObject bool) error { b := tx.Bucket(shardInfoBucket) if b == nil { diff --git a/pkg/local_object_storage/shard/container_stat.go b/pkg/local_object_storage/shard/container_stat.go new file mode 100644 index 000000000..c213aa7fc --- /dev/null +++ b/pkg/local_object_storage/shard/container_stat.go @@ -0,0 +1,63 @@ +package shard + +import ( + "context" + + meta "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/metabase" + "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +type ContainerStatPrm struct { + ContainerID []cid.ID + + Limit uint32 + StartFromContainerID *cid.ID +} + +type ContainerStat struct { + ContainerID cid.ID + SizeLogic uint64 + CountPhy, CountLogic, CountUser uint64 +} + +// ContainerStat returns object count and size for containers from metabase. +func (s *Shard) ContainerStat(ctx context.Context, prm ContainerStatPrm) ([]ContainerStat, error) { + _, span := tracing.StartSpanFromContext(ctx, "Shard.ContainerStat", + trace.WithAttributes( + attribute.String("shard_id", s.ID().String()), + attribute.Int("container_ids", len(prm.ContainerID)), + attribute.Int64("limit", int64(prm.Limit)), + attribute.Bool("start_from_container_id", prm.StartFromContainerID != nil), + )) + defer span.End() + + s.m.RLock() + defer s.m.RUnlock() + + if s.GetMode().NoMetabase() { + return nil, ErrDegradedMode + } + + res, err := s.metaBase.ContainerStat(ctx, meta.ContainerStatPrm{ + ContainerID: prm.ContainerID, + Limit: prm.Limit, + StartFromContainerID: prm.StartFromContainerID, + }) + if err != nil { + return nil, err + } + result := make([]ContainerStat, 0, len(res)) + for _, r := range res { + result = append(result, ContainerStat{ + ContainerID: r.ContainerID, + SizeLogic: r.SizeLogic, + CountPhy: r.CountPhy, + CountLogic: r.CountLogic, + CountUser: r.CountUser, + }) + } + return result, nil +} diff --git a/pkg/services/billing/server/list_containers.go b/pkg/services/billing/server/list_containers.go index e121e37f7..3882c879f 100644 --- a/pkg/services/billing/server/list_containers.go +++ b/pkg/services/billing/server/list_containers.go @@ -3,10 +3,135 @@ package server import ( "context" "errors" + "fmt" + containerApi "git.frostfs.info/TrueCloudLab/frostfs-api-go/v2/container" + containercore "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/container" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/local_object_storage/engine" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/billing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/client" + cid "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/container/id" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -func (s *Server) ListContainers(context.Context, *billing.ListContainersRequest) (*billing.ListContainersResponse, error) { - return nil, errors.New("not implemented") +const ( + defaultLimit = 1000 + maxLimit = 10000 +) + +var ( + errInvalidContainerIDLenght = errors.New("count of container ID array must be lower or equal 10 000 items") + errInvalidLimit = errors.New("limit value must be lower or equal 10 000") +) + +func (s *Server) ListContainers(ctx context.Context, req *billing.ListContainersRequest) (*billing.ListContainersResponse, error) { + err := s.isValidRequest(req) + if err != nil { + return nil, status.Error(codes.PermissionDenied, err.Error()) + } + enginePrm, err := convertToEngineContainerStatPrm(req) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + res, err := s.se.ContainerStat(ctx, enginePrm) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + resp := &billing.ListContainersResponse{ + Body: &billing.ListContainersResponse_Body{ + Result: s.addContainerInfo(res.ContainerStats), + NextPageToken: containerListNextPageToken(res.ContainerStats, enginePrm.Limit), + Partial: res.Partial, + }, + } + + if err = SignMessage(s.key, resp); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + return resp, nil +} + +func (s *Server) addContainerInfo(engineStats []engine.ContainerStat) []*billing.ListContainersResponse_Body_ContainerInfo { + var result []*billing.ListContainersResponse_Body_ContainerInfo + + for _, engineStat := range engineStats { + containerInfo := &billing.ListContainersResponse_Body_ContainerInfo{ + ContainerId: engineStat.ContainerID[:], + Count: &billing.ListContainersResponse_Body_ContainerInfo_Count{ + Phy: engineStat.CountPhy, + Logic: engineStat.CountLogic, + User: engineStat.CountUser, + }, + Size: &billing.ListContainersResponse_Body_ContainerInfo_Size{ + Logic: engineStat.SizeLogic, + }, + ContainerStatus: billing.ListContainersResponse_Body_ContainerInfo_UNDEFINED, + } + + cnr, err := s.cnrSrc.Get(engineStat.ContainerID) + if err != nil { + if client.IsErrContainerNotFound(err) { + existed, errWasRemoved := containercore.WasRemoved(s.cnrSrc, engineStat.ContainerID) + if errWasRemoved == nil && existed { + containerInfo.ContainerStatus = billing.ListContainersResponse_Body_ContainerInfo_DELETED + } + } + } else { + containerInfo.ContainerStatus = billing.ListContainersResponse_Body_ContainerInfo_AVAILABLE + containerInfo.Attributes = &billing.ListContainersResponse_Body_ContainerInfo_Attributes{ + OwnerWallet: cnr.Value.Owner().WalletBytes(), + Zone: cnr.Value.Attribute(containerApi.SysAttributeZone), + } + } + result = append(result, containerInfo) + } + + return result +} + +func containerListNextPageToken(engineStats []engine.ContainerStat, limit uint32) []byte { + if uint32(len(engineStats)) <= limit { + return nil + } + return engineStats[len(engineStats)-1].ContainerID[:] +} + +func convertToEngineContainerStatPrm(req *billing.ListContainersRequest) (engine.ContainerStatPrm, error) { + var result engine.ContainerStatPrm + + if len(req.GetBody().GetContainerId()) > 10000 { + return result, errInvalidContainerIDLenght + } + if len(req.GetBody().GetContainerId()) > 0 { + for idx, contIDBytes := range req.GetBody().GetContainerId() { + var contID cid.ID + if err := contID.Decode(contIDBytes); err != nil { + return result, fmt.Errorf("failed to decode container ID at index %d: %w", idx, err) + } + result.ContainerID = append(result.ContainerID, contID) + } + return result, nil + } + + result.Limit = defaultLimit + if req.GetBody().GetLimit() > maxLimit { + return result, errInvalidLimit + } + if req.GetBody().GetLimit() > 0 { + result.Limit = req.GetBody().GetLimit() + } + + if len(req.GetBody().GetNextPageToken()) > 0 { + var contID cid.ID + if err := contID.Decode(req.GetBody().GetNextPageToken()); err != nil { + return result, fmt.Errorf("invalid next page token: %w", err) + } + result.StartFromContainerID = &contID + } + + return result, nil } -- 2.45.2