From 560c60513e1b2b9f3ad0502b826740a53c328ccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=80=D0=B8=D1=8F=20=D0=9C=D0=B0=D0=BB=D1=8B?= =?UTF-8?q?=D0=B3=D0=B8=D0=BD=D0=B0?= Date: Tue, 18 Jul 2023 13:06:38 +0300 Subject: [PATCH] support multipart upload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Мария Малыгина --- go.mod | 31 +++---- go.sum | Bin 82853 -> 75707 bytes internal/s3/client.go | 34 +++++++ scenarios/run_scenarios.md | 177 +++++++++++++++++++++++++------------ scenarios/s3_multipart.js | 108 ++++++++++++++++++++++ 5 files changed, 280 insertions(+), 70 deletions(-) create mode 100644 scenarios/s3_multipart.js diff --git a/go.mod b/go.mod index 810a01b..b6e5bcd 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,10 @@ require ( git.frostfs.info/TrueCloudLab/frostfs-s3-gw v0.27.0-rc.2 git.frostfs.info/TrueCloudLab/frostfs-sdk-go v0.0.0-20230705125206-769f6eec0565 git.frostfs.info/TrueCloudLab/tzhash v1.8.0 - github.com/aws/aws-sdk-go-v2 v1.18.1 - github.com/aws/aws-sdk-go-v2/config v1.18.27 - github.com/aws/aws-sdk-go-v2/service/s3 v1.36.0 + github.com/aws/aws-sdk-go-v2 v1.19.0 + github.com/aws/aws-sdk-go-v2/config v1.18.28 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 + github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 github.com/dop251/goja v0.0.0-20230626124041-ba8a63e79201 github.com/go-loremipsum/loremipsum v1.1.3 github.com/google/uuid v1.3.0 @@ -33,19 +34,19 @@ require ( github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/aws/aws-sdk-go v1.44.296 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.27 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 // indirect github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bluele/gcache v0.0.2 // indirect diff --git a/go.sum b/go.sum index 6631eeba1bf696b7b757686d3bee136c83c16209..5eb2496f94be43933fa3a5a56a12dbb8c0602cf8 100644 GIT binary patch delta 2096 zcmZvcJ^A(hy?e7+xlf4`spZ+`Ro z&+cD)`-Lak*Ppy-UwZOS``aJ+$F=s?AAcL6FQMBWd@qccDbS!a< zF8wOq4NPMvi@cvU-2o7`i)TM|aeD(_!Uwkf+B0SQ$4}lzsY`@922BN-TMQcyo>~M} z=Dke<(Y~1teItl@v*^Rlf`)U%P4ev_=vMoKn;*P)WI;E*j8p3vkq9Dyfz-faDw*ek z1auW{mY2x{LR3_qu^>GsQ@rLdfH_6%ZhaUk!014m6C4*$#EJF;i`bu-EtJH+t$Y@Q)BA0^`n$jYuH7)0fdW4lt0%LUN2vV`apP`3Aj_D> z3qW0WQyJ-b)ec0HOx>*J%aC*MSR|nYojFm}zWnTS>>35*MY3@1;xv5tl0=T?7?NUz zqBk%LmFSW(sG?-18c7W0b!sg_f>ujE4f%%KqzCgI;s3;(m2aZav=*v#Dgv{B$B0q9 zL3}0PExWIU=tRZ5+VoP4<0Ynj4-8+Dgkq|ytENdRpv5E6-yd4@Q3m{CK>4pwYeYv29M^X5PQplRQo^DF z1*;uGB8_lp4O8zs@wpY4n?cr%Y{&H|&@=34;N@;Uuv4q9ut~hs*At74M3QU&{36tb zcb{%=-2EGjp56*Ju&G(-AtNITSW9+hVv!`2J$G?}wlp>N+_JH1#%vVmME@3eoI;_q z=DAu5Y=n<=$%$<>+)?~OZ#=xa>&0Xh8UNRaeLT>;EL{!57wIxE4RHBs8$epc^5Q&{N z93;M;<(n-Ht@S|87fD~NoI+n@Ml%r+%JJC9bR(Kj!R?{p9eRx(cZ-r&kFm0d#1Yq@ zgV@Yb>#RTTO~pkzn8ZrjER%d-rsYDIa}GFmkMtM+N1y$IvTcS#U4&M0*g;8kE3OK! zu{#A50{GITMo3zb+N5D72y+Z+KaX6X$R&&(tFMjjhFnR)R9VdtShWgU6Swtf#Katg z4+3sIC7e#T>@sxt9!nqDGkw>YL2IL$V#F|8Y_+xB37*+u}u|VMm2^F%AH5RWc$_qrZgq5{Rsg(E zn7)rFX5AqxWHfhqxXC&&jPVerox}(5ed0#%&l$L*gt6wa76YMlFp~#^J_!;f&<8ly zh80kB!dS_VZ*I^(3un`S~( z$1R=JkF3_a7p!ed2|OKK2@P O_o2W351GgFryLfDH?ZKq`ewbioFMTG+5D3!L};zLJrul9d(d zcmC&`=RD^;&+})0`!m1)TTgubv(GQEYxp5_jX($rVGs(#5QgFpA=vUP*s)v;!Qb%V zx9izB>Ty;S@&*#j6@_aHoZM!0(Cj@eviC{~s!Jyd*6aN#d*-dv7oUH6`~MmS_sMmX z?9TRW$fP8kl7XsIYERuR6x7@n*&HtE1ZbUTNn$Kyse16y#DsE3R#CRl_Nf``fy`CE%Wdj8%^ zKiS&p_g>hXzVeZ$ZhrLqvrj<8*|-5zvo7yn4@bbQw~~FKoY-YbS&ZL&3tB_ zzVM0HP9J>vJ(##g&*IP))>MI12&Hc-uI6>76sEP}E+KWw_)gj`WCWDwIOm_3`>+4f zS^JD$YL|_+7)=RJ*G#c(U6*CI0RnQ8k%H}bz1IPS(+EwJ<@MnHCx}}U)HQro2TIrD zNE}KSn>3fx{%8@l7%GkDELpB(@eWyKNyfU_eDM4AryirH7hWmBGq3LE*dh&az84Du zGo?Bwq+In5j#LH)lBm#DQC7NkY@4(8>!eH0@Q3aDZieT z6FEox;7(Zh*w4`edZBF|g4xW6+8xLMrh%hY4`n@zml{TdB_?cGci|zjPKzOS>$>o< zXCU$%Gt<+R*hEF4afC9}(qfV+u{BM_cs|Z#LlOFI#S&}A=$XeLNnQv;QU@6OD4e!# z7U-spkBrF0QwQ7Sffh?Z3r_{KuQ5b!Zu#ir&mqV)be6_epqw3@)kF+h{;X1<^==b9;5VtUWn7iHY7J&!0qNm!ixmR9)nQJ<;`}a&gnGFl*PI_ z1cx1V=X~_>dv}{8i`GEr&m)EHsvxQs23dNz)ULqQyfbrU*y0;(oY)6#9X^u2M_%rb zX`+{i!BW9&J+c`r$bl^?9NZ}8?NC>|R@I$Du`WzBDQ@{+eD%d4YU~{KWI^q-YU)mB zvz=!UQ~Y?eq~&-5%oq=WHgktc(p3`$9(1=-13E;FT|`}h1rC}(;;)y=LO()Kgml8D zJ0#mN#};;~Ys;gRsmIE59*MsU-Aaq?9JCK*&g9rJ_GvNZRpLPQTbosOQsMRU$%rS! zHLF7WW01HM=3rmcWpE@SE@2${IowZ<#*V5B?|@9Tzz#-Me~6{XwT1%!%;}4--5Xl= zIp%7z-6upe6?iy?$=G&k)giogQ?1Zel(SkdVMJhK&e5nrcxchD-J{^kg`NPMSyp@# z9BUL0SHhwjO`U?lJkpNUq=)fdSt3UvgUxf&&IUdT-iFNJ>ukwI3yYnk?rxq`Dpmu# z8LO=mq?l)ZGFRv|9=6?fXbESRKT2LkJnUMHa~;8jMrDBVMm%o8&2il)Io(94C(OE9 zOqUaA(-^n9!TlHRbe*`Uxw)FmWGrE1vPdN?grW!xnxxPOnNF6xThtkU^%>Fz?0=X4tjafBy}W*wUg zs7d^d6$sKQM(bjI-pcUsJF60ox$2^$$nnRq#*H8V2bh>J1DKtcM6PlKT`JMUt>E~) zm0${nFC-yp1r#c5gI!yA%%WyEHE$@{Yt~6C9pZ7<5S+21M=Ek|d`Lv3QTrxb%(op~B*nRm5%RK6LfjrUf|i3uevA{x7EotG!}k6b zK!j;|=!?7sj%=2D(QR_68ne!00=gS8ECTamlZ|Mj!rLscQEwi#1y=>sId{iRQ z91>x`=MC`7{|8@$d;~|f<7H!3Lw$E)HGYSgtI}9*X_;2MBaH(n0jzxDQy zjUczI_*nK%4Bda#yWy zmH+XLXWqB)Dy3}}1i>(&B<&1?mCb3L@3@RMRuPbidT>avNw$h}j(g0*2z@T_-c-c} zTBG?vQWh{aBW+AyKvn0hxbadkYQ)L-yw>!wynIYj2z`<4p5%g6uxU&p-8$PHkIV>c zH!sV&w-z)VtoQ);Q-2Fd_}O*#u2oEiY<#G0X2EJkXEUbab3Z(**^P zYZ={?Y`7Xtks^(w)iS(WoUc9j2!h^?!xaaPkEUqkJ*+5EMokck*Jol_R8@UU`?XyQ zfXb)V7a<=ct$27_q8bSAb-$$QsGyKW~{Zz48b`b zJ7d&1v;EaA1w&}9J`5V^N&D0a-4XS#;MsdBu9Jv-Hy>av?mcSfw?J2?D$G+|8Mlp%61Gm0h2nj^vE=4po6L zPLu5J_*Xym%$+cjX)BVSgd_;@GHC78iEmW2Ui!l^U^CJ7qdZz1cf1%=(5>(Q5|_dp zw#jGgqK`Hsa9H3n8%^gn>QH@Z%cK3=MZH5YmPKZZ-)38R^T}bAZ?g@w<%Wi0)Uub+ zz7OG41%Sz}9a`ps8}FC#o^UY`C$=kuxm6xedO$CNK6kEJl5ahQ(;I8P6EI2>Bd zydQ(_%l0mXI&o2mBjo@9TdnQcfB_RTI}q}M_mdS+O~|P$t?A+J`u#UwAL{FNk=-dM z1FW-_n7Y_`-d?H$Y2J9R;#HZR|zfKoXH@AiY2BY!H5soW5S+I4+pJ4P;Fa2aorz0xg{0s8x6L-^ZxGCiM zsuRmYrqR=GuJ3m~5MqZ|cC|X1+LgLD(PN8FedhFi>cNdj{qRNN@|qR{c3!R^cE6Oj z+GdBF>q)AOSjw1;#^N0GK|8es3LKZp={fQB8(RMQhsHPz!4Qg}N`d^CKG0KWnyeeKIQ^0R=_}&&S!aCn=k^~w1>N?=qT?nbx9Js0 zF*d!bP0|#WvQ#xgDqqWXPi=VK0Tyy!6(>G?jlArNlnr%@6tU{5=GTV~1Qb>nZc&Pq z)@?FeKE;|NCh*a<{pN-bfAAsk@q-ot!7vIF81m2>zDNi|+@X-p736g)zpdBH;!w_U z7taiS7HeZxuNG<3_Y|#iYmV;>7%ooF<)7Amy1|S57WC2&B$lKwX`5;5RZ>fKXY49o za3~^**}Q5U>A<*l>fAh^zx)J3-n~wMweLf--lC^)y?_@xHlY0@A+39MWk;o>2_mEv zanzN=(aUTed~m={?=<6+-@JSSG@GaHdPCB-u*>?wlqc3|EAcgCl`eCNfR-+1Th!|!0%$m2k91cPvl#HokYZUn6 zH(&j7a$kfI5`iEDCLTIFlyDq~#IW7&!Yb=c(Kh8Xs991uoMCE(thEV{nvYcH+1CN9W^~5Wo`08H!2lc9Sg|RYU|2ZDyoGKzV#>9uf75g zudT!o3p6xkc!%OP?INy?xuI~j}~m528loS58PgM zAXc?1J!%ZEG+;nHS}cjHLgPc~p4IWw{qMf=0T?>{^Y><_>+e2!^QG^71$p<6pSgPF z7Y;l=iH5fqtRM5T(F$T44-ZQ|4}x79&LnVNZPfZGlcxS|_UzT?&d^8S{pPb*zxv+& zXRq#E-5DWlpr0;B2Yqs6q;#1B4r}GiE;q;6a<8jXvRa$g&ghE$>8t 0) { + scenarios.write = { + executor: 'constant-vus', + vus: write_vu_count, + duration: `${duration}s`, + exec: 'obj_write_multipart', + gracefulStop: '5s', + }; +} + +export const options = { + scenarios, + setupTimeout: '5s', +}; + +export function setup() { + const total_vu_count = write_vu_count * write_multipart_vu_count; + + console.log(`Pregenerated buckets: ${bucket_list.length}`); + console.log(`Pregenerated read object size: ${read_size}`); + console.log(`Pregenerated total objects: ${obj_list.length}`); + console.log(`Writing VUs: ${write_vu_count}`); + console.log(`Writing multipart VUs: ${write_multipart_vu_count}`); + console.log(`Total VUs: ${total_vu_count}`); +} + +export function teardown(data) { + if (obj_registry) { + obj_registry.close(); + } +} + +export function handleSummary(data) { + return { + 'stdout': textSummary(data, {indent: ' ', enableColors: false}), + [summary_json]: JSON.stringify(data), + }; +} + +const write_multipart_part_size = 1024 * parseInt(__ENV.WRITE_OBJ_PART_SIZE || '0') + +export function obj_write_multipart() { + if (__ENV.SLEEP_WRITE) { + sleep(__ENV.SLEEP_WRITE); + } + + const key = __ENV.OBJ_NAME || uuidv4(); + const bucket = bucket_list[Math.floor(Math.random() * bucket_list.length)]; + + const {payload, hash} = generator.genPayload(registry_enabled); + const resp = s3_client.multipart(bucket, key, write_multipart_part_size, payload, write_multipart_vu_count); + if (!resp.success) { + log.withFields({bucket: bucket, key: key}).error(resp.error); + return; + } + + if (obj_registry) { + obj_registry.addObject("", "", bucket, key, hash); + } +} +