TrueCloudLab/xk6-frostfs
42
1
Fork 20
Code Issues 35 Pull requests 2 Releases Activity Actions

[#57] preset: Use temporary file for payload

Browse source 
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
Evgenii Stratonikov 2023-05-03 14:33:16 +03:00
parent da01f4bc2a
commit 2e918c57d5
3 changed files with 11 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
scenarios/preset/helpers/cmd.py
View file

@ -19,10 +19,9 @@ def execute_cmd(cmd_line):
return output, success return output, success
def random_payload(payload_filepath, size): def random_payload(file, size):
with open('%s' % payload_filepath, 'w+b') as fout: file.write(os.urandom(1024 * int(size)))
fout.write(os.urandom(1024 * int(size))) file.flush()
class ProgressBar: class ProgressBar:
@staticmethod @staticmethod

7
scenarios/preset/preset_grpc.py
View file

@ -4,6 +4,7 @@ import argparse
import json import json
import random import random
import sys import sys
import tempfile
from argparse import Namespace from argparse import Namespace
from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ProcessPoolExecutor
@ -39,7 +40,6 @@ print(args)
def main(): def main():
container_list = [] container_list = []
objects_list = [] objects_list = []
payload_filepath = '/tmp/data_file'
endpoints = args.endpoint.split(',') endpoints = args.endpoint.split(',')
@ -71,13 +71,14 @@ def main():
sys.exit(ERROR_NO_CONTAINERS) sys.exit(ERROR_NO_CONTAINERS)
print(f"Upload objects to each container: {args.preload_obj} ") print(f"Upload objects to each container: {args.preload_obj} ")
random_payload(payload_filepath, args.size) payload_file = tempfile.NamedTemporaryFile()
random_payload(payload_file, args.size)
print(" > Create random payload: Completed") print(" > Create random payload: Completed")
for container in container_list: for container in container_list:
print(f" > Upload objects for container {container}") print(f" > Upload objects for container {container}")
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor: with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
objects_runs = {executor.submit(upload_object, container, payload_filepath, objects_runs = {executor.submit(upload_object, container, payload_file.name,
endpoints[random.randrange(len(endpoints))], wallet, wallet_config): _ for _ in range(int(args.preload_obj))} endpoints[random.randrange(len(endpoints))], wallet, wallet_config): _ for _ in range(int(args.preload_obj))}
for run in objects_runs: for run in objects_runs:

7
scenarios/preset/preset_s3.py
View file

@ -3,6 +3,7 @@
import argparse import argparse
import json import json
import sys import sys
import tempfile
from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ProcessPoolExecutor
from helpers.cmd import random_payload from helpers.cmd import random_payload
@ -32,7 +33,6 @@ MAX_WORKERS = 50
def main(): def main():
bucket_list = [] bucket_list = []
objects_list = [] objects_list = []
payload_filepath = '/tmp/data_file'
ignore_errors = True if args.ignore_errors else False ignore_errors = True if args.ignore_errors else False
workers = int(args.workers) workers = int(args.workers)
@ -62,13 +62,14 @@ def main():
sys.exit(ERROR_NO_BUCKETS) sys.exit(ERROR_NO_BUCKETS)
print(f"Upload objects to each bucket: {args.preload_obj} ") print(f"Upload objects to each bucket: {args.preload_obj} ")
random_payload(payload_filepath, args.size) payload_file = tempfile.NamedTemporaryFile()
random_payload(payload_file, args.size)
print(" > Create random payload: Completed") print(" > Create random payload: Completed")
for bucket in bucket_list: for bucket in bucket_list:
print(f" > Upload objects for bucket {bucket}") print(f" > Upload objects for bucket {bucket}")
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor: with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
objects_runs = {executor.submit(upload_object, bucket, payload_filepath, objects_runs = {executor.submit(upload_object, bucket, payload_file.name,
args.endpoint): _ for _ in range(int(args.preload_obj))} args.endpoint): _ for _ in range(int(args.preload_obj))}
for run in objects_runs: for run in objects_runs: