TrueCloudLab/xk6-frostfs
42
1
Fork 20
Code Issues 35 Pull requests 2 Releases Activity Actions

preset: use temporary file for payload #58

Merged
fyrchik merged 1 commit from fyrchik/xk6-frostfs:fix-preset-tempfile into master 2023-05-22 09:06:10 +00:00
Conversation 2 Commits 1 Files changed 3 +11 -10
3 changed files with 11 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
scenarios/preset/helpers/cmd.py
View file

@ -19,10 +19,9 @@ def execute_cmd(cmd_line):
return output, success return output, success
def random_payload(payload_filepath, size): def random_payload(file, size):
with open('%s' % payload_filepath, 'w+b') as fout: file.write(os.urandom(1024 * int(size)))
fout.write(os.urandom(1024 * int(size))) file.flush()
class ProgressBar: class ProgressBar:
@staticmethod @staticmethod

7
scenarios/preset/preset_grpc.py
View file

@ -4,6 +4,7 @@ import argparse
import json import json
import random import random
import sys import sys
import tempfile
from argparse import Namespace from argparse import Namespace
from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ProcessPoolExecutor
@ -39,7 +40,6 @@ print(args)
def main(): def main():
container_list = [] container_list = []
objects_list = [] objects_list = []
payload_filepath = '/tmp/data_file'
endpoints = args.endpoint.split(',') endpoints = args.endpoint.split(',')
@ -71,13 +71,14 @@ def main():
sys.exit(ERROR_NO_CONTAINERS) sys.exit(ERROR_NO_CONTAINERS)
print(f"Upload objects to each container: {args.preload_obj} ") print(f"Upload objects to each container: {args.preload_obj} ")
random_payload(payload_filepath, args.size) payload_file = tempfile.NamedTemporaryFile()
random_payload(payload_file, args.size)
print(" > Create random payload: Completed") print(" > Create random payload: Completed")
for container in container_list: for container in container_list:
print(f" > Upload objects for container {container}") print(f" > Upload objects for container {container}")
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor: with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
objects_runs = {executor.submit(upload_object, container, payload_filepath, objects_runs = {executor.submit(upload_object, container, payload_file.name,
endpoints[random.randrange(len(endpoints))], wallet, wallet_config): _ for _ in range(int(args.preload_obj))} endpoints[random.randrange(len(endpoints))], wallet, wallet_config): _ for _ in range(int(args.preload_obj))}
for run in objects_runs: for run in objects_runs:

7
scenarios/preset/preset_s3.py
View file

@ -3,6 +3,7 @@
import argparse import argparse
import json import json
import sys import sys
import tempfile
anikeev-yadro commented 2023-05-11 16:22:52 +00:00
Outdated
Review
Copy link

Need to add this line to preset_grpc.py too.

root@tatlinobject:/home/service/xk6-frostfs-preset# ./scenarios/preset/preset_grpc.py --size 64 --containers 1 --out /var/data/tmp/grpc_reset1005-111.json --endpoint 10.78.70.148:8080 --preload_obj 1 --wallet scenarios/files/wallet.json --config scenarios/files/empty_pass.yaml
Namespace(size='64', containers='1', out='/var/data/tmp/grpc_reset1005-111.json', preload_obj='1', wallet='scenarios/files/wallet.json', config='scenarios/files/empty_pass.yaml', policy='REP 2 IN X CBF 2 SELECT 2 FROM * AS X', endpoint='10.78.70.148:8080', update=None, ignore_errors=None, workers=50)
Create containers: 1
Created container: 43m16EUiDmMyDKMShXjRX5xTijhLDL7vL5KFzrYkqAY4
Create containers: Completed
 > Containers: ['43m16EUiDmMyDKMShXjRX5xTijhLDL7vL5KFzrYkqAY4']
Upload objects to each container: 1
Traceback (most recent call last):
  File "/home/service/xk6-frostfs-preset/./scenarios/preset/preset_grpc.py", line 106, in <module>
    main()
  File "/home/service/xk6-frostfs-preset/./scenarios/preset/preset_grpc.py", line 73, in main
    payload_file = tempfile.NamedTemporaryFile()
NameError: name 'tempfile' is not defined
Need to add this line to preset_grpc.py too. ``` root@tatlinobject:/home/service/xk6-frostfs-preset# ./scenarios/preset/preset_grpc.py --size 64 --containers 1 --out /var/data/tmp/grpc_reset1005-111.json --endpoint 10.78.70.148:8080 --preload_obj 1 --wallet scenarios/files/wallet.json --config scenarios/files/empty_pass.yaml Namespace(size='64', containers='1', out='/var/data/tmp/grpc_reset1005-111.json', preload_obj='1', wallet='scenarios/files/wallet.json', config='scenarios/files/empty_pass.yaml', policy='REP 2 IN X CBF 2 SELECT 2 FROM * AS X', endpoint='10.78.70.148:8080', update=None, ignore_errors=None, workers=50) Create containers: 1 Created container: 43m16EUiDmMyDKMShXjRX5xTijhLDL7vL5KFzrYkqAY4 Create containers: Completed > Containers: ['43m16EUiDmMyDKMShXjRX5xTijhLDL7vL5KFzrYkqAY4'] Upload objects to each container: 1 Traceback (most recent call last): File "/home/service/xk6-frostfs-preset/./scenarios/preset/preset_grpc.py", line 106, in <module> main() File "/home/service/xk6-frostfs-preset/./scenarios/preset/preset_grpc.py", line 73, in main payload_file = tempfile.NamedTemporaryFile() NameError: name 'tempfile' is not defined ```
fyrchik commented 2023-05-12 07:35:01 +00:00
Outdated
Review
Copy link

Fixed

Fixed
from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ProcessPoolExecutor
from helpers.cmd import random_payload from helpers.cmd import random_payload
@ -32,7 +33,6 @@ MAX_WORKERS = 50
def main(): def main():
bucket_list = [] bucket_list = []
objects_list = [] objects_list = []
payload_filepath = '/tmp/data_file'
ignore_errors = True if args.ignore_errors else False ignore_errors = True if args.ignore_errors else False
workers = int(args.workers) workers = int(args.workers)
@ -62,13 +62,14 @@ def main():
sys.exit(ERROR_NO_BUCKETS) sys.exit(ERROR_NO_BUCKETS)
print(f"Upload objects to each bucket: {args.preload_obj} ") print(f"Upload objects to each bucket: {args.preload_obj} ")
random_payload(payload_filepath, args.size) payload_file = tempfile.NamedTemporaryFile()
random_payload(payload_file, args.size)
print(" > Create random payload: Completed") print(" > Create random payload: Completed")
for bucket in bucket_list: for bucket in bucket_list:
print(f" > Upload objects for bucket {bucket}") print(f" > Upload objects for bucket {bucket}")
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor: with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
objects_runs = {executor.submit(upload_object, bucket, payload_filepath, objects_runs = {executor.submit(upload_object, bucket, payload_file.name,
args.endpoint): _ for _ in range(int(args.preload_obj))} args.endpoint): _ for _ in range(int(args.preload_obj))}
for run in objects_runs: for run in objects_runs: