From ed25e553b22880cae6f62ff72d44646ab659158e Mon Sep 17 00:00:00 2001 From: "a.chetaev" Date: Thu, 17 Nov 2022 20:08:20 +0300 Subject: [PATCH] [#38] Move common function to helpers for preset Signed-off-by: a.chetaev --- scenarios/preset/check_objects_in_preset.py | 31 ++------ scenarios/preset/helpers/aws_cli.py | 50 +++++++++++++ scenarios/preset/helpers/cmd.py | 23 ++++++ scenarios/preset/helpers/neofs_cli.py | 61 +++++++++++++++ scenarios/preset/preset_grpc.py | 83 ++++----------------- scenarios/preset/preset_s3.py | 83 +++------------------ 6 files changed, 163 insertions(+), 168 deletions(-) create mode 100644 scenarios/preset/helpers/aws_cli.py create mode 100644 scenarios/preset/helpers/cmd.py create mode 100644 scenarios/preset/helpers/neofs_cli.py diff --git a/scenarios/preset/check_objects_in_preset.py b/scenarios/preset/check_objects_in_preset.py index aa62626..4c03592 100755 --- a/scenarios/preset/check_objects_in_preset.py +++ b/scenarios/preset/check_objects_in_preset.py @@ -2,13 +2,12 @@ import argparse import json -import shlex -from subprocess import check_output, CalledProcessError, STDOUT + +from helpers.neofs_cli import get_object parser = argparse.ArgumentParser() parser.add_argument('--endpoint', help='Node address') parser.add_argument('--preset_json', help='JSON file path with preset') -parser.add_argument('--print_success', help='Print objects that was successfully read', default=False) args = parser.parse_args() @@ -26,36 +25,16 @@ def main(): oid = obj.get('object') cid = obj.get('container') - cmd_line = f"neofs-cli object get -r {args.endpoint} -g" \ - f" --cid {cid} --oid {oid} --file /dev/null" + rst = get_object(cid, oid, args.endpoint, "/dev/null") - output, success = execute_cmd(cmd_line) - - if success: + if rst: success_objs += 1 - if args.print_success: - print(f'Object: {oid} from {cid}: {"Ok" if success else "False"}') else: failed_objs += 1 - print(f'Object: {oid} from {cid}: {"Ok" if success else "False"}') print(f'Success objects: {success_objs}') print(f'Failed objects: {failed_objs}') -def execute_cmd(cmd_line): - cmd_args = shlex.split(cmd_line) - - try: - output = check_output(cmd_args, stderr=STDOUT).decode() - success = True - - except CalledProcessError as e: - output = e.output.decode() - success = False - - return output, success - - if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scenarios/preset/helpers/aws_cli.py b/scenarios/preset/helpers/aws_cli.py new file mode 100644 index 0000000..6964fb9 --- /dev/null +++ b/scenarios/preset/helpers/aws_cli.py @@ -0,0 +1,50 @@ +import uuid + +from helpers.cmd import execute_cmd + + +def create_bucket(endpoint, versioning, location): + bucket_create_marker = False + + if location: + location = f"--create-bucket-configuration 'LocationConstraint={location}'" + bucket_name = str(uuid.uuid4()) + + cmd_line = f"aws --no-verify-ssl s3api create-bucket --bucket {bucket_name} " \ + f"--endpoint http://{endpoint} {location}" + cmd_line_ver = f"aws --no-verify-ssl s3api put-bucket-versioning --bucket {bucket_name} " \ + f"--versioning-configuration Status=Enabled --endpoint http://{endpoint} " + + out, success = execute_cmd(cmd_line) + + if not success: + if "succeeded and you already own it" in out: + bucket_create_marker = True + else: + print(f" > Bucket {bucket_name} has not been created.") + else: + bucket_create_marker = True + print(f"cmd: {cmd_line}") + + if bucket_create_marker and versioning == "True": + out, success = execute_cmd(cmd_line_ver) + if not success: + print(f" > Bucket versioning has not been applied for bucket {bucket_name}.") + else: + print(f" > Bucket versioning has been applied.") + + return bucket_name + + +def upload_object(bucket, payload_filepath, endpoint): + object_name = str(uuid.uuid4()) + + cmd_line = f"aws s3api put-object --bucket {bucket} --key {object_name} " \ + f"--body {payload_filepath} --endpoint http://{endpoint}" + out, success = execute_cmd(cmd_line) + + if not success: + print(f" > Object {object_name} has not been uploaded.") + return False + else: + return object_name diff --git a/scenarios/preset/helpers/cmd.py b/scenarios/preset/helpers/cmd.py new file mode 100644 index 0000000..229d8b1 --- /dev/null +++ b/scenarios/preset/helpers/cmd.py @@ -0,0 +1,23 @@ +import os +import shlex + +from subprocess import check_output, CalledProcessError, STDOUT + + +def execute_cmd(cmd_line): + cmd_args = shlex.split(cmd_line) + + try: + output = check_output(cmd_args, stderr=STDOUT).decode() + success = True + + except CalledProcessError as e: + output = e.output.decode() + success = False + + return output, success + + +def random_payload(payload_filepath, size): + with open('%s' % payload_filepath, 'w+b') as fout: + fout.write(os.urandom(1024 * int(size))) diff --git a/scenarios/preset/helpers/neofs_cli.py b/scenarios/preset/helpers/neofs_cli.py new file mode 100644 index 0000000..b45038d --- /dev/null +++ b/scenarios/preset/helpers/neofs_cli.py @@ -0,0 +1,61 @@ +from helpers.cmd import execute_cmd + + +def create_container(endpoint, policy): + cmd_line = f"neofs-cli --rpc-endpoint {endpoint} container create -g" \ + f" --policy '{policy}' --basic-acl public-read-write --await" + + output, success = execute_cmd(cmd_line) + + if not success: + print(f" > Container has not been created:\n{output}") + return False + else: + try: + fst_str = output.split('\n')[0] + except Exception: + print(f"Got empty output: {output}") + return False + splitted = fst_str.split(": ") + if len(splitted) != 2: + raise ValueError(f"no CID was parsed from command output: \t{fst_str}") + + print(f"Created container: {splitted[1]}") + + return splitted[1] + + +def upload_object(container, payload_filepath, endpoint): + object_name = "" + cmd_line = f"neofs-cli --rpc-endpoint {endpoint} object put -g --file {payload_filepath} " \ + f"--cid {container} --no-progress" + out, success = execute_cmd(cmd_line) + + if not success: + print(f" > Object {object_name} has not been uploaded:\n{out}") + return False + else: + try: + # taking second string from command output + snd_str = out.split('\n')[1] + except Exception: + print(f"Got empty input: {out}") + return False + splitted = snd_str.split(": ") + if len(splitted) != 2: + raise Exception(f"no OID was parsed from command output: \t{snd_str}") + return splitted[1] + + +def get_object(cid, oid, endpoint, out_filepath): + cmd_line = f"neofs-cli object get -r {endpoint} -g --cid {cid} --oid {oid} " \ + f"--file {out_filepath}" + + out, success = execute_cmd(cmd_line) + + if not success: + print(f" > Failed to get object {oid} from container {cid} \r\n" + f" > Error: {out}") + return False + + return True diff --git a/scenarios/preset/preset_grpc.py b/scenarios/preset/preset_grpc.py index a56a2c8..8635464 100755 --- a/scenarios/preset/preset_grpc.py +++ b/scenarios/preset/preset_grpc.py @@ -2,10 +2,12 @@ import argparse import json -import os -import shlex +from argparse import Namespace + from concurrent.futures import ProcessPoolExecutor -from subprocess import check_output, CalledProcessError, STDOUT + +from helpers.cmd import random_payload +from helpers.neofs_cli import create_container, upload_object parser = argparse.ArgumentParser() parser.add_argument('--size', help='Upload objects size in kb') @@ -15,12 +17,12 @@ parser.add_argument('--preload_obj', help='Number of pre-loaded objects') parser.add_argument( "--policy", help="Container placement policy", - default="REP 1 IN X CBF 1 SELECT 1 FROM * AS X" + default="REP 2 IN X CBF 2 SELECT 2 FROM * AS X" ) parser.add_argument('--endpoint', help='Node address') parser.add_argument('--update', help='Save existed containers') -args = parser.parse_args() +args: Namespace = parser.parse_args() print(args) @@ -37,10 +39,11 @@ def main(): else: print(f"Create containers: {args.containers}") with ProcessPoolExecutor(max_workers=10) as executor: - containers_runs = {executor.submit(create_container): _ for _ in range(int(args.containers))} + containers_runs = {executor.submit(create_container, args.endpoint, args.policy): _ for _ in + range(int(args.containers))} for run in containers_runs: - if run.result() is not None: + if run.result(): container_list.append(run.result()) print("Create containers: Completed") @@ -50,17 +53,17 @@ def main(): return print(f"Upload objects to each container: {args.preload_obj} ") - random_payload(payload_filepath) + random_payload(payload_filepath, args.size) print(" > Create random payload: Completed") for container in container_list: print(f" > Upload objects for container {container}") with ProcessPoolExecutor(max_workers=50) as executor: - objects_runs = {executor.submit(upload_object, container, payload_filepath): _ for _ in + objects_runs = {executor.submit(upload_object, container, payload_filepath, args.endpoint): _ for _ in range(int(args.preload_obj))} for run in objects_runs: - if run.result() is not None: + if run.result(): objects_struct.append({'container': container, 'object': run.result()}) print(f" > Upload objects for container {container}: Completed") @@ -76,65 +79,5 @@ def main(): print(f" > Total Objects has been created: {len(objects_struct)}.") -def random_payload(payload_filepath): - with open('%s' % payload_filepath, 'w+b') as fout: - fout.write(os.urandom(1024 * int(args.size))) - - -def execute_cmd(cmd_line): - args = shlex.split(cmd_line) - output = "" - try: - output = check_output(args, stderr=STDOUT).decode() - success = True - - except CalledProcessError as e: - output = e.output.decode() - success = False - - return output, success - - -def create_container(): - cmd_line = f"neofs-cli --rpc-endpoint {args.endpoint} container create -g --policy '{args.policy}' --basic-acl public-read-write --await" - output, success = execute_cmd(cmd_line) - - if not success: - print(f" > Container has not been created:\n{output}") - else: - try: - fst_str = output.split('\n')[0] - except Exception: - print(f"Got empty output: {output}") - return - splitted = fst_str.split(": ") - if len(splitted) != 2: - raise ValueError(f"no CID was parsed from command output: \t{fst_str}") - - print(f"Created container: {splitted[1]}") - - return splitted[1] - - -def upload_object(container, payload_filepath): - object_name = "" - cmd_line = f"neofs-cli --rpc-endpoint {args.endpoint} object put -g --file {payload_filepath} --cid {container} --no-progress" - out, success = execute_cmd(cmd_line) - - if not success: - print(f" > Object {object_name} has not been uploaded:\n{out}") - else: - try: - # taking second string from command output - snd_str = out.split('\n')[1] - except: - print(f"Got empty input: {out}") - return - splitted = snd_str.split(": ") - if len(splitted) != 2: - raise Exception(f"no OID was parsed from command output: \t{snd_str}") - return splitted[1] - - if __name__ == "__main__": main() diff --git a/scenarios/preset/preset_s3.py b/scenarios/preset/preset_s3.py index e2bdc7d..d1a374d 100755 --- a/scenarios/preset/preset_s3.py +++ b/scenarios/preset/preset_s3.py @@ -2,11 +2,10 @@ import argparse import json -import os -import shlex -import uuid from concurrent.futures import ProcessPoolExecutor -from subprocess import check_output, CalledProcessError, STDOUT + +from helpers.cmd import random_payload +from helpers.aws_cli import create_bucket, upload_object parser = argparse.ArgumentParser() @@ -15,8 +14,9 @@ parser.add_argument('--buckets', help='Number of buckets to create.') parser.add_argument('--out', help='JSON file with output.') parser.add_argument('--preload_obj', help='Number of pre-loaded objects.') parser.add_argument('--endpoint', help='S3 Gateway address.') -parser.add_argument('--update', help='True/False, False by default. Save existed buckets from target file (--out). New buckets will not be created.') -parser.add_argument('--location', help='AWS location. Will be empty, if has not be declared.') +parser.add_argument('--update', help='True/False, False by default. Save existed buckets from target file (--out). ' + 'New buckets will not be created.') +parser.add_argument('--location', help='AWS location. Will be empty, if has not be declared.', default="") parser.add_argument('--versioning', help='True/False, False by default.') args = parser.parse_args() @@ -38,7 +38,8 @@ def main(): print(f"Create buckets: {args.buckets}") with ProcessPoolExecutor(max_workers=10) as executor: - buckets_runs = {executor.submit(create_bucket): _ for _ in range(int(args.buckets))} + buckets_runs = {executor.submit(create_bucket, args.endpoint, args.versioning, + args.location): _ for _ in range(int(args.buckets))} for run in buckets_runs: if run.result() is not None: @@ -49,13 +50,14 @@ def main(): print(f" > Buckets: {bucket_list}") print(f"Upload objects to each bucket: {args.preload_obj} ") - random_payload(payload_filepath) + random_payload(payload_filepath, args.size) print(" > Create random payload: Completed") for bucket in bucket_list: print(f" > Upload objects for bucket {bucket}") with ProcessPoolExecutor(max_workers=50) as executor: - objects_runs = {executor.submit(upload_object, bucket, payload_filepath): _ for _ in range(int(args.preload_obj))} + objects_runs = {executor.submit(upload_object, bucket, payload_filepath, + args.endpoint): _ for _ in range(int(args.preload_obj))} for run in objects_runs: if run.result() is not None: @@ -74,68 +76,5 @@ def main(): print(f" > Total Objects has been created: {len(objects_struct)}.") -def random_payload(payload_filepath): - with open('%s' % payload_filepath, 'w+b') as fout: - fout.write(os.urandom(1024 * int(args.size))) - - -def execute_cmd(cmd_line): - args = shlex.split(cmd_line) - output = "" - try: - output = check_output(args, stderr=STDOUT).decode() - success = True - - except CalledProcessError as e: - output = e.output.decode() - success = False - - return output, success - - -def create_bucket(): - bucket_create_marker = False - - location = "" - if args.location: - location = f"--create-bucket-configuration 'LocationConstraint={args.location}'" - bucket_name = str(uuid.uuid4()) - - cmd_line = f"aws --no-verify-ssl s3api create-bucket --bucket {bucket_name} --endpoint http://{args.endpoint} {location}" - cmd_line_ver = f"aws --no-verify-ssl s3api put-bucket-versioning --bucket {bucket_name} --versioning-configuration Status=Enabled --endpoint http://{args.endpoint} " - - out, success = execute_cmd(cmd_line) - - if not success: - if "succeeded and you already own it" in out: - bucket_create_marker = True - else: - print(f" > Bucket {bucket_name} has not been created.") - else: - bucket_create_marker = True - print(f"cmd: {cmd_line}") - - if bucket_create_marker and args.versioning == "True": - out, success = execute_cmd(cmd_line_ver) - if not success: - print(f" > Bucket versioning has not been applied for bucket {bucket_name}.") - else: - print(f" > Bucket versioning has been applied.") - - return bucket_name - - -def upload_object(bucket, payload_filepath): - object_name = str(uuid.uuid4()) - - cmd_line = f"aws s3api put-object --bucket {bucket} --key {object_name} --body {payload_filepath} --endpoint http://{args.endpoint}" - out, success = execute_cmd(cmd_line) - - if not success: - print(f" > Object {object_name} has not been uploaded.") - else: - return object_name - - if __name__ == "__main__": main()