From 9b9db46a07c44739926dc195c2c9df90ed76665f Mon Sep 17 00:00:00 2001 From: "a.berezin" Date: Tue, 2 Jul 2024 20:45:31 +0300 Subject: [PATCH] [#152] Allow to set mix of policies for containers and buckets Signed-off-by: a.berezin --- scenarios/preset/helpers/aws_cli.py | 7 ++++--- scenarios/preset/helpers/frostfs_cli.py | 2 +- scenarios/preset/preset_grpc.py | 13 ++++++++----- scenarios/preset/preset_s3.py | 19 +++++++++++-------- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/scenarios/preset/helpers/aws_cli.py b/scenarios/preset/helpers/aws_cli.py index 5a85a7f..f59d629 100644 --- a/scenarios/preset/helpers/aws_cli.py +++ b/scenarios/preset/helpers/aws_cli.py @@ -4,15 +4,16 @@ from helpers.cmd import execute_cmd, log def create_bucket(endpoint, versioning, location, acl, no_verify_ssl): + configuration = "" if location: - location = f"--create-bucket-configuration 'LocationConstraint={location}'" + configuration = f"--create-bucket-configuration 'LocationConstraint={location}'" if acl: acl = f"--acl {acl}" bucket_name = str(uuid.uuid4()) no_verify_ssl_str = "--no-verify-ssl" if no_verify_ssl else "" cmd_line = f"aws {no_verify_ssl_str} s3api create-bucket --bucket {bucket_name} " \ - f"--endpoint {endpoint} {location} {acl} " + f"--endpoint {endpoint} {configuration} {acl} " cmd_line_ver = f"aws {no_verify_ssl_str} s3api put-bucket-versioning --bucket {bucket_name} " \ f"--versioning-configuration Status=Enabled --endpoint {endpoint} {acl} " @@ -33,7 +34,7 @@ def create_bucket(endpoint, versioning, location, acl, no_verify_ssl): else: log(f"Bucket versioning has been applied for bucket {bucket_name}", endpoint) - log(f"Created bucket: {bucket_name}", endpoint) + log(f"Created bucket: {bucket_name} ({location})", endpoint) return bucket_name diff --git a/scenarios/preset/helpers/frostfs_cli.py b/scenarios/preset/helpers/frostfs_cli.py index f49f30c..60c88bf 100644 --- a/scenarios/preset/helpers/frostfs_cli.py +++ b/scenarios/preset/helpers/frostfs_cli.py @@ -34,7 +34,7 @@ def create_container(endpoint, policy, wallet_path, config, acl, local=False, de raise ValueError(f"no CID was parsed from command output:\t{fst_str}") cid = splitted[1] - log(f"Created container {cid}", endpoint) + log(f"Created container: {cid} ({policy})", endpoint) if not local: return cid diff --git a/scenarios/preset/preset_grpc.py b/scenarios/preset/preset_grpc.py index 0f3e8fe..ac27abe 100755 --- a/scenarios/preset/preset_grpc.py +++ b/scenarios/preset/preset_grpc.py @@ -15,6 +15,7 @@ from helpers.frostfs_cli import create_container, upload_object ERROR_WRONG_CONTAINERS_COUNT = 1 ERROR_WRONG_OBJECTS_COUNT = 2 MAX_WORKERS = 50 +DEFAULT_POLICY = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" parser = argparse.ArgumentParser() parser.add_argument('--size', help='Upload objects size in kb') @@ -25,8 +26,8 @@ parser.add_argument('--wallet', help='Wallet file path') parser.add_argument('--config', help='Wallet config file path') parser.add_argument( "--policy", - help="Container placement policy", - default="REP 2 IN X CBF 2 SELECT 2 FROM * AS X" + help=f"Container placement policy. Default is {DEFAULT_POLICY}", + action="append" ) parser.add_argument('--endpoint', help='Nodes addresses separated by comma.') parser.add_argument('--update', help='Save existed containers') @@ -46,6 +47,8 @@ def main(): objects_list = [] endpoints = args.endpoint.split(',') + if not args.policy: + args.policy = [DEFAULT_POLICY] wallet = args.wallet wallet_config = args.config @@ -63,9 +66,9 @@ def main(): containers_count = int(args.containers) print(f"Create containers: {containers_count}") with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor: - containers_runs = [executor.submit(create_container, endpoint, args.policy, wallet, wallet_config, args.acl, args.local) - for _, endpoint in - zip(range(containers_count), cycle(endpoints))] + containers_runs = [executor.submit(create_container, endpoint, policy, wallet, wallet_config, args.acl, args.local) + for _, endpoint, policy in + zip(range(containers_count), cycle(endpoints), cycle(args.policy))] for run in containers_runs: container_id = run.result() diff --git a/scenarios/preset/preset_s3.py b/scenarios/preset/preset_s3.py index 637dac6..ec9edc4 100755 --- a/scenarios/preset/preset_s3.py +++ b/scenarios/preset/preset_s3.py @@ -11,6 +11,11 @@ from concurrent.futures import ProcessPoolExecutor from helpers.cmd import random_payload from helpers.aws_cli import create_bucket, upload_object +ERROR_WRONG_CONTAINERS_COUNT = 1 +ERROR_WRONG_OBJECTS_COUNT = 2 +MAX_WORKERS = 50 +DEFAULT_LOCATION = "" + parser = argparse.ArgumentParser() parser.add_argument('--size', help='Upload objects size in kb.') @@ -20,7 +25,7 @@ parser.add_argument('--preload_obj', help='Number of pre-loaded objects.') parser.add_argument('--endpoint', help='S3 Gateways addresses separated by comma.') parser.add_argument('--update', help='True/False, False by default. Save existed buckets from target file (--out). ' 'New buckets will not be created.') -parser.add_argument('--location', help='AWS location. Will be empty, if has not be declared.', default="") +parser.add_argument('--location', help=f'AWS location constraint. Default is "{DEFAULT_LOCATION}"', action="append") parser.add_argument('--versioning', help='True/False, False by default.') parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true') parser.add_argument('--no-verify-ssl', help='Ignore SSL verifications', action='store_true') @@ -32,10 +37,6 @@ parser.add_argument('--acl', help='Bucket ACL. Default is private. Expected valu args = parser.parse_args() print(args) -ERROR_WRONG_CONTAINERS_COUNT = 1 -ERROR_WRONG_OBJECTS_COUNT = 2 -MAX_WORKERS = 50 - def main(): buckets = [] objects_list = [] @@ -43,6 +44,8 @@ def main(): no_verify_ssl = args.no_verify_ssl endpoints = args.endpoint.split(',') + if not args.location: + args.location = [DEFAULT_LOCATION] workers = int(args.workers) objects_per_bucket = int(args.preload_obj) @@ -59,9 +62,9 @@ def main(): print(f"Create buckets: {buckets_count}") with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor: - buckets_runs = [executor.submit(create_bucket, endpoint, args.versioning, args.location, args.acl, no_verify_ssl) - for _, endpoint in - zip(range(buckets_count), cycle(endpoints))] + buckets_runs = [executor.submit(create_bucket, endpoint, args.versioning, location, args.acl, no_verify_ssl) + for _, endpoint, location in + zip(range(buckets_count), cycle(endpoints), cycle(args.location))] for run in buckets_runs: bucket_name = run.result() -- 2.45.2