TrueCloudLab/xk6-frostfs
42
1
Fork 20
Code Issues 33 Pull requests 1 Releases Activity Actions

[#152] Allow to set mix of policies for containers and buckets #152

Merged
fyrchik merged 1 commit from abereziny/xk6-frostfs:feature-multi-load-policy into master 2024-07-03 07:59:42 +00:00
Conversation 0 Commits 1 Files changed 4 +24 -17
4 changed files with 24 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
scenarios/preset/helpers/aws_cli.py
View file

@ -4,15 +4,16 @@ from helpers.cmd import execute_cmd, log
def create_bucket(endpoint, versioning, location, acl, no_verify_ssl):
configuration = ""
if location:
location = f"--create-bucket-configuration 'LocationConstraint={location}'"
configuration = f"--create-bucket-configuration 'LocationConstraint={location}'"
if acl:
acl = f"--acl {acl}"
bucket_name = str(uuid.uuid4())
no_verify_ssl_str = "--no-verify-ssl" if no_verify_ssl else ""
cmd_line = f"aws {no_verify_ssl_str} s3api create-bucket --bucket {bucket_name} " \
f"--endpoint {endpoint} {location} {acl} "
f"--endpoint {endpoint} {configuration} {acl} "
cmd_line_ver = f"aws {no_verify_ssl_str} s3api put-bucket-versioning --bucket {bucket_name} " \
f"--versioning-configuration Status=Enabled --endpoint {endpoint} {acl} "
@ -33,7 +34,7 @@ def create_bucket(endpoint, versioning, location, acl, no_verify_ssl):
else:
log(f"Bucket versioning has been applied for bucket {bucket_name}", endpoint)
log(f"Created bucket: {bucket_name}", endpoint)
log(f"Created bucket: {bucket_name} ({location})", endpoint)
return bucket_name

2
scenarios/preset/helpers/frostfs_cli.py
View file

@ -34,7 +34,7 @@ def create_container(endpoint, policy, wallet_path, config, acl, local=False, de
raise ValueError(f"no CID was parsed from command output:\t{fst_str}")
cid = splitted[1]
log(f"Created container {cid}", endpoint)
log(f"Created container: {cid} ({policy})", endpoint)
if not local:
return cid

13
scenarios/preset/preset_grpc.py
View file

@ -15,6 +15,7 @@ from helpers.frostfs_cli import create_container, upload_object
ERROR_WRONG_CONTAINERS_COUNT = 1
ERROR_WRONG_OBJECTS_COUNT = 2
MAX_WORKERS = 50
DEFAULT_POLICY = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
parser = argparse.ArgumentParser()
parser.add_argument('--size', help='Upload objects size in kb')
@ -25,8 +26,8 @@ parser.add_argument('--wallet', help='Wallet file path')
parser.add_argument('--config', help='Wallet config file path')
parser.add_argument(
"--policy",
fyrchik marked this conversation as resolved Outdated
fyrchik commented 2024-07-02 15:15:04 +00:00
Outdated
Review
Copy link

It breaks policies with commas in attributes being filtered.
Does parser module support list natively?

Some CLI frameworks parse --policy 1 --policy 2 as a list [1,2] -- this would be a better interface IMO

It breaks policies with commas in attributes being filtered. Does `parser` module support list natively? Some CLI frameworks parse `--policy 1 --policy 2` as a list `[1,2]` -- this would be a better interface IMO
fyrchik commented 2024-07-02 15:24:21 +00:00
Outdated
Review
Copy link
parser.add_argument('-l','--list', action='append', help='<Required> Set flag', required=True)
# Use like:
# python arg.py -l 1234 -l 2345 -l 3456 -l 4567

from https://stackoverflow.com/questions/15753701/how-can-i-pass-a-list-as-a-command-line-argument-with-argparse

``` parser.add_argument('-l','--list', action='append', help='<Required> Set flag', required=True) # Use like: # python arg.py -l 1234 -l 2345 -l 3456 -l 4567 ``` from https://stackoverflow.com/questions/15753701/how-can-i-pass-a-list-as-a-command-line-argument-with-argparse
abereziny commented 2024-07-02 17:04:18 +00:00
Outdated
Review
Copy link

Well, it requires elimination of default=REP 2 for backward compatibility, because action=append cannot work with str, and if we define default as default=["REP 2"], it will be there always.

Testing is in progress with action='append'

Well, it requires elimination of `default=REP 2` for backward compatibility, because `action=append` cannot work with str, and if we define default as `default=["REP 2"]`, it will be there always. Testing is in progress with `action='append'`
abereziny commented 2024-07-02 17:49:18 +00:00
Outdated
Review
Copy link

Reworked with action=append

Reworked with `action=append`
fyrchik commented 2024-07-03 05:04:36 +00:00
Outdated
Review
Copy link

The default can be made explicit in code (like if len is zero, use default)

The default can be made explicit in code (like if len is zero, use default)
help="Container placement policy",
default="REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
help=f"Container placement policy. Default is {DEFAULT_POLICY}",
action="append"
)
parser.add_argument('--endpoint', help='Nodes addresses separated by comma.')
parser.add_argument('--update', help='Save existed containers')
@ -46,6 +47,8 @@ def main():
objects_list = []
endpoints = args.endpoint.split(',')
if not args.policy:
args.policy = [DEFAULT_POLICY]
wallet = args.wallet
wallet_config = args.config
@ -63,9 +66,9 @@ def main():
containers_count = int(args.containers)
print(f"Create containers: {containers_count}")
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
containers_runs = [executor.submit(create_container, endpoint, args.policy, wallet, wallet_config, args.acl, args.local)
for _, endpoint in
zip(range(containers_count), cycle(endpoints))]
containers_runs = [executor.submit(create_container, endpoint, policy, wallet, wallet_config, args.acl, args.local)
for _, endpoint, policy in
zip(range(containers_count), cycle(endpoints), cycle(args.policy))]
for run in containers_runs:
container_id = run.result()

19
scenarios/preset/preset_s3.py
View file

@ -11,6 +11,11 @@ from concurrent.futures import ProcessPoolExecutor
from helpers.cmd import random_payload
from helpers.aws_cli import create_bucket, upload_object
ERROR_WRONG_CONTAINERS_COUNT = 1
ERROR_WRONG_OBJECTS_COUNT = 2
MAX_WORKERS = 50
DEFAULT_LOCATION = ""
parser = argparse.ArgumentParser()
parser.add_argument('--size', help='Upload objects size in kb.')
@ -20,7 +25,7 @@ parser.add_argument('--preload_obj', help='Number of pre-loaded objects.')
parser.add_argument('--endpoint', help='S3 Gateways addresses separated by comma.')
parser.add_argument('--update', help='True/False, False by default. Save existed buckets from target file (--out). '
'New buckets will not be created.')
parser.add_argument('--location', help='AWS location. Will be empty, if has not be declared.', default="")
parser.add_argument('--location', help=f'AWS location constraint. Default is "{DEFAULT_LOCATION}"', action="append")
parser.add_argument('--versioning', help='True/False, False by default.')
parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
parser.add_argument('--no-verify-ssl', help='Ignore SSL verifications', action='store_true')
@ -32,10 +37,6 @@ parser.add_argument('--acl', help='Bucket ACL. Default is private. Expected valu
args = parser.parse_args()
print(args)
ERROR_WRONG_CONTAINERS_COUNT = 1
ERROR_WRONG_OBJECTS_COUNT = 2
MAX_WORKERS = 50
def main():
buckets = []
objects_list = []
@ -43,6 +44,8 @@ def main():
no_verify_ssl = args.no_verify_ssl
endpoints = args.endpoint.split(',')
if not args.location:
args.location = [DEFAULT_LOCATION]
workers = int(args.workers)
objects_per_bucket = int(args.preload_obj)
@ -59,9 +62,9 @@ def main():
print(f"Create buckets: {buckets_count}")
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
buckets_runs = [executor.submit(create_bucket, endpoint, args.versioning, args.location, args.acl, no_verify_ssl)
for _, endpoint in
zip(range(buckets_count), cycle(endpoints))]
buckets_runs = [executor.submit(create_bucket, endpoint, args.versioning, location, args.acl, no_verify_ssl)
for _, endpoint, location in
zip(range(buckets_count), cycle(endpoints), cycle(args.location))]
for run in buckets_runs:
bucket_name = run.result()