TrueCloudLab/xk6-frostfs
40
0
Fork 19
Code Issues 35 Pull requests 1 Releases Activity Actions

[#81] remove schema from preset_s3 and k6 load s3 scenarios

Browse source
This commit is contained in:
Andrey Berezin 2023-07-13 15:50:03 +03:00 committed by Andrey Berezin
parent ff6814e15d
commit 6182d47b43
7 changed files with 29 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -99,13 +99,13 @@ Credentials are taken from default AWS configuration files and ENVs.
```js
import s3 from 'k6/x/frostfs/s3';
const s3_cli = s3.connect("http://s3.frostfs.devenv:8080")
const s3_cli = s3.connect("https://s3.frostfs.devenv:8080")
```
You can also provide additional options:
```js
import s3 from 'k6/x/frostfs/s3';
const s3_cli = s3.connect("http://s3.frostfs.devenv:8080", {'no_verify_ssl': 'true', 'timeout': '60s'})
const s3_cli = s3.connect("https://s3.frostfs.devenv:8080", {'no_verify_ssl': 'true', 'timeout': '60s'})
```
* `no_verify_ss` - Bool. If `true` - skip verifying the s3 certificate chain and host name (useful if s3 uses self-signed certificates)

20
scenarios/preset/helpers/aws_cli.py
View file

@ -3,15 +3,15 @@ import uuid
from helpers.cmd import execute_cmd
def create_bucket(endpoint, versioning, location):
def create_bucket(endpoint, versioning, location, no_verify_ssl):
if location:
location = f"--create-bucket-configuration 'LocationConstraint={location}'"
bucket_name = str(uuid.uuid4())
cmd_line = f"aws --no-verify-ssl s3api create-bucket --bucket {bucket_name} " \
f"--endpoint http://{endpoint} {location}"
cmd_line_ver = f"aws --no-verify-ssl s3api put-bucket-versioning --bucket {bucket_name} " \
f"--versioning-configuration Status=Enabled --endpoint http://{endpoint} "
no_verify_ssl_str = "--no-verify-ssl" if no_verify_ssl else ""
cmd_line = f"aws {no_verify_ssl_str} s3api create-bucket --bucket {bucket_name} " \
f"--endpoint {endpoint} {location}"
cmd_line_ver = f"aws {no_verify_ssl_str} s3api put-bucket-versioning --bucket {bucket_name} " \
f"--versioning-configuration Status=Enabled --endpoint {endpoint} "
out, success = execute_cmd(cmd_line)
@ -32,11 +32,11 @@ def create_bucket(endpoint, versioning, location):
return bucket_name
def upload_object(bucket, payload_filepath, endpoint):
def upload_object(bucket, payload_filepath, endpoint, no_verify_ssl):
object_name = str(uuid.uuid4())
cmd_line = f"aws --no-verify-ssl s3api put-object --bucket {bucket} --key {object_name} " \
f"--body {payload_filepath} --endpoint http://{endpoint}"
no_verify_ssl_str = "--no-verify-ssl" if no_verify_ssl else ""
cmd_line = f"aws {no_verify_ssl_str} s3api put-object --bucket {bucket} --key {object_name} " \
f"--body {payload_filepath} --endpoint {endpoint}"
out, success = execute_cmd(cmd_line)
if not success:

5
scenarios/preset/preset_grpc.py
View file

@ -10,7 +10,6 @@ import time
from argparse import Namespace
from concurrent.futures import ProcessPoolExecutor
from helpers.cmd import random_payload
from helpers.frostfs_cli import create_container, upload_object
@ -32,7 +31,7 @@ parser.add_argument(
)
parser.add_argument('--endpoint', help='Nodes addresses separated by comma.')
parser.add_argument('--update', help='Save existed containers')
parser.add_argument('--ignore-errors', help='Ignore preset errors')
parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50)
parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), '
'Default = 8', default=8)
@ -52,7 +51,7 @@ def main():
workers = int(args.workers)
objects_per_container = int(args.preload_obj)
ignore_errors = True if args.ignore_errors else False
ignore_errors = args.ignore_errors
if args.update:
# Open file
with open(args.out) as f:

10
scenarios/preset/preset_s3.py
View file

@ -22,7 +22,8 @@ parser.add_argument('--update', help='True/False, False by default. Save existed
'New buckets will not be created.')
parser.add_argument('--location', help='AWS location. Will be empty, if has not be declared.', default="")
parser.add_argument('--versioning', help='True/False, False by default.')
parser.add_argument('--ignore-errors', help='Ignore preset errors')
parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
parser.add_argument('--no-verify-ssl', help='Ignore SSL verifications', action='store_true')
parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50)
parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), '
'Default = 8', default=8)
@ -37,7 +38,8 @@ MAX_WORKERS = 50
def main():
buckets = []
objects_list = []
ignore_errors = True if args.ignore_errors else False
ignore_errors = args.ignore_errors
no_verify_ssl = args.no_verify_ssl
endpoints = args.endpoint.split(',')
@ -56,7 +58,7 @@ def main():
print(f"Create buckets: {buckets_count}")
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
buckets_runs = [executor.submit(create_bucket, endpoint, args.versioning, args.location)
buckets_runs = [executor.submit(create_bucket, endpoint, args.versioning, args.location, no_verify_ssl)
for _, endpoint in
zip(range(buckets_count), cycle(endpoints))]
@ -85,7 +87,7 @@ def main():
total_objects = objects_per_bucket * buckets_count
with ProcessPoolExecutor(max_workers=min(MAX_WORKERS, workers)) as executor:
objects_runs = [executor.submit(upload_object, bucket, payload_file.name, endpoint)
objects_runs = [executor.submit(upload_object, bucket, payload_file.name, endpoint, no_verify_ssl)
for _, bucket, endpoint in
zip(range(total_objects), cycle(buckets), cycle(endpoints))]

4
scenarios/s3.js
View file

@ -21,10 +21,12 @@ const bucket_list = new SharedArray('bucket_list', function () {
const read_size = JSON.parse(open(__ENV.PREGEN_JSON)).obj_size;
const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
const no_verify_ssl = __ENV.NO_VERIFY_SSL || "true";
const connection_args = {no_verify_ssl: no_verify_ssl}
// Select random S3 endpoint for current VU
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)];
const s3_client = s3.connect(`http://${s3_endpoint}`);
const s3_client = s3.connect(s3_endpoint, connection_args);
const log = logging.new().withField("endpoint", s3_endpoint);
const registry_enabled = !!__ENV.REGISTRY_FILE;

4
scenarios/s3_car.js
View file

@ -24,7 +24,9 @@ const summary_json = __ENV.SUMMARY_JSON || "/tmp/summary.json";
// Select random S3 endpoint for current VU
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)];
const s3_client = s3.connect(`http://${s3_endpoint}`);
const no_verify_ssl = __ENV.NO_VERIFY_SSL || "true";
const connection_args = {no_verify_ssl: no_verify_ssl}
const s3_client = s3.connect(s3_endpoint, connection_args);
const log = logging.new().withField("endpoint", s3_endpoint);
const registry_enabled = !!__ENV.REGISTRY_FILE;

4
scenarios/verify.js
View file

@ -43,10 +43,12 @@ if (__ENV.GRPC_ENDPOINTS) {
// Connect to random S3 endpoint
let s3_client = undefined;
if (__ENV.S3_ENDPOINTS) {
const no_verify_ssl = __ENV.NO_VERIFY_SSL || "true";
const connection_args = {no_verify_ssl: no_verify_ssl}
const s3_endpoints = __ENV.S3_ENDPOINTS.split(',');
const s3_endpoint = s3_endpoints[Math.floor(Math.random() * s3_endpoints.length)];
log = log.withField("endpoint", s3_endpoint);
s3_client = s3.connect(`http://${s3_endpoint}`);
s3_client = s3.connect(s3_endpoint, connection_args);
}
// We will attempt to verify every object in "created" status. The scenario will execute