TrueCloudLab
/
xk6-frostfs
38
0
Fork 15
Code Issues 30 Pull Requests Actions 4 Releases Activity

[#91] Improve logging for preset 

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
pull/92/head
Andrey Berezin 2023-08-08 18:57:06 +03:00
parent 5b1793f248
commit 86f5dfa47a
5 changed files with 44 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
scenarios/preset/helpers/aws_cli.py
View File

@ -1,6 +1,6 @@
import uuid
from helpers.cmd import execute_cmd
from helpers.cmd import execute_cmd, log
def create_bucket(endpoint, versioning, location, no_verify_ssl):
@ -13,22 +13,24 @@ def create_bucket(endpoint, versioning, location, no_verify_ssl):
cmd_line_ver = f"aws {no_verify_ssl_str} s3api put-bucket-versioning --bucket {bucket_name} " \
f"--versioning-configuration Status=Enabled --endpoint {endpoint} "
out, success = execute_cmd(cmd_line)
output, success = execute_cmd(cmd_line)
if not success and "succeeded and you already own it" not in out:
print(f" > Bucket {bucket_name} has not been created:\n{out}")
if not success and "succeeded and you already own it" not in output:
log(f"{cmd_line}\n"
f"Bucket {bucket_name} has not been created:\n"
f"Error: {output}", endpoint)
return False
print(f"cmd: {cmd_line}")
if versioning == "True":
out, success = execute_cmd(cmd_line_ver)
output, success = execute_cmd(cmd_line_ver)
if not success:
print(f" > Bucket versioning has not been applied for bucket {bucket_name}:\n{out}")
log(f"{cmd_line_ver}\n"
f"Bucket versioning has not been applied for bucket {bucket_name}\n"
f"Error: {output}", endpoint)
else:
print(f" > Bucket versioning has been applied.")
log(f"Bucket versioning has been applied for bucket {bucket_name}", endpoint)
print(f"Created bucket: {bucket_name} via endpoint {endpoint}")
log(f"Created bucket: {bucket_name}", endpoint)
return bucket_name
@ -37,10 +39,12 @@ def upload_object(bucket, payload_filepath, endpoint, no_verify_ssl):
no_verify_ssl_str = "--no-verify-ssl" if no_verify_ssl else ""
cmd_line = f"aws {no_verify_ssl_str} s3api put-object --bucket {bucket} --key {object_name} " \
f"--body {payload_filepath} --endpoint {endpoint}"
out, success = execute_cmd(cmd_line)
output, success = execute_cmd(cmd_line)
if not success:
print(f" > Object {object_name} has not been uploaded.")
log(f"{cmd_line}\n"
f"Object {object_name} has not been uploaded\n"
f"Error: {output}", endpoint)
return False
return bucket, endpoint, object_name

5
scenarios/preset/helpers/cmd.py
View File

@ -1,9 +1,12 @@
import os
import shlex
import sys
from datetime import datetime
from subprocess import check_output, CalledProcessError, STDOUT
def log(message, endpoint):
time = datetime.utcnow()
print(f"{time} at {endpoint}: {message}")
def execute_cmd(cmd_line):
cmd_args = shlex.split(cmd_line)

36
scenarios/preset/helpers/frostfs_cli.py
View File

@ -1,7 +1,5 @@
import re
from helpers.cmd import execute_cmd
from helpers.cmd import execute_cmd, log
def create_container(endpoint, policy, wallet_file, wallet_config):
cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} container create --wallet {wallet_file} --config {wallet_config} " \
@ -10,19 +8,23 @@ def create_container(endpoint, policy, wallet_file, wallet_config):
output, success = execute_cmd(cmd_line)
if not success:
print(f" > Container has not been created:\n{output}")
log(f"{cmd_line}\n"
f"Container has not been created\n"
f"{output}", endpoint)
return False
try:
fst_str = output.split('\n')[0]
except Exception:
print(f"Got empty output: {output}")
log(f"{cmd_line}\n"
f"Incorrect output\n"
f"Output: {output or '<empty>'}", endpoint)
return False
splitted = fst_str.split(": ")
if len(splitted) != 2:
raise ValueError(f"no CID was parsed from command output: \t{fst_str}")
raise ValueError(f"no CID was parsed from command output:\t{fst_str}")
print(f"Created container: {splitted[1]} via endpoint {endpoint}")
log(f"Created container {splitted[1]}", endpoint)
return splitted[1]
@ -34,14 +36,18 @@ def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_con
output, success = execute_cmd(cmd_line)
if not success:
print(f" > Object {object_name} has not been uploaded:\n{output}")
log(f"{cmd_line}\n"
f"Object {object_name} has not been uploaded\n"
f"Error: {output}", endpoint)
return False
try:
# taking second string from command output
snd_str = output.split('\n')[1]
except Exception:
print(f"Got empty input: {output}")
log(f"{cmd_line}\n"
f"Incorrect output\n"
f"Output: {output or '<empty>'}", endpoint)
return False
splitted = snd_str.split(": ")
if len(splitted) != 2:
@ -56,8 +62,9 @@ def get_object(cid, oid, endpoint, out_filepath, wallet_file, wallet_config):
output, success = execute_cmd(cmd_line)
if not success:
print(f" > Failed to get object {output} from container {cid} \r\n"
f" > Error: {output}")
log(f"{cmd_line}\n"
f"Failed to get object {oid} from container {cid}\n"
f"Error: {output}", endpoint)
return False
return True
@ -69,13 +76,14 @@ def search_object_by_id(cid, oid, endpoint, wallet_file, wallet_config, ttl=2):
output, success = execute_cmd(cmd_line)
if not success:
print(f" > Failed to search object {oid} for container {cid} \r\n"
f" > Error: {output}")
log(f"{cmd_line}\n"
f"Failed to search object {oid} for container {cid}\n"
f"Error: {output}", endpoint)
return False
re_rst = re.search(r'Found (\d+) objects', output)
if not re_rst:
raise Exception("Failed to parce search results")
raise Exception("Failed to parse search results")
return re_rst.group(1)

3
scenarios/preset/preset_grpc.py
View File

@ -3,7 +3,6 @@
import argparse
from itertools import cycle
import json
import random
import sys
import tempfile
import time
@ -33,7 +32,7 @@ parser.add_argument('--endpoint', help='Nodes addresses separated by comma.')
parser.add_argument('--update', help='Save existed containers')
parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50)
parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), '
parser.add_argument('--sleep', help='Time to sleep between containers creation and objects upload (in seconds), '
'Default = 8', default=8)
args: Namespace = parser.parse_args()

2
scenarios/preset/preset_s3.py
View File

@ -25,7 +25,7 @@ parser.add_argument('--versioning', help='True/False, False by default.')
parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
parser.add_argument('--no-verify-ssl', help='Ignore SSL verifications', action='store_true')
parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50)
parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), '
parser.add_argument('--sleep', help='Time to sleep between buckets creation and objects upload (in seconds), '
'Default = 8', default=8)
args = parser.parse_args()