From bf884936a7f9998828844daaefa5158263debdaf Mon Sep 17 00:00:00 2001 From: Andrey Berezin Date: Tue, 8 Aug 2023 18:57:06 +0300 Subject: [PATCH] [#91] Improve logging for preset Signed-off-by: Andrey Berezin --- scenarios/preset/helpers/aws_cli.py | 28 ++++++++++--------- scenarios/preset/helpers/cmd.py | 5 +++- scenarios/preset/helpers/frostfs_cli.py | 36 +++++++++++++++---------- scenarios/preset/preset_grpc.py | 3 +-- scenarios/preset/preset_s3.py | 2 +- 5 files changed, 44 insertions(+), 30 deletions(-) diff --git a/scenarios/preset/helpers/aws_cli.py b/scenarios/preset/helpers/aws_cli.py index bb514bc..8dd23c4 100644 --- a/scenarios/preset/helpers/aws_cli.py +++ b/scenarios/preset/helpers/aws_cli.py @@ -1,6 +1,6 @@ import uuid -from helpers.cmd import execute_cmd +from helpers.cmd import execute_cmd, log def create_bucket(endpoint, versioning, location, no_verify_ssl): @@ -13,22 +13,24 @@ def create_bucket(endpoint, versioning, location, no_verify_ssl): cmd_line_ver = f"aws {no_verify_ssl_str} s3api put-bucket-versioning --bucket {bucket_name} " \ f"--versioning-configuration Status=Enabled --endpoint {endpoint} " - out, success = execute_cmd(cmd_line) + output, success = execute_cmd(cmd_line) - if not success and "succeeded and you already own it" not in out: - print(f" > Bucket {bucket_name} has not been created:\n{out}") + if not success and "succeeded and you already own it" not in output: + log(f"{cmd_line}\n" + f"Bucket {bucket_name} has not been created:\n" + f"Error: {output}", endpoint) return False - - print(f"cmd: {cmd_line}") if versioning == "True": - out, success = execute_cmd(cmd_line_ver) + output, success = execute_cmd(cmd_line_ver) if not success: - print(f" > Bucket versioning has not been applied for bucket {bucket_name}:\n{out}") + log(f"{cmd_line_ver}\n" + f"Bucket versioning has not been applied for bucket {bucket_name}\n" + f"Error: {output}", endpoint) else: - print(f" > Bucket versioning has been applied.") + log(f"Bucket versioning has been applied for bucket {bucket_name}", endpoint) - print(f"Created bucket: {bucket_name} via endpoint {endpoint}") + log(f"Created bucket: {bucket_name}", endpoint) return bucket_name @@ -37,10 +39,12 @@ def upload_object(bucket, payload_filepath, endpoint, no_verify_ssl): no_verify_ssl_str = "--no-verify-ssl" if no_verify_ssl else "" cmd_line = f"aws {no_verify_ssl_str} s3api put-object --bucket {bucket} --key {object_name} " \ f"--body {payload_filepath} --endpoint {endpoint}" - out, success = execute_cmd(cmd_line) + output, success = execute_cmd(cmd_line) if not success: - print(f" > Object {object_name} has not been uploaded.") + log(f"{cmd_line}\n" + f"Object {object_name} has not been uploaded\n" + f"Error: {output}", endpoint) return False return bucket, endpoint, object_name diff --git a/scenarios/preset/helpers/cmd.py b/scenarios/preset/helpers/cmd.py index 54efb43..fea18a5 100644 --- a/scenarios/preset/helpers/cmd.py +++ b/scenarios/preset/helpers/cmd.py @@ -1,9 +1,12 @@ import os import shlex import sys - +from datetime import datetime from subprocess import check_output, CalledProcessError, STDOUT +def log(message, endpoint): + time = datetime.utcnow() + print(f"{time} at {endpoint}: {message}") def execute_cmd(cmd_line): cmd_args = shlex.split(cmd_line) diff --git a/scenarios/preset/helpers/frostfs_cli.py b/scenarios/preset/helpers/frostfs_cli.py index f8bf4a0..71331a3 100644 --- a/scenarios/preset/helpers/frostfs_cli.py +++ b/scenarios/preset/helpers/frostfs_cli.py @@ -1,7 +1,5 @@ import re - -from helpers.cmd import execute_cmd - +from helpers.cmd import execute_cmd, log def create_container(endpoint, policy, wallet_file, wallet_config): cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} container create --wallet {wallet_file} --config {wallet_config} " \ @@ -10,19 +8,23 @@ def create_container(endpoint, policy, wallet_file, wallet_config): output, success = execute_cmd(cmd_line) if not success: - print(f" > Container has not been created:\n{output}") + log(f"{cmd_line}\n" + f"Container has not been created\n" + f"{output}", endpoint) return False try: fst_str = output.split('\n')[0] except Exception: - print(f"Got empty output: {output}") + log(f"{cmd_line}\n" + f"Incorrect output\n" + f"Output: {output or ''}", endpoint) return False splitted = fst_str.split(": ") if len(splitted) != 2: - raise ValueError(f"no CID was parsed from command output: \t{fst_str}") + raise ValueError(f"no CID was parsed from command output:\t{fst_str}") - print(f"Created container: {splitted[1]} via endpoint {endpoint}") + log(f"Created container {splitted[1]}", endpoint) return splitted[1] @@ -34,14 +36,18 @@ def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_con output, success = execute_cmd(cmd_line) if not success: - print(f" > Object {object_name} has not been uploaded:\n{output}") + log(f"{cmd_line}\n" + f"Object {object_name} has not been uploaded\n" + f"Error: {output}", endpoint) return False try: # taking second string from command output snd_str = output.split('\n')[1] except Exception: - print(f"Got empty input: {output}") + log(f"{cmd_line}\n" + f"Incorrect output\n" + f"Output: {output or ''}", endpoint) return False splitted = snd_str.split(": ") if len(splitted) != 2: @@ -56,8 +62,9 @@ def get_object(cid, oid, endpoint, out_filepath, wallet_file, wallet_config): output, success = execute_cmd(cmd_line) if not success: - print(f" > Failed to get object {output} from container {cid} \r\n" - f" > Error: {output}") + log(f"{cmd_line}\n" + f"Failed to get object {oid} from container {cid}\n" + f"Error: {output}", endpoint) return False return True @@ -69,13 +76,14 @@ def search_object_by_id(cid, oid, endpoint, wallet_file, wallet_config, ttl=2): output, success = execute_cmd(cmd_line) if not success: - print(f" > Failed to search object {oid} for container {cid} \r\n" - f" > Error: {output}") + log(f"{cmd_line}\n" + f"Failed to search object {oid} for container {cid}\n" + f"Error: {output}", endpoint) return False re_rst = re.search(r'Found (\d+) objects', output) if not re_rst: - raise Exception("Failed to parce search results") + raise Exception("Failed to parse search results") return re_rst.group(1) diff --git a/scenarios/preset/preset_grpc.py b/scenarios/preset/preset_grpc.py index 5459b9c..89e74f9 100755 --- a/scenarios/preset/preset_grpc.py +++ b/scenarios/preset/preset_grpc.py @@ -3,7 +3,6 @@ import argparse from itertools import cycle import json -import random import sys import tempfile import time @@ -33,7 +32,7 @@ parser.add_argument('--endpoint', help='Nodes addresses separated by comma.') parser.add_argument('--update', help='Save existed containers') parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true') parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50) -parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), ' +parser.add_argument('--sleep', help='Time to sleep between containers creation and objects upload (in seconds), ' 'Default = 8', default=8) args: Namespace = parser.parse_args() diff --git a/scenarios/preset/preset_s3.py b/scenarios/preset/preset_s3.py index 1f756cf..0ce6683 100755 --- a/scenarios/preset/preset_s3.py +++ b/scenarios/preset/preset_s3.py @@ -25,7 +25,7 @@ parser.add_argument('--versioning', help='True/False, False by default.') parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true') parser.add_argument('--no-verify-ssl', help='Ignore SSL verifications', action='store_true') parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50) -parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), ' +parser.add_argument('--sleep', help='Time to sleep between buckets creation and objects upload (in seconds), ' 'Default = 8', default=8) args = parser.parse_args()