[#91] Improve logging for preset

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>

scenarios/preset/helpers/aws_cli.py

@@ -1,6 +1,6 @@
 import uuid
 
-from helpers.cmd import execute_cmd
+from helpers.cmd import execute_cmd, log
 
 
 def create_bucket(endpoint, versioning, location, no_verify_ssl):
@@ -13,22 +13,24 @@ def create_bucket(endpoint, versioning, location, no_verify_ssl):
     cmd_line_ver = f"aws {no_verify_ssl_str} s3api put-bucket-versioning --bucket {bucket_name} " \
                    f"--versioning-configuration Status=Enabled --endpoint {endpoint} "
 
-    out, success = execute_cmd(cmd_line)
+    output, success = execute_cmd(cmd_line)
 
-    if not success and "succeeded and you already own it" not in out:
+    if not success and "succeeded and you already own it" not in output:
-        print(f" > Bucket {bucket_name} has not been created:\n{out}")
+        log(f"{cmd_line}\n"
+            f"Bucket {bucket_name} has not been created:\n"
+            f"Error: {output}", endpoint)
         return False
 
-    print(f"cmd: {cmd_line}")
-
     if versioning == "True":
-        out, success = execute_cmd(cmd_line_ver)
+        output, success = execute_cmd(cmd_line_ver)
         if not success:
-            print(f" > Bucket versioning has not been applied for bucket {bucket_name}:\n{out}")
+            log(f"{cmd_line_ver}\n"
+                f"Bucket versioning has not been applied for bucket {bucket_name}\n"
+                f"Error: {output}", endpoint)
         else:
-            print(f" > Bucket versioning has been applied.")
+            log(f"Bucket versioning has been applied for bucket {bucket_name}", endpoint)
     
-    print(f"Created bucket: {bucket_name} via endpoint {endpoint}")
+    log(f"Created bucket: {bucket_name}", endpoint)
     return bucket_name
 
 
@@ -37,10 +39,12 @@ def upload_object(bucket, payload_filepath, endpoint, no_verify_ssl):
     no_verify_ssl_str = "--no-verify-ssl" if no_verify_ssl else ""
     cmd_line = f"aws {no_verify_ssl_str} s3api put-object --bucket {bucket} --key {object_name} " \
                f"--body {payload_filepath} --endpoint {endpoint}"
-    out, success = execute_cmd(cmd_line)
+    output, success = execute_cmd(cmd_line)
 
     if not success:
-        print(f" > Object {object_name} has not been uploaded.")
+        log(f"{cmd_line}\n"
+            f"Object {object_name} has not been uploaded\n"
+            f"Error: {output}", endpoint)
         return False
     
     return bucket, endpoint, object_name

scenarios/preset/helpers/cmd.py

@@ -1,9 +1,12 @@
 import os
 import shlex
 import sys
-
+from datetime import datetime
 from subprocess import check_output, CalledProcessError, STDOUT
 
+def log(message, endpoint):
+    time = datetime.utcnow()
+    print(f"{time} at {endpoint}: {message}")
 
 def execute_cmd(cmd_line):
     cmd_args = shlex.split(cmd_line)

scenarios/preset/helpers/frostfs_cli.py

@@ -1,7 +1,5 @@
 import re
-
+from helpers.cmd import execute_cmd, log
-from helpers.cmd import execute_cmd
-
 
 def create_container(endpoint, policy, wallet_file, wallet_config):
     cmd_line = f"frostfs-cli --rpc-endpoint {endpoint} container create --wallet {wallet_file} --config {wallet_config} " \
@@ -10,19 +8,23 @@ def create_container(endpoint, policy, wallet_file, wallet_config):
     output, success = execute_cmd(cmd_line)
 
     if not success:
-        print(f" > Container has not been created:\n{output}")
+        log(f"{cmd_line}\n"
+            f"Container has not been created\n"
+            f"{output}", endpoint)
         return False
     
     try:
         fst_str = output.split('\n')[0]
     except Exception:
-        print(f"Got empty output: {output}")
+        log(f"{cmd_line}\n"
+            f"Incorrect output\n"
+            f"Output: {output or '<empty>'}", endpoint)
         return False
     splitted = fst_str.split(": ")
     if len(splitted) != 2:
         raise ValueError(f"no CID was parsed from command output:\t{fst_str}")
 
-    print(f"Created container: {splitted[1]} via endpoint {endpoint}")
+    log(f"Created container {splitted[1]}", endpoint)
 
     return splitted[1]
 
@@ -34,14 +36,18 @@ def upload_object(container, payload_filepath, endpoint, wallet_file, wallet_con
     output, success = execute_cmd(cmd_line)
 
     if not success:
-        print(f" > Object {object_name} has not been uploaded:\n{output}")
+        log(f"{cmd_line}\n"
+            f"Object {object_name} has not been uploaded\n"
+            f"Error: {output}", endpoint)
         return False
 
     try:
         # taking second string from command output
         snd_str = output.split('\n')[1]
     except Exception:
-        print(f"Got empty input: {output}")
+        log(f"{cmd_line}\n"
+            f"Incorrect output\n"
+            f"Output: {output or '<empty>'}", endpoint)
         return False
     splitted = snd_str.split(": ")
     if len(splitted) != 2:
@@ -56,8 +62,9 @@ def get_object(cid, oid, endpoint, out_filepath, wallet_file, wallet_config):
     output, success = execute_cmd(cmd_line)
 
     if not success:
-        print(f" > Failed to get object {output} from container {cid} \r\n"
+        log(f"{cmd_line}\n"
-              f" > Error: {output}")
+            f"Failed to get object {oid} from container {cid}\n"
+            f"Error: {output}", endpoint)
         return False
 
     return True
@@ -69,13 +76,14 @@ def search_object_by_id(cid, oid, endpoint, wallet_file, wallet_config, ttl=2):
     output, success = execute_cmd(cmd_line)
 
     if not success:
-        print(f" > Failed to search object {oid} for container {cid} \r\n"
+        log(f"{cmd_line}\n"
-              f" > Error: {output}")
+            f"Failed to search object {oid} for container {cid}\n"
+            f"Error: {output}", endpoint)
         return False
 
     re_rst = re.search(r'Found (\d+) objects', output)
 
     if not re_rst:
-        raise Exception("Failed to parce search results")
+        raise Exception("Failed to parse search results")
 
     return re_rst.group(1)

scenarios/preset/preset_grpc.py

@@ -3,7 +3,6 @@
 import argparse
 from itertools import cycle
 import json
-import random
 import sys
 import tempfile
 import time
@@ -33,7 +32,7 @@ parser.add_argument('--endpoint', help='Nodes addresses separated by comma.')
 parser.add_argument('--update', help='Save existed containers')
 parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
 parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50)
-parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), '
+parser.add_argument('--sleep', help='Time to sleep between containers creation and objects upload (in seconds), '
                                     'Default = 8', default=8)
 
 args: Namespace = parser.parse_args()

scenarios/preset/preset_s3.py

@@ -25,7 +25,7 @@ parser.add_argument('--versioning', help='True/False, False by default.')
 parser.add_argument('--ignore-errors', help='Ignore preset errors', action='store_true')
 parser.add_argument('--no-verify-ssl', help='Ignore SSL verifications', action='store_true')
 parser.add_argument('--workers', help='Count of workers in preset. Max = 50, Default = 50', default=50)
-parser.add_argument('--sleep', help='Time to sleep between container creation and object PUT (in seconds), '
+parser.add_argument('--sleep', help='Time to sleep between buckets creation and objects upload (in seconds), '
                                     'Default = 8', default=8)
 
 args = parser.parse_args()