[#334] Disable automatic retries in S3 clients

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>

pytest_tests/steps/s3_gate_base.py

@@ -1,12 +1,41 @@
+import json
+import logging
 import os
+import re
+import uuid
+from typing import Optional
 
 import allure
+import boto3
 import pytest
+import urllib3
+from botocore.config import Config
+from botocore.exceptions import ClientError
+from cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd
+from common import (
+    NEOFS_AUTHMATE_EXEC,
+    NEOFS_ENDPOINT,
+    S3_GATE,
+    S3_GATE_WALLET_PASS,
+    S3_GATE_WALLET_PATH,
+)
+from data_formatters import get_wallet_public_key
 from python_keywords.container import list_containers
 
-from steps import s3_gate_bucket
 from steps.aws_cli_client import AwsCliClient
 
+# Disable warnings on self-signed certificate which the
+# boto library produces on requests to S3-gate in dev-env
+urllib3.disable_warnings()
+
+logger = logging.getLogger("NeoLogger")
+CREDENTIALS_CREATE_TIMEOUT = "1m"
+
+# Number of attempts that S3 clients will attempt per each request (1 means single attempt
+# without any retries)
+MAX_REQUEST_ATTEMPTS = 1
+RETRY_MODE = "standard"
+
 
 class TestS3GateBase:
     s3_client = None
@@ -23,19 +52,95 @@ class TestS3GateBase:
             access_key_id,
             secret_access_key,
             owner_private_key,
-        ) = s3_gate_bucket.init_s3_credentials(wallet, s3_bearer_rules_file=s3_bearer_rules_file)
+        ) = init_s3_credentials(wallet, s3_bearer_rules_file=s3_bearer_rules_file)
         containers_list = list_containers(wallet)
         assert cid in containers_list, f"Expected cid {cid} in {containers_list}"
 
         if request.param == "aws cli":
-            try:
-                client = AwsCliClient(access_key_id, secret_access_key)
-            except Exception as err:
-                if "command was not found or was not executable" in str(err):
-                    pytest.skip("AWS CLI was not found")
-                else:
-                    raise RuntimeError("Error on creating instance for AwsCliClient") from err
+            client = configure_cli_client(access_key_id, secret_access_key)
         else:
-            client = s3_gate_bucket.config_s3_client(access_key_id, secret_access_key)
+            client = configure_boto3_client(access_key_id, secret_access_key)
         TestS3GateBase.s3_client = client
         TestS3GateBase.wallet = wallet
+
+
+@allure.step("Init S3 Credentials")
+def init_s3_credentials(wallet_path: str, s3_bearer_rules_file: Optional[str] = None):
+    bucket = str(uuid.uuid4())
+    s3_bearer_rules = s3_bearer_rules_file or "robot/resources/files/s3_bearer_rules.json"
+    gate_public_key = get_wallet_public_key(S3_GATE_WALLET_PATH, S3_GATE_WALLET_PASS)
+    cmd = (
+        f"{NEOFS_AUTHMATE_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} "
+        f"issue-secret --wallet {wallet_path} --gate-public-key={gate_public_key} "
+        f"--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} "
+        f"--bearer-rules {s3_bearer_rules}"
+    )
+    logger.info(f"Executing command: {cmd}")
+
+    try:
+        output = _run_with_passwd(cmd)
+        logger.info(f"Command completed with output: {output}")
+
+        # output contains some debug info and then several JSON structures, so we find each
+        # JSON structure by curly brackets (naive approach, but works while JSON is not nested)
+        # and then we take JSON containing secret_access_key
+        json_blocks = re.findall(r"\{.*?\}", output, re.DOTALL)
+        for json_block in json_blocks:
+            try:
+                parsed_json_block = json.loads(json_block)
+                if "secret_access_key" in parsed_json_block:
+                    return (
+                        parsed_json_block["container_id"],
+                        bucket,
+                        parsed_json_block["access_key_id"],
+                        parsed_json_block["secret_access_key"],
+                        parsed_json_block["owner_private_key"],
+                    )
+            except json.JSONDecodeError:
+                raise AssertionError(f"Could not parse info from output\n{output}")
+        raise AssertionError(f"Could not find AWS credentials in output:\n{output}")
+
+    except Exception as exc:
+        raise RuntimeError(f"Failed to init s3 credentials because of error\n{exc}") from exc
+
+
+@allure.step("Configure S3 client (boto3)")
+def configure_boto3_client(access_key_id: str, secret_access_key: str):
+    try:
+        session = boto3.session.Session()
+        config = Config(
+            retries={
+                "max_attempts": MAX_REQUEST_ATTEMPTS,
+                "mode": RETRY_MODE,
+            }
+        )
+
+        s3_client = session.client(
+            service_name="s3",
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            config=config,
+            endpoint_url=S3_GATE,
+            verify=False,
+        )
+        return s3_client
+    except ClientError as err:
+        raise Exception(
+            f'Error Message: {err.response["Error"]["Message"]}\n'
+            f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
+        ) from err
+
+
+@allure.step("Configure S3 client (aws cli)")
+def configure_cli_client(access_key_id: str, secret_access_key: str):
+    try:
+        client = AwsCliClient()
+        _configure_aws_cli("aws configure", access_key_id, secret_access_key)
+        _cmd_run(f"aws configure set max_attempts {MAX_REQUEST_ATTEMPTS}")
+        _cmd_run(f"aws configure set retry_mode {RETRY_MODE}")
+        return client
+    except Exception as err:
+        if "command was not found or was not executable" in str(err):
+            pytest.skip("AWS CLI was not found")
+        else:
+            raise RuntimeError("Error while configuring AwsCliClient") from err