Add GenericCli utility

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>

src/frostfs_testlib/cli/__init__.py

@@ -1,4 +1,5 @@
 from frostfs_testlib.cli.frostfs_adm import FrostfsAdm
 from frostfs_testlib.cli.frostfs_authmate import FrostfsAuthmate
 from frostfs_testlib.cli.frostfs_cli import FrostfsCli
+from frostfs_testlib.cli.generic_cli import GenericCli
 from frostfs_testlib.cli.neogo import NeoGo, NetworkType

src/frostfs_testlib/cli/generic_cli.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+from frostfs_testlib.hosting.interfaces import Host
+from frostfs_testlib.shell.interfaces import CommandOptions, Shell
+
+
+class GenericCli(object):
+    def __init__(self, cli_name: str, host: Host) -> None:
+        self.host = host
+        self.cli_name = cli_name
+
+    def __call__(
+        self,
+        args: Optional[str] = "",
+        pipes: Optional[str] = "",
+        shell: Optional[Shell] = None,
+        options: Optional[CommandOptions] = None,
+    ):
+        if not shell:
+            shell = self.host.get_shell()
+
+        cli_config = self.host.get_cli_config(self.cli_name, True)
+        extra_args = ""
+        exec_path = self.cli_name
+        if cli_config:
+            extra_args = " ".join(cli_config.extra_args)
+            exec_path = cli_config.exec_path
+
+        cmd = f"{exec_path} {args} {extra_args} {pipes}"
+        return shell.exec(cmd, options)

src/frostfs_testlib/hosting/config.py

@@ -10,9 +10,7 @@ class ParsedAttributes:
     def parse(cls, attributes: dict[str, Any]):
         # Pick attributes supported by the class
         field_names = set(field.name for field in fields(cls))
-        supported_attributes = {
+        supported_attributes = {key: value for key, value in attributes.items() if key in field_names}
-            key: value for key, value in attributes.items() if key in field_names
-        }
         return cls(**supported_attributes)
 
 
@@ -29,6 +27,7 @@ class CLIConfig:
     name: str
     exec_path: str
     attributes: dict[str, str] = field(default_factory=dict)
+    extra_args: list[str] = field(default_factory=list)
 
 
 @dataclass

src/frostfs_testlib/hosting/interfaces.py

@@ -54,7 +54,7 @@ class Host(ABC):
             raise ValueError(f"Unknown service name: '{service_name}'")
         return service_config
 
-    def get_cli_config(self, cli_name: str) -> CLIConfig:
+    def get_cli_config(self, cli_name: str, allow_empty: bool = False) -> CLIConfig:
         """Returns config of CLI tool with specified name.
 
         The CLI must be located on this host.
@@ -66,7 +66,7 @@ class Host(ABC):
             Config of the CLI tool.
         """
         cli_config = self._cli_config_by_name.get(cli_name)
-        if cli_config is None:
+        if cli_config is None and not allow_empty:
             raise ValueError(f"Unknown CLI name: '{cli_name}'")
         return cli_config
 

src/frostfs_testlib/steps/cli/container.py

@@ -8,7 +8,7 @@ from typing import Optional, Union
 import requests
 
 from frostfs_testlib import reporter
-from frostfs_testlib.cli import FrostfsCli
+from frostfs_testlib.cli import FrostfsCli, GenericCli
 from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC
 from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
 from frostfs_testlib.shell import Shell
@@ -345,8 +345,8 @@ def _parse_cid(output: str) -> str:
 
 @reporter.step("Search container by name")
 def search_container_by_name(name: str, node: ClusterNode):
-    node_shell = node.host.get_shell()
+    curl = GenericCli("curl", node.host)
-    output = node_shell.exec(f"curl -I HEAD http://127.0.0.1:8084/{name}")
+    output = curl(f"-I http://127.0.0.1:8084/{name}")
     pattern = r"X-Container-Id: (\S+)"
     cid = re.findall(pattern, output.stdout)
     if cid:

src/frostfs_testlib/steps/http/http_gate.py

@@ -11,13 +11,14 @@ from urllib.parse import quote_plus
 import requests
 
 from frostfs_testlib import reporter
+from frostfs_testlib.cli import GenericCli
 from frostfs_testlib.resources.common import SIMPLE_OBJECT_SIZE
 from frostfs_testlib.s3.aws_cli_client import command_options
 from frostfs_testlib.shell import Shell
 from frostfs_testlib.shell.local_shell import LocalShell
 from frostfs_testlib.steps.cli.object import get_object
 from frostfs_testlib.steps.storage_policy import get_nodes_without_object
-from frostfs_testlib.storage.cluster import StorageNode
+from frostfs_testlib.storage.cluster import ClusterNode, StorageNode
 from frostfs_testlib.testing.test_control import retry
 from frostfs_testlib.utils.file_utils import get_file_hash
 
@@ -31,8 +32,7 @@ local_shell = LocalShell()
 def get_via_http_gate(
     cid: str,
     oid: str,
-    endpoint: str,
+    node: ClusterNode,
-    http_hostname: str,
     request_path: Optional[str] = None,
     timeout: Optional[int] = 300,
 ):
@@ -40,18 +40,19 @@ def get_via_http_gate(
     This function gets given object from HTTP gate
     cid:          container id to get object from
     oid:          object ID
-    endpoint:     http gate endpoint
+    node:         node to make request
-    http_hostname: http host name on the node
     request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}]
     """
 
     # if `request_path` parameter omitted, use default
     if request_path is None:
-        request = f"{endpoint}/get/{cid}/{oid}"
+        request = f"{node.http_gate.get_endpoint()}/get/{cid}/{oid}"
     else:
-        request = f"{endpoint}{request_path}"
+        request = f"{node.http_gate.get_endpoint()}{request_path}"
 
-    resp = requests.get(request, headers={"Host": http_hostname}, stream=True, timeout=timeout, verify=False)
+    resp = requests.get(
+        request, headers={"Host": node.storage_node.get_http_hostname()[0]}, stream=True, timeout=timeout, verify=False
+    )
 
     if not resp.ok:
         raise Exception(
@@ -72,15 +73,14 @@ def get_via_http_gate(
 
 
 @reporter.step("Get via Zip HTTP Gate")
-def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, http_hostname: str, timeout: Optional[int] = 300):
+def get_via_zip_http_gate(cid: str, prefix: str, node: ClusterNode, timeout: Optional[int] = 300):
     """
     This function gets given object from HTTP gate
     cid:      container id to get object from
     prefix:   common prefix
-    endpoint: http gate endpoint
+    node:     node to make request
-    http_hostname: http host name on the node
     """
-    request = f"{endpoint}/zip/{cid}/{prefix}"
+    request = f"{node.http_gate.get_endpoint()}/zip/{cid}/{prefix}"
     resp = requests.get(request, stream=True, timeout=timeout, verify=False)
 
     if not resp.ok:
@@ -109,8 +109,7 @@ def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str, http_hostname: s
 def get_via_http_gate_by_attribute(
     cid: str,
     attribute: dict,
-    endpoint: str,
+    node: ClusterNode,
-    http_hostname: str,
     request_path: Optional[str] = None,
     timeout: Optional[int] = 300,
 ):
@@ -126,11 +125,13 @@ def get_via_http_gate_by_attribute(
     attr_value = quote_plus(str(attribute.get(attr_name)))
     # if `request_path` parameter ommited, use default
     if request_path is None:
-        request = f"{endpoint}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}"
+        request = f"{node.http_gate.get_endpoint()}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}"
     else:
-        request = f"{endpoint}{request_path}"
+        request = f"{node.http_gate.get_endpoint()}{request_path}"
 
-    resp = requests.get(request, stream=True, timeout=timeout, verify=False, headers={"Host": http_hostname})
+    resp = requests.get(
+        request, stream=True, timeout=timeout, verify=False, headers={"Host": node.storage_node.get_http_hostname()[0]}
+    )
 
     if not resp.ok:
         raise Exception(
@@ -247,19 +248,18 @@ def upload_via_http_gate_curl(
 
 @retry(max_attempts=3, sleep_interval=1)
 @reporter.step("Get via HTTP Gate using Curl")
-def get_via_http_curl(cid: str, oid: str, endpoint: str, http_hostname: str) -> str:
+def get_via_http_curl(cid: str, oid: str, node: ClusterNode) -> str:
     """
     This function gets given object from HTTP gate using curl utility.
     cid:      CID to get object from
     oid:      object OID
-    endpoint: http gate endpoint
+    node:     node for request
-    http_hostname: http host name of the node
     """
-    request = f"{endpoint}/get/{cid}/{oid}"
+    request = f"{node.http_gate.get_endpoint()}/get/{cid}/{oid}"
     file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}")
 
-    cmd = f'curl -k -H "Host: {http_hostname}"  {request} > {file_path}'
+    curl = GenericCli("curl", node.host)
-    local_shell.exec(cmd)
+    curl(f'-k -H "Host: {node.storage_node.get_http_hostname()[0]}"', f"{request} > {file_path}", shell=local_shell)
 
     return file_path
 
@@ -274,12 +274,11 @@ def _attach_allure_step(request: str, status_code: int, req_type="GET"):
 def try_to_get_object_and_expect_error(
     cid: str,
     oid: str,
+    node: ClusterNode,
     error_pattern: str,
-    endpoint: str,
-    http_hostname: str,
 ) -> None:
     try:
-        get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
+        get_via_http_gate(cid=cid, oid=oid, node=node)
         raise AssertionError(f"Expected error on getting object with cid: {cid}")
     except Exception as err:
         match = error_pattern.casefold() in str(err).casefold()
@@ -292,13 +291,10 @@ def get_object_by_attr_and_verify_hashes(
     file_name: str,
     cid: str,
     attrs: dict,
-    endpoint: str,
+    node: ClusterNode,
-    http_hostname: str,
 ) -> None:
-    got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
+    got_file_path_http = get_via_http_gate(cid=cid, oid=oid, node=node)
-    got_file_path_http_attr = get_via_http_gate_by_attribute(
+    got_file_path_http_attr = get_via_http_gate_by_attribute(cid=cid, attribute=attrs, node=node)
-        cid=cid, attribute=attrs, endpoint=endpoint, http_hostname=http_hostname
-    )
     assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr)
 
 
@@ -309,8 +305,7 @@ def verify_object_hash(
     cid: str,
     shell: Shell,
     nodes: list[StorageNode],
-    endpoint: str,
+    request_node: ClusterNode,
-    http_hostname: str,
     object_getter=None,
 ) -> None:
 
@@ -336,7 +331,7 @@ def verify_object_hash(
         shell=shell,
         endpoint=random_node.get_rpc_endpoint(),
     )
-    got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint, http_hostname=http_hostname)
+    got_file_path_http = object_getter(cid=cid, oid=oid, node=request_node)
 
     assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
 
@@ -365,10 +360,9 @@ def attr_into_str_header_curl(attrs: dict) -> list:
 def try_to_get_object_via_passed_request_and_expect_error(
     cid: str,
     oid: str,
+    node: ClusterNode,
     error_pattern: str,
-    endpoint: str,
     http_request_path: str,
-    http_hostname: str,
     attrs: Optional[dict] = None,
 ) -> None:
     try:
@@ -376,17 +370,15 @@ def try_to_get_object_via_passed_request_and_expect_error(
             get_via_http_gate(
                 cid=cid,
                 oid=oid,
-                endpoint=endpoint,
+                node=node,
                 request_path=http_request_path,
-                http_hostname=http_hostname,
             )
         else:
             get_via_http_gate_by_attribute(
                 cid=cid,
                 attribute=attrs,
-                endpoint=endpoint,
+                node=node,
                 request_path=http_request_path,
-                http_hostname=http_hostname,
             )
         raise AssertionError(f"Expected error on getting object with cid: {cid}")
     except Exception as err:

src/frostfs_testlib/storage/controllers/cluster_state_controller.py

@@ -326,6 +326,8 @@ class ClusterStateController:
 
     @reporter.step("Restore blocked nodes")
     def restore_all_traffic(self):
+        if not self.dropped_traffic:
+            return
         parallel(self._restore_traffic_to_node, self.dropped_traffic)
 
     @run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)

src/frostfs_testlib/storage/dataclasses/frostfs_services.py

@@ -5,6 +5,7 @@ from frostfs_testlib.storage.constants import ConfigAttributes
 from frostfs_testlib.storage.dataclasses.node_base import NodeBase
 from frostfs_testlib.storage.dataclasses.shard import Shard
 
+
 class InnerRing(NodeBase):
     """
     Class represents inner ring node in a cluster
@@ -17,11 +18,7 @@ class InnerRing(NodeBase):
 
     def service_healthcheck(self) -> bool:
         health_metric = "frostfs_ir_ir_health"
-        output = (
+        output = self.host.get_shell().exec(f"curl -s localhost:6662 | grep {health_metric} | sed 1,2d").stdout
-            self.host.get_shell()
-            .exec(f"curl -s localhost:6662 | grep {health_metric} | sed 1,2d")
-            .stdout
-        )
         return health_metric in output
 
     def get_netmap_cleaner_threshold(self) -> str:
@@ -50,11 +47,7 @@ class S3Gate(NodeBase):
 
     def service_healthcheck(self) -> bool:
         health_metric = "frostfs_s3_gw_state_health"
-        output = (
+        output = self.host.get_shell().exec(f"curl -s localhost:8086 | grep {health_metric} | sed 1,2d").stdout
-            self.host.get_shell()
-            .exec(f"curl -s localhost:8086 | grep {health_metric} | sed 1,2d")
-            .stdout
-        )
         return health_metric in output
 
     @property
@@ -72,11 +65,7 @@ class HTTPGate(NodeBase):
 
     def service_healthcheck(self) -> bool:
         health_metric = "frostfs_http_gw_state_health"
-        output = (
+        output = self.host.get_shell().exec(f"curl -s localhost:5662 | grep {health_metric}  | sed 1,2d").stdout
-            self.host.get_shell()
-            .exec(f"curl -s localhost:5662 | grep {health_metric}  | sed 1,2d")
-            .stdout
-        )
         return health_metric in output
 
     @property
@@ -135,11 +124,7 @@ class StorageNode(NodeBase):
 
     def service_healthcheck(self) -> bool:
         health_metric = "frostfs_node_state_health"
-        output = (
+        output = self.host.get_shell().exec(f"curl -s localhost:6672 | grep {health_metric} | sed 1,2d").stdout
-            self.host.get_shell()
-            .exec(f"curl -s localhost:6672 | grep {health_metric} | sed 1,2d")
-            .stdout
-        )
         return health_metric in output
 
     def get_shard_config_path(self) -> str:
@@ -174,10 +159,10 @@ class StorageNode(NodeBase):
     def get_storage_config(self) -> str:
         return self.host.get_storage_config(self.name)
 
-    def get_http_hostname(self) -> str:
+    def get_http_hostname(self) -> list[str]:
         return self._get_attribute(ConfigAttributes.HTTP_HOSTNAME)
 
-    def get_s3_hostname(self) -> str:
+    def get_s3_hostname(self) -> list[str]:
         return self._get_attribute(ConfigAttributes.S3_HOSTNAME)
 
     def delete_blobovnicza(self):