Add await

Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>

pytest_tests/testsuites/conftest.py

@@ -8,7 +8,9 @@ import allure
 import pytest
 import yaml
 from dateutil import parser
+from frostfs_testlib.healthcheck.interfaces import Healthcheck
 from frostfs_testlib.hosting import Hosting
+from frostfs_testlib.plugins import load_plugin
 from frostfs_testlib.reporter import AllureHandler, get_reporter
 from frostfs_testlib.resources.common import (
     ASSETS_DIR,
@@ -182,8 +184,18 @@ def s3_policy(request: pytest.FixtureRequest):
 
 
 @pytest.fixture(scope="session")
-def cluster_state_controller(client_shell: Shell, cluster: Cluster) -> ClusterStateController:
+@allure.title("[Session] Create healthcheck object")
-    controller = ClusterStateController(client_shell, cluster)
+def healthcheck(cluster: Cluster) -> Healthcheck:
+    healthcheck_cls = load_plugin(
+        "frostfs.testlib.healthcheck", cluster.cluster_nodes[0].host.config.healthcheck_plugin_name
+    )
+
+    return healthcheck_cls()
+
+
+@pytest.fixture(scope="session")
+def cluster_state_controller(client_shell: Shell, cluster: Cluster, healthcheck: Healthcheck) -> ClusterStateController:
+    controller = ClusterStateController(client_shell, cluster, healthcheck)
     yield controller
 
 

pytest_tests/testsuites/container/test_policy.py

@@ -1485,11 +1485,16 @@ class TestPolicy(ClusterTestBase):
         with allure.step(f"Check the object appearance"):
             netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
             netmap = self.get_netmap_param(netmap)
+            list_of_location = []
             for node in resulting_copies:
                 node_address = node.get_rpc_endpoint().split(":")[0]
-                assert expected_params["location"][0] == netmap[node_address]["location"] or (
+                list_of_location.append(netmap[node_address]["location"])
-                    expected_params["location"][1] == netmap[node_address]["location"]
+
-                ), f"The node is selected from the wrong location. Expected {expected_params} and got {netmap[node_address]}"
+            assert (
+                expected_params["location"][0] in list_of_location
+                and expected_params["location"][1] in list_of_location
+                and len(resulting_copies) > 2
+            ), f"The node is selected from the wrong location. Expected {expected_params} and got {netmap[node_address]}"
 
             self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
 

pytest_tests/testsuites/failovers/test_failover_storage.py

@@ -5,11 +5,9 @@ from time import sleep
 
 import allure
 import pytest
-from frostfs_testlib.hosting import Host
 from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
 from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
 from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
-from frostfs_testlib.shell import CommandOptions
 from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
 from frostfs_testlib.steps.cli.object import get_object, put_object_to_random_node
 from frostfs_testlib.steps.node_management import (
@@ -21,7 +19,7 @@ from frostfs_testlib.steps.node_management import (
     wait_for_node_to_be_ready,
 )
 from frostfs_testlib.steps.s3 import s3_helper
-from frostfs_testlib.storage.cluster import Cluster, ClusterNode, StorageNode
+from frostfs_testlib.storage.cluster import Cluster, ClusterNode, S3Gate, StorageNode
 from frostfs_testlib.storage.controllers import ClusterStateController, ShardsWatcher
 from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
 from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
@@ -57,26 +55,11 @@ def after_run_return_all_stopped_hosts(cluster_state_controller: ClusterStateCon
     cluster_state_controller.start_stopped_hosts()
 
 
-@allure.step("Return all stopped storage services after test")
+@allure.step("Return all stopped services after test")
 @pytest.fixture(scope="function")
 def after_run_return_all_stopped_services(cluster_state_controller: ClusterStateController):
     yield
-    cluster_state_controller.start_stopped_storage_services()
+    cluster_state_controller.start_all_stopped_services()
-
-
-@allure.step("Return all stopped S3 GateWay services after test")
-@pytest.fixture(scope="function")
-def after_run_return_all_stopped_s3(cluster_state_controller: ClusterStateController):
-    yield
-    cluster_state_controller.start_stopped_s3_gates()
-
-
-def panic_reboot_host(host: Host) -> None:
-    shell = host.get_shell()
-    shell.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"')
-
-    options = CommandOptions(close_stdin=True, timeout=1, check=False)
-    shell.exec('sudo sh -c "echo b > /proc/sysrq-trigger"', options)
 
 
 @pytest.mark.failover
@@ -209,22 +192,21 @@ class TestFailoverStorage(ClusterTestBase):
         s3_client: S3ClientWrapper,
         simple_object_size: ObjectSize,
         cluster_state_controller: ClusterStateController,
-        after_run_return_all_stopped_s3,
         after_run_return_all_stopped_services,
     ):
         default_node = self.cluster.cluster_nodes[0]
 
         with allure.step("Turn S3 GW off on default node"):
-            cluster_state_controller.stop_s3_gate(default_node)
+            cluster_state_controller.stop_service_of_type(default_node, S3Gate)
 
         with allure.step("Turn off storage on default node"):
-            cluster_state_controller.stop_storage_service(default_node)
+            cluster_state_controller.stop_service_of_type(default_node, StorageNode)
 
         with allure.step("Turn on S3 GW on default node"):
-            cluster_state_controller.start_s3_gate(default_node)
+            cluster_state_controller.start_service_of_type(default_node, S3Gate)
 
         with allure.step("Turn on storage on default node"):
-            cluster_state_controller.start_storage_service(default_node)
+            cluster_state_controller.start_service_of_type(default_node, StorageNode)
 
         with allure.step("Create bucket with REP 1 SELECT 1 policy"):
             bucket = s3_client.create_bucket(
@@ -240,13 +222,13 @@ class TestFailoverStorage(ClusterTestBase):
         with allure.step("Turn off all storage nodes except default"):
             for node in self.cluster.cluster_nodes[1:]:
                 with allure.step(f"Stop storage service on node: {node}"):
-                    cluster_state_controller.stop_storage_service(node)
+                    cluster_state_controller.stop_service_of_type(node, StorageNode)
 
         with allure.step("Check that object is available"):
             s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
 
         with allure.step("Start storage nodes"):
-            cluster_state_controller.start_stopped_storage_services()
+            cluster_state_controller.start_all_stopped_services()
 
 
 @pytest.mark.failover
@@ -320,7 +302,7 @@ class TestEmptyMap(ClusterTestBase):
     def empty_map_stop_service_teardown(self, cluster_state_controller: ClusterStateController):
         yield
         with allure.step("Return all storage nodes to network map"):
-            cluster_state_controller.start_stopped_storage_services()
+            cluster_state_controller.start_all_stopped_services()
             for node in stopped_nodes:
                 check_node_in_map(node, shell=self.shell, alive_node=node)
 
@@ -365,7 +347,7 @@ class TestEmptyMap(ClusterTestBase):
             s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_objects)
 
         with allure.step("Stop all storage nodes"):
-            cluster_state_controller.stop_all_storage_services()
+            cluster_state_controller.stop_services_of_type(StorageNode)
 
         with allure.step("Remove all nodes from network map"):
             remove_nodes_from_map_morph(
@@ -383,7 +365,7 @@ class TestEmptyMap(ClusterTestBase):
         first_node = self.cluster.cluster_nodes[0].service(StorageNode)
 
         with allure.step("Start first node and check network map"):
-            cluster_state_controller.start_storage_service(self.cluster.cluster_nodes[0])
+            cluster_state_controller.start_service_of_type(self.cluster.cluster_nodes[0], StorageNode)
             wait_for_node_to_be_ready(first_node)
 
             for check_node in self.cluster.storage_nodes:
@@ -392,7 +374,7 @@ class TestEmptyMap(ClusterTestBase):
         for node in self.cluster.cluster_nodes[1:]:
             storage_node = node.service(StorageNode)
 
-            cluster_state_controller.start_storage_service(node)
+            cluster_state_controller.start_service_of_type(node, StorageNode)
             wait_for_node_to_be_ready(storage_node)
 
             sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
@@ -420,9 +402,7 @@ class TestEmptyMap(ClusterTestBase):
             object_versions.append(put_object)
 
         with allure.step("Stop all storage nodes"):
-            for node in self.cluster.cluster_nodes:
+            cluster_state_controller.stop_services_of_type(StorageNode)
-                with allure.step(f"Stop storage service on node: {node}"):
-                    cluster_state_controller.stop_storage_service(node)
 
         with allure.step("Delete blobovnicza and fstree from all nodes"):
             for node in self.cluster.storage_nodes:
@@ -430,7 +410,7 @@ class TestEmptyMap(ClusterTestBase):
                 node.delete_fstree()
 
         with allure.step("Start all storage nodes"):
-            cluster_state_controller.start_stopped_storage_services()
+            cluster_state_controller.start_all_stopped_services()
 
         # need to get Delete Marker first
         with allure.step("Delete the object from the bucket"):
@@ -462,9 +442,7 @@ class TestEmptyMap(ClusterTestBase):
             s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
 
         with allure.step("Stop all storage nodes"):
-            for node in self.cluster.cluster_nodes:
+            cluster_state_controller.stop_services_of_type(StorageNode)
-                with allure.step(f"Stop storage service on node: {node}"):
-                    cluster_state_controller.stop_storage_service(node)
 
         with allure.step("Delete blobovnicza and fstree from all nodes"):
             for node in self.cluster.storage_nodes:
@@ -472,7 +450,7 @@ class TestEmptyMap(ClusterTestBase):
                 node.delete_fstree()
 
         with allure.step("Start all storage nodes"):
-            cluster_state_controller.start_stopped_storage_services()
+            cluster_state_controller.start_all_stopped_services()
 
         with allure.step("Delete the object from the bucket"):
             s3_client.delete_object(bucket, file_name)
@@ -507,16 +485,14 @@ class TestEmptyMap(ClusterTestBase):
             s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
 
         with allure.step("Stop all storage nodes"):
-            for node in self.cluster.cluster_nodes:
+            cluster_state_controller.stop_services_of_type(StorageNode)
-                with allure.step(f"Stop storage service on node: {node}"):
-                    cluster_state_controller.stop_storage_service(node)
 
         with allure.step("Delete pilorama.db from all nodes"):
             for node in self.cluster.storage_nodes:
                 node.delete_pilorama()
 
         with allure.step("Start all storage nodes"):
-            cluster_state_controller.start_stopped_storage_services()
+            cluster_state_controller.start_all_stopped_services()
 
         with allure.step("Check list objects first time"):
             objects_list = s3_client.list_objects(bucket)
@@ -578,7 +554,7 @@ class TestStorageDataLoss(ClusterTestBase):
             )
 
         with allure.step("Stop storage services on all nodes"):
-            cluster_state_controller.stop_all_storage_services()
+            cluster_state_controller.stop_services_of_type(StorageNode)
 
         with allure.step("Delete metabase from all nodes"):
             for node in cluster_state_controller.cluster.storage_nodes:
@@ -594,7 +570,11 @@ class TestStorageDataLoss(ClusterTestBase):
                         storage_node.save_config(config, config_file_path)
 
         with allure.step("Start storage services on all nodes"):
-            cluster_state_controller.start_stopped_storage_services()
+            cluster_state_controller.start_all_stopped_services()
+
+        with allure.step("Wait for tree rebalance"):
+            # TODO: Use product metric when we have proper ones for this check
+            sleep(30)
 
         with allure.step("Delete objects from bucket"):
             with allure.step("Delete simple object from bucket"):
@@ -655,13 +635,13 @@ class TestStorageDataLoss(ClusterTestBase):
             shards_watcher.take_shards_snapshot()
 
         with allure.step(f"Stop storage service on node {node_under_test}"):
-            cluster_state_controller.stop_storage_service(node_under_test)
+            cluster_state_controller.stop_service_of_type(node_under_test, StorageNode)
 
         with allure.step(f"Delete write cache from node {node_under_test}"):
             node_under_test.storage_node.delete_write_cache()
 
         with allure.step(f"Start storage service on node {node_under_test}"):
-            cluster_state_controller.start_storage_service(node_under_test)
+            cluster_state_controller.start_all_stopped_services()
 
         with allure.step("Objects should be available"):
             for storage_object in storage_objects:
@@ -710,7 +690,7 @@ class TestStorageDataLoss(ClusterTestBase):
 
         with allure.step("Stop one node and wait for rebalance connection of s3 gate to storage service"):
             current_node = self.cluster.cluster_nodes[0]
-            cluster_state_controller.stop_storage_service(current_node)
+            cluster_state_controller.stop_service_of_type(current_node, StorageNode)
             # waiting for rebalance connection of s3 gate to storage service
             sleep(60)
 
@@ -752,15 +732,13 @@ class TestStorageDataLoss(ClusterTestBase):
             piloramas_list_before_removing = self.get_piloramas_list(node_to_check)
 
         with allure.step("Stop all storage nodes"):
-            for node in self.cluster.cluster_nodes:
+            cluster_state_controller.stop_services_of_type(StorageNode)
-                with allure.step(f"Stop storage service on node: {node}"):
-                    cluster_state_controller.stop_storage_service(node)
 
         with allure.step("Delete pilorama.db from one node"):
             node_to_check.delete_pilorama()
 
         with allure.step("Start all storage nodes"):
-            cluster_state_controller.start_stopped_storage_services()
+            cluster_state_controller.start_all_stopped_services()
 
         with allure.step("Tick epoch to trigger sync and then wait for 1 minute"):
             self.tick_epochs(1)

pytest_tests/testsuites/services/http_gate/test_http_gate.py

@@ -165,19 +165,17 @@ class TestHttpPut(ClusterTestBase):
         name="download by attributes",
     )
     @pytest.mark.skip("Skipped due to deprecated PUT via http")
-    @allure.title("Put over HTTP, Get over HTTP with header")
+    @allure.title("Put over HTTP, Get over HTTP with {id} header")
     @pytest.mark.parametrize(
-        "attributes",
+        "attributes,id",
         [
-            {"fileName": "simple_obj_filename"},
+            ({"fileName": "simple_obj_filename"}, "simple"),
-            {"file-Name": "simple obj filename"},
+            ({"file-Name": "simple obj filename"}, "hyphen"),
-            {"cat%jpeg": "cat%jpeg"},
+            ({"cat%jpeg": "cat%jpeg"}, "percent"),
         ],
         ids=["simple", "hyphen", "percent"],
     )
-    def test_put_http_get_http_with_headers(
+    def test_put_http_get_http_with_headers(self, attributes: dict, simple_object_size: ObjectSize, id: str):
-        self, attributes: dict, simple_object_size: ObjectSize, request: pytest.FixtureRequest
-    ):
         """
         Test that object can be downloaded using different attributes in HTTP header.
 
@@ -190,7 +188,6 @@ class TestHttpPut(ClusterTestBase):
         Expected result:
         Hashes must be the same.
         """
-        allure.dynamic.title(f"Put over HTTP, Get over HTTP with {request.node.callspec.id} header")
         cid = create_container(
             self.wallet,
             shell=self.shell,

pytest_tests/testsuites/services/http_gate/test_http_headers.py

@@ -203,6 +203,7 @@ class Test_http_headers(ClusterTestBase):
                 cid=storage_object_1.cid,
                 shell=self.shell,
                 endpoint=self.cluster.default_rpc_endpoint,
+                await_mode=True,
             )
             self.tick_epoch()
             wait_for_container_deletion(