From d0660d626bfd14c9c8707b7baf8fff88e5d9040c Mon Sep 17 00:00:00 2001 From: Yaroslava Lukoyanova Date: Fri, 26 May 2023 07:49:23 +0300 Subject: [PATCH] Fixture for restore stopped storage nodes in test_failover_storage --- pytest_tests/testsuites/conftest.py | 7 +++ .../failovers/test_failover_storage.py | 53 +++++++++---------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/pytest_tests/testsuites/conftest.py b/pytest_tests/testsuites/conftest.py index 28b7a09..3ac87fa 100644 --- a/pytest_tests/testsuites/conftest.py +++ b/pytest_tests/testsuites/conftest.py @@ -23,6 +23,7 @@ from frostfs_testlib.steps.cli.object import get_netmap_netinfo from frostfs_testlib.steps.node_management import storage_node_healthcheck from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.storage.cluster import Cluster +from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController from frostfs_testlib.storage.dataclasses.wallet import WalletFactory from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.utils import env_utils, version_utils @@ -149,6 +150,12 @@ def s3_policy(request: pytest.FixtureRequest): return policy +@pytest.fixture(scope="session") +def cluster_state_controller(client_shell: Shell, cluster: Cluster) -> ClusterStateController: + controller = ClusterStateController(client_shell, cluster) + yield controller + + @allure.step("[Class]: Create S3 client") @pytest.fixture(scope="class") def s3_client( diff --git a/pytest_tests/testsuites/failovers/test_failover_storage.py b/pytest_tests/testsuites/failovers/test_failover_storage.py index a916be7..1a5b6b0 100644 --- a/pytest_tests/testsuites/failovers/test_failover_storage.py +++ b/pytest_tests/testsuites/failovers/test_failover_storage.py @@ -22,6 +22,7 @@ from frostfs_testlib.steps.node_management import ( ) from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.storage.cluster import Cluster, StorageNode +from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.utils import datetime_utils from frostfs_testlib.utils.failover_utils import ( @@ -60,6 +61,11 @@ def return_stopped_hosts(shell: Shell, cluster: Cluster) -> None: @pytest.mark.failover class TestFailoverStorage(ClusterTestBase): + @pytest.fixture(scope="function", autouse=True) + def start_stopped_services(self, cluster_state_controller: ClusterStateController): + yield + cluster_state_controller.start_stopped_storage_services() + @allure.title("Lose and return storage node's host") @pytest.mark.parametrize("hard_reboot", [True, False]) @pytest.mark.failover_reboot @@ -362,7 +368,10 @@ class TestEmptyMap(ClusterTestBase): @allure.title("Test S3 Object loss from fstree/blobovnicza, versioning is enabled") def test_s3_fstree_blobovnicza_loss_versioning_on( - self, s3_client: S3ClientWrapper, simple_object_size + self, + s3_client: S3ClientWrapper, + simple_object_size: int, + cluster_state_controller: ClusterStateController, ): bucket = s3_client.create_bucket() s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED) @@ -377,10 +386,9 @@ class TestEmptyMap(ClusterTestBase): object_versions.append(put_object) with allure.step("Stop all storage nodes"): - for node in self.cluster.storage_nodes: + for node in self.cluster.cluster_nodes: with allure.step(f"Stop storage service on node: {node}"): - node.stop_service() - stopped_nodes.append(node) + cluster_state_controller.stop_storage_service(node) with allure.step("Delete blobovnicza and fstree from all nodes"): for node in self.cluster.storage_nodes: @@ -388,12 +396,7 @@ class TestEmptyMap(ClusterTestBase): node.delete_fstree() with allure.step("Start all storage nodes"): - for node in list(stopped_nodes): - with allure.step(f"Start node {node}"): - node.start_service() - stopped_nodes.remove(node) - with allure.step(f"Waiting status ready for node {node}"): - wait_for_node_to_be_ready(node) + cluster_state_controller.start_stopped_storage_services() # need to get Delete Marker first with allure.step("Delete the object from the bucket"): @@ -410,7 +413,10 @@ class TestEmptyMap(ClusterTestBase): @allure.title("Test S3 Object loss from fstree/blobovnicza, versioning is disabled") def test_s3_fstree_blobovnicza_loss_versioning_off( - self, s3_client: S3ClientWrapper, simple_object_size + self, + s3_client: S3ClientWrapper, + simple_object_size: int, + cluster_state_controller: ClusterStateController, ): bucket = s3_client.create_bucket() @@ -422,10 +428,9 @@ class TestEmptyMap(ClusterTestBase): s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name]) with allure.step("Stop all storage nodes"): - for node in self.cluster.storage_nodes: + for node in self.cluster.cluster_nodes: with allure.step(f"Stop storage service on node: {node}"): - node.stop_service() - stopped_nodes.append(node) + cluster_state_controller.stop_storage_service(node) with allure.step("Delete blobovnicza and fstree from all nodes"): for node in self.cluster.storage_nodes: @@ -433,12 +438,7 @@ class TestEmptyMap(ClusterTestBase): node.delete_fstree() with allure.step("Start all storage nodes"): - for node in list(stopped_nodes): - with allure.step(f"Start node {node}"): - node.start_service() - stopped_nodes.remove(node) - with allure.step(f"Waiting status ready for node {node}"): - wait_for_node_to_be_ready(node) + cluster_state_controller.start_stopped_storage_services() with allure.step("Delete the object from the bucket"): s3_client.delete_object(bucket, file_name) @@ -460,6 +460,7 @@ class TestEmptyMap(ClusterTestBase): s3_client: S3ClientWrapper, simple_object_size: int, versioning_status: VersioningStatus, + cluster_state_controller: ClusterStateController, ): bucket = s3_client.create_bucket() if versioning_status: @@ -473,22 +474,16 @@ class TestEmptyMap(ClusterTestBase): s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name]) with allure.step("Stop all storage nodes"): - for node in self.cluster.storage_nodes: + for node in self.cluster.cluster_nodes: with allure.step(f"Stop storage service on node: {node}"): - node.stop_service() - stopped_nodes.append(node) + cluster_state_controller.stop_storage_service(node) with allure.step("Delete pilorama.db from all nodes"): for node in self.cluster.storage_nodes: node.delete_pilorama() with allure.step("Start all storage nodes"): - for node in list(stopped_nodes): - with allure.step(f"Start node {node}"): - node.start_service() - stopped_nodes.remove(node) - with allure.step(f"Waiting status ready for node {node}"): - wait_for_node_to_be_ready(node) + cluster_state_controller.start_stopped_storage_services() with allure.step("Check list objects first time"): objects_list = s3_client.list_objects(bucket)