diff --git a/pytest_tests/testsuites/failovers/test_failover_storage.py b/pytest_tests/testsuites/failovers/test_failover_storage.py index 1abca02..d126fb4 100644 --- a/pytest_tests/testsuites/failovers/test_failover_storage.py +++ b/pytest_tests/testsuites/failovers/test_failover_storage.py @@ -65,6 +65,13 @@ def after_run_return_all_stopped_services(cluster_state_controller: ClusterState cluster_state_controller.start_stopped_storage_services() +@allure.step("Return all stopped S3 GateWay services after test") +@pytest.fixture(scope="function") +def after_run_return_all_stopped_s3(cluster_state_controller: ClusterStateController): + yield + cluster_state_controller.start_stopped_s3_gate() + + def panic_reboot_host(host: Host) -> None: shell = host.get_shell() shell.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"') @@ -212,6 +219,49 @@ class TestFailoverStorage(ClusterTestBase): ) assert get_file_hash(source_file_path) == get_file_hash(got_file_path) + @allure.title("Do not ignore unhealthy tree endpoints") + def test_unhealthy_tree( + self, + s3_client: S3ClientWrapper, + simple_object_size: int, + cluster_state_controller: ClusterStateController, + after_run_return_all_stopped_s3, + after_run_return_all_stopped_services, + ): + default_node = self.cluster.cluster_nodes[0] + default_s3gate = self.cluster.s3_gates[0] + + with allure.step("Turn S3 GW off on default node"): + default_s3gate.stop_service() + + with allure.step("Turn off storage on default node"): + cluster_state_controller.stop_storage_service(default_node) + + with allure.step("Turn on S3 GW on default node"): + default_s3gate.start_service() + + with allure.step("Turn on storage on default node"): + cluster_state_controller.start_stopped_storage_services() + + with allure.step("Create bucket with REP 1 SELECT 1 policy"): + bucket = s3_client.create_bucket( + location_constraint="load-1-1", + ) + + file_path = generate_file(simple_object_size) + file_name = s3_helper.object_key_from_file_path(file_path) + with allure.step("Put object into bucket"): + put_object = s3_client.put_object(bucket, file_path) + s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name]) + + with allure.step("Turn off all storage nodes except default"): + for node in self.cluster.cluster_nodes[1:]: + with allure.step(f"Stop storage service on node: {node}"): + cluster_state_controller.stop_storage_service(node) + + with allure.step("Check that object is available"): + s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name]) + def pytest_generate_tests(metafunc: pytest.Metafunc): if "s3_client" in metafunc.fixturenames: