From 27cf9bb1bd8a688cc59f6c1d4223c92536d49869 Mon Sep 17 00:00:00 2001 From: Andrey Berezin Date: Tue, 30 May 2023 16:42:27 +0300 Subject: [PATCH] Add metabase loss test --- .../failovers/test_failover_storage.py | 93 ++++++++++++++++++- requirements.txt | 4 +- 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/pytest_tests/testsuites/failovers/test_failover_storage.py b/pytest_tests/testsuites/failovers/test_failover_storage.py index 1a5b6b0..3ab542b 100644 --- a/pytest_tests/testsuites/failovers/test_failover_storage.py +++ b/pytest_tests/testsuites/failovers/test_failover_storage.py @@ -1,5 +1,4 @@ import logging -import os from time import sleep import allure @@ -22,26 +21,43 @@ from frostfs_testlib.steps.node_management import ( ) from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.storage.cluster import Cluster, StorageNode -from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController +from frostfs_testlib.storage.controllers import ClusterStateController from frostfs_testlib.testing.cluster_test_base import ClusterTestBase +from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.utils import datetime_utils from frostfs_testlib.utils.failover_utils import ( wait_all_storage_nodes_returned, wait_object_replication, ) +from frostfs_testlib.utils.file_keeper import FileKeeper from frostfs_testlib.utils.file_utils import generate_file, get_file_hash logger = logging.getLogger("NeoLogger") stopped_nodes: list[StorageNode] = [] +@pytest.fixture(scope="function") +@allure.title("Provide File Keeper") +def file_keeper(): + keeper = FileKeeper() + yield keeper + keeper.restore_files() + + @allure.step("Return all stopped hosts") @pytest.fixture(scope="function", autouse=True) -def after_run_return_all_stopped_hosts(client_shell: Shell, cluster: Cluster): - yield +def after_run_return_all_stopped_hosts(client_shell: Shell, cluster: Cluster) -> str: + yield "After this test stopped services will be started automatically via fixture" return_stopped_hosts(client_shell, cluster) +@allure.step("Return all stopped storage services after test") +@pytest.fixture(scope="function") +def after_run_return_all_stopped_services(cluster_state_controller: ClusterStateController): + yield + cluster_state_controller.start_stopped_storage_services() + + def panic_reboot_host(host: Host) -> None: shell = host.get_shell() shell.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"') @@ -495,3 +511,72 @@ class TestEmptyMap(ClusterTestBase): with allure.step("Delete bucket"): s3_client.delete_bucket(bucket) + + +@pytest.mark.failover +@pytest.mark.failover_data_loss +class TestStorageDataLoss(ClusterTestBase): + @allure.title( + "After metabase loss on all nodes operations on objects and buckets should be still available via S3" + ) + @pytest.mark.metabase_loss + def test_metabase_loss( + self, + s3_client: S3ClientWrapper, + simple_object_size: int, + complex_object_size: int, + cluster_state_controller: ClusterStateController, + after_run_return_all_stopped_services: str, + file_keeper: FileKeeper, + ): + allure.dynamic.description(after_run_return_all_stopped_services) + + with allure.step("Create bucket"): + bucket = s3_client.create_bucket() + + with allure.step("Put objects into bucket"): + simple_object_path = generate_file(simple_object_size) + simple_object_key = s3_helper.object_key_from_file_path(simple_object_path) + + complex_object_path = generate_file(complex_object_size) + complex_object_key = s3_helper.object_key_from_file_path(complex_object_path) + + s3_client.put_object(bucket, simple_object_path) + s3_client.put_object(bucket, complex_object_path) + + with allure.step("Check objects are in bucket"): + s3_helper.check_objects_in_bucket( + s3_client, bucket, expected_objects=[simple_object_key, complex_object_key] + ) + + with allure.step("Stop storage services on all nodes"): + cluster_state_controller.stop_all_storage_services() + + with allure.step("Delete metabase from all nodes"): + for node in cluster_state_controller.cluster.storage_nodes: + node.delete_metabase() + + with allure.step("Enable resync_metabase option for storage services"): + for storage_node in cluster_state_controller.cluster.storage_nodes: + with allure.step(f"Enable resync_metabase option for {storage_node}"): + config_file_path, config = storage_node.get_config() + if not config["storage"]["shard"]["default"]["resync_metabase"]: + file_keeper.add(storage_node, config_file_path) + config["storage"]["shard"]["default"]["resync_metabase"] = True + storage_node.save_config(config) + + with allure.step("Start storage services on all nodes"): + cluster_state_controller.start_stopped_storage_services() + + with allure.step("Delete objects from bucket"): + with allure.step("Delete simple object from bucket"): + with expect_not_raises(): + s3_client.delete_object(bucket, simple_object_key) + + with allure.step("Delete complex object from bucket"): + with expect_not_raises(): + s3_client.delete_object(bucket, complex_object_key) + + with allure.step("Delete bucket"): + with expect_not_raises(): + s3_client.delete_bucket(bucket) diff --git a/requirements.txt b/requirements.txt index d01cc1a..628ea64 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -allure-pytest==2.9.45 -allure-python-commons==2.9.45 +allure-pytest==2.13.2 +allure-python-commons==2.13.2 base58==2.1.0 boto3==1.16.33 botocore==1.19.33