Add metabase loss test

pytest_tests/testsuites/failovers/test_failover_storage.py

@@ -1,5 +1,4 @@
 import logging
-import os
 from time import sleep
 
 import allure
@@ -22,26 +21,43 @@ from frostfs_testlib.steps.node_management import (
 )
 from frostfs_testlib.steps.s3 import s3_helper
 from frostfs_testlib.storage.cluster import Cluster, StorageNode
-from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
+from frostfs_testlib.storage.controllers import ClusterStateController
 from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
+from frostfs_testlib.testing.test_control import expect_not_raises
 from frostfs_testlib.utils import datetime_utils
 from frostfs_testlib.utils.failover_utils import (
     wait_all_storage_nodes_returned,
     wait_object_replication,
 )
+from frostfs_testlib.utils.file_keeper import FileKeeper
 from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
 
 logger = logging.getLogger("NeoLogger")
 stopped_nodes: list[StorageNode] = []
 
 
+@pytest.fixture(scope="function")
+@allure.title("Provide File Keeper")
+def file_keeper():
+    keeper = FileKeeper()
+    yield keeper
+    keeper.restore_files()
+
+
 @allure.step("Return all stopped hosts")
 @pytest.fixture(scope="function", autouse=True)
-def after_run_return_all_stopped_hosts(client_shell: Shell, cluster: Cluster):
-    yield
+def after_run_return_all_stopped_hosts(client_shell: Shell, cluster: Cluster) -> str:
+    yield "After this test stopped services will be started automatically via fixture"
     return_stopped_hosts(client_shell, cluster)
 
 
+@allure.step("Return all stopped storage services after test")
+@pytest.fixture(scope="function")
+def after_run_return_all_stopped_services(cluster_state_controller: ClusterStateController):
+    yield
+    cluster_state_controller.start_stopped_storage_services()
+
+
 def panic_reboot_host(host: Host) -> None:
     shell = host.get_shell()
     shell.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"')
@@ -495,3 +511,72 @@ class TestEmptyMap(ClusterTestBase):
 
         with allure.step("Delete bucket"):
             s3_client.delete_bucket(bucket)
+
+
+@pytest.mark.failover
+@pytest.mark.failover_data_loss
+class TestStorageDataLoss(ClusterTestBase):
+    @allure.title(
+        "After metabase loss on all nodes operations on objects and buckets should be still available via S3"
+    )
+    @pytest.mark.metabase_loss
+    def test_metabase_loss(
+        self,
+        s3_client: S3ClientWrapper,
+        simple_object_size: int,
+        complex_object_size: int,
+        cluster_state_controller: ClusterStateController,
+        after_run_return_all_stopped_services: str,
+        file_keeper: FileKeeper,
+    ):
+        allure.dynamic.description(after_run_return_all_stopped_services)
+
+        with allure.step("Create bucket"):
+            bucket = s3_client.create_bucket()
+
+        with allure.step("Put objects into bucket"):
+            simple_object_path = generate_file(simple_object_size)
+            simple_object_key = s3_helper.object_key_from_file_path(simple_object_path)
+
+            complex_object_path = generate_file(complex_object_size)
+            complex_object_key = s3_helper.object_key_from_file_path(complex_object_path)
+
+            s3_client.put_object(bucket, simple_object_path)
+            s3_client.put_object(bucket, complex_object_path)
+
+        with allure.step("Check objects are in bucket"):
+            s3_helper.check_objects_in_bucket(
+                s3_client, bucket, expected_objects=[simple_object_key, complex_object_key]
+            )
+
+        with allure.step("Stop storage services on all nodes"):
+            cluster_state_controller.stop_all_storage_services()
+
+        with allure.step("Delete metabase from all nodes"):
+            for node in cluster_state_controller.cluster.storage_nodes:
+                node.delete_metabase()
+
+        with allure.step("Enable resync_metabase option for storage services"):
+            for storage_node in cluster_state_controller.cluster.storage_nodes:
+                with allure.step(f"Enable resync_metabase option for {storage_node}"):
+                    config_file_path, config = storage_node.get_config()
+                    if not config["storage"]["shard"]["default"]["resync_metabase"]:
+                        file_keeper.add(storage_node, config_file_path)
+                        config["storage"]["shard"]["default"]["resync_metabase"] = True
+                        storage_node.save_config(config)
+
+        with allure.step("Start storage services on all nodes"):
+            cluster_state_controller.start_stopped_storage_services()
+
+        with allure.step("Delete objects from bucket"):
+            with allure.step("Delete simple object from bucket"):
+                with expect_not_raises():
+                    s3_client.delete_object(bucket, simple_object_key)
+
+            with allure.step("Delete complex object from bucket"):
+                with expect_not_raises():
+                    s3_client.delete_object(bucket, complex_object_key)
+
+        with allure.step("Delete bucket"):
+            with expect_not_raises():
+                s3_client.delete_bucket(bucket)

requirements.txt

@@ -1,5 +1,5 @@
-allure-pytest==2.9.45
-allure-python-commons==2.9.45
+allure-pytest==2.13.2
+allure-python-commons==2.13.2
 base58==2.1.0
 boto3==1.16.33
 botocore==1.19.33