pytest_tests/pytest.ini

@@ -42,6 +42,7 @@ markers =
     failover_panic: tests for system recovery after panic reboot of a node
     failover_network: tests for network failure
     failover_reboot: tests for system recovery after reboot of a node
+    interfaces: tests down interface to system
     add_nodes: add nodes to cluster
     check_binaries: check frostfs installed binaries versions
     payments: tests for payment associated operations

pytest_tests/testsuites/failovers/test_failover_network.py

@@ -6,13 +6,20 @@ from time import sleep
 import allure
 import pytest
 from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
+from frostfs_testlib.s3 import AwsCliClient
 from frostfs_testlib.steps.cli.container import create_container
-from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object, put_object_to_random_node
+from frostfs_testlib.steps.cli.object import (
+    get_object,
+    get_object_nodes,
+    neo_go_query_height,
+    put_object,
+    put_object_to_random_node,
+)
 from frostfs_testlib.steps.storage_object import delete_objects
 from frostfs_testlib.storage.cluster import ClusterNode
 from frostfs_testlib.storage.controllers import ClusterStateController
 from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
-from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
+from frostfs_testlib.storage.dataclasses.storage_object_info import Interfaces, StorageObjectInfo
 from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
 from frostfs_testlib.utils.failover_utils import wait_all_storage_nodes_returned, wait_object_replication
 from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
@@ -31,6 +38,11 @@ OBJECT_ATTRIBUTES = [
 ]
 
 
+def pytest_generate_tests(metafunc: pytest.Metafunc):
+    if "s3_client" in metafunc.fixturenames:
+        metafunc.parametrize("s3_client", [AwsCliClient], indirect=True)
+
+
 @pytest.mark.failover
 @pytest.mark.failover_network
 class TestFailoverNetwork(ClusterTestBase):
@@ -49,7 +61,7 @@ class TestFailoverNetwork(ClusterTestBase):
 
     @pytest.fixture()
     @allure.title("Restore drop traffic to system")
-    def restore_traffic(self, cluster_state_controller: ClusterStateController):
+    def restore_down_interfaces(self, cluster_state_controller: ClusterStateController):
         yield
         cluster_state_controller.restore_interfaces()
 
@@ -190,7 +202,7 @@ class TestFailoverNetwork(ClusterTestBase):
         self,
         cluster_state_controller: ClusterStateController,
         default_wallet: str,
-        restore_traffic: None,
+        restore_down_interfaces: None,
         delete_file_after_test: None,
         storage_objects: list[StorageObjectInfo],
     ):
@@ -249,7 +261,7 @@ class TestFailoverNetwork(ClusterTestBase):
         self,
         cluster_state_controller: ClusterStateController,
         default_wallet: str,
-        restore_traffic: None,
+        restore_down_interfaces: None,
         delete_file_after_test: None,
         storage_objects: list[StorageObjectInfo],
         simple_object_size: ObjectSize,
@@ -322,3 +334,138 @@ class TestFailoverNetwork(ClusterTestBase):
         with allure.step("Restore interface and tick 1 epoch, wait 2 block"):
             cluster_state_controller.restore_interfaces()
             self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)
+
+    @pytest.mark.interfaces
+    @pytest.mark.baremetal
+    @pytest.mark.parametrize(
+        "block_interface, other_interface",
+        [(Interfaces.DATA_O, Interfaces.DATA_1), (Interfaces.DATA_1, Interfaces.DATA_O)],
+    )
+    @allure.title("Down data interfaces to all nodes(interface={block_interface})")
+    def test_down_data_interface(
+        self,
+        cluster_state_controller: ClusterStateController,
+        default_wallet: str,
+        simple_object_size: ObjectSize,
+        delete_file_after_test: None,
+        restore_down_interfaces: None,
+        block_interface: Interfaces,
+        other_interface: Interfaces,
+    ):
+        cluster_nodes = self.cluster.cluster_nodes
+        with allure.step(f"Block {block_interface.value} interfaces"):
+            cluster_state_controller.down_interface(cluster_nodes, block_interface.value)
+
+        with allure.step("Tick 1 epoch and wait 2 block for sync all nodes"):
+            self.tick_epochs(1, alive_node=cluster_nodes[0].storage_node, wait_block=2)
+
+        with allure.step("Create container"):
+            cid = create_container(
+                wallet=default_wallet,
+                shell=self.shell,
+                endpoint=f"{cluster_nodes[0].get_data_interface(other_interface.value)[0]}:8080",
+                rule="REP 4 CBF 1",
+            )
+
+        with allure.step("Put object"):
+            file_path = generate_file(simple_object_size.value)
+            file_wait_delete.append(file_path)
+
+            oid = put_object(
+                wallet=default_wallet,
+                path=file_path,
+                cid=cid,
+                shell=self.shell,
+                endpoint=f"{cluster_nodes[0].get_data_interface(other_interface.value)[0]}:8080",
+            )
+
+        with allure.step("Get object"):
+            file_get_path = get_object(
+                wallet=default_wallet,
+                cid=cid,
+                oid=oid,
+                shell=self.shell,
+                endpoint=f"{cluster_nodes[0].get_data_interface(other_interface.value)[0]}:8080",
+            )
+            file_wait_delete.append(file_get_path)
+
+        with allure.step("Restore interfaces all nodes"):
+            cluster_state_controller.restore_interfaces()
+            self.tick_epochs(1, alive_node=cluster_nodes[0].storage_node, wait_block=2)
+
+    @pytest.mark.interfaces
+    @pytest.mark.baremetal
+    @pytest.mark.parametrize("interface", [Interfaces.INTERNAL_0, Interfaces.INTERNAL_1])
+    @allure.title("Down internal interfaces to all nodes(interface={interface})")
+    def test_down_internal_interface(
+        self,
+        cluster_state_controller: ClusterStateController,
+        default_wallet: str,
+        simple_object_size: ObjectSize,
+        delete_file_after_test: None,
+        restore_down_interfaces: None,
+        interface: Interfaces,
+    ):
+        cluster_nodes = self.cluster.cluster_nodes
+        latest_block = {}
+
+        with allure.step("Get block all nodes"):
+            for cluster_node in cluster_nodes:
+                latest_block[cluster_node] = neo_go_query_height(
+                    shell=cluster_node.host.get_shell(), endpoint=cluster_node.morph_chain.get_http_endpoint()
+                )
+
+        with allure.step(f"Block {interface} interfaces"):
+            cluster_state_controller.down_interface(cluster_nodes, interface.value)
+
+        with allure.step("Tick 1 epoch and wait 2 block for sync all nodes"):
+            self.tick_epochs(1, alive_node=cluster_nodes[0].storage_node, wait_block=2)
+
+        with allure.step("Create container"):
+            cid = create_container(
+                wallet=default_wallet,
+                shell=self.shell,
+                endpoint=self.cluster.default_rpc_endpoint,
+                rule="REP 4 CBF 1",
+            )
+
+        with allure.step(f"Put object, after down {interface}"):
+            file_path = generate_file(simple_object_size.value)
+            file_wait_delete.append(file_path)
+
+            oid = put_object(
+                wallet=default_wallet,
+                path=file_path,
+                cid=cid,
+                shell=self.shell,
+                endpoint=self.cluster.default_rpc_endpoint,
+            )
+
+        with allure.step("Get object"):
+            file_get_path = get_object(
+                wallet=default_wallet,
+                cid=cid,
+                oid=oid,
+                shell=self.shell,
+                endpoint=self.cluster.default_rpc_endpoint,
+            )
+            file_wait_delete.append(file_get_path)
+
+        now_block = {}
+
+        with allure.step("Get actual block"):
+            for cluster_node in cluster_nodes:
+                now_block[cluster_node] = neo_go_query_height(
+                    shell=cluster_node.host.get_shell(), endpoint=cluster_node.morph_chain.get_http_endpoint()
+                )
+        with allure.step(f"Compare block"):
+            for cluster_node, items in now_block.items():
+                with allure.step(
+                    f"Node - {cluster_node.host_ip}, old block - {latest_block[cluster_node]['Latest block']}, "
+                    f"now block - {now_block[cluster_node]['Latest block']}"
+                ):
+                    assert latest_block[cluster_node]["Latest block"] < now_block[cluster_node]["Latest block"]
+
+        with allure.step("Restore interfaces all nodes"):
+            cluster_state_controller.restore_interfaces()
+            self.tick_epochs(1, alive_node=self.cluster.cluster_nodes[0].storage_node, wait_block=2)