Add network internal interface test

Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>

pytest_tests/testsuites/failovers/test_failover_network.py

@@ -7,7 +7,7 @@ import allure
 import pytest
 from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
 from frostfs_testlib.steps.cli.container import create_container
-from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object_to_random_node
+from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object, put_object_to_random_node
 from frostfs_testlib.steps.storage_object import delete_objects
 from frostfs_testlib.storage.cluster import ClusterNode
 from frostfs_testlib.storage.controllers import ClusterStateController
@@ -59,15 +59,20 @@ class TestFailoverNetwork(ClusterTestBase):
         yield
         for path in file_wait_delete:
             os.remove(path)
+        file_wait_delete.clear()
 
     @pytest.fixture()
-    def storage_objects(self, simple_object_size: ObjectSize, default_wallet: str) -> list[StorageObjectInfo]:
+    def storage_objects(
+        self,
+        simple_object_size: ObjectSize,
+        default_wallet: str,
+    ) -> list[StorageObjectInfo]:
 
         file_path = generate_file(simple_object_size.value)
         file_hash = get_file_hash(file_path)
 
         with allure.step("Create container"):
-            placement_rule = "REP 1"
+            placement_rule = "REP 1 CBF 1"
             cid = create_container(
                 wallet=default_wallet,
                 shell=self.shell,
@@ -179,6 +184,7 @@ class TestFailoverNetwork(ClusterTestBase):
             got_file_path = get_object(wallet, cid, oid, shell=self.shell, endpoint=new_nodes[0].get_rpc_endpoint())
             assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
 
+    @pytest.mark.interfaces
     @allure.title("Block DATA interface node")
     def test_block_data_interface(
         self,
@@ -201,10 +207,9 @@ class TestFailoverNetwork(ClusterTestBase):
             )
 
         with allure.step("Get data interface to node"):
-            data = "data"
             config_interfaces = list(nodes_with_object[0].host.config.interfaces.keys())
-            with allure.step(f"Get {data} in {config_interfaces}"):
-                data_interfaces = [interface for interface in config_interfaces if data in interface]
+            with allure.step(f"Get data in {config_interfaces}"):
+                data_interfaces = [interface for interface in config_interfaces if "data" in interface]
 
         with allure.step("Block data interfaces for node"):
             for interface in data_interfaces:
@@ -233,3 +238,87 @@ class TestFailoverNetwork(ClusterTestBase):
                 endpoint=nodes_without_an_object[0].storage_node.get_rpc_endpoint(),
             )
             file_wait_delete.append(input_file)
+
+        with allure.step("Restore interface and tick 1 epoch, wait 2 block"):
+            cluster_state_controller.restore_interfaces()
+            self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)
+
+    @pytest.mark.interfaces
+    @allure.title("Block INTERNAL interface node")
+    def test_block_internal_interface(
+        self,
+        cluster_state_controller: ClusterStateController,
+        default_wallet: str,
+        restore_traffic: None,
+        delete_file_after_test: None,
+        storage_objects: list[StorageObjectInfo],
+        simple_object_size: ObjectSize,
+    ):
+        storage_object = storage_objects[0]
+
+        with allure.step("Search nodes with object"):
+            nodes_with_object = get_object_nodes(
+                cluster=self.cluster,
+                wallet=default_wallet,
+                cid=storage_object.cid,
+                oid=storage_object.oid,
+                shell=self.shell,
+                endpoint=self.cluster.default_rpc_endpoint,
+            )
+
+        with allure.step("Get internal interface to node"):
+            config_interfaces = list(nodes_with_object[0].host.config.interfaces.keys())
+            with allure.step(f"Get internal in {config_interfaces}"):
+                internal_interfaces = [interface for interface in config_interfaces if "internal" in interface]
+
+        with allure.step("Block internal interfaces for node"):
+            for interface in internal_interfaces:
+                cluster_state_controller.down_interface(nodes=nodes_with_object, interface=interface)
+
+        with allure.step("Tick epoch and wait 2 block"):
+            nodes_without_an_object = list(set(self.cluster.cluster_nodes) - set(nodes_with_object))
+            self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)
+
+        with allure.step("Get object others node, expect false"):
+            with pytest.raises(RuntimeError, match="return code: 1"):
+                get_object(
+                    wallet=default_wallet,
+                    cid=storage_object.cid,
+                    oid=storage_object.oid,
+                    shell=self.shell,
+                    endpoint=nodes_without_an_object[0].storage_node.get_rpc_endpoint(),
+                )
+
+        with allure.step("Put object, others node, expect false"):
+            with pytest.raises(RuntimeError, match="return code: 1"):
+                put_object(
+                    wallet=default_wallet,
+                    path=storage_object.file_path,
+                    cid=storage_object.cid,
+                    shell=self.shell,
+                    endpoint=nodes_without_an_object[0].storage_node.get_rpc_endpoint(),
+                )
+
+        with allure.step(f"Get object nodes with object, expect true"):
+            input_file = get_object(
+                wallet=default_wallet,
+                cid=storage_object.cid,
+                oid=storage_object.oid,
+                shell=self.shell,
+                endpoint=nodes_with_object[0].storage_node.get_rpc_endpoint(),
+            )
+            file_wait_delete.append(input_file)
+
+        with allure.step(f"Put object nodes with object, expect true"):
+            temp_file_path = generate_file(simple_object_size.value)
+            _ = put_object(
+                wallet=default_wallet,
+                path=temp_file_path,
+                cid=storage_object.cid,
+                shell=self.shell,
+                endpoint=nodes_with_object[0].storage_node.get_rpc_endpoint(),
+            )
+            file_wait_delete.append(temp_file_path)
+        with allure.step("Restore interface and tick 1 epoch, wait 2 block"):
+            cluster_state_controller.restore_interfaces()
+            self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)

pytest_tests/testsuites/failovers/test_failover_server.py

@@ -7,11 +7,7 @@ import allure
 import pytest
 from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
 from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
-from frostfs_testlib.steps.cli.container import (
-    StorageContainer,
-    StorageContainerInfo,
-    create_container,
-)
+from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
 from frostfs_testlib.steps.cli.object import get_object
 from frostfs_testlib.steps.node_management import check_node_in_map, check_node_not_in_map
 from frostfs_testlib.storage.cluster import ClusterNode, StorageNode
@@ -63,9 +59,7 @@ class TestFailoverServer(ClusterTestBase):
             wallet = WalletInfo(path=default_wallet)
             storage_cont_info = StorageContainerInfo(id=cont_id, wallet_file=wallet)
             containers.append(
-                StorageContainer(
-                    storage_container_info=storage_cont_info, shell=self.shell, cluster=self.cluster
-                )
+                StorageContainer(storage_container_info=storage_cont_info, shell=self.shell, cluster=self.cluster)
             )
 
         return containers
@@ -168,11 +162,9 @@ class TestFailoverServer(ClusterTestBase):
             assert not corrupted_objects_list
 
         with allure.step(f"check {node_to_stop.storage_node} in map"):
-            self.wait_node_in_map(
-                node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]
-            )
+            self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
 
-        count_tick_epoch = int(alive_nodes[0].ir_node.get_netmap_cleaner_threshold()) + 2
+        count_tick_epoch = int(alive_nodes[0].ir_node.get_netmap_cleaner_threshold()) + 4
 
         with allure.step(f"Tick {count_tick_epoch} epoch, in {storage_nodes[0]} node"):
             for tick in range(count_tick_epoch):
@@ -183,13 +175,9 @@ class TestFailoverServer(ClusterTestBase):
             wait_for_host_offline(self.shell, node_to_stop.storage_node)
 
         with allure.step(f"Check {node_to_stop} in not map"):
-            self.wait_node_not_in_map(
-                node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]
-            )
+            self.wait_node_not_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
 
-        with allure.step(
-            f"Verify that there are no corrupted objects after {count_tick_epoch} epoch"
-        ):
+        with allure.step(f"Verify that there are no corrupted objects after {count_tick_epoch} epoch"):
             corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
             assert not corrupted_objects_list
 
@@ -222,9 +210,7 @@ class TestFailoverServer(ClusterTestBase):
             assert not corrupted_objects_list
 
         with allure.step(f"Check {node_to_stop} in map"):
-            self.wait_node_in_map(
-                node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]
-            )
+            self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
 
         cluster_state_controller.start_node_host(node_to_stop)