diff --git a/pytest_tests/testsuites/failovers/test_failover_network.py b/pytest_tests/testsuites/failovers/test_failover_network.py index 233b480..0b2eac9 100644 --- a/pytest_tests/testsuites/failovers/test_failover_network.py +++ b/pytest_tests/testsuites/failovers/test_failover_network.py @@ -7,7 +7,7 @@ import allure import pytest from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL from frostfs_testlib.steps.cli.container import create_container -from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object_to_random_node +from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object, put_object_to_random_node from frostfs_testlib.steps.storage_object import delete_objects from frostfs_testlib.storage.cluster import ClusterNode from frostfs_testlib.storage.controllers import ClusterStateController @@ -59,15 +59,20 @@ class TestFailoverNetwork(ClusterTestBase): yield for path in file_wait_delete: os.remove(path) + file_wait_delete.clear() @pytest.fixture() - def storage_objects(self, simple_object_size: ObjectSize, default_wallet: str) -> list[StorageObjectInfo]: + def storage_objects( + self, + simple_object_size: ObjectSize, + default_wallet: str, + ) -> list[StorageObjectInfo]: file_path = generate_file(simple_object_size.value) file_hash = get_file_hash(file_path) with allure.step("Create container"): - placement_rule = "REP 1" + placement_rule = "REP 1 CBF 1" cid = create_container( wallet=default_wallet, shell=self.shell, @@ -179,6 +184,7 @@ class TestFailoverNetwork(ClusterTestBase): got_file_path = get_object(wallet, cid, oid, shell=self.shell, endpoint=new_nodes[0].get_rpc_endpoint()) assert get_file_hash(source_file_path) == get_file_hash(got_file_path) + @pytest.mark.interfaces @allure.title("Block DATA interface node") def test_block_data_interface( self, @@ -201,10 +207,9 @@ class TestFailoverNetwork(ClusterTestBase): ) with allure.step("Get data interface to node"): - data = "data" config_interfaces = list(nodes_with_object[0].host.config.interfaces.keys()) - with allure.step(f"Get {data} in {config_interfaces}"): - data_interfaces = [interface for interface in config_interfaces if data in interface] + with allure.step(f"Get data in {config_interfaces}"): + data_interfaces = [interface for interface in config_interfaces if "data" in interface] with allure.step("Block data interfaces for node"): for interface in data_interfaces: @@ -233,3 +238,87 @@ class TestFailoverNetwork(ClusterTestBase): endpoint=nodes_without_an_object[0].storage_node.get_rpc_endpoint(), ) file_wait_delete.append(input_file) + + with allure.step("Restore interface and tick 1 epoch, wait 2 block"): + cluster_state_controller.restore_interfaces() + self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2) + + @pytest.mark.interfaces + @allure.title("Block INTERNAL interface node") + def test_block_internal_interface( + self, + cluster_state_controller: ClusterStateController, + default_wallet: str, + restore_traffic: None, + delete_file_after_test: None, + storage_objects: list[StorageObjectInfo], + simple_object_size: ObjectSize, + ): + storage_object = storage_objects[0] + + with allure.step("Search nodes with object"): + nodes_with_object = get_object_nodes( + cluster=self.cluster, + wallet=default_wallet, + cid=storage_object.cid, + oid=storage_object.oid, + shell=self.shell, + endpoint=self.cluster.default_rpc_endpoint, + ) + + with allure.step("Get internal interface to node"): + config_interfaces = list(nodes_with_object[0].host.config.interfaces.keys()) + with allure.step(f"Get internal in {config_interfaces}"): + internal_interfaces = [interface for interface in config_interfaces if "internal" in interface] + + with allure.step("Block internal interfaces for node"): + for interface in internal_interfaces: + cluster_state_controller.down_interface(nodes=nodes_with_object, interface=interface) + + with allure.step("Tick epoch and wait 2 block"): + nodes_without_an_object = list(set(self.cluster.cluster_nodes) - set(nodes_with_object)) + self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2) + + with allure.step("Get object others node, expect false"): + with pytest.raises(RuntimeError, match="return code: 1"): + get_object( + wallet=default_wallet, + cid=storage_object.cid, + oid=storage_object.oid, + shell=self.shell, + endpoint=nodes_without_an_object[0].storage_node.get_rpc_endpoint(), + ) + + with allure.step("Put object, others node, expect false"): + with pytest.raises(RuntimeError, match="return code: 1"): + put_object( + wallet=default_wallet, + path=storage_object.file_path, + cid=storage_object.cid, + shell=self.shell, + endpoint=nodes_without_an_object[0].storage_node.get_rpc_endpoint(), + ) + + with allure.step(f"Get object nodes with object, expect true"): + input_file = get_object( + wallet=default_wallet, + cid=storage_object.cid, + oid=storage_object.oid, + shell=self.shell, + endpoint=nodes_with_object[0].storage_node.get_rpc_endpoint(), + ) + file_wait_delete.append(input_file) + + with allure.step(f"Put object nodes with object, expect true"): + temp_file_path = generate_file(simple_object_size.value) + _ = put_object( + wallet=default_wallet, + path=temp_file_path, + cid=storage_object.cid, + shell=self.shell, + endpoint=nodes_with_object[0].storage_node.get_rpc_endpoint(), + ) + file_wait_delete.append(temp_file_path) + with allure.step("Restore interface and tick 1 epoch, wait 2 block"): + cluster_state_controller.restore_interfaces() + self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2) diff --git a/pytest_tests/testsuites/failovers/test_failover_server.py b/pytest_tests/testsuites/failovers/test_failover_server.py index 3f2e560..aa20306 100644 --- a/pytest_tests/testsuites/failovers/test_failover_server.py +++ b/pytest_tests/testsuites/failovers/test_failover_server.py @@ -7,11 +7,7 @@ import allure import pytest from frostfs_testlib.resources.common import MORPH_BLOCK_TIME from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL -from frostfs_testlib.steps.cli.container import ( - StorageContainer, - StorageContainerInfo, - create_container, -) +from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container from frostfs_testlib.steps.cli.object import get_object from frostfs_testlib.steps.node_management import check_node_in_map, check_node_not_in_map from frostfs_testlib.storage.cluster import ClusterNode, StorageNode @@ -63,9 +59,7 @@ class TestFailoverServer(ClusterTestBase): wallet = WalletInfo(path=default_wallet) storage_cont_info = StorageContainerInfo(id=cont_id, wallet_file=wallet) containers.append( - StorageContainer( - storage_container_info=storage_cont_info, shell=self.shell, cluster=self.cluster - ) + StorageContainer(storage_container_info=storage_cont_info, shell=self.shell, cluster=self.cluster) ) return containers @@ -168,11 +162,9 @@ class TestFailoverServer(ClusterTestBase): assert not corrupted_objects_list with allure.step(f"check {node_to_stop.storage_node} in map"): - self.wait_node_in_map( - node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0] - ) + self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]) - count_tick_epoch = int(alive_nodes[0].ir_node.get_netmap_cleaner_threshold()) + 2 + count_tick_epoch = int(alive_nodes[0].ir_node.get_netmap_cleaner_threshold()) + 4 with allure.step(f"Tick {count_tick_epoch} epoch, in {storage_nodes[0]} node"): for tick in range(count_tick_epoch): @@ -183,13 +175,9 @@ class TestFailoverServer(ClusterTestBase): wait_for_host_offline(self.shell, node_to_stop.storage_node) with allure.step(f"Check {node_to_stop} in not map"): - self.wait_node_not_in_map( - node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0] - ) + self.wait_node_not_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]) - with allure.step( - f"Verify that there are no corrupted objects after {count_tick_epoch} epoch" - ): + with allure.step(f"Verify that there are no corrupted objects after {count_tick_epoch} epoch"): corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects) assert not corrupted_objects_list @@ -222,9 +210,7 @@ class TestFailoverServer(ClusterTestBase): assert not corrupted_objects_list with allure.step(f"Check {node_to_stop} in map"): - self.wait_node_in_map( - node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0] - ) + self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]) cluster_state_controller.start_node_host(node_to_stop)