diff --git a/pytest_tests/pytest.ini b/pytest_tests/pytest.ini index 0b0be95..bd86ae6 100644 --- a/pytest_tests/pytest.ini +++ b/pytest_tests/pytest.ini @@ -42,6 +42,7 @@ markers = failover_panic: tests for system recovery after panic reboot of a node failover_network: tests for network failure failover_reboot: tests for system recovery after reboot of a node + interfaces: tests down interface to system add_nodes: add nodes to cluster check_binaries: check frostfs installed binaries versions payments: tests for payment associated operations diff --git a/pytest_tests/testsuites/failovers/test_failover_network.py b/pytest_tests/testsuites/failovers/test_failover_network.py index 0b2eac9..7bd911f 100644 --- a/pytest_tests/testsuites/failovers/test_failover_network.py +++ b/pytest_tests/testsuites/failovers/test_failover_network.py @@ -6,13 +6,20 @@ from time import sleep import allure import pytest from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL +from frostfs_testlib.s3 import AwsCliClient from frostfs_testlib.steps.cli.container import create_container -from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object, put_object_to_random_node +from frostfs_testlib.steps.cli.object import ( + get_object, + get_object_nodes, + neo_go_query_height, + put_object, + put_object_to_random_node, +) from frostfs_testlib.steps.storage_object import delete_objects from frostfs_testlib.storage.cluster import ClusterNode from frostfs_testlib.storage.controllers import ClusterStateController from frostfs_testlib.storage.dataclasses.object_size import ObjectSize -from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo +from frostfs_testlib.storage.dataclasses.storage_object_info import Interfaces, StorageObjectInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.utils.failover_utils import wait_all_storage_nodes_returned, wait_object_replication from frostfs_testlib.utils.file_utils import generate_file, get_file_hash @@ -31,6 +38,11 @@ OBJECT_ATTRIBUTES = [ ] +def pytest_generate_tests(metafunc: pytest.Metafunc): + if "s3_client" in metafunc.fixturenames: + metafunc.parametrize("s3_client", [AwsCliClient], indirect=True) + + @pytest.mark.failover @pytest.mark.failover_network class TestFailoverNetwork(ClusterTestBase): @@ -49,7 +61,7 @@ class TestFailoverNetwork(ClusterTestBase): @pytest.fixture() @allure.title("Restore drop traffic to system") - def restore_traffic(self, cluster_state_controller: ClusterStateController): + def restore_down_interfaces(self, cluster_state_controller: ClusterStateController): yield cluster_state_controller.restore_interfaces() @@ -190,7 +202,7 @@ class TestFailoverNetwork(ClusterTestBase): self, cluster_state_controller: ClusterStateController, default_wallet: str, - restore_traffic: None, + restore_down_interfaces: None, delete_file_after_test: None, storage_objects: list[StorageObjectInfo], ): @@ -249,7 +261,7 @@ class TestFailoverNetwork(ClusterTestBase): self, cluster_state_controller: ClusterStateController, default_wallet: str, - restore_traffic: None, + restore_down_interfaces: None, delete_file_after_test: None, storage_objects: list[StorageObjectInfo], simple_object_size: ObjectSize, @@ -322,3 +334,138 @@ class TestFailoverNetwork(ClusterTestBase): with allure.step("Restore interface and tick 1 epoch, wait 2 block"): cluster_state_controller.restore_interfaces() self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2) + + @pytest.mark.interfaces + @pytest.mark.baremetal + @pytest.mark.parametrize( + "block_interface, other_interface", + [(Interfaces.DATA_O, Interfaces.DATA_1), (Interfaces.DATA_1, Interfaces.DATA_O)], + ) + @allure.title("Down data interfaces to all nodes(interface={block_interface})") + def test_down_data_interface( + self, + cluster_state_controller: ClusterStateController, + default_wallet: str, + simple_object_size: ObjectSize, + delete_file_after_test: None, + restore_down_interfaces: None, + block_interface: Interfaces, + other_interface: Interfaces, + ): + cluster_nodes = self.cluster.cluster_nodes + with allure.step(f"Block {block_interface.value} interfaces"): + cluster_state_controller.down_interface(cluster_nodes, block_interface.value) + + with allure.step("Tick 1 epoch and wait 2 block for sync all nodes"): + self.tick_epochs(1, alive_node=cluster_nodes[0].storage_node, wait_block=2) + + with allure.step("Create container"): + cid = create_container( + wallet=default_wallet, + shell=self.shell, + endpoint=f"{cluster_nodes[0].get_data_interface(other_interface.value)[0]}:8080", + rule="REP 4 CBF 1", + ) + + with allure.step("Put object"): + file_path = generate_file(simple_object_size.value) + file_wait_delete.append(file_path) + + oid = put_object( + wallet=default_wallet, + path=file_path, + cid=cid, + shell=self.shell, + endpoint=f"{cluster_nodes[0].get_data_interface(other_interface.value)[0]}:8080", + ) + + with allure.step("Get object"): + file_get_path = get_object( + wallet=default_wallet, + cid=cid, + oid=oid, + shell=self.shell, + endpoint=f"{cluster_nodes[0].get_data_interface(other_interface.value)[0]}:8080", + ) + file_wait_delete.append(file_get_path) + + with allure.step("Restore interfaces all nodes"): + cluster_state_controller.restore_interfaces() + self.tick_epochs(1, alive_node=cluster_nodes[0].storage_node, wait_block=2) + + @pytest.mark.interfaces + @pytest.mark.baremetal + @pytest.mark.parametrize("interface", [Interfaces.INTERNAL_0, Interfaces.INTERNAL_1]) + @allure.title("Down internal interfaces to all nodes(interface={interface})") + def test_down_internal_interface( + self, + cluster_state_controller: ClusterStateController, + default_wallet: str, + simple_object_size: ObjectSize, + delete_file_after_test: None, + restore_down_interfaces: None, + interface: Interfaces, + ): + cluster_nodes = self.cluster.cluster_nodes + latest_block = {} + + with allure.step("Get block all nodes"): + for cluster_node in cluster_nodes: + latest_block[cluster_node] = neo_go_query_height( + shell=cluster_node.host.get_shell(), endpoint=cluster_node.morph_chain.get_http_endpoint() + ) + + with allure.step(f"Block {interface} interfaces"): + cluster_state_controller.down_interface(cluster_nodes, interface.value) + + with allure.step("Tick 1 epoch and wait 2 block for sync all nodes"): + self.tick_epochs(1, alive_node=cluster_nodes[0].storage_node, wait_block=2) + + with allure.step("Create container"): + cid = create_container( + wallet=default_wallet, + shell=self.shell, + endpoint=self.cluster.default_rpc_endpoint, + rule="REP 4 CBF 1", + ) + + with allure.step(f"Put object, after down {interface}"): + file_path = generate_file(simple_object_size.value) + file_wait_delete.append(file_path) + + oid = put_object( + wallet=default_wallet, + path=file_path, + cid=cid, + shell=self.shell, + endpoint=self.cluster.default_rpc_endpoint, + ) + + with allure.step("Get object"): + file_get_path = get_object( + wallet=default_wallet, + cid=cid, + oid=oid, + shell=self.shell, + endpoint=self.cluster.default_rpc_endpoint, + ) + file_wait_delete.append(file_get_path) + + now_block = {} + + with allure.step("Get actual block"): + for cluster_node in cluster_nodes: + now_block[cluster_node] = neo_go_query_height( + shell=cluster_node.host.get_shell(), endpoint=cluster_node.morph_chain.get_http_endpoint() + ) + with allure.step(f"Compare block"): + for cluster_node, items in now_block.items(): + with allure.step( + f"Node - {cluster_node.host_ip}, old block - {latest_block[cluster_node]['Latest block']}, " + f"now block - {now_block[cluster_node]['Latest block']}" + ): + assert latest_block[cluster_node]["Latest block"] < now_block[cluster_node]["Latest block"] + + with allure.step("Restore interfaces all nodes"): + cluster_state_controller.restore_interfaces() + self.tick_epochs(1, alive_node=self.cluster.cluster_nodes[0].storage_node, wait_block=2)