import logging import os.path import random import allure import pytest from frostfs_testlib import reporter from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object from frostfs_testlib.steps.node_management import check_node_in_map, check_node_not_in_map from frostfs_testlib.storage.cluster import ClusterNode, StorageNode from frostfs_testlib.storage.controllers import ClusterStateController from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.parallel import parallel from frostfs_testlib.testing.test_control import wait_for_success from frostfs_testlib.utils.failover_utils import wait_object_replication from frostfs_testlib.utils.file_utils import get_file_hash from pytest import FixtureRequest logger = logging.getLogger("NeoLogger") @pytest.mark.failover @pytest.mark.failover_server class TestFailoverServer(ClusterTestBase): @wait_for_success(max_wait_time=120, interval=1) def wait_node_not_in_map(self, *args, **kwargs): check_node_not_in_map(*args, **kwargs) @wait_for_success(max_wait_time=120, interval=1) def wait_node_in_map(self, *args, **kwargs): check_node_in_map(*args, **kwargs) @reporter.step("Create {count_containers} containers and {count_files} objects") @pytest.fixture def containers( self, request: FixtureRequest, default_wallet: WalletInfo, ) -> list[StorageContainer]: placement_rule = "REP 2 CBF 2 SELECT 2 FROM *" containers = [] for _ in range(request.param): cont_id = create_container( default_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, rule=placement_rule, basic_acl=PUBLIC_ACL, ) storage_cont_info = StorageContainerInfo(cont_id, default_wallet) containers.append(StorageContainer(storage_cont_info, self.shell, self.cluster)) return containers @reporter.step("Creation container") @pytest.fixture() def container(self, default_wallet: WalletInfo) -> StorageContainer: select = len(self.cluster.cluster_nodes) placement_rule = f"REP {select - 1} CBF 1 SELECT {select} FROM *" cont_id = create_container( default_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, rule=placement_rule, basic_acl=PUBLIC_ACL, ) storage_cont_info = StorageContainerInfo(cont_id, default_wallet) return StorageContainer(storage_cont_info, self.shell, self.cluster) @reporter.step("Create object and delete after test") @pytest.fixture(scope="class") def storage_objects( self, request: FixtureRequest, containers: list[StorageContainer], simple_object_size: ObjectSize, complex_object_size: ObjectSize, ) -> StorageObjectInfo: count_object = request.param object_sizes = [simple_object_size, complex_object_size] object_list: list[StorageObjectInfo] = [] for cont in containers: for _ in range(count_object): object_list.append(cont.generate_object(size=random.choice(object_sizes).value)) for storage_object in object_list: os.remove(storage_object.file_path) yield object_list @reporter.step("Select random node to stop and start it after test") @pytest.fixture def node_to_stop( self, node_under_test: ClusterNode, cluster_state_controller: ClusterStateController ) -> ClusterNode: yield node_under_test with reporter.step(f"start {node_under_test.storage_node}"): cluster_state_controller.start_stopped_hosts() @reporter.step("Upload object with nodes and compare") def get_corrupted_objects_list( self, nodes: list[StorageNode], storage_objects: list[StorageObjectInfo] ) -> list[StorageObjectInfo]: corrupted_objects = [] errors_get = [] for node in nodes: for storage_object in storage_objects: try: got_file_path = get_object( storage_object.wallet, storage_object.cid, storage_object.oid, endpoint=node.get_rpc_endpoint(), shell=self.shell, timeout="60s", ) if storage_object.file_hash != get_file_hash(got_file_path): corrupted_objects.append(storage_object) os.remove(got_file_path) except RuntimeError: errors_get.append(storage_object.oid) assert len(errors_get) == 0, f"Get failed - {errors_get}" return corrupted_objects def check_objects_replication( self, storage_objects: list[StorageObjectInfo], storage_nodes: list[StorageNode] ) -> None: for storage_object in storage_objects: wait_object_replication( storage_object.cid, storage_object.oid, 2, shell=self.shell, nodes=storage_nodes, sleep_interval=45, attempts=60, ) @pytest.fixture() def object_and_nodes( self, simple_object_size: ObjectSize, container: StorageContainer ) -> tuple[StorageObjectInfo, list[ClusterNode]]: object_info = container.generate_object(simple_object_size.value) object_nodes = get_object_nodes( cluster=self.cluster, cid=object_info.cid, oid=object_info.oid, alive_node=self.cluster.cluster_nodes[0] ) return object_info, object_nodes @pytest.fixture() def up_stop_nodes(self, cluster_state_controller: ClusterStateController): yield cluster_state_controller.start_stopped_hosts() @allure.title("Full shutdown node") @pytest.mark.parametrize("containers, storage_objects", [(5, 10)], indirect=True) def test_complete_node_shutdown( self, storage_objects: list[StorageObjectInfo], node_to_stop: ClusterNode, cluster_state_controller: ClusterStateController, ): with reporter.step(f"Remove {node_to_stop} from the list of nodes"): alive_nodes = list(set(self.cluster.cluster_nodes) - {node_to_stop}) storage_nodes = [cluster.storage_node for cluster in alive_nodes] with reporter.step("Tick epoch and wait for 2 blocks"): self.tick_epochs(1, storage_nodes[0], wait_block=2) with reporter.step(f"Stop node"): cluster_state_controller.stop_node_host(node=node_to_stop, mode="hard") with reporter.step("Verify that there are no corrupted objects"): corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects) assert not corrupted_objects_list with reporter.step(f"check {node_to_stop.storage_node} in map"): self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]) count_tick_epoch = int(alive_nodes[0].ir_node.get_netmap_cleaner_threshold()) + 4 with reporter.step(f"Tick {count_tick_epoch} epochs and wait for 2 blocks"): self.tick_epochs(count_tick_epoch, storage_nodes[0], wait_block=2) with reporter.step(f"Check {node_to_stop} in not map"): self.wait_node_not_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]) with reporter.step(f"Verify that there are no corrupted objects after {count_tick_epoch} epoch"): corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects) assert not corrupted_objects_list @allure.title("Temporarily disable a node") @pytest.mark.parametrize("containers, storage_objects", [(5, 10)], indirect=True) def test_temporarily_disable_a_node( self, storage_objects: list[StorageObjectInfo], node_to_stop: ClusterNode, cluster_state_controller: ClusterStateController, ): with reporter.step(f"Remove {node_to_stop} from the list of nodes"): storage_nodes = list(set(self.cluster.storage_nodes) - {node_to_stop.storage_node}) with reporter.step("Tick epoch and wait for 2 blocks"): self.tick_epochs(1, storage_nodes[0], wait_block=2) with reporter.step(f"Stop node"): cluster_state_controller.stop_node_host(node_to_stop, "hard") with reporter.step("Verify that there are no corrupted objects"): corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects) assert not corrupted_objects_list with reporter.step(f"Check {node_to_stop} in map"): self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]) cluster_state_controller.start_node_host(node_to_stop) with reporter.step("Verify that there are no corrupted objects"): corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects) assert not corrupted_objects_list @allure.title("Not enough nodes in the container with policy - 'REP 3 CBF 1 SELECT 4 FROM *'") def test_not_enough_nodes_in_container_rep_3( self, object_and_nodes: tuple[StorageObjectInfo, list[ClusterNode]], default_wallet: WalletInfo, cluster_state_controller: ClusterStateController, simple_file: str, up_stop_nodes: None, ): object_info, object_nodes = object_and_nodes endpoint_without_object = list(set(self.cluster.cluster_nodes) - set(object_nodes))[ 0 ].storage_node.get_rpc_endpoint() endpoint_with_object = object_nodes[0].storage_node.get_rpc_endpoint() with reporter.step("Stop all nodes with object except first one"): parallel(cluster_state_controller.stop_node_host, object_nodes[1:], mode="hard") with reporter.step(f"Get object from node without object"): get_object(default_wallet, object_info.cid, object_info.oid, self.shell, endpoint_without_object) with reporter.step(f"Get object from node with object"): get_object(default_wallet, object_info.cid, object_info.oid, self.shell, endpoint_with_object) with reporter.step(f"Put operation to node with object, expect error"): with pytest.raises(RuntimeError): put_object(default_wallet, simple_file, object_info.cid, self.shell, endpoint_with_object) @allure.title("Not enough nodes in the container with policy - 'REP 2 CBF 2 SELECT 4 FROM *'") def test_not_enough_nodes_in_container_rep_2( self, default_wallet: WalletInfo, cluster_state_controller: ClusterStateController, simple_file: str, up_stop_nodes: None, ): with reporter.step("Create container with full network map"): node_count = len(self.cluster.cluster_nodes) placement_rule = f"REP {node_count - 2} IN X CBF 2 SELECT {node_count} FROM * AS X" cid = create_container( default_wallet, self.shell, self.cluster.default_rpc_endpoint, rule=placement_rule, basic_acl=PUBLIC_ACL, ) with reporter.step("Put object"): oid = put_object(default_wallet, simple_file, cid, self.shell, self.cluster.default_rpc_endpoint) with reporter.step("Search nodes with object"): object_nodes = get_object_nodes(self.cluster, cid, oid, self.cluster.cluster_nodes[0]) with reporter.step("Choose node to stop"): node_to_stop = random.choice(object_nodes) alive_node_with_object = random.choice(list(set(object_nodes) - {node_to_stop})) alive_endpoint_with_object = alive_node_with_object.storage_node.get_rpc_endpoint() with reporter.step("Stop random node with object"): cluster_state_controller.stop_node_host(node_to_stop, "hard") with reporter.step("Put object to alive node with object"): oid_2 = put_object(default_wallet, simple_file, cid, self.shell, alive_endpoint_with_object) with reporter.step("Get object from alive node with object"): get_file = get_object(default_wallet, cid, oid_2, self.shell, alive_endpoint_with_object) os.remove(get_file) with reporter.step("Create container on alive node"): create_container( default_wallet, self.shell, alive_endpoint_with_object, rule=placement_rule, basic_acl=PUBLIC_ACL, )