import logging from time import sleep from typing import Optional import allure from common import NEOFS_NETMAP_DICT from python_keywords.node_management import node_healthcheck from storage_policy import get_nodes_with_object logger = logging.getLogger('NeoLogger') @allure.step('Wait for object replication') def wait_object_replication_on_nodes(wallet: str, cid: str, oid: str, expected_copies: int, excluded_nodes: Optional[list[str]] = None) -> list[str]: excluded_nodes = excluded_nodes or [] sleep_interval, attempts = 10, 18 nodes = [] for __attempt in range(attempts): nodes = get_nodes_with_object(wallet, cid, oid, skip_nodes=excluded_nodes) if len(nodes) == expected_copies: return nodes sleep(sleep_interval) raise AssertionError(f'Expected {expected_copies} copies of object, but found {len(nodes)}. ' f'Waiting time {sleep_interval * attempts}') @allure.step('Wait for storage node returned to cluster') def wait_all_storage_node_returned(): sleep_interval, attempts = 10, 12 for __attempt in range(attempts): if is_all_storage_node_returned(): return sleep(sleep_interval) raise AssertionError('Storage node(s) is broken') def is_all_storage_node_returned() -> bool: with allure.step('Run health check for all storage nodes'): for node_name in NEOFS_NETMAP_DICT.keys(): try: health_check = node_healthcheck(node_name) except Exception as err: logger.warning(f'Node healthcheck fails with error {err}') return False if health_check.health_status != 'READY' or health_check.network_status != 'ONLINE': return False return True