frostfs-testcases/robot/resources/lib/python_keywords/failover_utils.py

import logging
from time import sleep
from typing import Optional

import allure
from common import NEOFS_NETMAP_DICT
from python_keywords.node_management import node_healthcheck
from storage_policy import get_nodes_with_object

logger = logging.getLogger("NeoLogger")


@allure.step("Wait for object replication")
def wait_object_replication_on_nodes(
    wallet: str,
    cid: str,
    oid: str,
    expected_copies: int,
    excluded_nodes: Optional[list[str]] = None,
) -> list[str]:
    excluded_nodes = excluded_nodes or []
    sleep_interval, attempts = 10, 18
    nodes = []
    for __attempt in range(attempts):
        nodes = get_nodes_with_object(wallet, cid, oid, skip_nodes=excluded_nodes)
        if len(nodes) == expected_copies:
            return nodes
        sleep(sleep_interval)
    raise AssertionError(
        f"Expected {expected_copies} copies of object, but found {len(nodes)}. "
        f"Waiting time {sleep_interval * attempts}"
    )


@allure.step("Wait for storage node returned to cluster")
def wait_all_storage_node_returned():
    sleep_interval, attempts = 10, 12
    for __attempt in range(attempts):
        if is_all_storage_node_returned():
            return
        sleep(sleep_interval)
    raise AssertionError("Storage node(s) is broken")


def is_all_storage_node_returned() -> bool:
    with allure.step("Run health check for all storage nodes"):
        for node_name in NEOFS_NETMAP_DICT.keys():
            try:
                health_check = node_healthcheck(node_name)
            except Exception as err:
                logger.warning(f"Node healthcheck fails with error {err}")
                return False
            if health_check.health_status != "READY" or health_check.network_status != "ONLINE":
                return False
    return True