Add tests for node shutdown and object replication integrity

Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>

pytest_tests/helpers/failover_utils.py

@@ -2,11 +2,13 @@ import logging
 from time import sleep
 
 import allure
-from frostfs_testlib.shell import Shell
+from frostfs_testlib.hosting import Host
+from frostfs_testlib.shell import CommandOptions, Shell
 
 from pytest_tests.helpers.cluster import Cluster, StorageNode
 from pytest_tests.helpers.node_management import storage_node_healthcheck
 from pytest_tests.helpers.storage_policy import get_nodes_with_object
+from pytest_tests.helpers.test_control import retry
 
 logger = logging.getLogger("NeoLogger")
 
@@ -18,8 +20,9 @@ def wait_object_replication(
     expected_copies: int,
     shell: Shell,
     nodes: list[StorageNode],
+    sleep_interval: int = 15,
+    attempts: int = 20,
 ) -> list[StorageNode]:
-    sleep_interval, attempts = 15, 20
     nodes_with_object = []
     for _ in range(attempts):
         nodes_with_object = get_nodes_with_object(cid, oid, shell=shell, nodes=nodes)
@@ -53,3 +56,20 @@ def is_all_storage_nodes_returned(cluster: Cluster) -> bool:
             if health_check.health_status != "READY" or health_check.network_status != "ONLINE":
                 return False
     return True
+
+
+@allure.step("Ping node")
+def ping_host(shell: Shell, host: Host):
+    options = CommandOptions(check=False)
+    return shell.exec(f"ping {host.config.address} -c 1", options).return_code
+
+
+@retry(max_attempts=60, sleep_interval=5, expected_result=1)
+@allure.step("Waiting for host of {node} to go offline")
+def wait_for_host_offline(shell: Shell, node: StorageNode):
+    try:
+        # TODO: Quick solution for now, should be replaced by lib interactions
+        return ping_host(shell, node.host)
+    except Exception as err:
+        logger.warning(f"Host ping fails with error {err}")
+        return 0