Add metabase and write_cache operations

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>

src/frostfs_testlib/utils/failover_utils.py

@@ -28,8 +28,8 @@ def ping_host(shell: Shell, host: Host):
 
 @reporter.step_deco("Wait for storage nodes returned to cluster")
 def wait_all_storage_nodes_returned(shell: Shell, cluster: Cluster) -> None:
-    with reporter.step("Run health check for all storage nodes"):
-        for node in cluster.services(StorageNode):
+    for node in cluster.services(StorageNode):
+        with reporter.step(f"Run health check for storage at '{node}'"):
             wait_for_host_online(shell, node)
             wait_for_node_online(node)
 
@@ -56,7 +56,7 @@ def wait_for_host_offline(shell: Shell, node: StorageNode):
         return 0
 
 
-@retry(max_attempts=10, sleep_interval=15, expected_result=True)
+@retry(max_attempts=20, sleep_interval=30, expected_result=True)
 @reporter.step_deco("Waiting for node {node} to go online")
 def wait_for_node_online(node: StorageNode):
     try:

src/frostfs_testlib/utils/file_keeper.py

@@ -0,0 +1,50 @@
+from concurrent.futures import ThreadPoolExecutor
+
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.storage.dataclasses.node_base import NodeBase
+
+reporter = get_reporter()
+
+
+class FileKeeper:
+    """This class is responsible to make backup copy of modified file and restore when required (mostly after the test)"""
+
+    files_to_restore: dict[NodeBase, list[str]] = {}
+
+    @reporter.step_deco("Adding {file_to_restore} from node {node} to restore list")
+    def add(self, node: NodeBase, file_to_restore: str):
+        if node in self.files_to_restore and file_to_restore in self.files_to_restore[node]:
+            # Already added
+            return
+
+        if node not in self.files_to_restore:
+            self.files_to_restore[node] = []
+
+        if file_to_restore not in self.files_to_restore[node]:
+            self.files_to_restore[node].append(file_to_restore)
+
+        shell = node.host.get_shell()
+        shell.exec(f"cp {file_to_restore} {file_to_restore}.bak")
+
+    @reporter.step_deco("Restore files")
+    def restore_files(self):
+        nodes = self.files_to_restore.keys()
+        if not nodes:
+            return
+
+        with ThreadPoolExecutor(max_workers=len(nodes)) as executor:
+            results = executor.map(self._restore_files_on_node, nodes)
+
+        self.files_to_restore.clear()
+
+        for _ in results:
+            # Iterate through results for exception check if any
+            pass
+
+    @reporter.step_deco("Restore files on node {node}")
+    def _restore_files_on_node(self, node: NodeBase):
+        shell = node.host.get_shell()
+        for file_to_restore in self.files_to_restore[node]:
+            with reporter.step(f"Restore file {file_to_restore} on node {node}"):
+                shell.exec(f"cp {file_to_restore}.bak {file_to_restore}")
+                shell.exec(f"rm {file_to_restore}.bak")