From b72f6daeb7ea7bc44b4a75003629ff4b2ca6b89f Mon Sep 17 00:00:00 2001
From: Andrey Berezin <a.berezin@yadro.com>
Date: Fri, 20 Oct 2023 18:50:02 +0300
Subject: [PATCH] [#115] Make logs gathering parallel and in single command

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
---
 pytest_tests/testsuites/special/test_logs.py | 37 ++++++++++++--------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/pytest_tests/testsuites/special/test_logs.py b/pytest_tests/testsuites/special/test_logs.py
index 1b70aa88..a4cea1e6 100644
--- a/pytest_tests/testsuites/special/test_logs.py
+++ b/pytest_tests/testsuites/special/test_logs.py
@@ -4,7 +4,9 @@ from datetime import datetime
 
 import allure
 import pytest
-from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
+from frostfs_testlib.hosting import Host
+from frostfs_testlib.testing.cluster_test_base import Cluster
+from frostfs_testlib.testing.parallel import parallel
 
 
 def pytest_generate_tests(metafunc: pytest.Metafunc):
@@ -16,12 +18,12 @@ def pytest_generate_tests(metafunc: pytest.Metafunc):
     )
 
 
-class TestLogs(ClusterTestBase):
+class TestLogs:
     @allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}'")
     @pytest.mark.logs_after_session
     @pytest.mark.no_healthcheck
     def test_logs_after_session(
-        self, temp_directory: str, session_start_time: datetime, request: pytest.FixtureRequest
+        self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest
     ):
         """
         This test automatically added to any test run to check logs from cluster for critical errors.
@@ -34,18 +36,13 @@ class TestLogs(ClusterTestBase):
         # Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID
         issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas"
 
-        hosts_with_problems = []
-        for host in self.cluster.hosts:
-            with allure.step(f"Check logs on {host.config.address}"):
-                if host.is_message_in_logs(issues_regex, session_start_time, end_time):
-                    hosts_with_problems.append(host.config.address)
-                    host.dump_logs(
-                        logs_dir,
-                        since=session_start_time,
-                        until=end_time,
-                        filter_regex=issues_regex,
-                    )
+        futures = parallel(
+            self._collect_logs_on_host, cluster.hosts, logs_dir, issues_regex, session_start_time, end_time
+        )
 
+        hosts_with_problems = [
+            future.result() for future in futures if not future.exception() and future.result() is not None
+        ]
         if hosts_with_problems:
             self._attach_logs(logs_dir)
 
@@ -53,6 +50,18 @@ class TestLogs(ClusterTestBase):
             not hosts_with_problems
         ), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}"
 
+    def _collect_logs_on_host(self, host: Host, logs_dir: str, regex: str, since: datetime, until: datetime):
+        with allure.step(f"Get logs from {host.config.address}"):
+            logs = host.get_filtered_logs(regex, since, until)
+
+        if not logs:
+            return None
+
+        with open(os.path.join(logs_dir, f"{host.config.address}.log"), "w") as file:
+            file.write(logs)
+
+        return host.config.address
+
     def _attach_logs(self, logs_dir: str) -> None:
         # Zip all files and attach to Allure because it is more convenient to download a single
         # zip with all logs rather than mess with individual logs files per service or node