[#115] Make logs gathering parallel and in single command

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-10-20 18:50:02 +03:00
parent ad1254b4f3
commit b72f6daeb7

View file

@ -4,7 +4,9 @@ from datetime import datetime
import allure import allure
import pytest import pytest
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.hosting import Host
from frostfs_testlib.testing.cluster_test_base import Cluster
from frostfs_testlib.testing.parallel import parallel
def pytest_generate_tests(metafunc: pytest.Metafunc): def pytest_generate_tests(metafunc: pytest.Metafunc):
@ -16,12 +18,12 @@ def pytest_generate_tests(metafunc: pytest.Metafunc):
) )
class TestLogs(ClusterTestBase): class TestLogs:
@allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}'") @allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}'")
@pytest.mark.logs_after_session @pytest.mark.logs_after_session
@pytest.mark.no_healthcheck @pytest.mark.no_healthcheck
def test_logs_after_session( def test_logs_after_session(
self, temp_directory: str, session_start_time: datetime, request: pytest.FixtureRequest self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest
): ):
""" """
This test automatically added to any test run to check logs from cluster for critical errors. This test automatically added to any test run to check logs from cluster for critical errors.
@ -34,18 +36,13 @@ class TestLogs(ClusterTestBase):
# Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID # Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas" issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas"
hosts_with_problems = [] futures = parallel(
for host in self.cluster.hosts: self._collect_logs_on_host, cluster.hosts, logs_dir, issues_regex, session_start_time, end_time
with allure.step(f"Check logs on {host.config.address}"):
if host.is_message_in_logs(issues_regex, session_start_time, end_time):
hosts_with_problems.append(host.config.address)
host.dump_logs(
logs_dir,
since=session_start_time,
until=end_time,
filter_regex=issues_regex,
) )
hosts_with_problems = [
future.result() for future in futures if not future.exception() and future.result() is not None
]
if hosts_with_problems: if hosts_with_problems:
self._attach_logs(logs_dir) self._attach_logs(logs_dir)
@ -53,6 +50,18 @@ class TestLogs(ClusterTestBase):
not hosts_with_problems not hosts_with_problems
), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}" ), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}"
def _collect_logs_on_host(self, host: Host, logs_dir: str, regex: str, since: datetime, until: datetime):
with allure.step(f"Get logs from {host.config.address}"):
logs = host.get_filtered_logs(regex, since, until)
if not logs:
return None
with open(os.path.join(logs_dir, f"{host.config.address}.log"), "w") as file:
file.write(logs)
return host.config.address
def _attach_logs(self, logs_dir: str) -> None: def _attach_logs(self, logs_dir: str) -> None:
# Zip all files and attach to Allure because it is more convenient to download a single # Zip all files and attach to Allure because it is more convenient to download a single
# zip with all logs rather than mess with individual logs files per service or node # zip with all logs rather than mess with individual logs files per service or node