[#115] Make logs gathering parallel and in single command

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-10-20 18:50:02 +03:00
parent ad1254b4f3
commit b72f6daeb7

View file

@ -4,7 +4,9 @@ from datetime import datetime
import allure
import pytest
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.hosting import Host
from frostfs_testlib.testing.cluster_test_base import Cluster
from frostfs_testlib.testing.parallel import parallel
def pytest_generate_tests(metafunc: pytest.Metafunc):
@ -16,12 +18,12 @@ def pytest_generate_tests(metafunc: pytest.Metafunc):
)
class TestLogs(ClusterTestBase):
class TestLogs:
@allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}'")
@pytest.mark.logs_after_session
@pytest.mark.no_healthcheck
def test_logs_after_session(
self, temp_directory: str, session_start_time: datetime, request: pytest.FixtureRequest
self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest
):
"""
This test automatically added to any test run to check logs from cluster for critical errors.
@ -34,18 +36,13 @@ class TestLogs(ClusterTestBase):
# Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas"
hosts_with_problems = []
for host in self.cluster.hosts:
with allure.step(f"Check logs on {host.config.address}"):
if host.is_message_in_logs(issues_regex, session_start_time, end_time):
hosts_with_problems.append(host.config.address)
host.dump_logs(
logs_dir,
since=session_start_time,
until=end_time,
filter_regex=issues_regex,
)
futures = parallel(
self._collect_logs_on_host, cluster.hosts, logs_dir, issues_regex, session_start_time, end_time
)
hosts_with_problems = [
future.result() for future in futures if not future.exception() and future.result() is not None
]
if hosts_with_problems:
self._attach_logs(logs_dir)
@ -53,6 +50,18 @@ class TestLogs(ClusterTestBase):
not hosts_with_problems
), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}"
def _collect_logs_on_host(self, host: Host, logs_dir: str, regex: str, since: datetime, until: datetime):
with allure.step(f"Get logs from {host.config.address}"):
logs = host.get_filtered_logs(regex, since, until)
if not logs:
return None
with open(os.path.join(logs_dir, f"{host.config.address}.log"), "w") as file:
file.write(logs)
return host.config.address
def _attach_logs(self, logs_dir: str) -> None:
# Zip all files and attach to Allure because it is more convenient to download a single
# zip with all logs rather than mess with individual logs files per service or node