import os import shutil from datetime import datetime import allure import pytest from frostfs_testlib.testing.cluster_test_base import ClusterTestBase def pytest_generate_tests(metafunc: pytest.Metafunc): metafunc.fixturenames.append("repo") metafunc.fixturenames.append("markers") metafunc.parametrize( "repo, markers", [("frostfs-testcases", metafunc.config.option.markexpr)], ) class TestLogs(ClusterTestBase): @allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}'") @pytest.mark.logs_after_session @pytest.mark.no_healthcheck def test_logs_after_session( self, temp_directory: str, session_start_time: datetime, request: pytest.FixtureRequest ): """ This test automatically added to any test run to check logs from cluster for critical errors. """ end_time = datetime.utcnow() logs_dir = os.path.join(temp_directory, "logs") os.makedirs(logs_dir) # Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds" hosts_with_problems = [] for host in self.cluster.hosts: with allure.step(f"Check logs on {host.config.address}"): if host.is_message_in_logs(issues_regex, session_start_time, end_time): hosts_with_problems.append(host.config.address) host.dump_logs( logs_dir, since=session_start_time, until=end_time, filter_regex=issues_regex, ) if hosts_with_problems: self._attach_logs(logs_dir) assert ( not hosts_with_problems ), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}" def _attach_logs(self, logs_dir: str) -> None: # Zip all files and attach to Allure because it is more convenient to download a single # zip with all logs rather than mess with individual logs files per service or node logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir) allure.attach.file(logs_zip_file_path, name="logs.zip", extension="zip")