diff --git a/pytest.ini b/pytest.ini index d65541e..067a4ba 100644 --- a/pytest.ini +++ b/pytest.ini @@ -12,6 +12,7 @@ markers = smoke: test runs in smoke testrun # controlling markers order: manual control of test order + logs_after_session: Make the last test in session # functional markers maintenance: tests for change mode node container: tests for container creation diff --git a/pytest_tests/testsuites/special/test_frostfs_logs.py b/pytest_tests/testsuites/special/test_frostfs_logs.py index 5ad58dc..28c39d7 100644 --- a/pytest_tests/testsuites/special/test_frostfs_logs.py +++ b/pytest_tests/testsuites/special/test_frostfs_logs.py @@ -1,7 +1,7 @@ import os import shutil import time -from datetime import datetime +from datetime import datetime, timezone import allure import pytest @@ -20,23 +20,72 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): ) +@pytest.mark.logs_after_session class TestLogs: - @allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}'") @pytest.mark.order(1000) - def test_logs_after_session( + @allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}' - search errors") + def test_logs_search_errors( self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest ): - """ - This test automatically added to any test run to check logs from cluster for critical errors. - - """ - - end_time = datetime.utcnow() + end_time = datetime.now(timezone.utc) logs_dir = os.path.join(temp_directory, "logs") - os.makedirs(logs_dir) - # Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID - issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|wallet passwd|secret \bkey\b|access \bkey\b|cannot assign requested address" + if not os.path.exists(logs_dir): + os.makedirs(logs_dir) + + issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|cannot assign requested address|\bunable to process\b" exclude_filter = r"too many requests" + log_level_priority = "3" # will include 0-3 priority logs (0: emergency 1: alerts 2: critical 3: errors) + + time.sleep(2) + + futures = parallel( + self._collect_logs_on_host, + cluster.hosts, + logs_dir, + issues_regex, + session_start_time, + end_time, + exclude_filter, + priority=log_level_priority, + ) + + hosts_with_problems = [ + future.result() for future in futures if not future.exception() and future.result() is not None + ] + if hosts_with_problems: + self._attach_logs(logs_dir) + + assert ( + not hosts_with_problems + ), f"The following hosts contains critical errors in system logs: {', '.join(hosts_with_problems)}" + + @pytest.mark.order(1001) + @allure.title( + "Check logs from frostfs-testcases with marks '{request.config.option.markexpr}' - identify sensitive data" + ) + def test_logs_identify_sensitive_data( + self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest + ): + end_time = datetime.now(timezone.utc) + logs_dir = os.path.join(temp_directory, "logs") + if not os.path.exists(logs_dir): + os.makedirs(logs_dir) + + _regex = { + "authorization_basic": r"basic [a-zA-Z0-9=:_\+\/-]{16,100}", + "authorization_bearer": r"bearer [a-zA-Z0-9_\-\.=:_\+\/]{16,100}", + "access_token": r"\"access_token\":\"[0-9a-z]{16}\$[0-9a-f]{32}\"", + "api_token": r"\"api_token\":\"(xox[a-zA-Z]-[a-zA-Z0-9-]+)\"", + "yadro_access_token": r"[a-zA-Z0-9_-]*:[a-zA-Z0-9_\-]+@yadro\.com*", + "SSH_privKey": r"([-]+BEGIN [^\s]+ PRIVATE KEY[-]+[\s]*[^-]*[-]+END [^\s]+ PRIVATE KEY[-]+)", + "possible_Creds": r"(?i)(" + r"password\s*[`=:]+\s*[^\s]+|" + r"password is\s*[`=:]*\s*[^\s]+|" + r"passwd\s*[`=:]+\s*[^\s]+)", + } + + issues_regex = "|".join(_regex.values()) + exclude_filter = "COMMAND=" time.sleep(2) @@ -58,13 +107,22 @@ class TestLogs: assert ( not hosts_with_problems - ), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}" + ), f"The following hosts contains sensitive data in system logs: {', '.join(hosts_with_problems)}" def _collect_logs_on_host( - self, host: Host, logs_dir: str, regex: str, since: datetime, until: datetime, exclude_filter: str + self, + host: Host, + logs_dir: str, + regex: str, + since: datetime, + until: datetime, + exclude_filter: str, + priority: str = None, ): with reporter.step(f"Get logs from {host.config.address}"): - logs = host.get_filtered_logs(filter_regex=regex, since=since, until=until, exclude_filter=exclude_filter) + logs = host.get_filtered_logs( + filter_regex=regex, since=since, until=until, exclude_filter=exclude_filter, priority=priority + ) if not logs: return None