pytest.ini

@@ -12,6 +12,7 @@ markers =
     smoke: test runs in smoke testrun
 #   controlling markers
     no_healthcheck: skip healthcheck for this test
+    logs_after_session: Make the last test in session
 #   functional markers
     maintenance: tests for change mode node
     container: tests for container creation

pytest_tests/testsuites/special/test_logs.py

@@ -1,7 +1,7 @@
 import os
 import shutil
 import time
-from datetime import datetime
+from datetime import datetime, timezone
 
 import allure
 import pytest
@@ -20,24 +20,72 @@ def pytest_generate_tests(metafunc: pytest.Metafunc):
     )
 
 
+@pytest.mark.logs_after_session
 class TestLogs:
-    @allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}'")
+    @pytest.mark.order(1000)
-    @pytest.mark.logs_after_session
+    @allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}' - search errors")
-    @pytest.mark.no_healthcheck
+    def test_logs_search_errors(
-    def test_logs_after_session(
         self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest
     ):
-        """
+        end_time = datetime.now(timezone.utc)
-        This test automatically added to any test run to check logs from cluster for critical errors.
-
-        """
-
-        end_time = datetime.utcnow()
         logs_dir = os.path.join(temp_directory, "logs")
-        os.makedirs(logs_dir)
+        if not os.path.exists(logs_dir):
-        # Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID
+            os.makedirs(logs_dir)
-        issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|wallet passwd|secret \bkey\b|access \bkey\b|cannot assign requested address"
+
+        issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|cannot assign requested address|\bunable to process\b"
         exclude_filter = r"too many requests"
+        log_level_priority = "3"  # will include 0-3 priority logs (0: emergency 1: alerts 2: critical 3: errors)
+
+        time.sleep(2)
+
+        futures = parallel(
+            self._collect_logs_on_host,
+            cluster.hosts,
+            logs_dir,
+            issues_regex,
+            session_start_time,
+            end_time,
+            exclude_filter,
+            priority=log_level_priority,
+        )
+
+        hosts_with_problems = [
+            future.result() for future in futures if not future.exception() and future.result() is not None
+        ]
+        if hosts_with_problems:
+            self._attach_logs(logs_dir)
+
+        assert (
+            not hosts_with_problems
+        ), f"The following hosts contains critical errors in system logs: {', '.join(hosts_with_problems)}"
+
+    @pytest.mark.order(1001)
+    @allure.title(
+        "Check logs from frostfs-testcases with marks '{request.config.option.markexpr}' - identify sensitive data"
+    )
+    def test_logs_identify_sensitive_data(
+        self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest
+    ):
+        end_time = datetime.now(timezone.utc)
+        logs_dir = os.path.join(temp_directory, "logs")
+        if not os.path.exists(logs_dir):
+            os.makedirs(logs_dir)
+
+        _regex = {
+            "authorization_basic": r"basic [a-zA-Z0-9=:_\+\/-]{16,100}",
+            "authorization_bearer": r"bearer [a-zA-Z0-9_\-\.=:_\+\/]{16,100}",
+            "access_token": r"\"access_token\":\"[0-9a-z]{16}\$[0-9a-f]{32}\"",
+            "api_token": r"\"api_token\":\"(xox[a-zA-Z]-[a-zA-Z0-9-]+)\"",
+            "yadro_access_token": r"[a-zA-Z0-9_-]*:[a-zA-Z0-9_\-]+@yadro\.com*",
+            "SSH_privKey": r"([-]+BEGIN [^\s]+ PRIVATE KEY[-]+[\s]*[^-]*[-]+END [^\s]+ PRIVATE KEY[-]+)",
+            "possible_Creds": r"(?i)("
+            r"password\s*[`=:]+\s*[^\s]+|"
+            r"password is\s*[`=:]*\s*[^\s]+|"
+            r"passwd\s*[`=:]+\s*[^\s]+)",
+        }
+
+        issues_regex = "|".join(_regex.values())
+        exclude_filter = "COMMAND="
 
         time.sleep(2)
 
@@ -59,13 +107,22 @@ class TestLogs:
 
         assert (
             not hosts_with_problems
-        ), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}"
+        ), f"The following hosts contains sensitive data in system logs: {', '.join(hosts_with_problems)}"
 
     def _collect_logs_on_host(
-        self, host: Host, logs_dir: str, regex: str, since: datetime, until: datetime, exclude_filter: str
+        self,
+        host: Host,
+        logs_dir: str,
+        regex: str,
+        since: datetime,
+        until: datetime,
+        exclude_filter: str,
+        priority: str = None,
     ):
         with reporter.step(f"Get logs from {host.config.address}"):
-            logs = host.get_filtered_logs(filter_regex=regex, since=since, until=until, exclude_filter=exclude_filter)
+            logs = host.get_filtered_logs(
+                filter_regex=regex, since=since, until=until, exclude_filter=exclude_filter, priority=priority
+            )
 
         if not logs:
             return None