[#260] fix tests logs

This commit is contained in:
Ilyas Niyazov 2024-06-29 12:05:10 +03:00 committed by Andrey Berezin
parent cc440f9c12
commit 7d4792f49b
2 changed files with 74 additions and 15 deletions

View file

@ -12,6 +12,7 @@ markers =
smoke: test runs in smoke testrun
# controlling markers
order: manual control of test order
logs_after_session: Make the last test in session
# functional markers
maintenance: tests for change mode node
container: tests for container creation

View file

@ -1,7 +1,7 @@
import os
import shutil
import time
from datetime import datetime
from datetime import datetime, timezone
import allure
import pytest
@ -20,23 +20,72 @@ def pytest_generate_tests(metafunc: pytest.Metafunc):
)
@pytest.mark.logs_after_session
class TestLogs:
@allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}'")
@pytest.mark.order(1000)
def test_logs_after_session(
@allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}' - search errors")
def test_logs_search_errors(
self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest
):
"""
This test automatically added to any test run to check logs from cluster for critical errors.
"""
end_time = datetime.utcnow()
end_time = datetime.now(timezone.utc)
logs_dir = os.path.join(temp_directory, "logs")
os.makedirs(logs_dir)
# Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|wallet passwd|secret \bkey\b|access \bkey\b|cannot assign requested address"
if not os.path.exists(logs_dir):
os.makedirs(logs_dir)
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|cannot assign requested address|\bunable to process\b"
exclude_filter = r"too many requests"
log_level_priority = "3" # will include 0-3 priority logs (0: emergency 1: alerts 2: critical 3: errors)
time.sleep(2)
futures = parallel(
self._collect_logs_on_host,
cluster.hosts,
logs_dir,
issues_regex,
session_start_time,
end_time,
exclude_filter,
priority=log_level_priority,
)
hosts_with_problems = [
future.result() for future in futures if not future.exception() and future.result() is not None
]
if hosts_with_problems:
self._attach_logs(logs_dir)
assert (
not hosts_with_problems
), f"The following hosts contains critical errors in system logs: {', '.join(hosts_with_problems)}"
@pytest.mark.order(1001)
@allure.title(
"Check logs from frostfs-testcases with marks '{request.config.option.markexpr}' - identify sensitive data"
)
def test_logs_identify_sensitive_data(
self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest
):
end_time = datetime.now(timezone.utc)
logs_dir = os.path.join(temp_directory, "logs")
if not os.path.exists(logs_dir):
os.makedirs(logs_dir)
_regex = {
"authorization_basic": r"basic [a-zA-Z0-9=:_\+\/-]{16,100}",
"authorization_bearer": r"bearer [a-zA-Z0-9_\-\.=:_\+\/]{16,100}",
"access_token": r"\"access_token\":\"[0-9a-z]{16}\$[0-9a-f]{32}\"",
"api_token": r"\"api_token\":\"(xox[a-zA-Z]-[a-zA-Z0-9-]+)\"",
"yadro_access_token": r"[a-zA-Z0-9_-]*:[a-zA-Z0-9_\-]+@yadro\.com*",
"SSH_privKey": r"([-]+BEGIN [^\s]+ PRIVATE KEY[-]+[\s]*[^-]*[-]+END [^\s]+ PRIVATE KEY[-]+)",
"possible_Creds": r"(?i)("
r"password\s*[`=:]+\s*[^\s]+|"
r"password is\s*[`=:]*\s*[^\s]+|"
r"passwd\s*[`=:]+\s*[^\s]+)",
}
issues_regex = "|".join(_regex.values())
exclude_filter = "COMMAND="
time.sleep(2)
@ -58,13 +107,22 @@ class TestLogs:
assert (
not hosts_with_problems
), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}"
), f"The following hosts contains sensitive data in system logs: {', '.join(hosts_with_problems)}"
def _collect_logs_on_host(
self, host: Host, logs_dir: str, regex: str, since: datetime, until: datetime, exclude_filter: str
self,
host: Host,
logs_dir: str,
regex: str,
since: datetime,
until: datetime,
exclude_filter: str,
priority: str = None,
):
with reporter.step(f"Get logs from {host.config.address}"):
logs = host.get_filtered_logs(filter_regex=regex, since=since, until=until, exclude_filter=exclude_filter)
logs = host.get_filtered_logs(
filter_regex=regex, since=since, until=until, exclude_filter=exclude_filter, priority=priority
)
if not logs:
return None