forked from TrueCloudLab/frostfs-testcases
139 lines
4.9 KiB
Python
139 lines
4.9 KiB
Python
import os
|
|
import shutil
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
import allure
|
|
import pytest
|
|
from frostfs_testlib import reporter
|
|
from frostfs_testlib.hosting import Host
|
|
from frostfs_testlib.testing.cluster_test_base import Cluster
|
|
from frostfs_testlib.testing.parallel import parallel
|
|
|
|
|
|
def pytest_generate_tests(metafunc: pytest.Metafunc):
|
|
metafunc.fixturenames.append("repo")
|
|
metafunc.fixturenames.append("markers")
|
|
metafunc.parametrize(
|
|
"repo, markers",
|
|
[("frostfs-testcases", metafunc.config.option.markexpr)],
|
|
)
|
|
|
|
|
|
@pytest.mark.logs_after_session
|
|
class TestLogs:
|
|
@pytest.mark.order(1000)
|
|
@allure.title("Check logs from frostfs-testcases with marks '{request.config.option.markexpr}' - search errors")
|
|
def test_logs_search_errors(
|
|
self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest
|
|
):
|
|
end_time = datetime.now(timezone.utc)
|
|
logs_dir = os.path.join(temp_directory, "logs")
|
|
if not os.path.exists(logs_dir):
|
|
os.makedirs(logs_dir)
|
|
|
|
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|cannot assign requested address|\bunable to process\b"
|
|
exclude_filter = r"too many requests"
|
|
log_level_priority = "3" # will include 0-3 priority logs (0: emergency 1: alerts 2: critical 3: errors)
|
|
|
|
time.sleep(2)
|
|
|
|
futures = parallel(
|
|
self._collect_logs_on_host,
|
|
cluster.hosts,
|
|
logs_dir,
|
|
issues_regex,
|
|
session_start_time,
|
|
end_time,
|
|
exclude_filter,
|
|
priority=log_level_priority,
|
|
)
|
|
|
|
hosts_with_problems = [
|
|
future.result() for future in futures if not future.exception() and future.result() is not None
|
|
]
|
|
if hosts_with_problems:
|
|
self._attach_logs(logs_dir)
|
|
|
|
assert (
|
|
not hosts_with_problems
|
|
), f"The following hosts contains critical errors in system logs: {', '.join(hosts_with_problems)}"
|
|
|
|
@pytest.mark.order(1001)
|
|
@allure.title(
|
|
"Check logs from frostfs-testcases with marks '{request.config.option.markexpr}' - identify sensitive data"
|
|
)
|
|
def test_logs_identify_sensitive_data(
|
|
self, temp_directory: str, cluster: Cluster, session_start_time: datetime, request: pytest.FixtureRequest
|
|
):
|
|
end_time = datetime.now(timezone.utc)
|
|
logs_dir = os.path.join(temp_directory, "logs")
|
|
if not os.path.exists(logs_dir):
|
|
os.makedirs(logs_dir)
|
|
|
|
_regex = {
|
|
"authorization_basic": r"basic [a-zA-Z0-9=:_\+\/-]{16,100}",
|
|
"authorization_bearer": r"bearer [a-zA-Z0-9_\-\.=:_\+\/]{16,100}",
|
|
"access_token": r"\"access_token\":\"[0-9a-z]{16}\$[0-9a-f]{32}\"",
|
|
"api_token": r"\"api_token\":\"(xox[a-zA-Z]-[a-zA-Z0-9-]+)\"",
|
|
"yadro_access_token": r"[a-zA-Z0-9_-]*:[a-zA-Z0-9_\-]+@yadro\.com*",
|
|
"SSH_privKey": r"([-]+BEGIN [^\s]+ PRIVATE KEY[-]+[\s]*[^-]*[-]+END [^\s]+ PRIVATE KEY[-]+)",
|
|
"possible_Creds": r"(?i)("
|
|
r"password\s*[`=:]+\s*[^\s]+|"
|
|
r"password is\s*[`=:]+\s*[^\s]+|"
|
|
r"passwd\s*[`=:]+\s*[^\s]+)",
|
|
}
|
|
|
|
issues_regex = "|".join(_regex.values())
|
|
exclude_filter = r"COMMAND=\|--\sBoot\s"
|
|
|
|
time.sleep(2)
|
|
|
|
futures = parallel(
|
|
self._collect_logs_on_host,
|
|
cluster.hosts,
|
|
logs_dir,
|
|
issues_regex,
|
|
session_start_time,
|
|
end_time,
|
|
exclude_filter,
|
|
)
|
|
|
|
hosts_with_problems = [
|
|
future.result() for future in futures if not future.exception() and future.result() is not None
|
|
]
|
|
if hosts_with_problems:
|
|
self._attach_logs(logs_dir)
|
|
|
|
assert (
|
|
not hosts_with_problems
|
|
), f"The following hosts contains sensitive data in system logs: {', '.join(hosts_with_problems)}"
|
|
|
|
def _collect_logs_on_host(
|
|
self,
|
|
host: Host,
|
|
logs_dir: str,
|
|
regex: str,
|
|
since: datetime,
|
|
until: datetime,
|
|
exclude_filter: str,
|
|
priority: str = None,
|
|
):
|
|
with reporter.step(f"Get logs from {host.config.address}"):
|
|
logs = host.get_filtered_logs(
|
|
filter_regex=regex, since=since, until=until, exclude_filter=exclude_filter, priority=priority
|
|
)
|
|
|
|
if not logs:
|
|
return None
|
|
|
|
with open(os.path.join(logs_dir, f"{host.config.address}.log"), "w") as file:
|
|
file.write(logs)
|
|
|
|
return host.config.address
|
|
|
|
def _attach_logs(self, logs_dir: str) -> None:
|
|
# Zip all files and attach to Allure because it is more convenient to download a single
|
|
# zip with all logs rather than mess with individual logs files per service or node
|
|
logs_zip_file_path = shutil.make_archive(logs_dir, "zip", logs_dir)
|
|
reporter.attach(logs_zip_file_path, "logs.zip")
|