forked from TrueCloudLab/frostfs-testcases
Add fixture to collect logs after test execution
Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
parent
6d040c6834
commit
94d6ec6b12
2 changed files with 66 additions and 1 deletions
|
@ -3,6 +3,8 @@ import logging
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
|
from datetime import datetime
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
import docker
|
import docker
|
||||||
|
|
||||||
|
@ -20,6 +22,20 @@ class LocalDevEnvStorageServiceHelper:
|
||||||
"""
|
"""
|
||||||
Manages storage services running on local devenv.
|
Manages storage services running on local devenv.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# Names of all containers that are running neofs code
|
||||||
|
ALL_CONTAINERS = [
|
||||||
|
"s3_gate",
|
||||||
|
"http_gate",
|
||||||
|
"s03",
|
||||||
|
"s01",
|
||||||
|
"s02",
|
||||||
|
"s04",
|
||||||
|
"ir01",
|
||||||
|
"morph_chain",
|
||||||
|
"main_chain",
|
||||||
|
]
|
||||||
|
|
||||||
def stop_node(self, node_name: str, wait: bool = True) -> None:
|
def stop_node(self, node_name: str, wait: bool = True) -> None:
|
||||||
container_name = _get_storage_container_name(node_name)
|
container_name = _get_storage_container_name(node_name)
|
||||||
client = self._get_docker_client(node_name)
|
client = self._get_docker_client(node_name)
|
||||||
|
@ -59,6 +75,17 @@ class LocalDevEnvStorageServiceHelper:
|
||||||
def get_binaries_version(self) -> dict:
|
def get_binaries_version(self) -> dict:
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
def logs(self, since: Optional[datetime], until: Optional[datetime]) -> dict[str, str]:
|
||||||
|
# All containers are running on the same host, so we can use 1st node to collect all logs
|
||||||
|
first_node_name = next(iter(NEOFS_NETMAP_DICT))
|
||||||
|
client = self._get_docker_client(first_node_name)
|
||||||
|
|
||||||
|
logs_by_container = {}
|
||||||
|
for container_name in self.ALL_CONTAINERS:
|
||||||
|
logs = client.logs(container_name, since=since, until=until)
|
||||||
|
logs_by_container[container_name] = logs
|
||||||
|
return logs_by_container
|
||||||
|
|
||||||
def _get_container_by_name(self, node_name: str, container_name: str) -> dict:
|
def _get_container_by_name(self, node_name: str, container_name: str) -> dict:
|
||||||
client = self._get_docker_client(node_name)
|
client = self._get_docker_client(node_name)
|
||||||
containers = client.containers(all=True)
|
containers = client.containers(all=True)
|
||||||
|
@ -92,6 +119,14 @@ class LocalDevEnvStorageServiceHelper:
|
||||||
class CloudVmStorageServiceHelper:
|
class CloudVmStorageServiceHelper:
|
||||||
STORAGE_SERVICE = "neofs-storage.service"
|
STORAGE_SERVICE = "neofs-storage.service"
|
||||||
|
|
||||||
|
ALL_SERVICES = [
|
||||||
|
"neofs-http.service",
|
||||||
|
"neofs-s3.service",
|
||||||
|
"neofs-storage.service",
|
||||||
|
"neofs-ir.service",
|
||||||
|
"neofs-morph-cn.service",
|
||||||
|
]
|
||||||
|
|
||||||
def stop_node(self, node_name: str, wait: bool = True) -> None:
|
def stop_node(self, node_name: str, wait: bool = True) -> None:
|
||||||
with _create_ssh_client(node_name) as ssh_client:
|
with _create_ssh_client(node_name) as ssh_client:
|
||||||
cmd = f"sudo systemctl stop {self.STORAGE_SERVICE}"
|
cmd = f"sudo systemctl stop {self.STORAGE_SERVICE}"
|
||||||
|
@ -188,6 +223,22 @@ class CloudVmStorageServiceHelper:
|
||||||
f'(mismatch on node {node_name})'
|
f'(mismatch on node {node_name})'
|
||||||
return version_map
|
return version_map
|
||||||
|
|
||||||
|
def logs(self, since: Optional[datetime], until: Optional[datetime]) -> dict[str, str]:
|
||||||
|
logs_by_service_id = {}
|
||||||
|
for node_name in NEOFS_NETMAP_DICT:
|
||||||
|
with _create_ssh_client(node_name) as ssh_client:
|
||||||
|
for service_name in self.ALL_SERVICES:
|
||||||
|
filters = " ".join([
|
||||||
|
f"--since '{since:%Y-%m-%d %H:%M:%S}'" if since else "",
|
||||||
|
f"--until '{until:%Y-%m-%d %H:%M:%S}'" if until else "",
|
||||||
|
])
|
||||||
|
result = ssh_client.exec(f"journalctl -u {service_name} --no-pager {filters}")
|
||||||
|
logs = result.stdout
|
||||||
|
|
||||||
|
service_id = f"{node_name}_{service_name}"
|
||||||
|
logs_by_service_id[service_id] = logs
|
||||||
|
return logs_by_service_id
|
||||||
|
|
||||||
|
|
||||||
class RemoteDevEnvStorageServiceHelper(LocalDevEnvStorageServiceHelper):
|
class RemoteDevEnvStorageServiceHelper(LocalDevEnvStorageServiceHelper):
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -2,6 +2,7 @@ import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
import pytest
|
import pytest
|
||||||
|
@ -62,9 +63,22 @@ def _get_binaries_version_local(binaries: list) -> dict:
|
||||||
return env_out
|
return env_out
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='session', autouse=True)
|
||||||
|
@allure.title('Collect logs')
|
||||||
|
def collect_logs():
|
||||||
|
start_time = datetime.utcnow()
|
||||||
|
yield
|
||||||
|
end_time = datetime.utcnow()
|
||||||
|
|
||||||
|
helper = get_storage_service_helper()
|
||||||
|
logs_by_service_id = helper.logs(since=start_time, until=end_time)
|
||||||
|
for service_id, logs in logs_by_service_id.items():
|
||||||
|
allure.attach(logs, f"logs_{service_id}", allure.attachment_type.TEXT)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope='session', autouse=True)
|
@pytest.fixture(scope='session', autouse=True)
|
||||||
@allure.title('Run health check for all storage nodes')
|
@allure.title('Run health check for all storage nodes')
|
||||||
def run_health_check():
|
def run_health_check(collect_logs):
|
||||||
failed_nodes = []
|
failed_nodes = []
|
||||||
for node_name in NEOFS_NETMAP_DICT.keys():
|
for node_name in NEOFS_NETMAP_DICT.keys():
|
||||||
health_check = node_healthcheck(node_name)
|
health_check = node_healthcheck(node_name)
|
||||||
|
|
Loading…
Reference in a new issue