forked from TrueCloudLab/frostfs-testcases
92 lines
4.2 KiB
Python
92 lines
4.2 KiB
Python
import time
|
|
from datetime import datetime, timezone
|
|
|
|
import allure
|
|
import pytest
|
|
from frostfs_testlib import reporter
|
|
from frostfs_testlib.steps.metrics import get_metrics_value
|
|
from frostfs_testlib.storage.cluster import ClusterNode
|
|
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
|
|
from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager
|
|
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
|
|
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
|
from frostfs_testlib.testing.test_control import wait_for_success
|
|
|
|
|
|
class TestLogsMetrics(ClusterTestBase):
|
|
@pytest.fixture
|
|
def revert_all(self, cluster_state_controller: ClusterStateController):
|
|
yield
|
|
cluster_state_controller.manager(ConfigStateManager).revert_all()
|
|
|
|
def restart_storage_service(self, cluster_state_controller: ClusterStateController) -> datetime:
|
|
config_manager = cluster_state_controller.manager(ConfigStateManager)
|
|
config_manager.csc.stop_services_of_type(StorageNode)
|
|
config_manager.csc.start_services_of_type(StorageNode)
|
|
|
|
@wait_for_success(interval=10)
|
|
def check_metrics_in_node(self, cluster_node: ClusterNode, restart_time: datetime, log_priority: str = None, **metrics_greps):
|
|
current_time = datetime.now(timezone.utc)
|
|
counter_metrics = get_metrics_value(cluster_node, **metrics_greps)
|
|
counter_logs = self.get_logs_count_by_level(cluster_node, metrics_greps.get("level"), restart_time, current_time, log_priority)
|
|
if (counter_metrics + counter_logs) < 1000:
|
|
assert (
|
|
counter_logs == counter_metrics
|
|
), f"counter_logs: {counter_logs}, counter_metrics: {counter_metrics} in node: {cluster_node}"
|
|
else:
|
|
# for big values check permissible deviation
|
|
assert self.are_numbers_similar(
|
|
counter_logs, counter_metrics
|
|
), f"counter_logs: {counter_logs}, counter_metrics: {counter_metrics} in node: {cluster_node}"
|
|
|
|
@staticmethod
|
|
def are_numbers_similar(num1, num2, tolerance_percentage=1):
|
|
"""
|
|
if difference of numbers is less than permissible deviation than numbers are similar
|
|
"""
|
|
# Calculate the permissible deviation
|
|
average = (num1 + num2) / 2
|
|
tolerance = average * (tolerance_percentage / 100)
|
|
|
|
# Calculate the real difference
|
|
difference = abs(num1 - num2)
|
|
return difference <= tolerance
|
|
|
|
@staticmethod
|
|
def get_logs_count_by_level(node: ClusterNode, level: str, since: datetime, until: datetime, priority: str):
|
|
try:
|
|
result = node.host.get_filtered_logs(
|
|
filter_regex=level, unit="frostfs-storage", since=since, until=until, priority=priority, calc_count=True
|
|
)
|
|
return int(result)
|
|
except Exception as e:
|
|
return e
|
|
|
|
@allure.title("Metrics for the log counter")
|
|
def test_log_counter_metrics(self, cluster_state_controller: ClusterStateController, node_under_test: ClusterNode, revert_all):
|
|
with reporter.step("Restart frostfs-node for reset logs metrics"):
|
|
self.restart_storage_service(cluster_state_controller)
|
|
|
|
with reporter.step("wait until the number of logs in frostfs-storage stabilizes"):
|
|
restart_time = datetime.now(timezone.utc)
|
|
time.sleep(10)
|
|
|
|
with reporter.step(f"Check metrics count logs with level 'info'"):
|
|
self.check_metrics_in_node(
|
|
node_under_test,
|
|
restart_time=restart_time,
|
|
log_priority="6..6",
|
|
command="frostfs_node_logger_entry_count",
|
|
level="info",
|
|
dropped="false",
|
|
)
|
|
|
|
with reporter.step(f"Check metrics count logs with level 'error'"):
|
|
self.check_metrics_in_node(
|
|
node_under_test,
|
|
restart_time=restart_time,
|
|
# log_priority="3..3", TODO: add filter log_priority after https://j.yadro.com/browse/OBJECT-8581
|
|
command="frostfs_node_logger_entry_count",
|
|
level="error",
|
|
dropped="false",
|
|
)
|