diff --git a/src/frostfs_testlib/steps/metrics.py b/src/frostfs_testlib/steps/metrics.py index 29e49d4..4d40931 100644 --- a/src/frostfs_testlib/steps/metrics.py +++ b/src/frostfs_testlib/steps/metrics.py @@ -1,8 +1,10 @@ +import json import re +from datetime import datetime from frostfs_testlib import reporter -from frostfs_testlib.testing.test_control import wait_for_success from frostfs_testlib.storage.cluster import ClusterNode +from frostfs_testlib.testing.test_control import wait_for_success @reporter.step("Check metrics result") @@ -22,6 +24,24 @@ def check_metrics_counter( ), f"Expected: {counter_exp} {operator} Actual: {counter_act} in node: {cluster_node}" +@reporter.step("Check Victoria metrics agent metrics result") +def check_vma_metrics_counter(metrics_results: list[dict], bucket_name: str, operation: str, counter_exp: int = 0, operator: str = "=="): + with reporter.step(f"Get metrics value for bucket: {bucket_name}, operation: {operation}"): + counter_act = None + try: + for res in metrics_results: + metric = res["metric"] + if metric.get("bucket") == bucket_name and metric.get("operation") == operation and metric.get("cid"): + counter_act = int(res["values"][0][-1]) + break + except Exception as e: + counter_act = 0 + + with reporter.step("Check metric value"): + assert counter_act, f"Not found metric value for bucket: {bucket_name}, operation: {operation}" + assert eval(f"{counter_act} {operator} {counter_exp}"), f"Expected: {counter_exp} {operator} Actual: {counter_act}" + + @reporter.step("Get metrics value from node: {node}") def get_metrics_value(node: ClusterNode, parse_from_command: bool = False, **metrics_greps: str): try: @@ -36,6 +56,19 @@ def get_metrics_value(node: ClusterNode, parse_from_command: bool = False, **met return metrics_counter +@reporter.step("Get VM agent metrics") +@wait_for_success(max_wait_time=300, interval=30) +def get_vm_agent_metrics_values(node: ClusterNode, metric_name: str, since: datetime = None, **greps): + try: + command_result = node.metrics.storage.get_vma_metrics_result(metric_name, since, **greps) + result = json.loads(command_result) + except Exception as e: + ... + + assert len(result["data"]["result"]) > 0, "Metrics are not available" + return result + + @reporter.step("Parse metrics count and calc sum of result") def calc_metrics_count_from_stdout(metric_result_stdout: str, command: str = None): if command: diff --git a/src/frostfs_testlib/storage/cluster.py b/src/frostfs_testlib/storage/cluster.py index 9fcc4c9..52a4214 100644 --- a/src/frostfs_testlib/storage/cluster.py +++ b/src/frostfs_testlib/storage/cluster.py @@ -11,10 +11,10 @@ from frostfs_testlib.storage import get_service_registry from frostfs_testlib.storage.configuration.interfaces import ServiceConfigurationYml from frostfs_testlib.storage.constants import ConfigAttributes from frostfs_testlib.storage.dataclasses.frostfs_services import HTTPGate, InnerRing, MorphChain, S3Gate, StorageNode +from frostfs_testlib.storage.dataclasses.metrics import Metrics from frostfs_testlib.storage.dataclasses.node_base import NodeBase, ServiceClass from frostfs_testlib.storage.dataclasses.storage_object_info import Interfaces from frostfs_testlib.storage.service_registry import ServiceRegistry -from frostfs_testlib.storage.dataclasses.metrics import Metrics class ClusterNode: @@ -31,7 +31,11 @@ class ClusterNode: self.host = host self.id = id self.class_registry = get_service_registry() - self.metrics = Metrics(host=self.host, metrics_endpoint=self.storage_node.get_metrics_endpoint()) + self.metrics = Metrics( + host=self.host, + metrics_endpoint=self.storage_node.get_metrics_endpoint(), + vm_agent_endpoint=self.storage_node.get_vm_agent_endpoint(), + ) @property def host_ip(self): diff --git a/src/frostfs_testlib/storage/constants.py b/src/frostfs_testlib/storage/constants.py index 2cffd3a..20311ee 100644 --- a/src/frostfs_testlib/storage/constants.py +++ b/src/frostfs_testlib/storage/constants.py @@ -15,6 +15,7 @@ class ConfigAttributes: ENDPOINT_DATA_0_NS = "endpoint_data0_namespace" ENDPOINT_INTERNAL = "endpoint_internal0" ENDPOINT_PROMETHEUS = "endpoint_prometheus" + ENDPOINT_VM_AGENT = "endpoint_vm_agent" CONTROL_ENDPOINT = "control_endpoint" UN_LOCODE = "un_locode" diff --git a/src/frostfs_testlib/storage/dataclasses/metrics.py b/src/frostfs_testlib/storage/dataclasses/metrics.py index 81e757c..aec340f 100644 --- a/src/frostfs_testlib/storage/dataclasses/metrics.py +++ b/src/frostfs_testlib/storage/dataclasses/metrics.py @@ -1,20 +1,23 @@ +from datetime import datetime + from frostfs_testlib.hosting import Host from frostfs_testlib.shell.interfaces import CommandResult class Metrics: - def __init__(self, host: Host, metrics_endpoint: str) -> None: - self.storage = StorageMetrics(host, metrics_endpoint) - + def __init__(self, host: Host, metrics_endpoint: str, vm_agent_endpoint: str) -> None: + self.storage = StorageMetrics(host, metrics_endpoint, vm_agent_endpoint) class StorageMetrics: """ Class represents storage metrics in a cluster """ - def __init__(self, host: Host, metrics_endpoint: str) -> None: + + def __init__(self, host: Host, metrics_endpoint: str, vm_agent_endpoint: str) -> None: self.host = host self.metrics_endpoint = metrics_endpoint + self.vm_agent_endpoint = vm_agent_endpoint def get_metrics_search_by_greps(self, **greps) -> CommandResult: """ @@ -29,7 +32,19 @@ class StorageMetrics: additional_greps = " |grep ".join([grep_command for grep_command in greps.values()]) result = shell.exec(f"curl -s {self.metrics_endpoint} | grep {additional_greps}") return result - + + def get_vma_metrics_result(self, metric_name: str, since: datetime = None, **greps): + shell = self.host.get_shell(sudo=False) + if greps: + additional_filters = ",".join([f"{i}='{j}'" for i, j in greps.items()]) + metric_name += f"{{{additional_filters}}}" + command = f'curl {self.vm_agent_endpoint}/api/v1/query_range --data-urlencode "query={metric_name}"' + if since: + date_from = since.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + command += f' --data-urlencode "start={date_from}"' + result = shell.exec(command).stdout + return result + def get_all_metrics(self) -> CommandResult: shell = self.host.get_shell() result = shell.exec(f"curl -s {self.metrics_endpoint}") diff --git a/src/frostfs_testlib/storage/dataclasses/node_base.py b/src/frostfs_testlib/storage/dataclasses/node_base.py index 8291345..3ade84b 100644 --- a/src/frostfs_testlib/storage/dataclasses/node_base.py +++ b/src/frostfs_testlib/storage/dataclasses/node_base.py @@ -78,6 +78,9 @@ class NodeBase(HumanReadableABC): def get_metrics_endpoint(self) -> str: return self._get_attribute(ConfigAttributes.ENDPOINT_PROMETHEUS) + def get_vm_agent_endpoint(self) -> str: + return self._get_attribute(ConfigAttributes.ENDPOINT_VM_AGENT) + def stop_service(self, mask: bool = True): if mask: with reporter.step(f"Mask {self.name} service on {self.host.config.address}"): @@ -185,9 +188,7 @@ class NodeBase(HumanReadableABC): if attribute_name not in config.attributes: if default_attribute_name is None: - raise RuntimeError( - f"Service {self.name} has no {attribute_name} in config and fallback attribute isn't set either" - ) + raise RuntimeError(f"Service {self.name} has no {attribute_name} in config and fallback attribute isn't set either") return config.attributes[default_attribute_name] @@ -197,9 +198,7 @@ class NodeBase(HumanReadableABC): return self.host.get_service_config(self.name) def get_service_uptime(self, service: str) -> datetime: - result = self.host.get_shell().exec( - f"systemctl show {service} --property ActiveEnterTimestamp | cut -d '=' -f 2" - ) + result = self.host.get_shell().exec(f"systemctl show {service} --property ActiveEnterTimestamp | cut -d '=' -f 2") start_time = parser.parse(result.stdout.strip()) current_time = datetime.now(tz=timezone.utc) active_time = current_time - start_time