From 147ac009264ddf6b9c25b790e209f9555bd8c7c3 Mon Sep 17 00:00:00 2001 From: Ilyas Niyazov Date: Tue, 1 Oct 2024 08:16:24 +0300 Subject: [PATCH] [#295] add billing metrics methods --- src/frostfs_testlib/steps/metrics.py | 85 ++++++++++++++++++- src/frostfs_testlib/storage/cluster.py | 8 +- src/frostfs_testlib/storage/constants.py | 1 + .../storage/dataclasses/metrics.py | 25 ++++-- .../storage/dataclasses/node_base.py | 11 ++- 5 files changed, 116 insertions(+), 14 deletions(-) diff --git a/src/frostfs_testlib/steps/metrics.py b/src/frostfs_testlib/steps/metrics.py index 29e49d4..c2a9f98 100644 --- a/src/frostfs_testlib/steps/metrics.py +++ b/src/frostfs_testlib/steps/metrics.py @@ -1,8 +1,11 @@ +import json import re +from datetime import datetime +from typing import Optional from frostfs_testlib import reporter -from frostfs_testlib.testing.test_control import wait_for_success from frostfs_testlib.storage.cluster import ClusterNode +from frostfs_testlib.testing.test_control import wait_for_success @reporter.step("Check metrics result") @@ -22,6 +25,86 @@ def check_metrics_counter( ), f"Expected: {counter_exp} {operator} Actual: {counter_act} in node: {cluster_node}" +@reporter.step("Check billing metrics in node: {node}") +@wait_for_success(max_wait_time=300, interval=30) +def check_billing_metrics_in_node( + node: ClusterNode, + expected_metrics_results: list[dict], + bucket_name: Optional[str] = None, + include_cid: Optional[bool] = None, + **metrics_filters, +): + """ + expected_metrics_results = [{"operation": "PUT", "value": 7}, {"operation": "GET", "value": 3}] + expected_metrics_results = [{"direction": "IN", "value": 1000}, {"direction": "OUT", "value": 200}] + """ + metrics_bucket = get_vm_agent_metrics_values(node, **metrics_filters) + assert len(metrics_bucket["data"]["result"]) > 0, "Metrics are not available" + + result = set() + for exp_metric in expected_metrics_results: + result.add( + check_vma_metrics_counter( + metrics_results=metrics_bucket["data"]["result"], + bucket_name=bucket_name, + operation=exp_metric.get("operation"), + direction=exp_metric.get("direction"), + include_cid=include_cid, + counter_exp=exp_metric.get("value"), + ) + ) + + assert result == {True}, f"Not correct VM Agent metrics values in node: {node}" + + +@reporter.step("Check Victoria metrics agent metrics result") +def check_vma_metrics_counter( + metrics_results: list[dict], + bucket_name: Optional[str] = None, + operation: Optional[str] = None, + direction: Optional[str] = None, + include_cid: Optional[bool] = None, + counter_exp: Optional[int] = 0, + operator: Optional[str] = "==", +): + with reporter.step(f"Get metrics value for bucket: {bucket_name}, operation: {operation}"): + counter_act = None + try: + for res in metrics_results: + metric = res["metric"] + if bucket_name: + if not metric.get("bucket") == bucket_name: + continue + if operation: + if not metric.get("operation") == operation: + continue + if direction: + if not metric.get("direction") == direction: + continue + if include_cid: + if metric.get("cid") is None: + continue + # if metric.get("bucket") == bucket_name and metric.get("operation") == operation and metric.get("cid"): + counter_act = sum([int(value[-1]) for value in res["values"]]) + break + except Exception as e: + counter_act = 0 + + with reporter.step("Check metric value"): + return eval(f"{counter_act} {operator} {counter_exp}") + + +@reporter.step("Get VM agent metrics in node: {node}") +def get_vm_agent_metrics_values(node: ClusterNode, metric_name: str, since: datetime = None, **metrics_filters): + try: + command_result = node.metrics.storage.get_vma_metrics_result(metric_name, since, **metrics_filters) + result = json.loads(command_result) + except Exception as e: + result = {} + + return result + + @reporter.step("Get metrics value from node: {node}") def get_metrics_value(node: ClusterNode, parse_from_command: bool = False, **metrics_greps: str): try: diff --git a/src/frostfs_testlib/storage/cluster.py b/src/frostfs_testlib/storage/cluster.py index 9fcc4c9..52a4214 100644 --- a/src/frostfs_testlib/storage/cluster.py +++ b/src/frostfs_testlib/storage/cluster.py @@ -11,10 +11,10 @@ from frostfs_testlib.storage import get_service_registry from frostfs_testlib.storage.configuration.interfaces import ServiceConfigurationYml from frostfs_testlib.storage.constants import ConfigAttributes from frostfs_testlib.storage.dataclasses.frostfs_services import HTTPGate, InnerRing, MorphChain, S3Gate, StorageNode +from frostfs_testlib.storage.dataclasses.metrics import Metrics from frostfs_testlib.storage.dataclasses.node_base import NodeBase, ServiceClass from frostfs_testlib.storage.dataclasses.storage_object_info import Interfaces from frostfs_testlib.storage.service_registry import ServiceRegistry -from frostfs_testlib.storage.dataclasses.metrics import Metrics class ClusterNode: @@ -31,7 +31,11 @@ class ClusterNode: self.host = host self.id = id self.class_registry = get_service_registry() - self.metrics = Metrics(host=self.host, metrics_endpoint=self.storage_node.get_metrics_endpoint()) + self.metrics = Metrics( + host=self.host, + metrics_endpoint=self.storage_node.get_metrics_endpoint(), + vm_agent_endpoint=self.storage_node.get_vm_agent_endpoint(), + ) @property def host_ip(self): diff --git a/src/frostfs_testlib/storage/constants.py b/src/frostfs_testlib/storage/constants.py index 2cffd3a..20311ee 100644 --- a/src/frostfs_testlib/storage/constants.py +++ b/src/frostfs_testlib/storage/constants.py @@ -15,6 +15,7 @@ class ConfigAttributes: ENDPOINT_DATA_0_NS = "endpoint_data0_namespace" ENDPOINT_INTERNAL = "endpoint_internal0" ENDPOINT_PROMETHEUS = "endpoint_prometheus" + ENDPOINT_VM_AGENT = "endpoint_vm_agent" CONTROL_ENDPOINT = "control_endpoint" UN_LOCODE = "un_locode" diff --git a/src/frostfs_testlib/storage/dataclasses/metrics.py b/src/frostfs_testlib/storage/dataclasses/metrics.py index 81e757c..28d871f 100644 --- a/src/frostfs_testlib/storage/dataclasses/metrics.py +++ b/src/frostfs_testlib/storage/dataclasses/metrics.py @@ -1,20 +1,23 @@ +from datetime import datetime + from frostfs_testlib.hosting import Host from frostfs_testlib.shell.interfaces import CommandResult class Metrics: - def __init__(self, host: Host, metrics_endpoint: str) -> None: - self.storage = StorageMetrics(host, metrics_endpoint) - + def __init__(self, host: Host, metrics_endpoint: str, vm_agent_endpoint: str) -> None: + self.storage = StorageMetrics(host, metrics_endpoint, vm_agent_endpoint) class StorageMetrics: """ Class represents storage metrics in a cluster """ - def __init__(self, host: Host, metrics_endpoint: str) -> None: + + def __init__(self, host: Host, metrics_endpoint: str, vm_agent_endpoint: str) -> None: self.host = host self.metrics_endpoint = metrics_endpoint + self.vm_agent_endpoint = vm_agent_endpoint def get_metrics_search_by_greps(self, **greps) -> CommandResult: """ @@ -29,7 +32,19 @@ class StorageMetrics: additional_greps = " |grep ".join([grep_command for grep_command in greps.values()]) result = shell.exec(f"curl -s {self.metrics_endpoint} | grep {additional_greps}") return result - + + def get_vma_metrics_result(self, metric_name: str, since: datetime = None, **metrics_filters): + shell = self.host.get_shell(sudo=False) + if metrics_filters: + additional_filters = ",".join([f"{i}='{j}'" for i, j in metrics_filters.items()]) + metric_name += f"{{{additional_filters}}}" + command = f'curl {self.vm_agent_endpoint}/api/v1/query_range --data-urlencode "query={metric_name}"' + if since: + date_from = since.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + command += f' --data-urlencode "start={date_from}"' + result = shell.exec(command).stdout + return result + def get_all_metrics(self) -> CommandResult: shell = self.host.get_shell() result = shell.exec(f"curl -s {self.metrics_endpoint}") diff --git a/src/frostfs_testlib/storage/dataclasses/node_base.py b/src/frostfs_testlib/storage/dataclasses/node_base.py index 8291345..3ade84b 100644 --- a/src/frostfs_testlib/storage/dataclasses/node_base.py +++ b/src/frostfs_testlib/storage/dataclasses/node_base.py @@ -78,6 +78,9 @@ class NodeBase(HumanReadableABC): def get_metrics_endpoint(self) -> str: return self._get_attribute(ConfigAttributes.ENDPOINT_PROMETHEUS) + def get_vm_agent_endpoint(self) -> str: + return self._get_attribute(ConfigAttributes.ENDPOINT_VM_AGENT) + def stop_service(self, mask: bool = True): if mask: with reporter.step(f"Mask {self.name} service on {self.host.config.address}"): @@ -185,9 +188,7 @@ class NodeBase(HumanReadableABC): if attribute_name not in config.attributes: if default_attribute_name is None: - raise RuntimeError( - f"Service {self.name} has no {attribute_name} in config and fallback attribute isn't set either" - ) + raise RuntimeError(f"Service {self.name} has no {attribute_name} in config and fallback attribute isn't set either") return config.attributes[default_attribute_name] @@ -197,9 +198,7 @@ class NodeBase(HumanReadableABC): return self.host.get_service_config(self.name) def get_service_uptime(self, service: str) -> datetime: - result = self.host.get_shell().exec( - f"systemctl show {service} --property ActiveEnterTimestamp | cut -d '=' -f 2" - ) + result = self.host.get_shell().exec(f"systemctl show {service} --property ActiveEnterTimestamp | cut -d '=' -f 2") start_time = parser.parse(result.stdout.strip()) current_time = datetime.now(tz=timezone.utc) active_time = current_time - start_time