forked from TrueCloudLab/frostfs-testcases
[#255] refactore metrics tests
This commit is contained in:
parent
e65d19f056
commit
d08dbfa07d
6 changed files with 345 additions and 372 deletions
|
|
@ -1,21 +1,20 @@
|
|||
import random
|
||||
import re
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib import reporter
|
||||
from frostfs_testlib.healthcheck.interfaces import Healthcheck
|
||||
from frostfs_testlib.steps.cli.container import create_container, delete_container, get_container, list_containers
|
||||
from frostfs_testlib.steps.cli.container import create_container, get_container, list_containers
|
||||
from frostfs_testlib.steps.cli.object import get_object, head_object, put_object, search_object
|
||||
from frostfs_testlib.steps.cli.tree import get_tree_list
|
||||
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
|
||||
from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value
|
||||
from frostfs_testlib.storage.cluster import Cluster
|
||||
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
|
||||
from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager
|
||||
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
|
||||
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
|
||||
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
||||
from frostfs_testlib.testing.test_control import wait_for_success
|
||||
from frostfs_testlib.utils.file_utils import generate_file
|
||||
|
||||
|
||||
|
|
@ -27,25 +26,6 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
yield
|
||||
cluster_state_controller.manager(ConfigStateManager).revert_all()
|
||||
|
||||
@wait_for_success(interval=10)
|
||||
def check_metrics_in_node(self, cluster_node: ClusterNode, counter_exp: int, **metrics_greps: str):
|
||||
counter_act = self.get_metrics_value(cluster_node, **metrics_greps)
|
||||
assert counter_act == counter_exp, f"Expected: {counter_exp}, Actual: {counter_act} in node: {cluster_node}"
|
||||
|
||||
def get_metrics_value(self, node: ClusterNode, **metrics_greps: str):
|
||||
try:
|
||||
command_result = node.metrics.storage.get_metrics_search_by_greps(**metrics_greps)
|
||||
metrics_counter = self.calc_metrics_count_from_stdout(command_result.stdout)
|
||||
except RuntimeError as e:
|
||||
metrics_counter = 0
|
||||
|
||||
return metrics_counter
|
||||
|
||||
@staticmethod
|
||||
def calc_metrics_count_from_stdout(metric_result_stdout: str):
|
||||
result = re.findall(r"}\s(\d+)", metric_result_stdout)
|
||||
return sum(map(int, result))
|
||||
|
||||
@allure.title("GRPC metrics container operations")
|
||||
def test_grpc_metrics_container_operations(self, default_wallet: WalletInfo, cluster: Cluster):
|
||||
placement_policy = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
|
||||
|
|
@ -54,7 +34,7 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
node = random.choice(cluster.cluster_nodes)
|
||||
|
||||
with reporter.step("Get current gRPC metrics for method 'Put'"):
|
||||
metrics_counter_put = self.get_metrics_value(
|
||||
metrics_counter_put = get_metrics_value(
|
||||
node, command="grpc_server_handled_total", service="ContainerService", method="Put"
|
||||
)
|
||||
|
||||
|
|
@ -63,12 +43,16 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
|
||||
with reporter.step(f"Check gRPC metrics method 'Put', 'the counter should increase by 1'"):
|
||||
metrics_counter_put += 1
|
||||
self.check_metrics_in_node(
|
||||
node, metrics_counter_put, command="grpc_server_handled_total", service="ContainerService", method="Put"
|
||||
check_metrics_counter(
|
||||
[node],
|
||||
counter_exp=metrics_counter_put,
|
||||
command="grpc_server_handled_total",
|
||||
service="ContainerService",
|
||||
method="Put",
|
||||
)
|
||||
|
||||
with reporter.step("Get current gRPC metrics for method 'Get'"):
|
||||
metrics_counter_get = self.get_metrics_value(
|
||||
metrics_counter_get = get_metrics_value(
|
||||
node, command="grpc_server_handled_total", service="ContainerService", method="Get"
|
||||
)
|
||||
|
||||
|
|
@ -77,12 +61,16 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
|
||||
with reporter.step(f"Check gRPC metrics method=Get, 'the counter should increase by 1'"):
|
||||
metrics_counter_get += 1
|
||||
self.check_metrics_in_node(
|
||||
node, metrics_counter_get, command="grpc_server_handled_total", service="ContainerService", method="Get"
|
||||
check_metrics_counter(
|
||||
[node],
|
||||
counter_exp=metrics_counter_get,
|
||||
command="grpc_server_handled_total",
|
||||
service="ContainerService",
|
||||
method="Get",
|
||||
)
|
||||
|
||||
with reporter.step("Get current gRPC metrics for method 'List'"):
|
||||
metrics_counter_list = self.get_metrics_value(
|
||||
metrics_counter_list = get_metrics_value(
|
||||
node, command="grpc_server_handled_total", service="ContainerService", method="List"
|
||||
)
|
||||
|
||||
|
|
@ -91,17 +79,14 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
|
||||
with reporter.step(f"Check gRPC metrics method=List, 'the counter should increase by 1'"):
|
||||
metrics_counter_list += 1
|
||||
self.check_metrics_in_node(
|
||||
node,
|
||||
metrics_counter_list,
|
||||
check_metrics_counter(
|
||||
[node],
|
||||
counter_exp=metrics_counter_list,
|
||||
command="grpc_server_handled_total",
|
||||
service="ContainerService",
|
||||
method="List",
|
||||
)
|
||||
|
||||
with reporter.step("Delete container"):
|
||||
delete_container(default_wallet, cid, self.shell, self.cluster.default_rpc_endpoint)
|
||||
|
||||
@allure.title("GRPC metrics object operations")
|
||||
def test_grpc_metrics_object_operations(
|
||||
self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster, disable_policer
|
||||
|
|
@ -116,7 +101,7 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
|
||||
|
||||
with reporter.step("Get current gRPC metrics for method 'Put'"):
|
||||
metrics_counter_put = self.get_metrics_value(
|
||||
metrics_counter_put = get_metrics_value(
|
||||
node, command="grpc_server_handled_total", service="ObjectService", method="Put"
|
||||
)
|
||||
|
||||
|
|
@ -125,12 +110,16 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
|
||||
with reporter.step(f"Check gRPC metrics method 'Put', 'the counter should increase by 1'"):
|
||||
metrics_counter_put += 1
|
||||
self.check_metrics_in_node(
|
||||
node, metrics_counter_put, command="grpc_server_handled_total", service="ObjectService", method="Put"
|
||||
check_metrics_counter(
|
||||
[node],
|
||||
counter_exp=metrics_counter_put,
|
||||
command="grpc_server_handled_total",
|
||||
service="ObjectService",
|
||||
method="Put",
|
||||
)
|
||||
|
||||
with reporter.step("Get current gRPC metrics for method 'Get'"):
|
||||
metrics_counter_get = self.get_metrics_value(
|
||||
metrics_counter_get = get_metrics_value(
|
||||
node, command="grpc_server_handled_total", service="ObjectService", method="Get"
|
||||
)
|
||||
|
||||
|
|
@ -139,12 +128,16 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
|
||||
with reporter.step(f"Check gRPC metrics method=Get, 'the counter should increase by 1'"):
|
||||
metrics_counter_get += 1
|
||||
self.check_metrics_in_node(
|
||||
node, metrics_counter_get, command="grpc_server_handled_total", service="ObjectService", method="Get"
|
||||
check_metrics_counter(
|
||||
[node],
|
||||
counter_exp=metrics_counter_get,
|
||||
command="grpc_server_handled_total",
|
||||
service="ObjectService",
|
||||
method="Get",
|
||||
)
|
||||
|
||||
with reporter.step("Get current gRPC metrics for method 'Search'"):
|
||||
metrics_counter_search = self.get_metrics_value(
|
||||
metrics_counter_search = get_metrics_value(
|
||||
node, command="grpc_server_handled_total", service="ObjectService", method="Search"
|
||||
)
|
||||
|
||||
|
|
@ -153,16 +146,16 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
|
||||
with reporter.step(f"Check gRPC metrics method=Search, 'the counter should increase by 1'"):
|
||||
metrics_counter_search += 1
|
||||
self.check_metrics_in_node(
|
||||
node,
|
||||
metrics_counter_search,
|
||||
check_metrics_counter(
|
||||
[node],
|
||||
counter_exp=metrics_counter_search,
|
||||
command="grpc_server_handled_total",
|
||||
service="ObjectService",
|
||||
method="Search",
|
||||
)
|
||||
|
||||
with reporter.step("Get current gRPC metrics for method 'Head'"):
|
||||
metrics_counter_head = self.get_metrics_value(
|
||||
metrics_counter_head = get_metrics_value(
|
||||
node, command="grpc_server_handled_total", service="ObjectService", method="Head"
|
||||
)
|
||||
|
||||
|
|
@ -171,20 +164,21 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
|
||||
with reporter.step(f"Check gRPC metrics method=Head, 'the counter should increase by 1'"):
|
||||
metrics_counter_head += 1
|
||||
self.check_metrics_in_node(
|
||||
node, metrics_counter_head, command="grpc_server_handled_total", service="ObjectService", method="Head"
|
||||
check_metrics_counter(
|
||||
[node],
|
||||
counter_exp=metrics_counter_head,
|
||||
command="grpc_server_handled_total",
|
||||
service="ObjectService",
|
||||
method="Head",
|
||||
)
|
||||
|
||||
with reporter.step("Delete container"):
|
||||
delete_container(default_wallet, cid, self.shell, self.cluster.default_rpc_endpoint)
|
||||
|
||||
@allure.title("GRPC metrics Tree healthcheck")
|
||||
def test_grpc_metrics_tree_service(self, cluster: Cluster, healthcheck: Healthcheck):
|
||||
with reporter.step("Select random node"):
|
||||
node = random.choice(cluster.cluster_nodes)
|
||||
|
||||
with reporter.step("Get current gRPC metrics for Healthcheck"):
|
||||
metrics_counter = self.get_metrics_value(
|
||||
metrics_counter = get_metrics_value(
|
||||
node, command="grpc_server_handled_total", service="TreeService", method="Healthcheck"
|
||||
)
|
||||
|
||||
|
|
@ -192,10 +186,14 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
healthcheck.tree_healthcheck(node)
|
||||
|
||||
with reporter.step(f"Check gRPC metrics for Healthcheck, 'the counter should increase'"):
|
||||
metrics_counter_new = self.get_metrics_value(
|
||||
node, command="grpc_server_handled_total", service="TreeService", method="Healthcheck"
|
||||
check_metrics_counter(
|
||||
[node],
|
||||
">",
|
||||
metrics_counter,
|
||||
command="grpc_server_handled_total",
|
||||
service="TreeService",
|
||||
method="Healthcheck",
|
||||
)
|
||||
assert metrics_counter_new > metrics_counter, "the metrics has not increased"
|
||||
|
||||
@allure.title("GRPC metrics Tree list")
|
||||
def test_grpc_metrics_tree_list(self, default_wallet: WalletInfo, cluster: Cluster):
|
||||
|
|
@ -208,7 +206,7 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
|
||||
|
||||
with reporter.step("Get current gRPC metrics for Tree List"):
|
||||
metrics_counter = self.get_metrics_value(
|
||||
metrics_counter = get_metrics_value(
|
||||
node, command="grpc_server_handled_total", service="TreeService", method="TreeList"
|
||||
)
|
||||
|
||||
|
|
@ -217,9 +215,10 @@ class TestGRPCMetrics(ClusterTestBase):
|
|||
|
||||
with reporter.step(f"Check gRPC metrics for Tree List, 'the counter should increase by 1'"):
|
||||
metrics_counter += 1
|
||||
self.check_metrics_in_node(
|
||||
node, metrics_counter, command="grpc_server_handled_total", service="TreeService", method="TreeList"
|
||||
check_metrics_counter(
|
||||
[node],
|
||||
counter_exp=metrics_counter,
|
||||
command="grpc_server_handled_total",
|
||||
service="TreeService",
|
||||
method="TreeList",
|
||||
)
|
||||
|
||||
with reporter.step("Delete container"):
|
||||
delete_container(default_wallet, cid, self.shell, self.cluster.default_rpc_endpoint)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue