import random import allure import pytest from frostfs_testlib import reporter from frostfs_testlib.healthcheck.interfaces import Healthcheck from frostfs_testlib.steps.cli.container import create_container, get_container, list_containers from frostfs_testlib.steps.cli.object import get_object, head_object, put_object, search_object from frostfs_testlib.steps.cli.tree import get_tree_list from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value from frostfs_testlib.storage.cluster import Cluster from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.utils.file_utils import generate_file @pytest.mark.nightly @pytest.mark.metrics class TestGRPCMetrics(ClusterTestBase): @pytest.fixture def disable_policer(self, cluster_state_controller: ClusterStateController): config_manager = cluster_state_controller.manager(ConfigStateManager) config_manager.set_on_all_nodes(StorageNode, {"policer:unsafe_disable": "true"}) yield cluster_state_controller.manager(ConfigStateManager).revert_all() @allure.title("GRPC metrics container operations") def test_grpc_metrics_container_operations(self, default_wallet: WalletInfo, cluster: Cluster): placement_policy = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" with reporter.step("Select random node"): node = random.choice(cluster.cluster_nodes) with reporter.step("Get current gRPC metrics for method 'Put'"): metrics_counter_put = get_metrics_value(node, command="grpc_server_handled_total", service="ContainerService", method="Put") with reporter.step(f"Create container with policy {placement_policy}"): cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy) with reporter.step(f"Check gRPC metrics method 'Put', 'the counter should increase by 1'"): metrics_counter_put += 1 check_metrics_counter( [node], counter_exp=metrics_counter_put, command="grpc_server_handled_total", service="ContainerService", method="Put", ) with reporter.step("Get current gRPC metrics for method 'Get'"): metrics_counter_get = get_metrics_value(node, command="grpc_server_handled_total", service="ContainerService", method="Get") with reporter.step(f"Get container"): get_container(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint()) with reporter.step(f"Check gRPC metrics method=Get, 'the counter should increase by 1'"): metrics_counter_get += 1 check_metrics_counter( [node], counter_exp=metrics_counter_get, command="grpc_server_handled_total", service="ContainerService", method="Get", ) with reporter.step("Get current gRPC metrics for method 'List'"): metrics_counter_list = get_metrics_value(node, command="grpc_server_handled_total", service="ContainerService", method="List") with reporter.step(f"Get container list"): list_containers(default_wallet, self.shell, node.storage_node.get_rpc_endpoint()) with reporter.step(f"Check gRPC metrics method=List, 'the counter should increase by 1'"): metrics_counter_list += 1 check_metrics_counter( [node], counter_exp=metrics_counter_list, command="grpc_server_handled_total", service="ContainerService", method="List", ) @allure.title("GRPC metrics object operations") def test_grpc_metrics_object_operations( self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster, container: str, disable_policer ): file_path = generate_file(simple_object_size.value) with reporter.step("Select random node"): node = random.choice(cluster.cluster_nodes) with reporter.step("Get current gRPC metrics for method 'Put'"): metrics_counter_put = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Put") with reporter.step("Put object to selected node"): oid = put_object(default_wallet, file_path, container, self.shell, node.storage_node.get_rpc_endpoint()) with reporter.step(f"Check gRPC metrics method 'Put', 'the counter should increase by 1'"): metrics_counter_put += 1 check_metrics_counter( [node], counter_exp=metrics_counter_put, command="grpc_server_handled_total", service="ObjectService", method="Put", ) with reporter.step("Get current gRPC metrics for method 'Get'"): metrics_counter_get = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Get") with reporter.step(f"Get object"): get_object(default_wallet, container, oid, self.shell, node.storage_node.get_rpc_endpoint()) with reporter.step(f"Check gRPC metrics method=Get, 'the counter should increase by 1'"): metrics_counter_get += 1 check_metrics_counter( [node], counter_exp=metrics_counter_get, command="grpc_server_handled_total", service="ObjectService", method="Get", ) with reporter.step("Get current gRPC metrics for method 'Search'"): metrics_counter_search = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Search") with reporter.step(f"Search object"): search_object(default_wallet, container, self.shell, node.storage_node.get_rpc_endpoint()) with reporter.step(f"Check gRPC metrics method=Search, 'the counter should increase by 1'"): metrics_counter_search += 1 check_metrics_counter( [node], counter_exp=metrics_counter_search, command="grpc_server_handled_total", service="ObjectService", method="Search", ) with reporter.step("Get current gRPC metrics for method 'Head'"): metrics_counter_head = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Head") with reporter.step(f"Head object"): head_object(default_wallet, container, oid, self.shell, node.storage_node.get_rpc_endpoint()) with reporter.step(f"Check gRPC metrics method=Head, 'the counter should increase by 1'"): metrics_counter_head += 1 check_metrics_counter( [node], counter_exp=metrics_counter_head, command="grpc_server_handled_total", service="ObjectService", method="Head", ) @allure.title("GRPC metrics Tree healthcheck") def test_grpc_metrics_tree_service(self, cluster: Cluster, healthcheck: Healthcheck): with reporter.step("Select random node"): node = random.choice(cluster.cluster_nodes) with reporter.step("Get current gRPC metrics for Healthcheck"): metrics_counter = get_metrics_value(node, command="grpc_server_handled_total", service="TreeService", method="Healthcheck") with reporter.step("Query Tree healthcheck status"): healthcheck.tree_healthcheck(node) with reporter.step(f"Check gRPC metrics for Healthcheck, 'the counter should increase'"): check_metrics_counter( [node], ">", metrics_counter, command="grpc_server_handled_total", service="TreeService", method="Healthcheck", ) @allure.title("GRPC metrics Tree list") def test_grpc_metrics_tree_list(self, default_wallet: WalletInfo, cluster: Cluster): placement_policy = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" with reporter.step("Select random node"): node = random.choice(cluster.cluster_nodes) with reporter.step(f"Create container with policy {placement_policy}"): cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy) with reporter.step("Get current gRPC metrics for Tree List"): metrics_counter = get_metrics_value(node, command="grpc_server_handled_total", service="TreeService", method="TreeList") with reporter.step("Query Tree List"): get_tree_list(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint()) with reporter.step(f"Check gRPC metrics for Tree List, 'the counter should increase by 1'"): metrics_counter += 1 check_metrics_counter( [node], counter_exp=metrics_counter, command="grpc_server_handled_total", service="TreeService", method="TreeList", )