frostfs-testcases/pytest_tests/testsuites/metrics/test_grpc_metrics.py

import random

import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.healthcheck.interfaces import Healthcheck
from frostfs_testlib.steps.cli.container import create_container, get_container, list_containers
from frostfs_testlib.steps.cli.object import get_object, head_object, put_object, search_object
from frostfs_testlib.steps.cli.tree import get_tree_list
from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file


@pytest.mark.nightly
class TestGRPCMetrics(ClusterTestBase):
    @pytest.fixture
    def disable_policer(self, cluster_state_controller: ClusterStateController):
        config_manager = cluster_state_controller.manager(ConfigStateManager)
        config_manager.set_on_all_nodes(StorageNode, {"policer:unsafe_disable": "true"})
        yield
        cluster_state_controller.manager(ConfigStateManager).revert_all()

    @allure.title("GRPC metrics container operations")
    def test_grpc_metrics_container_operations(self, default_wallet: WalletInfo, cluster: Cluster):
        placement_policy = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"

        with reporter.step("Select random node"):
            node = random.choice(cluster.cluster_nodes)

        with reporter.step("Get current gRPC metrics for method 'Put'"):
            metrics_counter_put = get_metrics_value(node, command="grpc_server_handled_total", service="ContainerService", method="Put")

        with reporter.step(f"Create container with policy {placement_policy}"):
            cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)

        with reporter.step(f"Check gRPC metrics method 'Put', 'the counter should increase by 1'"):
            metrics_counter_put += 1
            check_metrics_counter(
                [node],
                counter_exp=metrics_counter_put,
                command="grpc_server_handled_total",
                service="ContainerService",
                method="Put",
            )

        with reporter.step("Get current gRPC metrics for method 'Get'"):
            metrics_counter_get = get_metrics_value(node, command="grpc_server_handled_total", service="ContainerService", method="Get")

        with reporter.step(f"Get container"):
            get_container(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint())

        with reporter.step(f"Check gRPC metrics method=Get, 'the counter should increase by 1'"):
            metrics_counter_get += 1
            check_metrics_counter(
                [node],
                counter_exp=metrics_counter_get,
                command="grpc_server_handled_total",
                service="ContainerService",
                method="Get",
            )

        with reporter.step("Get current gRPC metrics for method 'List'"):
            metrics_counter_list = get_metrics_value(node, command="grpc_server_handled_total", service="ContainerService", method="List")

        with reporter.step(f"Get container list"):
            list_containers(default_wallet, self.shell, node.storage_node.get_rpc_endpoint())

        with reporter.step(f"Check gRPC metrics method=List, 'the counter should increase by 1'"):
            metrics_counter_list += 1
            check_metrics_counter(
                [node],
                counter_exp=metrics_counter_list,
                command="grpc_server_handled_total",
                service="ContainerService",
                method="List",
            )

    @allure.title("GRPC metrics object operations")
    def test_grpc_metrics_object_operations(
        self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster, disable_policer
    ):
        file_path = generate_file(simple_object_size.value)
        placement_policy = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"

        with reporter.step("Select random node"):
            node = random.choice(cluster.cluster_nodes)

        with reporter.step(f"Create container with policy {placement_policy}"):
            cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)

        with reporter.step("Get current gRPC metrics for method 'Put'"):
            metrics_counter_put = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Put")

        with reporter.step("Put object to selected node"):
            oid = put_object(default_wallet, file_path, cid, self.shell, node.storage_node.get_rpc_endpoint())

        with reporter.step(f"Check gRPC metrics method 'Put', 'the counter should increase by 1'"):
            metrics_counter_put += 1
            check_metrics_counter(
                [node],
                counter_exp=metrics_counter_put,
                command="grpc_server_handled_total",
                service="ObjectService",
                method="Put",
            )

        with reporter.step("Get current gRPC metrics for method 'Get'"):
            metrics_counter_get = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Get")

        with reporter.step(f"Get object"):
            get_object(default_wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())

        with reporter.step(f"Check gRPC metrics method=Get, 'the counter should increase by 1'"):
            metrics_counter_get += 1
            check_metrics_counter(
                [node],
                counter_exp=metrics_counter_get,
                command="grpc_server_handled_total",
                service="ObjectService",
                method="Get",
            )

        with reporter.step("Get current gRPC metrics for method 'Search'"):
            metrics_counter_search = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Search")

        with reporter.step(f"Search object"):
            search_object(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint())

        with reporter.step(f"Check gRPC metrics method=Search, 'the counter should increase by 1'"):
            metrics_counter_search += 1
            check_metrics_counter(
                [node],
                counter_exp=metrics_counter_search,
                command="grpc_server_handled_total",
                service="ObjectService",
                method="Search",
            )

        with reporter.step("Get current gRPC metrics for method 'Head'"):
            metrics_counter_head = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Head")

        with reporter.step(f"Head object"):
            head_object(default_wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())

        with reporter.step(f"Check gRPC metrics method=Head, 'the counter should increase by 1'"):
            metrics_counter_head += 1
            check_metrics_counter(
                [node],
                counter_exp=metrics_counter_head,
                command="grpc_server_handled_total",
                service="ObjectService",
                method="Head",
            )

    @allure.title("GRPC metrics Tree healthcheck")
    def test_grpc_metrics_tree_service(self, cluster: Cluster, healthcheck: Healthcheck):
        with reporter.step("Select random node"):
            node = random.choice(cluster.cluster_nodes)

        with reporter.step("Get current gRPC metrics for Healthcheck"):
            metrics_counter = get_metrics_value(node, command="grpc_server_handled_total", service="TreeService", method="Healthcheck")

        with reporter.step("Query Tree healthcheck status"):
            healthcheck.tree_healthcheck(node)

        with reporter.step(f"Check gRPC metrics for Healthcheck, 'the counter should increase'"):
            check_metrics_counter(
                [node],
                ">",
                metrics_counter,
                command="grpc_server_handled_total",
                service="TreeService",
                method="Healthcheck",
            )

    @allure.title("GRPC metrics Tree list")
    def test_grpc_metrics_tree_list(self, default_wallet: WalletInfo, cluster: Cluster):
        placement_policy = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"

        with reporter.step("Select random node"):
            node = random.choice(cluster.cluster_nodes)

        with reporter.step(f"Create container with policy {placement_policy}"):
            cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)

        with reporter.step("Get current gRPC metrics for Tree List"):
            metrics_counter = get_metrics_value(node, command="grpc_server_handled_total", service="TreeService", method="TreeList")

        with reporter.step("Query Tree List"):
            get_tree_list(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint())

        with reporter.step(f"Check gRPC metrics for Tree List, 'the counter should increase by 1'"):
            metrics_counter += 1
            check_metrics_counter(
                [node],
                counter_exp=metrics_counter,
                command="grpc_server_handled_total",
                service="TreeService",
                method="TreeList",
            )