frostfs-testcases/pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py

import re

import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.steps.cli.object import delete_object, put_object_to_random_node
from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value
from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import wait_for_success
from frostfs_testlib.utils.file_utils import generate_file

from ...helpers.container_request import PUBLIC_WITH_POLICY, requires_container


@pytest.mark.nightly
@pytest.mark.metrics
class TestGarbageCollectorMetrics(ClusterTestBase):
    @wait_for_success(interval=10)
    def check_metrics_in_node(self, cluster_node: ClusterNode, counter_exp: int, **metrics_greps: str):
        counter_act = 0
        try:
            metric_result = cluster_node.metrics.storage.get_metrics_search_by_greps(**metrics_greps)
            counter_act += self.calc_metrics_count_from_stdout(metric_result.stdout)
        except RuntimeError as e:
            ...
        assert counter_act == counter_exp, f"Expected: {counter_exp}, Actual: {counter_act} in node: {cluster_node}"

    @staticmethod
    def calc_metrics_count_from_stdout(metric_result_stdout: str):
        result = re.findall(r"}\s(\d+)", metric_result_stdout)
        return sum(map(int, result))

    @allure.title("Garbage collector expire_at object")
    @requires_container(PUBLIC_WITH_POLICY("REP 2 IN X CBF 2 SELECT 2 FROM * AS X", short_name="REP 2"))
    def test_garbage_collector_metrics_expire_at_object(
        self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster, container: str
    ):
        file_path = generate_file(simple_object_size.value)
        metrics_step = 1

        with reporter.step("Get current garbage collector metrics for each nodes"):
            metrics_counter = {}
            for node in cluster.cluster_nodes:
                metrics_counter[node] = get_metrics_value(node, command="frostfs_node_garbage_collector_marked_for_removal_objects_total")

        with reporter.step("Put object to random node with expire_at"):
            current_epoch = self.get_epoch()
            oid = put_object_to_random_node(
                default_wallet,
                file_path,
                container,
                self.shell,
                cluster,
                expire_at=current_epoch + 1,
            )

        with reporter.step("Get object nodes"):
            object_storage_nodes = get_nodes_with_object(container, oid, self.shell, cluster.storage_nodes)
            object_nodes = [cluster_node for cluster_node in cluster.cluster_nodes if cluster_node.storage_node in object_storage_nodes]

        with reporter.step("Tick Epoch"):
            self.tick_epochs(epochs_to_tick=2, wait_block=2)

        with reporter.step(f"Check garbage collector metrics 'the counter should increase by {metrics_step}' in object nodes"):
            for node in object_nodes:
                metrics_counter[node] += metrics_step

            for node, counter in metrics_counter.items():
                check_metrics_counter(
                    [node],
                    counter_exp=counter,
                    command="frostfs_node_garbage_collector_marked_for_removal_objects_total",
                )

    @allure.title("Garbage collector delete object")
    @requires_container(PUBLIC_WITH_POLICY("REP 2 IN X CBF 2 SELECT 2 FROM * AS X", short_name="REP 2"))
    def test_garbage_collector_metrics_deleted_objects(
        self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster, container: str
    ):
        file_path = generate_file(simple_object_size.value)
        metrics_step = 1

        with reporter.step("Get current garbage collector metrics for each nodes"):
            metrics_counter = {}
            for node in cluster.cluster_nodes:
                metrics_counter[node] = get_metrics_value(node, command="frostfs_node_garbage_collector_deleted_objects_total")

        with reporter.step("Put object to random node"):
            oid = put_object_to_random_node(
                default_wallet,
                file_path,
                container,
                self.shell,
                cluster,
            )

        with reporter.step("Get object nodes"):
            object_storage_nodes = get_nodes_with_object(container, oid, self.shell, cluster.storage_nodes)
            object_nodes = [cluster_node for cluster_node in cluster.cluster_nodes if cluster_node.storage_node in object_storage_nodes]

        with reporter.step("Delete file, wait until gc remove object"):
            delete_object(default_wallet, container, oid, self.shell, node.storage_node.get_rpc_endpoint())

        with reporter.step(f"Check garbage collector metrics 'the counter should increase by {metrics_step}'"):
            for node in object_nodes:
                exp_metrics_counter = metrics_counter[node] + metrics_step
                check_metrics_counter(
                    [node], counter_exp=exp_metrics_counter, command="frostfs_node_garbage_collector_deleted_objects_total"
                )