From 042e17161945847ff509080b663c72b8858997dc Mon Sep 17 00:00:00 2001 From: Ilyas Niyazov Date: Tue, 28 May 2024 15:31:35 +0300 Subject: [PATCH] [#237] add test garbage collector metrics --- .../metrics/test_garbage_collector_metrics.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py diff --git a/pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py b/pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py new file mode 100644 index 00000000..752f6a98 --- /dev/null +++ b/pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py @@ -0,0 +1,113 @@ +import random +import re + +import allure +from frostfs_testlib import reporter +from frostfs_testlib.steps.cli.container import create_container, delete_container +from frostfs_testlib.steps.cli.object import delete_object, get_object_nodes, put_object, put_object_to_random_node +from frostfs_testlib.storage.cluster import Cluster, ClusterNode +from frostfs_testlib.storage.dataclasses.object_size import ObjectSize +from frostfs_testlib.storage.dataclasses.wallet import WalletInfo +from frostfs_testlib.testing.cluster_test_base import ClusterTestBase +from frostfs_testlib.testing.test_control import wait_for_success +from frostfs_testlib.utils.file_utils import generate_file + + +class TestGarbageCollectorMetrics(ClusterTestBase): + @wait_for_success(interval=10) + def check_metrics_in_node(self, cluster_node: ClusterNode, counter_exp: int, **metrics_greps: str): + counter_act = 0 + try: + metric_result = cluster_node.metrics.storage.get_metrics_search_by_greps(**metrics_greps) + counter_act += self.calc_metrics_count_from_stdout(metric_result.stdout) + except RuntimeError as e: + ... + assert counter_act == counter_exp, f"Expected: {counter_exp}, Actual: {counter_act} in node: {cluster_node}" + + @staticmethod + def calc_metrics_count_from_stdout(metric_result_stdout: str): + result = re.findall(r"}\s(\d+)", metric_result_stdout) + return sum(map(int, result)) + + @allure.title("Garbage collector expire_at object") + def test_garbage_collector_metrics_expire_at_object( + self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster + ): + file_path = generate_file(simple_object_size.value) + placement_policy = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" + metrics_step = 1 + + with reporter.step("Get current garbage collector metrics for each nodes"): + metrics_counter = {} + for node in cluster.cluster_nodes: + command_result = node.metrics.storage.get_metrics_search_by_greps( + command="frostfs_node_garbage_collector_marked_for_removal_objects_total" + ) + metrics_counter[node] = self.calc_metrics_count_from_stdout(command_result.stdout) + + with reporter.step(f"Create container with policy {placement_policy}"): + cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, placement_policy) + + with reporter.step("Put object to random node with expire_at"): + current_epoch = self.get_epoch() + oid = put_object_to_random_node( + default_wallet, + file_path, + cid, + self.shell, + cluster, + expire_at=current_epoch + 1, + ) + + with reporter.step("Get object nodes"): + object_nodes = get_object_nodes(cluster, cid, oid, cluster.cluster_nodes[0]) + + with reporter.step("Tick Epoch"): + self.tick_epochs(epochs_to_tick=2, wait_block=2) + + with reporter.step(f"Check garbage collector metrics 'the counter should increase by {metrics_step}'"): + for node in object_nodes: + metrics_counter[node] += metrics_step + + for node, counter in metrics_counter.items(): + self.check_metrics_in_node( + node, counter, command="frostfs_node_garbage_collector_marked_for_removal_objects_total" + ) + + with reporter.step("Delete container"): + delete_container(default_wallet, cid, self.shell, cluster.default_rpc_endpoint) + + @allure.title("Garbage collector delete object") + def test_garbage_collector_metrics_deleted_objects( + self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster + ): + file_path = generate_file(simple_object_size.value) + placement_policy = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" + metrics_step = 1 + + with reporter.step("Select random node"): + node = random.choice(cluster.cluster_nodes) + + with reporter.step("Get current garbage collector metrics for selected node"): + command_result = node.metrics.storage.get_metrics_search_by_greps( + command="frostfs_node_garbage_collector_deleted_objects_total" + ) + metrics_counter = self.calc_metrics_count_from_stdout(command_result.stdout) + + with reporter.step(f"Create container with policy {placement_policy}"): + cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy) + + with reporter.step("Put object to selected node"): + oid = put_object(default_wallet, file_path, cid, self.shell, node.storage_node.get_rpc_endpoint()) + + with reporter.step("Delete file, wait until gc remove object"): + delete_object(default_wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint()) + + with reporter.step(f"Check garbage collector metrics 'the counter should increase by {metrics_step}'"): + metrics_counter += metrics_step + self.check_metrics_in_node( + node, metrics_counter, command="frostfs_node_garbage_collector_deleted_objects_total" + ) + + with reporter.step("Delete container"): + delete_container(default_wallet, cid, self.shell, cluster.default_rpc_endpoint)