import math import time import allure import pytest from frostfs_testlib import reporter from frostfs_testlib.steps.cli.container import create_container, search_nodes_with_container from frostfs_testlib.steps.cli.object import delete_object, head_object, put_object_to_random_node from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value from frostfs_testlib.steps.storage_policy import get_nodes_with_object from frostfs_testlib.storage.cluster import Cluster from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.utils.file_utils import generate_file @pytest.mark.nightly @pytest.mark.container class TestContainerMetrics(ClusterTestBase): @allure.title("Container metrics (obj_size={object_size},policy={policy})") @pytest.mark.parametrize("placement_policy, policy", [("REP 2 IN X CBF 2 SELECT 2 FROM * AS X", "REP"), ("EC 1.1 CBF 1", "EC")]) def test_container_metrics( self, object_size: ObjectSize, max_object_size: int, default_wallet: WalletInfo, cluster: Cluster, placement_policy: str, policy: str, ): file_path = generate_file(object_size.value) copies = 2 if policy == "REP" else 1 object_chunks = 1 link_object = 0 with reporter.step(f"Create container with policy {placement_policy}"): cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, placement_policy) if object_size.value > max_object_size: object_chunks = math.ceil(object_size.value / max_object_size) link_object = len(search_nodes_with_container(default_wallet, cid, self.shell, cluster.default_rpc_endpoint, cluster)) with reporter.step("Put object to random node"): oid = put_object_to_random_node( wallet=default_wallet, path=file_path, cid=cid, shell=self.shell, cluster=cluster, ) with reporter.step("Get object nodes"): object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, cluster.storage_nodes) object_nodes = [cluster_node for cluster_node in cluster.cluster_nodes if cluster_node.storage_node in object_storage_nodes] with reporter.step("Check metric appears in node where the object is located"): count_metrics = (object_chunks * copies) + link_object if policy == "EC": count_metrics = (object_chunks * 2) + link_object check_metrics_counter(object_nodes, counter_exp=count_metrics, command="container_objects_total", cid=cid, type="phy") check_metrics_counter(object_nodes, counter_exp=count_metrics, command="container_objects_total", cid=cid, type="logic") check_metrics_counter(object_nodes, counter_exp=copies, command="container_objects_total", cid=cid, type="user") with reporter.step("Delete file, wait until gc remove object"): delete_object(default_wallet, cid, oid, self.shell, cluster.default_rpc_endpoint) with reporter.step(f"Check container metrics 'the counter should equal {len(object_nodes)}' in object nodes"): check_metrics_counter(object_nodes, counter_exp=len(object_nodes), command="container_objects_total", cid=cid, type="phy") check_metrics_counter(object_nodes, counter_exp=len(object_nodes), command="container_objects_total", cid=cid, type="logic") check_metrics_counter(object_nodes, counter_exp=0, command="container_objects_total", cid=cid, type="user") with reporter.step("Check metrics(Phy, Logic, User) in each nodes"): # Phy and Logic metrics are 4, because in rule 'CBF 2 SELECT 2 FROM', cbf2*sel2=4 expect_metrics = 4 if policy == "REP" else 2 check_metrics_counter(cluster.cluster_nodes, counter_exp=expect_metrics, command="container_objects_total", cid=cid, type="phy") check_metrics_counter( cluster.cluster_nodes, counter_exp=expect_metrics, command="container_objects_total", cid=cid, type="logic" ) check_metrics_counter(cluster.cluster_nodes, counter_exp=0, command="container_objects_total", cid=cid, type="user") @allure.title("Container size metrics (obj_size={object_size},policy={policy})") @pytest.mark.parametrize("placement_policy, policy", [("REP 2 IN X CBF 2 SELECT 2 FROM * AS X", "REP"), ("EC 1.1 CBF 1", "EC")]) def test_container_size_metrics( self, object_size: ObjectSize, default_wallet: WalletInfo, placement_policy: str, policy: str, ): file_path = generate_file(object_size.value) with reporter.step(f"Create container with policy {policy}"): cid = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint, placement_policy) with reporter.step("Put object to random node"): oid = put_object_to_random_node( wallet=default_wallet, path=file_path, cid=cid, shell=self.shell, cluster=self.cluster, ) with reporter.step("Get object nodes"): object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, self.cluster.storage_nodes) object_nodes = [ cluster_node for cluster_node in self.cluster.cluster_nodes if cluster_node.storage_node in object_storage_nodes ] with reporter.step("Check metric appears in all node where the object is located"): act_metric = sum( [get_metrics_value(node, command="frostfs_node_engine_container_size_bytes", cid=cid) for node in object_nodes] ) assert (act_metric // 2) == object_size.value with reporter.step("Delete file, wait until gc remove object"): id_tombstone = delete_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint) tombstone = head_object(default_wallet, cid, id_tombstone, self.shell, self.cluster.default_rpc_endpoint) with reporter.step(f"Check container size metrics"): act_metric = get_metrics_value(object_nodes[0], command="frostfs_node_engine_container_size_bytes", cid=cid) assert act_metric == int(tombstone["header"]["payloadLength"])