[#255] refactore metrics tests

pytest_tests/testsuites/conftest.py

@@ -354,15 +354,20 @@ def two_buckets(s3_client: S3ClientWrapper, request: pytest.FixtureRequest):
             s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
 
 
-@allure.title("[Autouse/Session] Check binary versions")
+@allure.title("[Autouse/Session] Collect binary versions")
 @pytest.fixture(scope="session", autouse=True)
-def check_binary_versions(hosting: Hosting, client_shell: Shell, request: pytest.FixtureRequest):
+def collect_binary_versions(hosting: Hosting, client_shell: Shell, request: pytest.FixtureRequest):
     local_versions = version_utils.get_local_binaries_versions(client_shell)
-    remote_versions, exсeptions_remote_binaries_versions = version_utils.get_remote_binaries_versions(hosting)
+    remote_versions = version_utils.get_remote_binaries_versions(hosting)
+    remote_versions_keys = list(remote_versions.keys())
 
     all_versions = {
         **local_versions,
-        **{binary_name: binary["version"] for binary_name, binary in remote_versions.items()},
+        **{
+            f"{name}_{remote_versions_keys.index(host) + 1:02d}": version
+            for host, versions in remote_versions.items()
+            for name, version in versions.items()
+        },
     }
 
     environment_dir = request.config.getoption("--alluredir")

pytest_tests/testsuites/metrics/test_container_metrics.py

@@ -1,47 +1,22 @@
 import math
-import re
 
 import allure
 import pytest
 from frostfs_testlib import reporter
-from frostfs_testlib.steps.cli.container import create_container, delete_container
-from frostfs_testlib.steps.cli.object import delete_object, get_object_nodes, put_object_to_random_node
-from frostfs_testlib.storage.cluster import Cluster, ClusterNode
+from frostfs_testlib.steps.cli.container import create_container
+from frostfs_testlib.steps.cli.object import delete_object, put_object_to_random_node
+from frostfs_testlib.steps.metrics import check_metrics_counter
+from frostfs_testlib.steps.storage_policy import get_nodes_with_object
+from frostfs_testlib.storage.cluster import Cluster
 from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
 from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
 from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
-from frostfs_testlib.testing.test_control import wait_for_success
 from frostfs_testlib.utils.file_utils import generate_file
 
 
+@pytest.mark.temporarymark
 @pytest.mark.container
 class TestContainerMetrics(ClusterTestBase):
-    @wait_for_success(interval=10)
-    def check_sum_counter_metrics_in_nodes(
-        self, cluster_nodes: list[ClusterNode], cid: str, phy_exp: int, logic_exp: int, user_exp: int
-    ):
-        counter_phy = 0
-        counter_logic = 0
-        counter_user = 0
-        for cluster_node in cluster_nodes:
-            metric_result = cluster_node.metrics.storage.get_metric_container(f"container_objects_total", cid)
-            counter_phy += self.get_count_metric_type_from_stdout(metric_result.stdout, "phy")
-            counter_logic += self.get_count_metric_type_from_stdout(metric_result.stdout, "logic")
-            counter_user += self.get_count_metric_type_from_stdout(metric_result.stdout, "user")
-
-        assert counter_phy == phy_exp, f"Expected metric Phy={phy_exp}, Actual: {counter_phy} in nodes: {cluster_nodes}"
-        assert (
-            counter_logic == logic_exp
-        ), f"Expected metric logic={logic_exp}, Actual: {counter_logic} in nodes: {cluster_nodes}"
-        assert (
-            counter_user == user_exp
-        ), f"Expected metric User={user_exp}, Actual: {counter_user} in nodes: {cluster_nodes}"
-
-    @staticmethod
-    def get_count_metric_type_from_stdout(metric_result_stdout: str, metric_type: str):
-        result = re.findall(rf'type="{metric_type}"}}\s(\d+)', metric_result_stdout)
-        return sum(map(int, result))
-
     @allure.title("Container metrics (obj_size={object_size})")
     def test_container_metrics(
         self, object_size: ObjectSize, max_object_size: int, default_wallet: WalletInfo, cluster: Cluster
@@ -57,15 +32,10 @@ class TestContainerMetrics(ClusterTestBase):
             link_object = 1
 
         with reporter.step(f"Create container with policy {placement_policy}"):
-            cid = create_container(
-                default_wallet,
-                rule=placement_policy,
-                shell=self.shell,
-                endpoint=self.cluster.default_rpc_endpoint,
-            )
+            cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, placement_policy)
 
         with reporter.step("Put object to random node"):
-            storage_object_id = put_object_to_random_node(
+            oid = put_object_to_random_node(
                 wallet=default_wallet,
                 path=file_path,
                 cid=cid,
@@ -73,25 +43,46 @@ class TestContainerMetrics(ClusterTestBase):
                 cluster=cluster,
             )
 
+        with reporter.step("Get object nodes"):
+            object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, cluster.storage_nodes)
+            object_nodes = [
+                cluster_node
+                for cluster_node in cluster.cluster_nodes
+                if cluster_node.storage_node in object_storage_nodes
+            ]
+
         with reporter.step("Check metric appears in node where the object is located"):
-            object_nodes = get_object_nodes(
-                cluster=cluster, cid=cid, oid=storage_object_id, alive_node=cluster.cluster_nodes[0]
+            count_metrics = (object_chunks + head_object + link_object) * copies
+            check_metrics_counter(
+                object_nodes, counter_exp=count_metrics, command="container_objects_total", cid=cid, type="phy"
             )
-            count_metrics_exp = (object_chunks + head_object + link_object) * copies
-            self.check_sum_counter_metrics_in_nodes(
-                object_nodes, cid, phy_exp=count_metrics_exp, logic_exp=count_metrics_exp, user_exp=copies
+            check_metrics_counter(
+                object_nodes, counter_exp=count_metrics, command="container_objects_total", cid=cid, type="logic"
+            )
+            check_metrics_counter(
+                object_nodes, counter_exp=copies, command="container_objects_total", cid=cid, type="user"
             )
 
         with reporter.step("Delete file, wait until gc remove object"):
-            delete_object(default_wallet, cid, storage_object_id, self.shell, self.cluster.default_rpc_endpoint)
-            count_metrics_exp = len(object_nodes)
-            self.check_sum_counter_metrics_in_nodes(
-                object_nodes, cid, phy_exp=count_metrics_exp, logic_exp=count_metrics_exp, user_exp=0
+            delete_object(default_wallet, cid, oid, self.shell, cluster.default_rpc_endpoint)
+
+        with reporter.step(f"Check container metrics 'the counter should equal {len(object_nodes)}' in object nodes"):
+            check_metrics_counter(
+                object_nodes, counter_exp=len(object_nodes), command="container_objects_total", cid=cid, type="phy"
             )
+            check_metrics_counter(
+                object_nodes, counter_exp=len(object_nodes), command="container_objects_total", cid=cid, type="logic"
+            )
+            check_metrics_counter(object_nodes, counter_exp=0, command="container_objects_total", cid=cid, type="user")
 
         with reporter.step("Check metrics(Phy, Logic, User) in each nodes"):
             # Phy and Logic metrics are 4, because in rule 'CBF 2 SELECT 2 FROM', cbf2*sel2=4
-            self.check_sum_counter_metrics_in_nodes(cluster.cluster_nodes, cid, phy_exp=4, logic_exp=4, user_exp=0)
-
-        with reporter.step("Delete container"):
-            delete_container(default_wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
+            check_metrics_counter(
+                cluster.cluster_nodes, counter_exp=4, command="container_objects_total", cid=cid, type="phy"
+            )
+            check_metrics_counter(
+                cluster.cluster_nodes, counter_exp=4, command="container_objects_total", cid=cid, type="logic"
+            )
+            check_metrics_counter(
+                cluster.cluster_nodes, counter_exp=0, command="container_objects_total", cid=cid, type="user"
+            )

pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py

@@ -2,9 +2,12 @@ import random
 import re
 
 import allure
+import pytest
 from frostfs_testlib import reporter
-from frostfs_testlib.steps.cli.container import create_container, delete_container
-from frostfs_testlib.steps.cli.object import delete_object, get_object_nodes, put_object, put_object_to_random_node
+from frostfs_testlib.steps.cli.container import create_container
+from frostfs_testlib.steps.cli.object import delete_object, put_object, put_object_to_random_node
+from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value
+from frostfs_testlib.steps.storage_policy import get_nodes_with_object
 from frostfs_testlib.storage.cluster import Cluster, ClusterNode
 from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
 from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
@@ -13,6 +16,7 @@ from frostfs_testlib.testing.test_control import wait_for_success
 from frostfs_testlib.utils.file_utils import generate_file
 
 
+@pytest.mark.temporarymark
 class TestGarbageCollectorMetrics(ClusterTestBase):
     @wait_for_success(interval=10)
     def check_metrics_in_node(self, cluster_node: ClusterNode, counter_exp: int, **metrics_greps: str):
@@ -40,10 +44,9 @@ class TestGarbageCollectorMetrics(ClusterTestBase):
         with reporter.step("Get current garbage collector metrics for each nodes"):
             metrics_counter = {}
             for node in cluster.cluster_nodes:
-                command_result = node.metrics.storage.get_metrics_search_by_greps(
-                    command="frostfs_node_garbage_collector_marked_for_removal_objects_total"
+                metrics_counter[node] = get_metrics_value(
+                    node, command="frostfs_node_garbage_collector_marked_for_removal_objects_total"
                 )
-                metrics_counter[node] = self.calc_metrics_count_from_stdout(command_result.stdout)
 
         with reporter.step(f"Create container with policy {placement_policy}"):
             cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, placement_policy)
@@ -60,23 +63,29 @@ class TestGarbageCollectorMetrics(ClusterTestBase):
             )
 
         with reporter.step("Get object nodes"):
-            object_nodes = get_object_nodes(cluster, cid, oid, cluster.cluster_nodes[0])
+            object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, cluster.storage_nodes)
+            object_nodes = [
+                cluster_node
+                for cluster_node in cluster.cluster_nodes
+                if cluster_node.storage_node in object_storage_nodes
+            ]
 
         with reporter.step("Tick Epoch"):
             self.tick_epochs(epochs_to_tick=2, wait_block=2)
 
-        with reporter.step(f"Check garbage collector metrics 'the counter should increase by {metrics_step}'"):
+        with reporter.step(
+            f"Check garbage collector metrics 'the counter should increase by {metrics_step}' in object nodes"
+        ):
             for node in object_nodes:
                 metrics_counter[node] += metrics_step
 
             for node, counter in metrics_counter.items():
-                self.check_metrics_in_node(
-                    node, counter, command="frostfs_node_garbage_collector_marked_for_removal_objects_total"
+                check_metrics_counter(
+                    [node],
+                    counter_exp=counter,
+                    command="frostfs_node_garbage_collector_marked_for_removal_objects_total",
                 )
 
-        with reporter.step("Delete container"):
-            delete_container(default_wallet, cid, self.shell, cluster.default_rpc_endpoint)
-
     @allure.title("Garbage collector delete object")
     def test_garbage_collector_metrics_deleted_objects(
         self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster
@@ -89,10 +98,7 @@ class TestGarbageCollectorMetrics(ClusterTestBase):
             node = random.choice(cluster.cluster_nodes)
 
         with reporter.step("Get current garbage collector metrics for selected node"):
-            command_result = node.metrics.storage.get_metrics_search_by_greps(
-                command="frostfs_node_garbage_collector_deleted_objects_total"
-            )
-            metrics_counter = self.calc_metrics_count_from_stdout(command_result.stdout)
+            metrics_counter = get_metrics_value(node, command="frostfs_node_garbage_collector_deleted_objects_total")
 
         with reporter.step(f"Create container with policy {placement_policy}"):
             cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
@@ -105,9 +111,6 @@ class TestGarbageCollectorMetrics(ClusterTestBase):
 
         with reporter.step(f"Check garbage collector metrics 'the counter should increase by {metrics_step}'"):
             metrics_counter += metrics_step
-            self.check_metrics_in_node(
-                node, metrics_counter, command="frostfs_node_garbage_collector_deleted_objects_total"
+            check_metrics_counter(
+                [node], counter_exp=metrics_counter, command="frostfs_node_garbage_collector_deleted_objects_total"
             )
-
-        with reporter.step("Delete container"):
-            delete_container(default_wallet, cid, self.shell, cluster.default_rpc_endpoint)

pytest_tests/testsuites/metrics/test_grpc_metrics.py

@@ -1,24 +1,24 @@
 import random
-import re
 
 import allure
 import pytest
 from frostfs_testlib import reporter
 from frostfs_testlib.healthcheck.interfaces import Healthcheck
-from frostfs_testlib.steps.cli.container import create_container, delete_container, get_container, list_containers
+from frostfs_testlib.steps.cli.container import create_container, get_container, list_containers
 from frostfs_testlib.steps.cli.object import get_object, head_object, put_object, search_object
 from frostfs_testlib.steps.cli.tree import get_tree_list
-from frostfs_testlib.storage.cluster import Cluster, ClusterNode
+from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value
+from frostfs_testlib.storage.cluster import Cluster
 from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
 from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager
 from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
 from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
 from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
 from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
-from frostfs_testlib.testing.test_control import wait_for_success
 from frostfs_testlib.utils.file_utils import generate_file
 
 
+@pytest.mark.temporarymark
 class TestGRPCMetrics(ClusterTestBase):
     @pytest.fixture
     def disable_policer(self, cluster_state_controller: ClusterStateController):
@@ -27,25 +27,6 @@ class TestGRPCMetrics(ClusterTestBase):
         yield
         cluster_state_controller.manager(ConfigStateManager).revert_all()
 
-    @wait_for_success(interval=10)
-    def check_metrics_in_node(self, cluster_node: ClusterNode, counter_exp: int, **metrics_greps: str):
-        counter_act = self.get_metrics_value(cluster_node, **metrics_greps)
-        assert counter_act == counter_exp, f"Expected: {counter_exp}, Actual: {counter_act} in node: {cluster_node}"
-
-    def get_metrics_value(self, node: ClusterNode, **metrics_greps: str):
-        try:
-            command_result = node.metrics.storage.get_metrics_search_by_greps(**metrics_greps)
-            metrics_counter = self.calc_metrics_count_from_stdout(command_result.stdout)
-        except RuntimeError as e:
-            metrics_counter = 0
-
-        return metrics_counter
-
-    @staticmethod
-    def calc_metrics_count_from_stdout(metric_result_stdout: str):
-        result = re.findall(r"}\s(\d+)", metric_result_stdout)
-        return sum(map(int, result))
-
     @allure.title("GRPC metrics container operations")
     def test_grpc_metrics_container_operations(self, default_wallet: WalletInfo, cluster: Cluster):
         placement_policy = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
@@ -54,7 +35,7 @@ class TestGRPCMetrics(ClusterTestBase):
             node = random.choice(cluster.cluster_nodes)
 
         with reporter.step("Get current gRPC metrics for method 'Put'"):
-            metrics_counter_put = self.get_metrics_value(
+            metrics_counter_put = get_metrics_value(
                 node, command="grpc_server_handled_total", service="ContainerService", method="Put"
             )
 
@@ -63,12 +44,16 @@ class TestGRPCMetrics(ClusterTestBase):
 
         with reporter.step(f"Check gRPC metrics method 'Put', 'the counter should increase by 1'"):
             metrics_counter_put += 1
-            self.check_metrics_in_node(
-                node, metrics_counter_put, command="grpc_server_handled_total", service="ContainerService", method="Put"
+            check_metrics_counter(
+                [node],
+                counter_exp=metrics_counter_put,
+                command="grpc_server_handled_total",
+                service="ContainerService",
+                method="Put",
             )
 
         with reporter.step("Get current gRPC metrics for method 'Get'"):
-            metrics_counter_get = self.get_metrics_value(
+            metrics_counter_get = get_metrics_value(
                 node, command="grpc_server_handled_total", service="ContainerService", method="Get"
             )
 
@@ -77,12 +62,16 @@ class TestGRPCMetrics(ClusterTestBase):
 
         with reporter.step(f"Check gRPC metrics method=Get, 'the counter should increase by 1'"):
             metrics_counter_get += 1
-            self.check_metrics_in_node(
-                node, metrics_counter_get, command="grpc_server_handled_total", service="ContainerService", method="Get"
+            check_metrics_counter(
+                [node],
+                counter_exp=metrics_counter_get,
+                command="grpc_server_handled_total",
+                service="ContainerService",
+                method="Get",
             )
 
         with reporter.step("Get current gRPC metrics for method 'List'"):
-            metrics_counter_list = self.get_metrics_value(
+            metrics_counter_list = get_metrics_value(
                 node, command="grpc_server_handled_total", service="ContainerService", method="List"
             )
 
@@ -91,17 +80,14 @@ class TestGRPCMetrics(ClusterTestBase):
 
         with reporter.step(f"Check gRPC metrics method=List, 'the counter should increase by 1'"):
             metrics_counter_list += 1
-            self.check_metrics_in_node(
-                node,
-                metrics_counter_list,
+            check_metrics_counter(
+                [node],
+                counter_exp=metrics_counter_list,
                 command="grpc_server_handled_total",
                 service="ContainerService",
                 method="List",
             )
 
-        with reporter.step("Delete container"):
-            delete_container(default_wallet, cid, self.shell, self.cluster.default_rpc_endpoint)
-
     @allure.title("GRPC metrics object operations")
     def test_grpc_metrics_object_operations(
         self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster, disable_policer
@@ -116,7 +102,7 @@ class TestGRPCMetrics(ClusterTestBase):
             cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
 
         with reporter.step("Get current gRPC metrics for method 'Put'"):
-            metrics_counter_put = self.get_metrics_value(
+            metrics_counter_put = get_metrics_value(
                 node, command="grpc_server_handled_total", service="ObjectService", method="Put"
             )
 
@@ -125,12 +111,16 @@ class TestGRPCMetrics(ClusterTestBase):
 
         with reporter.step(f"Check gRPC metrics method 'Put', 'the counter should increase by 1'"):
             metrics_counter_put += 1
-            self.check_metrics_in_node(
-                node, metrics_counter_put, command="grpc_server_handled_total", service="ObjectService", method="Put"
+            check_metrics_counter(
+                [node],
+                counter_exp=metrics_counter_put,
+                command="grpc_server_handled_total",
+                service="ObjectService",
+                method="Put",
             )
 
         with reporter.step("Get current gRPC metrics for method 'Get'"):
-            metrics_counter_get = self.get_metrics_value(
+            metrics_counter_get = get_metrics_value(
                 node, command="grpc_server_handled_total", service="ObjectService", method="Get"
             )
 
@@ -139,12 +129,16 @@ class TestGRPCMetrics(ClusterTestBase):
 
         with reporter.step(f"Check gRPC metrics method=Get, 'the counter should increase by 1'"):
             metrics_counter_get += 1
-            self.check_metrics_in_node(
-                node, metrics_counter_get, command="grpc_server_handled_total", service="ObjectService", method="Get"
+            check_metrics_counter(
+                [node],
+                counter_exp=metrics_counter_get,
+                command="grpc_server_handled_total",
+                service="ObjectService",
+                method="Get",
             )
 
         with reporter.step("Get current gRPC metrics for method 'Search'"):
-            metrics_counter_search = self.get_metrics_value(
+            metrics_counter_search = get_metrics_value(
                 node, command="grpc_server_handled_total", service="ObjectService", method="Search"
             )
 
@@ -153,16 +147,16 @@ class TestGRPCMetrics(ClusterTestBase):
 
         with reporter.step(f"Check gRPC metrics method=Search, 'the counter should increase by 1'"):
             metrics_counter_search += 1
-            self.check_metrics_in_node(
-                node,
-                metrics_counter_search,
+            check_metrics_counter(
+                [node],
+                counter_exp=metrics_counter_search,
                 command="grpc_server_handled_total",
                 service="ObjectService",
                 method="Search",
             )
 
         with reporter.step("Get current gRPC metrics for method 'Head'"):
-            metrics_counter_head = self.get_metrics_value(
+            metrics_counter_head = get_metrics_value(
                 node, command="grpc_server_handled_total", service="ObjectService", method="Head"
             )
 
@@ -171,20 +165,21 @@ class TestGRPCMetrics(ClusterTestBase):
 
         with reporter.step(f"Check gRPC metrics method=Head, 'the counter should increase by 1'"):
             metrics_counter_head += 1
-            self.check_metrics_in_node(
-                node, metrics_counter_head, command="grpc_server_handled_total", service="ObjectService", method="Head"
+            check_metrics_counter(
+                [node],
+                counter_exp=metrics_counter_head,
+                command="grpc_server_handled_total",
+                service="ObjectService",
+                method="Head",
             )
 
-        with reporter.step("Delete container"):
-            delete_container(default_wallet, cid, self.shell, self.cluster.default_rpc_endpoint)
-
     @allure.title("GRPC metrics Tree healthcheck")
     def test_grpc_metrics_tree_service(self, cluster: Cluster, healthcheck: Healthcheck):
         with reporter.step("Select random node"):
             node = random.choice(cluster.cluster_nodes)
 
         with reporter.step("Get current gRPC metrics for Healthcheck"):
-            metrics_counter = self.get_metrics_value(
+            metrics_counter = get_metrics_value(
                 node, command="grpc_server_handled_total", service="TreeService", method="Healthcheck"
             )
 
@@ -192,10 +187,14 @@ class TestGRPCMetrics(ClusterTestBase):
             healthcheck.tree_healthcheck(node)
 
         with reporter.step(f"Check gRPC metrics for Healthcheck, 'the counter should increase'"):
-            metrics_counter_new = self.get_metrics_value(
-                node, command="grpc_server_handled_total", service="TreeService", method="Healthcheck"
+            check_metrics_counter(
+                [node],
+                ">",
+                metrics_counter,
+                command="grpc_server_handled_total",
+                service="TreeService",
+                method="Healthcheck",
             )
-            assert metrics_counter_new > metrics_counter, "the metrics has not increased"
 
     @allure.title("GRPC metrics Tree list")
     def test_grpc_metrics_tree_list(self, default_wallet: WalletInfo, cluster: Cluster):
@@ -208,7 +207,7 @@ class TestGRPCMetrics(ClusterTestBase):
             cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
 
         with reporter.step("Get current gRPC metrics for Tree List"):
-            metrics_counter = self.get_metrics_value(
+            metrics_counter = get_metrics_value(
                 node, command="grpc_server_handled_total", service="TreeService", method="TreeList"
             )
 
@@ -217,9 +216,10 @@ class TestGRPCMetrics(ClusterTestBase):
 
         with reporter.step(f"Check gRPC metrics for Tree List, 'the counter should increase by 1'"):
             metrics_counter += 1
-            self.check_metrics_in_node(
-                node, metrics_counter, command="grpc_server_handled_total", service="TreeService", method="TreeList"
+            check_metrics_counter(
+                [node],
+                counter_exp=metrics_counter,
+                command="grpc_server_handled_total",
+                service="TreeService",
+                method="TreeList",
             )
-
-        with reporter.step("Delete container"):
-            delete_container(default_wallet, cid, self.shell, self.cluster.default_rpc_endpoint)

pytest_tests/testsuites/metrics/test_logs_metrics.py

@@ -1,11 +1,11 @@
 import random
 import re
-from datetime import datetime
+from datetime import datetime, timezone
 
 import allure
 import pytest
 from frostfs_testlib import reporter
-from frostfs_testlib.shell import Shell
+from frostfs_testlib.steps.metrics import get_metrics_value
 from frostfs_testlib.storage.cluster import Cluster, ClusterNode
 from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
 from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager
@@ -14,61 +14,43 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
 from frostfs_testlib.testing.test_control import wait_for_success
 
 
+@pytest.mark.temporarymark
 class TestLogsMetrics(ClusterTestBase):
     @pytest.fixture
-    def restart_storage_service(self, cluster_state_controller: ClusterStateController) -> str:
+    def restart_storage_service(self, cluster_state_controller: ClusterStateController) -> datetime:
         config_manager = cluster_state_controller.manager(ConfigStateManager)
         config_manager.csc.stop_services_of_type(StorageNode)
-        restart_time = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+        restart_time = datetime.now(timezone.utc)
         config_manager.csc.start_services_of_type(StorageNode)
         yield restart_time
 
         cluster_state_controller.manager(ConfigStateManager).revert_all()
 
     @wait_for_success(interval=10)
-    def check_metrics_in_node(self, cluster_node: ClusterNode, counter_exp: int, **metrics_greps: str):
-        counter_act = self.get_metrics_value(cluster_node, **metrics_greps)
+    def check_metrics_in_node(self, cluster_node: ClusterNode, restart_time: datetime, **metrics_greps):
+        counter_exp = self.get_count_logs_by_level(cluster_node, metrics_greps.get("level"), restart_time)
+        counter_act = get_metrics_value(cluster_node, **metrics_greps)
         assert counter_act == counter_exp, f"Expected: {counter_exp}, Actual: {counter_act} in node: {cluster_node}"
 
-    def get_metrics_value(self, node: ClusterNode, **metrics_greps: str):
-        try:
-            command_result = node.metrics.storage.get_metrics_search_by_greps(**metrics_greps)
-            metrics_counter = self.calc_metrics_count_from_stdout(command_result.stdout)
-        except RuntimeError as e:
-            metrics_counter = 0
-
-        return metrics_counter
-
     @staticmethod
-    def calc_metrics_count_from_stdout(metric_result_stdout: str):
-        result = re.findall(r"}\s(\d+)", metric_result_stdout)
-        return sum(map(int, result))
-
-    @staticmethod
-    def get_count_logs_by_level(shell: Shell, log_level: str, after_time: str):
+    def get_count_logs_by_level(cluster_node: ClusterNode, log_level: str, after_time: datetime):
         count_logs = 0
         try:
-            logs = shell.exec(f"journalctl -u frostfs-storage --grep='{log_level}' --since '{after_time}'")
-            result = re.findall(rf"Z\s+{log_level}\s+", logs.stdout)
+            logs = cluster_node.host.get_filtered_logs(log_level, unit="frostfs-storage", since=after_time)
+            result = re.findall(rf"Z\s+{log_level}\s+", logs)
             count_logs += len(result)
         except RuntimeError as e:
             ...
         return count_logs
 
     @allure.title("Metrics for the log counter")
-    def test_log_counter_metrics(self, cluster: Cluster, restart_storage_service: str):
+    def test_log_counter_metrics(self, cluster: Cluster, restart_storage_service: datetime):
         restart_time = restart_storage_service
         with reporter.step("Select random node"):
             node = random.choice(cluster.cluster_nodes)
 
-        with reporter.step("Get count logs from journalctl with level 'info'"):
-            count_logs_info = self.get_count_logs_by_level(node.host.get_shell(), "info", restart_time)
-
         with reporter.step(f"Check metrics count logs with level 'info'"):
-            self.check_metrics_in_node(node, count_logs_info, command="frostfs_node_logger_entry_count", level="info")
-
-        with reporter.step("Get count logs from journalctl with level 'error'"):
-            count_logs_error = self.get_count_logs_by_level(node.host.get_shell(), "error", restart_time)
+            self.check_metrics_in_node(node, restart_time, command="frostfs_node_logger_entry_count", level="info")
 
         with reporter.step(f"Check metrics count logs with level 'error'"):
-            self.check_metrics_in_node(node, count_logs_error, command="frostfs_node_logger_entry_count", level="error")
+            self.check_metrics_in_node(node, restart_time, command="frostfs_node_logger_entry_count", level="error")

pytest_tests/testsuites/metrics/test_object_metrics.py

@@ -5,92 +5,20 @@ import allure
 import pytest
 from frostfs_testlib import reporter
 from frostfs_testlib.steps.cli.container import create_container, delete_container, search_nodes_with_container
-from frostfs_testlib.steps.cli.object import (
-    delete_object,
-    get_object_nodes,
-    lock_object,
-    put_object,
-    put_object_to_random_node,
-)
+from frostfs_testlib.steps.cli.object import delete_object, lock_object, put_object, put_object_to_random_node
+from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value
+from frostfs_testlib.steps.storage_policy import get_nodes_with_object
 from frostfs_testlib.storage.cluster import Cluster, ClusterNode
 from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
 from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
 from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
 from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
-from frostfs_testlib.testing.test_control import wait_for_success
 from frostfs_testlib.utils.file_utils import generate_file
 
 
+@pytest.mark.temporarymark
 class TestObjectMetrics(ClusterTestBase):
-    @wait_for_success(interval=10)
-    def check_metrics_by_type(
-        self, cluster_nodes: list[ClusterNode], metric_command: str, grep_by: str, metric_type: str, counter_exp: int
-    ):
-        counter_act = 0
-        for cluster_node in cluster_nodes:
-            try:
-                metric_result = cluster_node.metrics.storage.get_metrics_search_by_greps(
-                    command=metric_command, grep_by=grep_by
-                )
-                counter_act += self.calc_metrics_count_from_stdout(metric_result.stdout, metric_type)
-            except RuntimeError as e:
-                ...
-        assert (
-            counter_act == counter_exp
-        ), f"Expected metric {metric_type}={counter_exp}, Actual: {counter_act} in nodes: {cluster_nodes}"
-
-    @staticmethod
-    def calc_metrics_count_from_stdout(metric_result_stdout: str, metric_type: str):
-        result = re.findall(rf'type="{metric_type}"}}\s(\d+)', metric_result_stdout)
-        return sum(map(int, result))
-
-    @wait_for_success(interval=10)
-    def check_object_metrics_total_and_container(
-        self, cluster_nodes: list[ClusterNode], cid: str, objects_metric_total: int, objects_metric_container: int
-    ):
-        self.check_metrics_by_type(
-            cluster_nodes,
-            "frostfs_node_engine_objects_total",
-            grep_by="user",
-            metric_type="user",
-            counter_exp=objects_metric_total,
-        )
-
-        objects_metric_container_act = 0
-        for node in cluster_nodes:
-            try:
-                metrics_container = node.metrics.storage.get_metrics_search_by_greps(
-                    command="frostfs_node_engine_container_objects_total", cid=cid, type="user"
-                )
-                objects_metric_container_act += self.calc_metrics_count_from_stdout(
-                    metrics_container.stdout, metric_type="user"
-                )
-            except RuntimeError as e:
-                ...
-        assert (
-            objects_metric_container_act == objects_metric_container
-        ), f"Expected {objects_metric_container} objects in container"
-
-    @wait_for_success(max_wait_time=120, interval=10)
-    def check_object_metrics_container(
-        self, cluster_nodes: list[ClusterNode], cid: str, objects_metric_container_exp: int
-    ):
-        objects_metric_container_act = 0
-        for node in cluster_nodes:
-            try:
-                metrics_container = node.metrics.storage.get_metrics_search_by_greps(
-                    command="frostfs_node_engine_container_objects_total", cid=cid, type="user"
-                )
-                objects_metric_container_act += self.calc_metrics_count_from_stdout(
-                    metrics_container.stdout, metric_type="user"
-                )
-            except RuntimeError as e:
-                ...
-        assert (
-            objects_metric_container_act == objects_metric_container_exp
-        ), f"Expected {objects_metric_container_exp} objects in container"
-
-    @allure.title("Object metrics of removed container")
+    @allure.title("Object metrics of removed container (obj_size={object_size})")
     def test_object_metrics_removed_container(
         self, object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster
     ):
@@ -99,28 +27,26 @@ class TestObjectMetrics(ClusterTestBase):
         copies = 2
 
         with reporter.step(f"Create container with policy {placement_policy}"):
-            cid = create_container(
-                default_wallet,
-                rule=placement_policy,
-                shell=self.shell,
-                endpoint=self.cluster.default_rpc_endpoint,
-            )
+            cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, placement_policy)
 
         with reporter.step("Put object to random node"):
-            storage_object_id = put_object_to_random_node(
-                wallet=default_wallet,
-                path=file_path,
-                cid=cid,
-                shell=self.shell,
-                cluster=cluster,
-            )
+            oid = put_object_to_random_node(default_wallet, file_path, cid, self.shell, cluster)
 
         with reporter.step("Check metric appears in node where the object is located"):
-            object_nodes = get_object_nodes(
-                cluster=cluster, cid=cid, oid=storage_object_id, alive_node=cluster.cluster_nodes[0]
-            )
+            object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, cluster.storage_nodes)
+            object_nodes = [
+                cluster_node
+                for cluster_node in cluster.cluster_nodes
+                if cluster_node.storage_node in object_storage_nodes
+            ]
 
-            self.check_metrics_by_type(object_nodes, "frostfs_node_engine_container_objects_total", cid, "user", copies)
+            check_metrics_counter(
+                object_nodes,
+                counter_exp=copies,
+                command="frostfs_node_engine_container_objects_total",
+                cid=cid,
+                type="user",
+            )
 
         with reporter.step("Delete container"):
             delete_container(default_wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
@@ -129,13 +55,20 @@ class TestObjectMetrics(ClusterTestBase):
             self.tick_epochs(epochs_to_tick=2, wait_block=2)
 
         with reporter.step("Check metrics of removed containers doesn't appear in the storage node"):
-            self.check_metrics_by_type(object_nodes, "frostfs_node_engine_container_objects_total", cid, "user", 0)
+            check_metrics_counter(
+                object_nodes, counter_exp=0, command="frostfs_node_engine_container_objects_total", cid=cid, type="user"
+            )
+            check_metrics_counter(
+                object_nodes, counter_exp=0, command="frostfs_node_engine_container_size_byte", cid=cid
+            )
 
             for node in object_nodes:
-                with pytest.raises(RuntimeError):
-                    node.metrics.storage.get_metric_container(f"frostfs_node_engine_container_size_byte", cid)
+                all_metrics = node.metrics.storage.get_all_metrics()
+                assert (
+                    cid not in all_metrics.stdout
+                ), "metrics of removed containers shouldn't appear in the storage node"
 
-    @allure.title("Object metrics, locked object, (policy={placement_policy})")
+    @allure.title("Object metrics, locked object (obj_size={object_size}, policy={placement_policy})")
     @pytest.mark.parametrize(
         "placement_policy", ["REP 1 IN X CBF 1 SELECT 1 FROM * AS X", "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"]
     )
@@ -146,12 +79,7 @@ class TestObjectMetrics(ClusterTestBase):
         metric_step = int(re.search(r"REP\s(\d+)", placement_policy).group(1))
 
         with reporter.step(f"Create container with policy {placement_policy}"):
-            cid = create_container(
-                wallet=default_wallet,
-                rule=placement_policy,
-                shell=self.shell,
-                endpoint=self.cluster.default_rpc_endpoint,
-            )
+            cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, placement_policy)
 
         with reporter.step("Search container nodes"):
             container_nodes = search_nodes_with_container(
@@ -165,34 +93,51 @@ class TestObjectMetrics(ClusterTestBase):
         with reporter.step("Get current metrics for metric_type=user"):
             objects_metric_counter = 0
             for node in container_nodes:
-                metric_objects_total = node.metrics.storage.get_metrics_search_by_greps(
-                    command="frostfs_node_engine_objects_total", type="user"
-                )
-                objects_metric_counter += self.calc_metrics_count_from_stdout(
-                    metric_objects_total.stdout, metric_type="user"
+                objects_metric_counter += get_metrics_value(
+                    node, command="frostfs_node_engine_objects_total", type="user"
                 )
 
         with reporter.step("Put object to container node"):
             oid = put_object(
-                wallet=default_wallet,
-                path=file_path,
-                cid=cid,
-                shell=self.shell,
-                endpoint=container_nodes[0].storage_node.get_rpc_endpoint(),
+                default_wallet, file_path, cid, self.shell, container_nodes[0].storage_node.get_rpc_endpoint()
             )
 
         with reporter.step(f"Check metric user 'the counter should increase by {metric_step}'"):
             objects_metric_counter += metric_step
-            self.check_object_metrics_total_and_container(container_nodes, cid, objects_metric_counter, metric_step)
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=objects_metric_counter,
+                command="frostfs_node_engine_objects_total",
+                type="user",
+            )
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=metric_step,
+                command="frostfs_node_engine_container_objects_total",
+                cid=cid,
+                type="user",
+            )
 
         with reporter.step("Delete object"):
             delete_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
 
         with reporter.step(f"Check metric user 'the counter should decrease by {metric_step}'"):
             objects_metric_counter -= metric_step
-            self.check_object_metrics_total_and_container(container_nodes, cid, objects_metric_counter, 0)
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=objects_metric_counter,
+                command="frostfs_node_engine_objects_total",
+                type="user",
+            )
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=0,
+                command="frostfs_node_engine_container_objects_total",
+                cid=cid,
+                type="user",
+            )
 
-        with reporter.step("Put object and lock it"):
+        with reporter.step("Put object and lock it to next epoch"):
             oid = put_object(
                 default_wallet, file_path, cid, self.shell, container_nodes[0].storage_node.get_rpc_endpoint()
             )
@@ -208,18 +153,47 @@ class TestObjectMetrics(ClusterTestBase):
 
         with reporter.step(f"Check metric user 'the counter should increase by {metric_step}'"):
             objects_metric_counter += metric_step
-            self.check_object_metrics_total_and_container(container_nodes, cid, objects_metric_counter, metric_step)
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=objects_metric_counter,
+                command="frostfs_node_engine_objects_total",
+                type="user",
+            )
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=metric_step,
+                command="frostfs_node_engine_container_objects_total",
+                cid=cid,
+                type="user",
+            )
 
         with reporter.step(f"Wait until remove locking 'the counter doesn't change'"):
             self.tick_epochs(epochs_to_tick=2)
-            self.check_object_metrics_total_and_container(container_nodes, cid, objects_metric_counter, metric_step)
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=objects_metric_counter,
+                command="frostfs_node_engine_objects_total",
+                type="user",
+            )
 
         with reporter.step("Delete object"):
             delete_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
 
         with reporter.step(f"Check metric user 'the counter should decrease by {metric_step}'"):
             objects_metric_counter -= metric_step
-            self.check_object_metrics_total_and_container(container_nodes, cid, objects_metric_counter, 0)
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=objects_metric_counter,
+                command="frostfs_node_engine_objects_total",
+                type="user",
+            )
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=0,
+                command="frostfs_node_engine_container_objects_total",
+                cid=cid,
+                type="user",
+            )
 
         with reporter.step("Put object with expire_at"):
             current_epoch = self.get_epoch()
@@ -234,23 +208,43 @@ class TestObjectMetrics(ClusterTestBase):
 
         with reporter.step(f"Check metric user 'the counter should increase by {metric_step}'"):
             objects_metric_counter += metric_step
-            self.check_object_metrics_total_and_container(container_nodes, cid, objects_metric_counter, metric_step)
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=objects_metric_counter,
+                command="frostfs_node_engine_objects_total",
+                type="user",
+            )
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=metric_step,
+                command="frostfs_node_engine_container_objects_total",
+                cid=cid,
+                type="user",
+            )
 
         with reporter.step("Tick Epoch"):
             self.tick_epochs(epochs_to_tick=2)
 
         with reporter.step(f"Check metric user 'the counter should decrease by {metric_step}'"):
             objects_metric_counter -= metric_step
-            self.check_object_metrics_total_and_container(container_nodes, cid, objects_metric_counter, 0)
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=objects_metric_counter,
+                command="frostfs_node_engine_objects_total",
+                type="user",
+            )
+            check_metrics_counter(
+                container_nodes,
+                counter_exp=0,
+                command="frostfs_node_engine_container_objects_total",
+                cid=cid,
+                type="user",
+            )
 
-        with reporter.step("Delete container"):
-            delete_container(default_wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
-
-    @allure.title("Object metrics, stop the node")
+    @allure.title("Object metrics, stop the node (obj_size={object_size})")
     def test_object_metrics_stop_node(
         self,
         object_size: ObjectSize,
-        max_object_size: int,
         default_wallet: WalletInfo,
         cluster_state_controller: ClusterStateController,
     ):
@@ -259,60 +253,70 @@ class TestObjectMetrics(ClusterTestBase):
         copies = 2
 
         with reporter.step(f"Create container with policy {placement_policy}"):
-            cid = create_container(
-                wallet=default_wallet,
-                rule=placement_policy,
-                shell=self.shell,
-                endpoint=self.cluster.default_rpc_endpoint,
-            )
+            cid = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint, placement_policy)
 
-        with reporter.step("Search container nodes"):
-            container_nodes = search_nodes_with_container(
-                wallet=default_wallet,
+        with reporter.step(f"Check object metrics in container 'should be zero'"):
+            check_metrics_counter(
+                self.cluster.cluster_nodes,
+                counter_exp=0,
+                command="frostfs_node_engine_container_objects_total",
+                type="user",
                 cid=cid,
-                shell=self.shell,
-                endpoint=self.cluster.default_rpc_endpoint,
-                cluster=self.cluster,
             )
 
-        with reporter.step("Get current metrics for container nodes"):
-            objects_metric_counter = 0
-            for node in container_nodes:
-                metric_objects_total = node.metrics.storage.get_metrics_search_by_greps(
-                    command="frostfs_node_engine_objects_total", type="user"
-                )
-                objects_metric_counter += self.calc_metrics_count_from_stdout(
-                    metric_objects_total.stdout, metric_type="user"
+        with reporter.step("Get current metrics for each nodes"):
+            objects_metric_counter: dict[ClusterNode:int] = {}
+            for node in self.cluster.cluster_nodes:
+                objects_metric_counter[node] = get_metrics_value(
+                    node, command="frostfs_node_engine_objects_total", type="user"
                 )
 
-        with reporter.step("Put object to container node"):
-            oid = put_object(
-                wallet=default_wallet,
-                path=file_path,
-                cid=cid,
-                shell=self.shell,
-                endpoint=container_nodes[0].storage_node.get_rpc_endpoint(),
-            )
+        with reporter.step("Put object"):
+            oid = put_object(default_wallet, file_path, cid, self.shell, self.cluster.default_rpc_endpoint)
 
-        with reporter.step(f"Check metric in container nodes 'the counter should increase by {copies}'"):
-            objects_metric_counter += copies
-            self.check_object_metrics_total_and_container(container_nodes, cid, objects_metric_counter, copies)
+        with reporter.step("Get object nodes"):
+            object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, self.cluster.storage_nodes)
+            object_nodes = [
+                cluster_node
+                for cluster_node in self.cluster.cluster_nodes
+                if cluster_node.storage_node in object_storage_nodes
+            ]
+
+        with reporter.step(f"Check metrics in object nodes 'the counter should increase by {copies}'"):
+            counter_exp = sum(objects_metric_counter[node] for node in object_nodes) + copies
+            check_metrics_counter(
+                object_nodes, counter_exp=counter_exp, command="frostfs_node_engine_objects_total", type="user"
+            )
+            check_metrics_counter(
+                object_nodes,
+                counter_exp=copies,
+                command="frostfs_node_engine_container_objects_total",
+                type="user",
+                cid=cid,
+            )
 
         with reporter.step(f"Select node to stop"):
-            node_to_stop = container_nodes[0]
-            alive_nodes = [node for node in container_nodes if node != node_to_stop]
+            node_to_stop = random.choice(object_nodes)
+            alive_nodes = set(object_nodes).difference({node_to_stop})
 
         with reporter.step(f"Stop the node, wait until the object is replicated to another node"):
             cluster_state_controller.stop_node_host(node_to_stop, "hard")
+            objects_metric_counter[node_to_stop] += 1
 
-        with reporter.step(f"Check metric in alive nodes 'the counter should increase by 1'"):
-            self.check_object_metrics_container(alive_nodes, cid, copies)
+        with reporter.step(f"Check metric in alive nodes 'the counter should increase'"):
+            counter_exp = sum(objects_metric_counter[node] for node in alive_nodes)
+            check_metrics_counter(
+                alive_nodes, ">=", counter_exp, command="frostfs_node_engine_objects_total", type="user"
+            )
 
         with reporter.step("Start node"):
             cluster_state_controller.start_node_host(node_to_stop)
 
-        with reporter.step(f"Check metric in container nodes 'the counter doesn't change'"):
-            self.check_object_metrics_total_and_container(container_nodes, cid, objects_metric_counter, copies)
-
-        with reporter.step("Delete container"):
-            delete_container(default_wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
+        with reporter.step(f"Check metric in restarted node, 'the counter doesn't change'"):
+            check_metrics_counter(
+                object_nodes,
+                counter_exp=copies,
+                command="frostfs_node_engine_container_objects_total",
+                type="user",
+                cid=cid,
+            )

pytest_tests/testsuites/metrics/test_shard_metrics.py

@@ -4,11 +4,13 @@ import re
 import allure
 import pytest
 from frostfs_testlib import reporter
+from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
 from frostfs_testlib.resources.wellknown_acl import EACL_PUBLIC_READ_WRITE
-from frostfs_testlib.shell import Shell
-from frostfs_testlib.steps.cli.container import create_container, delete_container
-from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object
+from frostfs_testlib.steps.cli.container import create_container
+from frostfs_testlib.steps.cli.object import get_object, put_object
+from frostfs_testlib.steps.metrics import check_metrics_counter
 from frostfs_testlib.steps.node_management import node_shard_list, node_shard_set_mode
+from frostfs_testlib.steps.storage_policy import get_nodes_with_object
 from frostfs_testlib.storage.cluster import Cluster, ClusterNode
 from frostfs_testlib.storage.controllers import ShardsWatcher
 from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
@@ -17,6 +19,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
 from frostfs_testlib.utils.file_utils import generate_file
 
 
+@pytest.mark.temporarymark
 class TestShardMetrics(ClusterTestBase):
     @pytest.fixture()
     @allure.title("Get two shards for set mode")
@@ -45,28 +48,13 @@ class TestShardMetrics(ClusterTestBase):
             watcher.set_shard_mode(shard["shard_id"], mode="read-write")
         watcher.await_for_all_shards_status(status="read-write")
 
-    @wait_for_success(interval=10)
-    def check_metrics_in_node(self, cluster_node: ClusterNode, counter_exp: int, **metrics_greps: str):
-        counter_act = 0
-        try:
-            metric_result = cluster_node.metrics.storage.get_metrics_search_by_greps(**metrics_greps)
-            counter_act += self.calc_metrics_count_from_stdout(metric_result.stdout)
-        except RuntimeError as e:
-            ...
-        assert counter_act == counter_exp, f"Expected: {counter_exp}, Actual: {counter_act} in node: {cluster_node}"
-
     @staticmethod
-    def calc_metrics_count_from_stdout(metric_result_stdout: str):
-        result = re.findall(r"}\s(\d+)", metric_result_stdout)
-        return sum(map(int, result))
-
-    @staticmethod
-    def get_error_count_from_logs(shell: Shell, object_path: str, object_name: str):
+    def get_error_count_from_logs(cluster_node: ClusterNode, object_path: str, object_name: str):
         error_count = 0
         try:
-            logs = shell.exec(f"journalctl -u frostfs-storage --grep='error count' --no-pager")
+            logs = cluster_node.host.get_filtered_logs("error count", unit="frostfs-storage")
             # search error logs for current object
-            for error_line in logs.stdout.split("\n"):
+            for error_line in logs.split("\n"):
                 if object_path in error_line and object_name in error_line:
                     result = re.findall(r'"error\scount":\s(\d+)', error_line)
                     error_count += sum(map(int, result))
@@ -106,17 +94,21 @@ class TestShardMetrics(ClusterTestBase):
             node_shard_set_mode(node.storage_node, shard1, "read-only")
 
         with reporter.step(f"Check shard metrics, 'the mode will change to 'READ_ONLY'"):
-            self.check_metrics_in_node(
-                node, metrics_counter, command="frostfs_node_engine_mode_info", mode="READ_ONLY", shard_id=shard1
+            check_metrics_counter(
+                [node],
+                counter_exp=metrics_counter,
+                command="frostfs_node_engine_mode_info",
+                mode="READ_ONLY",
+                shard_id=shard1,
             )
 
         with reporter.step("Shard2 set to mode 'degraded-read-only'"):
             node_shard_set_mode(node.storage_node, shard2, "degraded-read-only")
 
         with reporter.step(f"Check shard metrics, 'the mode will change to 'DEGRADED_READ_ONLY'"):
-            self.check_metrics_in_node(
-                node,
-                metrics_counter,
+            check_metrics_counter(
+                [node],
+                counter_exp=metrics_counter,
                 command="frostfs_node_engine_mode_info",
                 mode="DEGRADED_READ_ONLY",
                 shard_id=shard2,
@@ -126,9 +118,14 @@ class TestShardMetrics(ClusterTestBase):
             for shard in [shard1, shard2]:
                 node_shard_set_mode(node.storage_node, shard, "read-write")
 
-            with reporter.step(f"Check shard metrics, 'the mode will change to 'READ_WRITE'"):
-                self.check_metrics_in_node(
-                    node, metrics_counter, command="frostfs_node_engine_mode_info", mode="READ_WRITE", shard_id=shard
+        with reporter.step(f"Check shard metrics, 'the mode will change to 'READ_WRITE'"):
+            for shard in [shard1, shard2]:
+                check_metrics_counter(
+                    [node],
+                    counter_exp=metrics_counter,
+                    command="frostfs_node_engine_mode_info",
+                    mode="READ_WRITE",
+                    shard_id=shard,
                 )
 
     @allure.title("Metric for error count on shard")
@@ -141,17 +138,22 @@ class TestShardMetrics(ClusterTestBase):
             cid = create_container(
                 wallet=default_wallet,
                 shell=self.shell,
-                endpoint=self.cluster.default_rpc_endpoint,
+                endpoint=cluster.default_rpc_endpoint,
                 rule="REP 1 CBF 1",
                 basic_acl=EACL_PUBLIC_READ_WRITE,
             )
 
         with reporter.step("Put object"):
-            oid = put_object(default_wallet, file_path, cid, self.shell, self.cluster.default_rpc_endpoint)
+            oid = put_object(default_wallet, file_path, cid, self.shell, cluster.default_rpc_endpoint)
 
         with reporter.step("Get object nodes"):
-            object_nodes = get_object_nodes(cluster, cid, oid, cluster.cluster_nodes[0])
-            node = object_nodes[0]
+            object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, cluster.storage_nodes)
+            object_nodes = [
+                cluster_node
+                for cluster_node in cluster.cluster_nodes
+                if cluster_node.storage_node in object_storage_nodes
+            ]
+            node = random.choice(object_nodes)
 
         with reporter.step("Search object in system."):
             object_path, object_name = self.get_object_path_and_name_file(oid, cid, node)
@@ -160,7 +162,7 @@ class TestShardMetrics(ClusterTestBase):
             node.host.get_shell().exec(f"chmod a-r {object_path}/{object_name}")
 
         with reporter.step("Get object, expect error"):
-            with pytest.raises(RuntimeError):
+            with pytest.raises(RuntimeError, match=OBJECT_NOT_FOUND):
                 get_object(
                     wallet=default_wallet,
                     cid=cid,
@@ -170,10 +172,7 @@ class TestShardMetrics(ClusterTestBase):
                 )
 
         with reporter.step(f"Get shard error count from logs"):
-            counter = self.get_error_count_from_logs(node.host.get_shell(), object_path, object_name)
+            counter = self.get_error_count_from_logs(node, object_path, object_name)
 
         with reporter.step(f"Check shard error metrics"):
-            self.check_metrics_in_node(node, counter, command="frostfs_node_engine_errors_total")
-
-        with reporter.step("Delete container"):
-            delete_container(default_wallet, cid, self.shell, cluster.default_rpc_endpoint)
+            check_metrics_counter([node], counter_exp=counter, command="frostfs_node_engine_errors_total")

pytest_tests/testsuites/services/test_binaries.py

@@ -1,77 +1,42 @@
 import logging
-import os
-from http import HTTPStatus
-from re import fullmatch, match
+from re import fullmatch
 
 import allure
 import pytest
-import requests
 from frostfs_testlib import reporter
 from frostfs_testlib.hosting import Hosting
-from frostfs_testlib.resources.common import ASSETS_DIR
-from frostfs_testlib.utils.env_utils import read_env_properties, save_env_properties
 from frostfs_testlib.utils.version_utils import get_remote_binaries_versions
-from pytest import FixtureRequest
 
 logger = logging.getLogger("NeoLogger")
 
+VERSION_REGEX = r"^([a-zA-Z0-9]*/)?\d+\.\d+\.\d+(-.*)?(?<!dirty)"
+VERSION_ERROR_MSG = "{name} [{host}]: Actual version doesn't conform to format '0.0.0-000-aaaaaaa': {version}"
+
+
+def _check_version_format(version):
+    return fullmatch(VERSION_REGEX, version)
+
 
 @allure.title("Check binaries versions")
 @pytest.mark.check_binaries
-def test_binaries_versions(request: FixtureRequest, hosting: Hosting):
+def test_binaries_versions(hosting: Hosting):
     """
     Compare binaries versions from external source (url) and deployed on servers.
     """
     with reporter.step("Get binaries versions from servers"):
-        got_versions, exсeptions_remote_binaries_versions = get_remote_binaries_versions(hosting)
+        versions_by_host = get_remote_binaries_versions(hosting)
 
-    environment_dir = request.config.getoption("--alluredir") or ASSETS_DIR
-    env_file = os.path.join(environment_dir, "environment.properties")
-    env_properties = read_env_properties(env_file)
-
-    # compare versions from servers and file
     exсeptions = []
-    additional_env_properties = {}
 
-    for binary_name, binary in got_versions.items():
-        version = binary["version"]
-        requires_check = binary["check"]
-        if requires_check and not fullmatch(r"^\d+\.\d+\.\d+(-.*)?(?<!dirty)", version):
-            exсeptions.append(f"{binary_name}: Actual version doesn't conform to format '0.0.0-000-aaaaaaa': {version}")
+    last_host, versions_on_last_host = versions_by_host.popitem()
+    for name, version in versions_on_last_host.items():
+        for host, versions_on_host in versions_by_host.items():
+            if versions_on_host[name] != version:
+                exсeptions.append(f"Binary of {name} has inconsistent version {versions_on_host[name]} on host {host}")
+            if not _check_version_format(versions_on_host[name]):
+                exсeptions.append(VERSION_ERROR_MSG.format(name=name, host=host, version=version))
 
-        # If some binary was not listed in the env properties file, let's add it
-        # so that we have full information about versions in allure report
-        if env_properties and binary_name not in env_properties:
-            additional_env_properties[binary_name] = version
+        if not _check_version_format(version):
+            exсeptions.append(VERSION_ERROR_MSG.format(name=name, host=last_host, version=version))
 
-    if env_properties and additional_env_properties:
-        save_env_properties(env_file, additional_env_properties)
-
-    exсeptions.extend(exсeptions_remote_binaries_versions)
-
-    # create clear beautiful error with aggregation info
-    if exсeptions:
-        msg = "\n".join(exсeptions)
-        raise AssertionError(f"Found binaries with unexpected versions:\n{msg}")
-
-
-@reporter.step("Download versions info from {url}")
-def download_versions_info(url: str) -> dict:
-    binaries_to_version = {}
-
-    response = requests.get(url)
-
-    assert response.status_code == HTTPStatus.OK, f"Got {response.status_code} code. Content {response.json()}"
-
-    content = response.text
-    assert content, f"Expected file with content, got {response}"
-
-    for line in content.split("\n"):
-        m = match("(.*)=(.*)", line)
-        if not m:
-            logger.warning(f"Could not get binary/version from {line}")
-            continue
-        bin_name, bin_version = m.group(1), m.group(2)
-        binaries_to_version[bin_name] = bin_version
-
-    return binaries_to_version
+    assert not exсeptions, "\n".join(exсeptions)