[#372] Added metrics test multipart object

Signed-off-by: Ilyas Niyazov <i.niyazov@yadro.com>

pytest_tests/testsuites/metrics/test_container_metrics.py

@@ -12,13 +12,14 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
 from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
 from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
 from frostfs_testlib.testing.parallel import parallel
+from frostfs_testlib.testing.test_control import wait_for_success
 from frostfs_testlib.utils.file_utils import TestFile, generate_file
 
 from ...helpers.container_request import PUBLIC_WITH_POLICY, REP_2_1_4_PUBLIC, ContainerRequest, requires_container
 from ...helpers.utility import are_numbers_similar
 
 
-@pytest.mark.order(-5)
+@pytest.mark.order(-10)
 @pytest.mark.nightly
 @pytest.mark.metrics
 class TestContainerMetrics(ClusterTestBase):
@@ -35,6 +36,14 @@ class TestContainerMetrics(ClusterTestBase):
         except Exception as e:
             return None
 
+    @wait_for_success(max_wait_time=300, interval=30)
+    def check_metrics_value_by_approx(self, cluster_nodes: list[ClusterNode], metric_name: str, cid: str, expected_value: int, copies: int):
+        futures = parallel(get_metrics_value, cluster_nodes, command=metric_name, cid=cid)
+        metric_values = [future.result() for future in futures if future.result()]
+        actual_value = sum(metric_values) // copies
+
+        assert are_numbers_similar(actual_value, expected_value, tolerance_percentage=2), "metric container size bytes value not correct"
+
     @allure.title("Container metrics (obj_size={object_size}, policy={container_request})")
     @pytest.mark.parametrize(
         "container_request, copies",
@@ -119,10 +128,13 @@ class TestContainerMetrics(ClusterTestBase):
             ]
 
         with reporter.step("Check metric appears in all node where the object is located"):
-            act_metric = sum(
+            self.check_metrics_value_by_approx(
-                [get_metrics_value(node, command="frostfs_node_engine_container_size_bytes", cid=container) for node in object_nodes]
+                object_nodes,
+                metric_name="frostfs_node_engine_container_size_bytes",
+                cid=container,
+                expected_value=object_size.value,
+                copies=2,  # for policy REP 2, actual metric value divide by 2
             )
-            assert (act_metric // 2) == object_size.value
 
         with reporter.step("Delete file, wait until gc remove object"):
             id_tombstone = delete_object(default_wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint)
@@ -144,15 +156,13 @@ class TestContainerMetrics(ClusterTestBase):
             oids = [future.result() for future in futures]
 
         with reporter.step("Check metric appears in all nodes"):
-            metric_values = [
+            self.check_metrics_value_by_approx(
-                get_metrics_value(node, command="frostfs_node_engine_container_size_bytes", cid=container)
+                self.cluster.cluster_nodes,
-                for node in self.cluster.cluster_nodes
+                metric_name="frostfs_node_engine_container_size_bytes",
-            ]
+                cid=container,
-            actual_value = sum(metric_values) // 2  # for policy REP 2, value divide by 2
+                expected_value=object_size.value * objects_count,
-            expected_value = object_size.value * objects_count
+                copies=2,  # for policy REP 2, actual metric value divide by 2
-            assert are_numbers_similar(
+            )
-                actual_value, expected_value, tolerance_percentage=2
-            ), "metric container size bytes value not correct"
 
         with reporter.step("Delete file, wait until gc remove object"):
             tombstones_size = 0

pytest_tests/testsuites/metrics/test_epoch_metrics.py

@@ -1,13 +1,13 @@
 import allure
-from frostfs_testlib.testing import parallel
 import pytest
 from frostfs_testlib import reporter
 from frostfs_testlib.steps.metrics import get_metrics_value
-from frostfs_testlib.storage.cluster import ClusterNode, Cluster
+from frostfs_testlib.storage.cluster import Cluster, ClusterNode
+from frostfs_testlib.testing import parallel
 from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
 
 
-@pytest.mark.order(-7)
+@pytest.mark.order(-12)
 @pytest.mark.nightly
 @pytest.mark.metrics
 class TestEpochMetrics(ClusterTestBase):
@@ -28,11 +28,11 @@ class TestEpochMetrics(ClusterTestBase):
         with reporter.step("Check that the metric values are the same in all nodes"):
             assert len(set(metrics_results)) == 1, f"Metric {metric_name} values aren't same in all nodes"
             assert len(metrics_results) == len(cluster.cluster_nodes), "Metrics are not available in some nodes"
-        
+
         with reporter.step("Tick epoch"):
             self.tick_epoch(wait_block=2)
-  
+
-        with reporter.step('Check that metric value increase'):
+        with reporter.step("Check that metric value increase"):
             futures = parallel(self.get_metrics_search_by_greps_parallel, cluster.cluster_nodes, command=metric_name)
             new_metrics_results = [future.result() for future in futures if future.result() is not None]
 

pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py

@@ -16,7 +16,7 @@ from frostfs_testlib.utils.file_utils import generate_file
 from ...helpers.container_request import PUBLIC_WITH_POLICY, requires_container
 
 
-@pytest.mark.order(-9)
+@pytest.mark.order(-13)
 @pytest.mark.nightly
 @pytest.mark.metrics
 class TestGarbageCollectorMetrics(ClusterTestBase):

pytest_tests/testsuites/metrics/test_grpc_metrics.py

@@ -18,7 +18,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
 from frostfs_testlib.utils.file_utils import generate_file
 
 
-@pytest.mark.order(-6)
+@pytest.mark.order(-11)
 @pytest.mark.nightly
 @pytest.mark.metrics
 class TestGRPCMetrics(ClusterTestBase):

pytest_tests/testsuites/metrics/test_logs_metrics.py

@@ -14,7 +14,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
 from frostfs_testlib.testing.test_control import wait_for_success
 
 
-@pytest.mark.order(-10)
+@pytest.mark.order(-14)
 @pytest.mark.nightly
 @pytest.mark.metrics
 class TestLogsMetrics(ClusterTestBase):
@@ -30,13 +30,13 @@ class TestLogsMetrics(ClusterTestBase):
         config_manager.csc.start_services_of_type(StorageNode)
         return restart_time
 
-    @wait_for_success(interval=10)
+    @wait_for_success(max_wait_time=300, interval=30)
     def check_metrics_in_node(self, cluster_node: ClusterNode, restart_time: datetime, log_priority: str = None, **metrics_greps):
         current_time = datetime.now(timezone.utc)
         counter_metrics = get_metrics_value(cluster_node, **metrics_greps)
         counter_logs = self.get_count_logs_by_level(cluster_node, metrics_greps.get("level"), restart_time, current_time, log_priority)
         assert counter_logs == pytest.approx(
-            counter_metrics, rel=0.02
+            counter_metrics, rel=0.05
         ), f"counter_logs: {counter_logs}, counter_metrics: {counter_metrics} in node: {cluster_node}"
 
     @staticmethod

pytest_tests/testsuites/metrics/test_object_metrics.py

@@ -4,6 +4,8 @@ import re
 import allure
 import pytest
 from frostfs_testlib import reporter
+from frostfs_testlib.clients.s3.interfaces import BucketContainerResolver, S3ClientWrapper
+from frostfs_testlib.steps import s3_helper
 from frostfs_testlib.steps.cli.container import delete_container, search_nodes_with_container
 from frostfs_testlib.steps.cli.object import delete_object, lock_object, put_object, put_object_to_random_node
 from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value
@@ -12,12 +14,12 @@ from frostfs_testlib.storage.cluster import Cluster, ClusterNode
 from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
 from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
 from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
-from frostfs_testlib.utils.file_utils import TestFile
+from frostfs_testlib.utils.file_utils import TestFile, generate_file, split_file
 
 from ...helpers.container_request import PUBLIC_WITH_POLICY, ContainerRequest, requires_container
 
 
-@pytest.mark.order(-11)
+@pytest.mark.order(-16)
 @pytest.mark.nightly
 @pytest.mark.metrics
 class TestObjectMetrics(ClusterTestBase):
@@ -52,8 +54,12 @@ class TestObjectMetrics(ClusterTestBase):
             )
             check_metrics_counter(object_nodes, counter_exp=0, command="frostfs_node_engine_container_size_byte", cid=container)
 
+        with reporter.step("Check removed containers shouldn't appear in the storage node"):
             for node in object_nodes:
-                all_metrics = node.metrics.storage.get_metrics_search_by_greps(command="frostfs_node_engine_container_size_byte")
+                try:
+                    all_metrics = node.metrics.storage.get_metrics_search_by_greps(command="frostfs_node_engine_container_size_byte")
+                except:
+                    continue
                 assert container not in all_metrics.stdout, "metrics of removed containers shouldn't appear in the storage node"
 
     @allure.title("Object metrics, locked object (obj_size={object_size}, policy={container_request})")
@@ -155,7 +161,7 @@ class TestObjectMetrics(ClusterTestBase):
             self.tick_epochs(epochs_to_tick=2)
             check_metrics_counter(
                 container_nodes,
-                operator=">=",
+                operator="<=",
                 counter_exp=objects_metric_counter,
                 command="frostfs_node_engine_objects_total",
                 type="user",
@@ -297,3 +303,48 @@ class TestObjectMetrics(ClusterTestBase):
                 type="user",
                 cid=container,
             )
+
+    @allure.title("Multipart object metrics, PUT over S3, check part counts (s3_client={s3_client}, parts_count={parts_count})")
+    @pytest.mark.parametrize("parts_count", [4, 5, 10])
+    def test_multipart_object_metrics_check_parts_count(
+        self, s3_client: S3ClientWrapper, bucket_container_resolver: BucketContainerResolver, parts_count: int
+    ):
+        parts = []
+        object_size_10_mb = 10 * 1024 * 1024
+        original_size = object_size_10_mb * parts_count
+
+        with reporter.step("Create public container"):
+            bucket = s3_client.create_bucket()
+
+        with reporter.step("Generate original object and split it into parts"):
+            original_file = generate_file(original_size)
+            file_parts = split_file(original_file, parts_count)
+            object_key = s3_helper.object_key_from_file_path(original_file)
+
+        with reporter.step("Create multipart and upload parts"):
+            upload_id = s3_client.create_multipart_upload(bucket, object_key)
+            for part_id, file_path in enumerate(file_parts, start=1):
+                etag = s3_client.upload_part(bucket, object_key, upload_id, part_id, file_path)
+                parts.append((part_id, etag))
+
+        with reporter.step("Check all parts are visible in bucket"):
+            got_parts = s3_client.list_parts(bucket, object_key, upload_id)
+            assert len(got_parts) == len(file_parts), f"Expected {parts_count} parts, got:\n{got_parts}"
+
+        with reporter.step("Complete multipart upload"):
+            s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
+
+        with reporter.step(f"Get related container_id for bucket"):
+            cid = bucket_container_resolver.resolve(self.cluster.cluster_nodes[0], bucket)
+
+        with reporter.step(f"Check metric count object should be equal count parts of multipart object"):
+            # plus 1, because creating an additional object when calling CompleteMultipart
+            # multiply by 2 because default location constraint is REP 2
+            expected_user_metric = (parts_count + 1) * 2
+            check_metrics_counter(
+                self.cluster.cluster_nodes,
+                counter_exp=expected_user_metric,
+                command="frostfs_node_engine_container_objects_total",
+                type="user",
+                cid=cid,
+            )

pytest_tests/testsuites/metrics/test_shard_metrics.py

@@ -19,7 +19,7 @@ from frostfs_testlib.utils.file_utils import generate_file
 from ...helpers.container_request import PUBLIC_WITH_POLICY, requires_container
 
 
-@pytest.mark.order(-8)
+@pytest.mark.order(-15)
 @pytest.mark.nightly
 @pytest.mark.metrics
 class TestShardMetrics(ClusterTestBase):