diff --git a/pytest_tests/testsuites/metrics/test_container_metrics.py b/pytest_tests/testsuites/metrics/test_container_metrics.py index 2477e1f..4978cca 100644 --- a/pytest_tests/testsuites/metrics/test_container_metrics.py +++ b/pytest_tests/testsuites/metrics/test_container_metrics.py @@ -12,13 +12,14 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.parallel import parallel +from frostfs_testlib.testing.test_control import wait_for_success from frostfs_testlib.utils.file_utils import TestFile, generate_file from ...helpers.container_request import PUBLIC_WITH_POLICY, REP_2_1_4_PUBLIC, ContainerRequest, requires_container from ...helpers.utility import are_numbers_similar -@pytest.mark.order(-5) +@pytest.mark.order(-10) @pytest.mark.nightly @pytest.mark.metrics class TestContainerMetrics(ClusterTestBase): @@ -35,6 +36,14 @@ class TestContainerMetrics(ClusterTestBase): except Exception as e: return None + @wait_for_success(max_wait_time=300, interval=30) + def check_metrics_value_by_approx(self, cluster_nodes: list[ClusterNode], metric_name: str, cid: str, expected_value: int, copies: int): + futures = parallel(get_metrics_value, cluster_nodes, command=metric_name, cid=cid) + metric_values = [future.result() for future in futures if future.result()] + actual_value = sum(metric_values) // copies + + assert are_numbers_similar(actual_value, expected_value, tolerance_percentage=2), "metric container size bytes value not correct" + @allure.title("Container metrics (obj_size={object_size}, policy={container_request})") @pytest.mark.parametrize( "container_request, copies", @@ -119,10 +128,13 @@ class TestContainerMetrics(ClusterTestBase): ] with reporter.step("Check metric appears in all node where the object is located"): - act_metric = sum( - [get_metrics_value(node, command="frostfs_node_engine_container_size_bytes", cid=container) for node in object_nodes] + self.check_metrics_value_by_approx( + object_nodes, + metric_name="frostfs_node_engine_container_size_bytes", + cid=container, + expected_value=object_size.value, + copies=2, # for policy REP 2, actual metric value divide by 2 ) - assert (act_metric // 2) == object_size.value with reporter.step("Delete file, wait until gc remove object"): id_tombstone = delete_object(default_wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint) @@ -144,15 +156,13 @@ class TestContainerMetrics(ClusterTestBase): oids = [future.result() for future in futures] with reporter.step("Check metric appears in all nodes"): - metric_values = [ - get_metrics_value(node, command="frostfs_node_engine_container_size_bytes", cid=container) - for node in self.cluster.cluster_nodes - ] - actual_value = sum(metric_values) // 2 # for policy REP 2, value divide by 2 - expected_value = object_size.value * objects_count - assert are_numbers_similar( - actual_value, expected_value, tolerance_percentage=2 - ), "metric container size bytes value not correct" + self.check_metrics_value_by_approx( + self.cluster.cluster_nodes, + metric_name="frostfs_node_engine_container_size_bytes", + cid=container, + expected_value=object_size.value * objects_count, + copies=2, # for policy REP 2, actual metric value divide by 2 + ) with reporter.step("Delete file, wait until gc remove object"): tombstones_size = 0 diff --git a/pytest_tests/testsuites/metrics/test_epoch_metrics.py b/pytest_tests/testsuites/metrics/test_epoch_metrics.py index 8693f6a..d845090 100644 --- a/pytest_tests/testsuites/metrics/test_epoch_metrics.py +++ b/pytest_tests/testsuites/metrics/test_epoch_metrics.py @@ -1,13 +1,13 @@ import allure -from frostfs_testlib.testing import parallel import pytest from frostfs_testlib import reporter from frostfs_testlib.steps.metrics import get_metrics_value -from frostfs_testlib.storage.cluster import ClusterNode, Cluster +from frostfs_testlib.storage.cluster import Cluster, ClusterNode +from frostfs_testlib.testing import parallel from frostfs_testlib.testing.cluster_test_base import ClusterTestBase -@pytest.mark.order(-7) +@pytest.mark.order(-12) @pytest.mark.nightly @pytest.mark.metrics class TestEpochMetrics(ClusterTestBase): @@ -28,11 +28,11 @@ class TestEpochMetrics(ClusterTestBase): with reporter.step("Check that the metric values are the same in all nodes"): assert len(set(metrics_results)) == 1, f"Metric {metric_name} values aren't same in all nodes" assert len(metrics_results) == len(cluster.cluster_nodes), "Metrics are not available in some nodes" - + with reporter.step("Tick epoch"): self.tick_epoch(wait_block=2) - - with reporter.step('Check that metric value increase'): + + with reporter.step("Check that metric value increase"): futures = parallel(self.get_metrics_search_by_greps_parallel, cluster.cluster_nodes, command=metric_name) new_metrics_results = [future.result() for future in futures if future.result() is not None] diff --git a/pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py b/pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py index 8b75979..e8f5521 100644 --- a/pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py +++ b/pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py @@ -16,7 +16,7 @@ from frostfs_testlib.utils.file_utils import generate_file from ...helpers.container_request import PUBLIC_WITH_POLICY, requires_container -@pytest.mark.order(-9) +@pytest.mark.order(-13) @pytest.mark.nightly @pytest.mark.metrics class TestGarbageCollectorMetrics(ClusterTestBase): diff --git a/pytest_tests/testsuites/metrics/test_grpc_metrics.py b/pytest_tests/testsuites/metrics/test_grpc_metrics.py index aa89b2d..7bb9406 100644 --- a/pytest_tests/testsuites/metrics/test_grpc_metrics.py +++ b/pytest_tests/testsuites/metrics/test_grpc_metrics.py @@ -18,7 +18,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.utils.file_utils import generate_file -@pytest.mark.order(-6) +@pytest.mark.order(-11) @pytest.mark.nightly @pytest.mark.metrics class TestGRPCMetrics(ClusterTestBase): diff --git a/pytest_tests/testsuites/metrics/test_logs_metrics.py b/pytest_tests/testsuites/metrics/test_logs_metrics.py index 096dd10..e699079 100644 --- a/pytest_tests/testsuites/metrics/test_logs_metrics.py +++ b/pytest_tests/testsuites/metrics/test_logs_metrics.py @@ -14,7 +14,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.test_control import wait_for_success -@pytest.mark.order(-10) +@pytest.mark.order(-14) @pytest.mark.nightly @pytest.mark.metrics class TestLogsMetrics(ClusterTestBase): @@ -30,13 +30,13 @@ class TestLogsMetrics(ClusterTestBase): config_manager.csc.start_services_of_type(StorageNode) return restart_time - @wait_for_success(interval=10) + @wait_for_success(max_wait_time=300, interval=30) def check_metrics_in_node(self, cluster_node: ClusterNode, restart_time: datetime, log_priority: str = None, **metrics_greps): current_time = datetime.now(timezone.utc) counter_metrics = get_metrics_value(cluster_node, **metrics_greps) counter_logs = self.get_count_logs_by_level(cluster_node, metrics_greps.get("level"), restart_time, current_time, log_priority) assert counter_logs == pytest.approx( - counter_metrics, rel=0.02 + counter_metrics, rel=0.05 ), f"counter_logs: {counter_logs}, counter_metrics: {counter_metrics} in node: {cluster_node}" @staticmethod diff --git a/pytest_tests/testsuites/metrics/test_object_metrics.py b/pytest_tests/testsuites/metrics/test_object_metrics.py index 020720e..2268b9c 100644 --- a/pytest_tests/testsuites/metrics/test_object_metrics.py +++ b/pytest_tests/testsuites/metrics/test_object_metrics.py @@ -4,6 +4,8 @@ import re import allure import pytest from frostfs_testlib import reporter +from frostfs_testlib.clients.s3.interfaces import BucketContainerResolver, S3ClientWrapper +from frostfs_testlib.steps import s3_helper from frostfs_testlib.steps.cli.container import delete_container, search_nodes_with_container from frostfs_testlib.steps.cli.object import delete_object, lock_object, put_object, put_object_to_random_node from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value @@ -12,12 +14,12 @@ from frostfs_testlib.storage.cluster import Cluster, ClusterNode from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase -from frostfs_testlib.utils.file_utils import TestFile +from frostfs_testlib.utils.file_utils import TestFile, generate_file, split_file from ...helpers.container_request import PUBLIC_WITH_POLICY, ContainerRequest, requires_container -@pytest.mark.order(-11) +@pytest.mark.order(-16) @pytest.mark.nightly @pytest.mark.metrics class TestObjectMetrics(ClusterTestBase): @@ -52,8 +54,12 @@ class TestObjectMetrics(ClusterTestBase): ) check_metrics_counter(object_nodes, counter_exp=0, command="frostfs_node_engine_container_size_byte", cid=container) + with reporter.step("Check removed containers shouldn't appear in the storage node"): for node in object_nodes: - all_metrics = node.metrics.storage.get_metrics_search_by_greps(command="frostfs_node_engine_container_size_byte") + try: + all_metrics = node.metrics.storage.get_metrics_search_by_greps(command="frostfs_node_engine_container_size_byte") + except: + continue assert container not in all_metrics.stdout, "metrics of removed containers shouldn't appear in the storage node" @allure.title("Object metrics, locked object (obj_size={object_size}, policy={container_request})") @@ -155,7 +161,7 @@ class TestObjectMetrics(ClusterTestBase): self.tick_epochs(epochs_to_tick=2) check_metrics_counter( container_nodes, - operator=">=", + operator="<=", counter_exp=objects_metric_counter, command="frostfs_node_engine_objects_total", type="user", @@ -297,3 +303,48 @@ class TestObjectMetrics(ClusterTestBase): type="user", cid=container, ) + + @allure.title("Multipart object metrics, PUT over S3, check part counts (s3_client={s3_client}, parts_count={parts_count})") + @pytest.mark.parametrize("parts_count", [4, 5, 10]) + def test_multipart_object_metrics_check_parts_count( + self, s3_client: S3ClientWrapper, bucket_container_resolver: BucketContainerResolver, parts_count: int + ): + parts = [] + object_size_10_mb = 10 * 1024 * 1024 + original_size = object_size_10_mb * parts_count + + with reporter.step("Create public container"): + bucket = s3_client.create_bucket() + + with reporter.step("Generate original object and split it into parts"): + original_file = generate_file(original_size) + file_parts = split_file(original_file, parts_count) + object_key = s3_helper.object_key_from_file_path(original_file) + + with reporter.step("Create multipart and upload parts"): + upload_id = s3_client.create_multipart_upload(bucket, object_key) + for part_id, file_path in enumerate(file_parts, start=1): + etag = s3_client.upload_part(bucket, object_key, upload_id, part_id, file_path) + parts.append((part_id, etag)) + + with reporter.step("Check all parts are visible in bucket"): + got_parts = s3_client.list_parts(bucket, object_key, upload_id) + assert len(got_parts) == len(file_parts), f"Expected {parts_count} parts, got:\n{got_parts}" + + with reporter.step("Complete multipart upload"): + s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts) + + with reporter.step(f"Get related container_id for bucket"): + cid = bucket_container_resolver.resolve(self.cluster.cluster_nodes[0], bucket) + + with reporter.step(f"Check metric count object should be equal count parts of multipart object"): + # plus 1, because creating an additional object when calling CompleteMultipart + # multiply by 2 because default location constraint is REP 2 + expected_user_metric = (parts_count + 1) * 2 + check_metrics_counter( + self.cluster.cluster_nodes, + counter_exp=expected_user_metric, + command="frostfs_node_engine_container_objects_total", + type="user", + cid=cid, + ) diff --git a/pytest_tests/testsuites/metrics/test_shard_metrics.py b/pytest_tests/testsuites/metrics/test_shard_metrics.py index b841734..ef51848 100644 --- a/pytest_tests/testsuites/metrics/test_shard_metrics.py +++ b/pytest_tests/testsuites/metrics/test_shard_metrics.py @@ -19,7 +19,7 @@ from frostfs_testlib.utils.file_utils import generate_file from ...helpers.container_request import PUBLIC_WITH_POLICY, requires_container -@pytest.mark.order(-8) +@pytest.mark.order(-15) @pytest.mark.nightly @pytest.mark.metrics class TestShardMetrics(ClusterTestBase):