d.zayakin/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#372] Added metrics test multipart object

Browse source 
Signed-off-by: Ilyas Niyazov <i.niyazov@yadro.com>
This commit is contained in:
Ilyas Niyazov 2025-03-06 09:23:29 +03:00 • committed by Ilyas Niyazov
parent ffd3e7ade3
commit 24301f4e8c
7 changed files with 90 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
pytest_tests/testsuites/metrics/test_container_metrics.py
View file

@ -12,13 +12,14 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.testing.test_control import wait_for_success
from frostfs_testlib.utils.file_utils import TestFile, generate_file
from ...helpers.container_request import PUBLIC_WITH_POLICY, REP_2_1_4_PUBLIC, ContainerRequest, requires_container
from ...helpers.utility import are_numbers_similar
@pytest.mark.order(-5)
@pytest.mark.order(-10)
@pytest.mark.nightly
@pytest.mark.metrics
class TestContainerMetrics(ClusterTestBase):
@ -35,6 +36,14 @@ class TestContainerMetrics(ClusterTestBase):
except Exception as e:
return None
@wait_for_success(max_wait_time=300, interval=30)
def check_metrics_value_by_approx(self, cluster_nodes: list[ClusterNode], metric_name: str, cid: str, expected_value: int, copies: int):
futures = parallel(get_metrics_value, cluster_nodes, command=metric_name, cid=cid)
metric_values = [future.result() for future in futures if future.result()]
actual_value = sum(metric_values) // copies
assert are_numbers_similar(actual_value, expected_value, tolerance_percentage=2), "metric container size bytes value not correct"
@allure.title("Container metrics (obj_size={object_size}, policy={container_request})")
@pytest.mark.parametrize(
"container_request, copies",
@ -119,10 +128,13 @@ class TestContainerMetrics(ClusterTestBase):
]
with reporter.step("Check metric appears in all node where the object is located"):
act_metric = sum(
[get_metrics_value(node, command="frostfs_node_engine_container_size_bytes", cid=container) for node in object_nodes]
self.check_metrics_value_by_approx(
object_nodes,
metric_name="frostfs_node_engine_container_size_bytes",
cid=container,
expected_value=object_size.value,
copies=2, # for policy REP 2, actual metric value divide by 2
)
assert (act_metric // 2) == object_size.value
with reporter.step("Delete file, wait until gc remove object"):
id_tombstone = delete_object(default_wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint)
@ -144,15 +156,13 @@ class TestContainerMetrics(ClusterTestBase):
oids = [future.result() for future in futures]
with reporter.step("Check metric appears in all nodes"):
metric_values = [
get_metrics_value(node, command="frostfs_node_engine_container_size_bytes", cid=container)
for node in self.cluster.cluster_nodes
]
actual_value = sum(metric_values) // 2 # for policy REP 2, value divide by 2
expected_value = object_size.value * objects_count
assert are_numbers_similar(
actual_value, expected_value, tolerance_percentage=2
), "metric container size bytes value not correct"
self.check_metrics_value_by_approx(
self.cluster.cluster_nodes,
metric_name="frostfs_node_engine_container_size_bytes",
cid=container,
expected_value=object_size.value * objects_count,
copies=2, # for policy REP 2, actual metric value divide by 2
)
with reporter.step("Delete file, wait until gc remove object"):
tombstones_size = 0

12
pytest_tests/testsuites/metrics/test_epoch_metrics.py
View file

@ -1,13 +1,13 @@
import allure
from frostfs_testlib.testing import parallel
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.steps.metrics import get_metrics_value
from frostfs_testlib.storage.cluster import ClusterNode, Cluster
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.testing import parallel
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
@pytest.mark.order(-7)
@pytest.mark.order(-12)
@pytest.mark.nightly
@pytest.mark.metrics
class TestEpochMetrics(ClusterTestBase):
@ -28,11 +28,11 @@ class TestEpochMetrics(ClusterTestBase):
with reporter.step("Check that the metric values are the same in all nodes"):
assert len(set(metrics_results)) == 1, f"Metric {metric_name} values aren't same in all nodes"
assert len(metrics_results) == len(cluster.cluster_nodes), "Metrics are not available in some nodes"
with reporter.step("Tick epoch"):
self.tick_epoch(wait_block=2)
with reporter.step('Check that metric value increase'):
with reporter.step("Check that metric value increase"):
futures = parallel(self.get_metrics_search_by_greps_parallel, cluster.cluster_nodes, command=metric_name)
new_metrics_results = [future.result() for future in futures if future.result() is not None]

2
pytest_tests/testsuites/metrics/test_garbage_collector_metrics.py
View file

@ -16,7 +16,7 @@ from frostfs_testlib.utils.file_utils import generate_file
from ...helpers.container_request import PUBLIC_WITH_POLICY, requires_container
@pytest.mark.order(-9)
@pytest.mark.order(-13)
@pytest.mark.nightly
@pytest.mark.metrics
class TestGarbageCollectorMetrics(ClusterTestBase):

2
pytest_tests/testsuites/metrics/test_grpc_metrics.py
View file

@ -18,7 +18,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.order(-6)
@pytest.mark.order(-11)
@pytest.mark.nightly
@pytest.mark.metrics
class TestGRPCMetrics(ClusterTestBase):

6
pytest_tests/testsuites/metrics/test_logs_metrics.py
View file

@ -14,7 +14,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import wait_for_success
@pytest.mark.order(-10)
@pytest.mark.order(-14)
@pytest.mark.nightly
@pytest.mark.metrics
class TestLogsMetrics(ClusterTestBase):
@ -30,13 +30,13 @@ class TestLogsMetrics(ClusterTestBase):
config_manager.csc.start_services_of_type(StorageNode)
return restart_time
@wait_for_success(interval=10)
@wait_for_success(max_wait_time=300, interval=30)
def check_metrics_in_node(self, cluster_node: ClusterNode, restart_time: datetime, log_priority: str = None, **metrics_greps):
current_time = datetime.now(timezone.utc)
counter_metrics = get_metrics_value(cluster_node, **metrics_greps)
counter_logs = self.get_count_logs_by_level(cluster_node, metrics_greps.get("level"), restart_time, current_time, log_priority)
assert counter_logs == pytest.approx(
counter_metrics, rel=0.02
counter_metrics, rel=0.05
), f"counter_logs: {counter_logs}, counter_metrics: {counter_metrics} in node: {cluster_node}"
@staticmethod

59
pytest_tests/testsuites/metrics/test_object_metrics.py
View file

@ -4,6 +4,8 @@ import re
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.clients.s3.interfaces import BucketContainerResolver, S3ClientWrapper
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.steps.cli.container import delete_container, search_nodes_with_container
from frostfs_testlib.steps.cli.object import delete_object, lock_object, put_object, put_object_to_random_node
from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value
@ -12,12 +14,12 @@ from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import TestFile
from frostfs_testlib.utils.file_utils import TestFile, generate_file, split_file
from ...helpers.container_request import PUBLIC_WITH_POLICY, ContainerRequest, requires_container
@pytest.mark.order(-11)
@pytest.mark.order(-16)
@pytest.mark.nightly
@pytest.mark.metrics
class TestObjectMetrics(ClusterTestBase):
@ -52,8 +54,12 @@ class TestObjectMetrics(ClusterTestBase):
)
check_metrics_counter(object_nodes, counter_exp=0, command="frostfs_node_engine_container_size_byte", cid=container)
with reporter.step("Check removed containers shouldn't appear in the storage node"):
for node in object_nodes:
all_metrics = node.metrics.storage.get_metrics_search_by_greps(command="frostfs_node_engine_container_size_byte")
try:
all_metrics = node.metrics.storage.get_metrics_search_by_greps(command="frostfs_node_engine_container_size_byte")
except:
continue
assert container not in all_metrics.stdout, "metrics of removed containers shouldn't appear in the storage node"
@allure.title("Object metrics, locked object (obj_size={object_size}, policy={container_request})")
@ -155,7 +161,7 @@ class TestObjectMetrics(ClusterTestBase):
self.tick_epochs(epochs_to_tick=2)
check_metrics_counter(
container_nodes,
operator=">=",
operator="<=",
counter_exp=objects_metric_counter,
command="frostfs_node_engine_objects_total",
type="user",
@ -297,3 +303,48 @@ class TestObjectMetrics(ClusterTestBase):
type="user",
cid=container,
)
@allure.title("Multipart object metrics, PUT over S3, check part counts (s3_client={s3_client}, parts_count={parts_count})")
@pytest.mark.parametrize("parts_count", [4, 5, 10])
def test_multipart_object_metrics_check_parts_count(
self, s3_client: S3ClientWrapper, bucket_container_resolver: BucketContainerResolver, parts_count: int
):
parts = []
object_size_10_mb = 10 * 1024 * 1024
original_size = object_size_10_mb * parts_count
with reporter.step("Create public container"):
bucket = s3_client.create_bucket()
with reporter.step("Generate original object and split it into parts"):
original_file = generate_file(original_size)
file_parts = split_file(original_file, parts_count)
object_key = s3_helper.object_key_from_file_path(original_file)
with reporter.step("Create multipart and upload parts"):
upload_id = s3_client.create_multipart_upload(bucket, object_key)
for part_id, file_path in enumerate(file_parts, start=1):
etag = s3_client.upload_part(bucket, object_key, upload_id, part_id, file_path)
parts.append((part_id, etag))
with reporter.step("Check all parts are visible in bucket"):
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
assert len(got_parts) == len(file_parts), f"Expected {parts_count} parts, got:\n{got_parts}"
with reporter.step("Complete multipart upload"):
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
with reporter.step(f"Get related container_id for bucket"):
cid = bucket_container_resolver.resolve(self.cluster.cluster_nodes[0], bucket)
with reporter.step(f"Check metric count object should be equal count parts of multipart object"):
# plus 1, because creating an additional object when calling CompleteMultipart
# multiply by 2 because default location constraint is REP 2
expected_user_metric = (parts_count + 1) * 2
check_metrics_counter(
self.cluster.cluster_nodes,
counter_exp=expected_user_metric,
command="frostfs_node_engine_container_objects_total",
type="user",
cid=cid,
)

2
pytest_tests/testsuites/metrics/test_shard_metrics.py
View file

@ -19,7 +19,7 @@ from frostfs_testlib.utils.file_utils import generate_file
from ...helpers.container_request import PUBLIC_WITH_POLICY, requires_container
@pytest.mark.order(-8)
@pytest.mark.order(-15)
@pytest.mark.nightly
@pytest.mark.metrics
class TestShardMetrics(ClusterTestBase):