[#243] add test garbage collector metrics
This commit is contained in:
parent
a7dea66608
commit
c930496716
1 changed files with 113 additions and 0 deletions
|
@ -0,0 +1,113 @@
|
||||||
|
import random
|
||||||
|
import re
|
||||||
|
|
||||||
|
import allure
|
||||||
|
from frostfs_testlib import reporter
|
||||||
|
from frostfs_testlib.steps.cli.container import create_container, delete_container
|
||||||
|
from frostfs_testlib.steps.cli.object import delete_object, get_object_nodes, put_object, put_object_to_random_node
|
||||||
|
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
|
||||||
|
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||||
|
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
|
||||||
|
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
||||||
|
from frostfs_testlib.testing.test_control import wait_for_success
|
||||||
|
from frostfs_testlib.utils.file_utils import generate_file
|
||||||
|
|
||||||
|
|
||||||
|
class TestGarbageCollectorMetrics(ClusterTestBase):
|
||||||
|
@wait_for_success(interval=10)
|
||||||
|
def check_metrics_in_node(self, cluster_node: ClusterNode, counter_exp: int, **metrics_greps: str):
|
||||||
|
counter_act = 0
|
||||||
|
try:
|
||||||
|
metric_result = cluster_node.metrics.storage.get_metrics_search_by_greps(**metrics_greps)
|
||||||
|
counter_act += self.calc_metrics_count_from_stdout(metric_result.stdout)
|
||||||
|
except RuntimeError as e:
|
||||||
|
...
|
||||||
|
assert counter_act == counter_exp, f"Expected: {counter_exp}, Actual: {counter_act} in node: {cluster_node}"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def calc_metrics_count_from_stdout(metric_result_stdout: str):
|
||||||
|
result = re.findall(r"}\s(\d+)", metric_result_stdout)
|
||||||
|
return sum(map(int, result))
|
||||||
|
|
||||||
|
@allure.title("Garbage collector expire_at object")
|
||||||
|
def test_garbage_collector_metrics_expire_at_object(
|
||||||
|
self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster
|
||||||
|
):
|
||||||
|
file_path = generate_file(simple_object_size.value)
|
||||||
|
placement_policy = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
|
||||||
|
metrics_step = 1
|
||||||
|
|
||||||
|
with reporter.step("Get current garbage collector metrics for each nodes"):
|
||||||
|
metrics_counter = {}
|
||||||
|
for node in cluster.cluster_nodes:
|
||||||
|
command_result = node.metrics.storage.get_metrics_search_by_greps(
|
||||||
|
command="frostfs_node_garbage_collector_marked_for_removal_objects_total"
|
||||||
|
)
|
||||||
|
metrics_counter[node] = self.calc_metrics_count_from_stdout(command_result.stdout)
|
||||||
|
|
||||||
|
with reporter.step(f"Create container with policy {placement_policy}"):
|
||||||
|
cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, placement_policy)
|
||||||
|
|
||||||
|
with reporter.step("Put object to random node with expire_at"):
|
||||||
|
current_epoch = self.get_epoch()
|
||||||
|
oid = put_object_to_random_node(
|
||||||
|
default_wallet,
|
||||||
|
file_path,
|
||||||
|
cid,
|
||||||
|
self.shell,
|
||||||
|
cluster,
|
||||||
|
expire_at=current_epoch + 1,
|
||||||
|
)
|
||||||
|
|
||||||
|
with reporter.step("Get object nodes"):
|
||||||
|
object_nodes = get_object_nodes(cluster, cid, oid, cluster.cluster_nodes[0])
|
||||||
|
|
||||||
|
with reporter.step("Tick Epoch"):
|
||||||
|
self.tick_epochs(epochs_to_tick=2, wait_block=2)
|
||||||
|
|
||||||
|
with reporter.step(f"Check garbage collector metrics 'the counter should increase by {metrics_step}'"):
|
||||||
|
for node in object_nodes:
|
||||||
|
metrics_counter[node] += metrics_step
|
||||||
|
|
||||||
|
for node, counter in metrics_counter.items():
|
||||||
|
self.check_metrics_in_node(
|
||||||
|
node, counter, command="frostfs_node_garbage_collector_marked_for_removal_objects_total"
|
||||||
|
)
|
||||||
|
|
||||||
|
with reporter.step("Delete container"):
|
||||||
|
delete_container(default_wallet, cid, self.shell, cluster.default_rpc_endpoint)
|
||||||
|
|
||||||
|
@allure.title("Garbage collector delete object")
|
||||||
|
def test_garbage_collector_metrics_deleted_objects(
|
||||||
|
self, simple_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster
|
||||||
|
):
|
||||||
|
file_path = generate_file(simple_object_size.value)
|
||||||
|
placement_policy = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
|
||||||
|
metrics_step = 1
|
||||||
|
|
||||||
|
with reporter.step("Select random node"):
|
||||||
|
node = random.choice(cluster.cluster_nodes)
|
||||||
|
|
||||||
|
with reporter.step("Get current garbage collector metrics for selected node"):
|
||||||
|
command_result = node.metrics.storage.get_metrics_search_by_greps(
|
||||||
|
command="frostfs_node_garbage_collector_deleted_objects_total"
|
||||||
|
)
|
||||||
|
metrics_counter = self.calc_metrics_count_from_stdout(command_result.stdout)
|
||||||
|
|
||||||
|
with reporter.step(f"Create container with policy {placement_policy}"):
|
||||||
|
cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
|
||||||
|
|
||||||
|
with reporter.step("Put object to selected node"):
|
||||||
|
oid = put_object(default_wallet, file_path, cid, self.shell, node.storage_node.get_rpc_endpoint())
|
||||||
|
|
||||||
|
with reporter.step("Delete file, wait until gc remove object"):
|
||||||
|
delete_object(default_wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())
|
||||||
|
|
||||||
|
with reporter.step(f"Check garbage collector metrics 'the counter should increase by {metrics_step}'"):
|
||||||
|
metrics_counter += metrics_step
|
||||||
|
self.check_metrics_in_node(
|
||||||
|
node, metrics_counter, command="frostfs_node_garbage_collector_deleted_objects_total"
|
||||||
|
)
|
||||||
|
|
||||||
|
with reporter.step("Delete container"):
|
||||||
|
delete_container(default_wallet, cid, self.shell, cluster.default_rpc_endpoint)
|
Loading…
Reference in a new issue