TrueCloudLab/frostfs-testcases
26
0
Fork 21
Code Issues 2 Pull requests 3 Releases Activity Actions

[#162] Added exception in log analyzer

Browse source 
Added exception of error 'Too many requests' in log analyzer

Signed-off-by: Mikhail Kadilov <m.kadilov@yadro.com>
This commit is contained in:
Mikhail Kadilov 2023-12-01 19:08:42 +03:00
parent da06c09ed0
commit 61c4463a95
41 changed files with 1015 additions and 984 deletions
Download patch file Download diff file Expand all files Collapse all files

66
pytest_tests/testsuites/failovers/test_failover_server.py
View file

@ -1,10 +1,10 @@
import logging
import os.path
import random
import time
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
@ -36,7 +36,7 @@ class TestFailoverServer(ClusterTestBase):
def wait_node_in_map(self, *args, **kwargs):
check_node_in_map(*args, **kwargs)
@allure.step("Create {count_containers} containers and {count_files} objects")
@reporter.step("Create {count_containers} containers and {count_files} objects")
@pytest.fixture
def containers(
self,
@ -64,7 +64,7 @@ class TestFailoverServer(ClusterTestBase):
return containers
@allure.step("Creation container")
@reporter.step("Creation container")
@pytest.fixture()
def container(self, default_wallet: str) -> StorageContainer:
select = len(self.cluster.cluster_nodes)
@ -80,7 +80,7 @@ class TestFailoverServer(ClusterTestBase):
storage_cont_info = StorageContainerInfo(id=cont_id, wallet_file=wallet)
return StorageContainer(storage_container_info=storage_cont_info, shell=self.shell, cluster=self.cluster)
@allure.step("Create object and delete after test")
@reporter.step("Create object and delete after test")
@pytest.fixture(scope="class")
def storage_objects(
self,
@ -101,16 +101,16 @@ class TestFailoverServer(ClusterTestBase):
yield object_list
@allure.step("Select random node to stop and start it after test")
@reporter.step("Select random node to stop and start it after test")
@pytest.fixture
def node_to_stop(
self, node_under_test: ClusterNode, cluster_state_controller: ClusterStateController
) -> ClusterNode:
yield node_under_test
with allure.step(f"start {node_under_test.storage_node}"):
with reporter.step(f"start {node_under_test.storage_node}"):
cluster_state_controller.start_stopped_hosts()
@allure.step("Upload object with nodes and compare")
@reporter.step("Upload object with nodes and compare")
def get_corrupted_objects_list(
self, nodes: list[StorageNode], storage_objects: list[StorageObjectInfo]
) -> list[StorageObjectInfo]:
@ -179,34 +179,34 @@ class TestFailoverServer(ClusterTestBase):
cluster_state_controller: ClusterStateController,
):
with allure.step(f"Remove {node_to_stop} from the list of nodes"):
with reporter.step(f"Remove {node_to_stop} from the list of nodes"):
alive_nodes = list(set(self.cluster.cluster_nodes) - {node_to_stop})
storage_nodes = [cluster.storage_node for cluster in alive_nodes]
with allure.step("Tick epoch and wait for 2 blocks"):
with reporter.step("Tick epoch and wait for 2 blocks"):
self.tick_epochs(1, storage_nodes[0], wait_block=2)
with allure.step(f"Stop node"):
with reporter.step(f"Stop node"):
cluster_state_controller.stop_node_host(node=node_to_stop, mode="hard")
with allure.step("Verify that there are no corrupted objects"):
with reporter.step("Verify that there are no corrupted objects"):
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
assert not corrupted_objects_list
with allure.step(f"check {node_to_stop.storage_node} in map"):
with reporter.step(f"check {node_to_stop.storage_node} in map"):
self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
count_tick_epoch = int(alive_nodes[0].ir_node.get_netmap_cleaner_threshold()) + 4
with allure.step(f"Tick {count_tick_epoch} epochs and wait for 2 blocks"):
with reporter.step(f"Tick {count_tick_epoch} epochs and wait for 2 blocks"):
self.tick_epochs(count_tick_epoch, storage_nodes[0], wait_block=2)
with allure.step(f"Check {node_to_stop} in not map"):
with reporter.step(f"Check {node_to_stop} in not map"):
self.wait_node_not_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
with allure.step(f"Verify that there are no corrupted objects after {count_tick_epoch} epoch"):
with reporter.step(f"Verify that there are no corrupted objects after {count_tick_epoch} epoch"):
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
assert not corrupted_objects_list
@ -218,25 +218,25 @@ class TestFailoverServer(ClusterTestBase):
node_to_stop: ClusterNode,
cluster_state_controller: ClusterStateController,
):
with allure.step(f"Remove {node_to_stop} from the list of nodes"):
with reporter.step(f"Remove {node_to_stop} from the list of nodes"):
storage_nodes = list(set(self.cluster.storage_nodes) - {node_to_stop.storage_node})
with allure.step("Tick epoch and wait for 2 blocks"):
with reporter.step("Tick epoch and wait for 2 blocks"):
self.tick_epochs(1, storage_nodes[0], wait_block=2)
with allure.step(f"Stop node"):
with reporter.step(f"Stop node"):
cluster_state_controller.stop_node_host(node=node_to_stop, mode="hard")
with allure.step("Verify that there are no corrupted objects"):
with reporter.step("Verify that there are no corrupted objects"):
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
assert not corrupted_objects_list
with allure.step(f"Check {node_to_stop} in map"):
with reporter.step(f"Check {node_to_stop} in map"):
self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
cluster_state_controller.start_node_host(node_to_stop)
with allure.step("Verify that there are no corrupted objects"):
with reporter.step("Verify that there are no corrupted objects"):
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
assert not corrupted_objects_list
@ -253,12 +253,12 @@ class TestFailoverServer(ClusterTestBase):
object_info, object_nodes = object_and_nodes
node_not_object = list(set(self.cluster.cluster_nodes) - set(object_nodes))[0]
with allure.step("Stop all nodes with object, except 1"):
with reporter.step("Stop all nodes with object, except 1"):
for cluster_node in object_nodes[1:]:
cluster_state_controller.stop_node_host(node=cluster_node, mode="hard")
with allure.step("Get object"):
with allure.step(f"Get operation to {node_not_object} where it does not exist, expect success"):
with reporter.step("Get object"):
with reporter.step(f"Get operation to {node_not_object} where it does not exist, expect success"):
get_object(
wallet=default_wallet,
cid=object_info.cid,
@ -266,7 +266,7 @@ class TestFailoverServer(ClusterTestBase):
shell=self.shell,
endpoint=node_not_object.storage_node.get_rpc_endpoint(),
)
with allure.step(f"Get operation to {object_nodes[0]} with object, expect success"):
with reporter.step(f"Get operation to {object_nodes[0]} with object, expect success"):
get_object(
wallet=default_wallet,
cid=object_info.cid,
@ -275,7 +275,7 @@ class TestFailoverServer(ClusterTestBase):
endpoint=object_nodes[0].storage_node.get_rpc_endpoint(),
)
with allure.step(f"Put operation to node with object, expect error"):
with reporter.step(f"Put operation to node with object, expect error"):
with pytest.raises(RuntimeError):
put_object(
wallet=default_wallet,
@ -293,7 +293,7 @@ class TestFailoverServer(ClusterTestBase):
simple_file: str,
up_stop_nodes: None,
):
with allure.step("Creating a container with a full network map"):
with reporter.step("Creating a container with a full network map"):
select = len(self.cluster.cluster_nodes)
placement_rule = f"REP {select - 2} IN X CBF 2 SELECT {select} FROM * AS X"
cid_1 = create_container(
@ -303,7 +303,7 @@ class TestFailoverServer(ClusterTestBase):
rule=placement_rule,
basic_acl=PUBLIC_ACL,
)
with allure.step("Put object"):
with reporter.step("Put object"):
oid = put_object(
wallet=default_wallet,
path=simple_file,
@ -311,7 +311,7 @@ class TestFailoverServer(ClusterTestBase):
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with allure.step("Search nodes with object"):
with reporter.step("Search nodes with object"):
object_nodes = get_object_nodes(
cluster=self.cluster,
wallet=default_wallet,
@ -320,9 +320,9 @@ class TestFailoverServer(ClusterTestBase):
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with allure.step("Turn off random node with object"):
with reporter.step("Turn off random node with object"):
cluster_state_controller.stop_node_host(node=random.choice(object_nodes[1:]), mode="hard")
with allure.step("Checking PUT operation"):
with reporter.step("Checking PUT operation"):
oid_2 = put_object(
wallet=default_wallet,
path=simple_file,
@ -330,7 +330,7 @@ class TestFailoverServer(ClusterTestBase):
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with allure.step("Checking GET operation"):
with reporter.step("Checking GET operation"):
get_file = get_object(
wallet=default_wallet,
cid=cid_1,
@ -339,7 +339,7 @@ class TestFailoverServer(ClusterTestBase):
endpoint=object_nodes[0].storage_node.get_rpc_endpoint(),
)
os.remove(get_file)
with allure.step("Checking creating container"):
with reporter.step("Checking creating container"):
_ = create_container(
default_wallet,
shell=self.shell,