Add network internal interface test

Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
This commit is contained in:
Dmitriy Zayakin 2023-10-12 10:30:13 +03:00
parent b3c24828d3
commit adfd6854f8
2 changed files with 102 additions and 27 deletions

View file

@ -7,7 +7,7 @@ import allure
import pytest
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object_to_random_node
from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object, put_object_to_random_node
from frostfs_testlib.steps.storage_object import delete_objects
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.controllers import ClusterStateController
@ -59,15 +59,20 @@ class TestFailoverNetwork(ClusterTestBase):
yield
for path in file_wait_delete:
os.remove(path)
file_wait_delete.clear()
@pytest.fixture()
def storage_objects(self, simple_object_size: ObjectSize, default_wallet: str) -> list[StorageObjectInfo]:
def storage_objects(
self,
simple_object_size: ObjectSize,
default_wallet: str,
) -> list[StorageObjectInfo]:
file_path = generate_file(simple_object_size.value)
file_hash = get_file_hash(file_path)
with allure.step("Create container"):
placement_rule = "REP 1"
placement_rule = "REP 1 CBF 1"
cid = create_container(
wallet=default_wallet,
shell=self.shell,
@ -179,6 +184,7 @@ class TestFailoverNetwork(ClusterTestBase):
got_file_path = get_object(wallet, cid, oid, shell=self.shell, endpoint=new_nodes[0].get_rpc_endpoint())
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
@pytest.mark.interfaces
@allure.title("Block DATA interface node")
def test_block_data_interface(
self,
@ -201,10 +207,9 @@ class TestFailoverNetwork(ClusterTestBase):
)
with allure.step("Get data interface to node"):
data = "data"
config_interfaces = list(nodes_with_object[0].host.config.interfaces.keys())
with allure.step(f"Get {data} in {config_interfaces}"):
data_interfaces = [interface for interface in config_interfaces if data in interface]
with allure.step(f"Get data in {config_interfaces}"):
data_interfaces = [interface for interface in config_interfaces if "data" in interface]
with allure.step("Block data interfaces for node"):
for interface in data_interfaces:
@ -233,3 +238,87 @@ class TestFailoverNetwork(ClusterTestBase):
endpoint=nodes_without_an_object[0].storage_node.get_rpc_endpoint(),
)
file_wait_delete.append(input_file)
with allure.step("Restore interface and tick 1 epoch, wait 2 block"):
cluster_state_controller.restore_interfaces()
self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)
@pytest.mark.interfaces
@allure.title("Block INTERNAL interface node")
def test_block_internal_interface(
self,
cluster_state_controller: ClusterStateController,
default_wallet: str,
restore_traffic: None,
delete_file_after_test: None,
storage_objects: list[StorageObjectInfo],
simple_object_size: ObjectSize,
):
storage_object = storage_objects[0]
with allure.step("Search nodes with object"):
nodes_with_object = get_object_nodes(
cluster=self.cluster,
wallet=default_wallet,
cid=storage_object.cid,
oid=storage_object.oid,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with allure.step("Get internal interface to node"):
config_interfaces = list(nodes_with_object[0].host.config.interfaces.keys())
with allure.step(f"Get internal in {config_interfaces}"):
internal_interfaces = [interface for interface in config_interfaces if "internal" in interface]
with allure.step("Block internal interfaces for node"):
for interface in internal_interfaces:
cluster_state_controller.down_interface(nodes=nodes_with_object, interface=interface)
with allure.step("Tick epoch and wait 2 block"):
nodes_without_an_object = list(set(self.cluster.cluster_nodes) - set(nodes_with_object))
self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)
with allure.step("Get object others node, expect false"):
with pytest.raises(RuntimeError, match="return code: 1"):
get_object(
wallet=default_wallet,
cid=storage_object.cid,
oid=storage_object.oid,
shell=self.shell,
endpoint=nodes_without_an_object[0].storage_node.get_rpc_endpoint(),
)
with allure.step("Put object, others node, expect false"):
with pytest.raises(RuntimeError, match="return code: 1"):
put_object(
wallet=default_wallet,
path=storage_object.file_path,
cid=storage_object.cid,
shell=self.shell,
endpoint=nodes_without_an_object[0].storage_node.get_rpc_endpoint(),
)
with allure.step(f"Get object nodes with object, expect true"):
input_file = get_object(
wallet=default_wallet,
cid=storage_object.cid,
oid=storage_object.oid,
shell=self.shell,
endpoint=nodes_with_object[0].storage_node.get_rpc_endpoint(),
)
file_wait_delete.append(input_file)
with allure.step(f"Put object nodes with object, expect true"):
temp_file_path = generate_file(simple_object_size.value)
_ = put_object(
wallet=default_wallet,
path=temp_file_path,
cid=storage_object.cid,
shell=self.shell,
endpoint=nodes_with_object[0].storage_node.get_rpc_endpoint(),
)
file_wait_delete.append(temp_file_path)
with allure.step("Restore interface and tick 1 epoch, wait 2 block"):
cluster_state_controller.restore_interfaces()
self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)

View file

@ -7,11 +7,7 @@ import allure
import pytest
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import (
StorageContainer,
StorageContainerInfo,
create_container,
)
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
from frostfs_testlib.steps.cli.object import get_object
from frostfs_testlib.steps.node_management import check_node_in_map, check_node_not_in_map
from frostfs_testlib.storage.cluster import ClusterNode, StorageNode
@ -63,9 +59,7 @@ class TestFailoverServer(ClusterTestBase):
wallet = WalletInfo(path=default_wallet)
storage_cont_info = StorageContainerInfo(id=cont_id, wallet_file=wallet)
containers.append(
StorageContainer(
storage_container_info=storage_cont_info, shell=self.shell, cluster=self.cluster
)
StorageContainer(storage_container_info=storage_cont_info, shell=self.shell, cluster=self.cluster)
)
return containers
@ -168,11 +162,9 @@ class TestFailoverServer(ClusterTestBase):
assert not corrupted_objects_list
with allure.step(f"check {node_to_stop.storage_node} in map"):
self.wait_node_in_map(
node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]
)
self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
count_tick_epoch = int(alive_nodes[0].ir_node.get_netmap_cleaner_threshold()) + 2
count_tick_epoch = int(alive_nodes[0].ir_node.get_netmap_cleaner_threshold()) + 4
with allure.step(f"Tick {count_tick_epoch} epoch, in {storage_nodes[0]} node"):
for tick in range(count_tick_epoch):
@ -183,13 +175,9 @@ class TestFailoverServer(ClusterTestBase):
wait_for_host_offline(self.shell, node_to_stop.storage_node)
with allure.step(f"Check {node_to_stop} in not map"):
self.wait_node_not_in_map(
node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]
)
self.wait_node_not_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
with allure.step(
f"Verify that there are no corrupted objects after {count_tick_epoch} epoch"
):
with allure.step(f"Verify that there are no corrupted objects after {count_tick_epoch} epoch"):
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
assert not corrupted_objects_list
@ -222,9 +210,7 @@ class TestFailoverServer(ClusterTestBase):
assert not corrupted_objects_list
with allure.step(f"Check {node_to_stop} in map"):
self.wait_node_in_map(
node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0]
)
self.wait_node_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
cluster_state_controller.start_node_host(node_to_stop)