[#154] Add new test
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
This commit is contained in:
parent
fbefa422e8
commit
269caf8e03
6 changed files with 833 additions and 677 deletions
|
|
@ -1,12 +1,14 @@
|
|||
import logging
|
||||
import os.path
|
||||
import random
|
||||
import time
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
|
||||
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
|
||||
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
|
||||
from frostfs_testlib.steps.cli.object import get_object
|
||||
from frostfs_testlib.steps.cli.object import get_object, get_object_nodes, put_object
|
||||
from frostfs_testlib.steps.node_management import check_node_in_map, check_node_not_in_map
|
||||
from frostfs_testlib.storage.cluster import ClusterNode, StorageNode
|
||||
from frostfs_testlib.storage.controllers import ClusterStateController
|
||||
|
|
@ -15,6 +17,7 @@ from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjec
|
|||
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
|
||||
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
||||
from frostfs_testlib.testing.test_control import wait_for_success
|
||||
from frostfs_testlib.utils import datetime_utils
|
||||
from frostfs_testlib.utils.failover_utils import wait_object_replication
|
||||
from frostfs_testlib.utils.file_utils import get_file_hash
|
||||
from pytest import FixtureRequest
|
||||
|
|
@ -61,6 +64,22 @@ class TestFailoverServer(ClusterTestBase):
|
|||
|
||||
return containers
|
||||
|
||||
@allure.step("Creation container")
|
||||
@pytest.fixture()
|
||||
def container(self, default_wallet: str) -> StorageContainer:
|
||||
select = len(self.cluster.cluster_nodes)
|
||||
placement_rule = f"REP {select - 1} CBF 1 SELECT {select} FROM *"
|
||||
cont_id = create_container(
|
||||
default_wallet,
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
rule=placement_rule,
|
||||
basic_acl=PUBLIC_ACL,
|
||||
)
|
||||
wallet = WalletInfo(path=default_wallet)
|
||||
storage_cont_info = StorageContainerInfo(id=cont_id, wallet_file=wallet)
|
||||
return StorageContainer(storage_container_info=storage_cont_info, shell=self.shell, cluster=self.cluster)
|
||||
|
||||
@allure.step("Create object and delete after test")
|
||||
@pytest.fixture(scope="class")
|
||||
def storage_objects(
|
||||
|
|
@ -96,20 +115,25 @@ class TestFailoverServer(ClusterTestBase):
|
|||
self, nodes: list[StorageNode], storage_objects: list[StorageObjectInfo]
|
||||
) -> list[StorageObjectInfo]:
|
||||
corrupted_objects = []
|
||||
errors_get = []
|
||||
for node in nodes:
|
||||
for storage_object in storage_objects:
|
||||
got_file_path = get_object(
|
||||
storage_object.wallet_file_path,
|
||||
storage_object.cid,
|
||||
storage_object.oid,
|
||||
endpoint=node.get_rpc_endpoint(),
|
||||
shell=self.shell,
|
||||
timeout="60s",
|
||||
)
|
||||
if storage_object.file_hash != get_file_hash(got_file_path):
|
||||
corrupted_objects.append(storage_object)
|
||||
os.remove(got_file_path)
|
||||
try:
|
||||
got_file_path = get_object(
|
||||
storage_object.wallet_file_path,
|
||||
storage_object.cid,
|
||||
storage_object.oid,
|
||||
endpoint=node.get_rpc_endpoint(),
|
||||
shell=self.shell,
|
||||
timeout="60s",
|
||||
)
|
||||
if storage_object.file_hash != get_file_hash(got_file_path):
|
||||
corrupted_objects.append(storage_object)
|
||||
os.remove(got_file_path)
|
||||
except RuntimeError:
|
||||
errors_get.append(storage_object.oid)
|
||||
|
||||
assert len(errors_get) == 0, f"Get failed - {errors_get}"
|
||||
return corrupted_objects
|
||||
|
||||
def check_objects_replication(
|
||||
|
|
@ -126,6 +150,26 @@ class TestFailoverServer(ClusterTestBase):
|
|||
attempts=60,
|
||||
)
|
||||
|
||||
@pytest.fixture()
|
||||
def object_and_nodes(
|
||||
self, simple_object_size: ObjectSize, container: StorageContainer, default_wallet: str
|
||||
) -> tuple[StorageObjectInfo, list[ClusterNode]]:
|
||||
object_info = container.generate_object(simple_object_size.value)
|
||||
object_nodes = get_object_nodes(
|
||||
cluster=self.cluster,
|
||||
wallet=default_wallet,
|
||||
cid=object_info.cid,
|
||||
oid=object_info.oid,
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
return object_info, object_nodes
|
||||
|
||||
@pytest.fixture()
|
||||
def up_stop_nodes(self, cluster_state_controller: ClusterStateController):
|
||||
yield
|
||||
cluster_state_controller.start_stopped_hosts()
|
||||
|
||||
@allure.title("Full shutdown node")
|
||||
@pytest.mark.parametrize("containers, storage_objects", [(5, 10)], indirect=True)
|
||||
def test_complete_node_shutdown(
|
||||
|
|
@ -157,7 +201,7 @@ class TestFailoverServer(ClusterTestBase):
|
|||
count_tick_epoch = int(alive_nodes[0].ir_node.get_netmap_cleaner_threshold()) + 4
|
||||
|
||||
with allure.step(f"Tick {count_tick_epoch} epochs and wait for 2 blocks"):
|
||||
self.tick_epochs(count_tick_epoch, wait_block=2)
|
||||
self.tick_epochs(count_tick_epoch, storage_nodes[0], wait_block=2)
|
||||
|
||||
with allure.step(f"Check {node_to_stop} in not map"):
|
||||
self.wait_node_not_in_map(node_to_stop.storage_node, self.shell, alive_node=storage_nodes[0])
|
||||
|
|
@ -195,3 +239,111 @@ class TestFailoverServer(ClusterTestBase):
|
|||
with allure.step("Verify that there are no corrupted objects"):
|
||||
corrupted_objects_list = self.get_corrupted_objects_list(storage_nodes, storage_objects)
|
||||
assert not corrupted_objects_list
|
||||
|
||||
@allure.title("Not enough nodes in the container with policy - 'REP 3 CBF 1 SELECT 4 FROM *'")
|
||||
def test_not_enough_nodes_in_container_rep_3(
|
||||
self,
|
||||
container: list[StorageContainer],
|
||||
object_and_nodes: tuple[StorageObjectInfo, list[ClusterNode]],
|
||||
default_wallet: str,
|
||||
cluster_state_controller: ClusterStateController,
|
||||
simple_file: str,
|
||||
up_stop_nodes: None,
|
||||
):
|
||||
object_info, object_nodes = object_and_nodes
|
||||
node_not_object = list(set(self.cluster.cluster_nodes) - set(object_nodes))[0]
|
||||
|
||||
with allure.step("Stop all nodes with object, except 1"):
|
||||
for cluster_node in object_nodes[1:]:
|
||||
cluster_state_controller.stop_node_host(node=cluster_node, mode="hard")
|
||||
|
||||
with allure.step("Get object"):
|
||||
with allure.step(f"Get operation to {node_not_object} where it does not exist, expect success"):
|
||||
get_object(
|
||||
wallet=default_wallet,
|
||||
cid=object_info.cid,
|
||||
oid=object_info.oid,
|
||||
shell=self.shell,
|
||||
endpoint=node_not_object.storage_node.get_rpc_endpoint(),
|
||||
)
|
||||
with allure.step(f"Get operation to {object_nodes[0]} with object, expect success"):
|
||||
get_object(
|
||||
wallet=default_wallet,
|
||||
cid=object_info.cid,
|
||||
oid=object_info.oid,
|
||||
shell=self.shell,
|
||||
endpoint=object_nodes[0].storage_node.get_rpc_endpoint(),
|
||||
)
|
||||
|
||||
with allure.step(f"Put operation to node with object, expect error"):
|
||||
with pytest.raises(RuntimeError):
|
||||
put_object(
|
||||
wallet=default_wallet,
|
||||
path=simple_file,
|
||||
cid=object_info.cid,
|
||||
shell=self.shell,
|
||||
endpoint=node_not_object.storage_node.get_rpc_endpoint(),
|
||||
)
|
||||
|
||||
@allure.title("Not enough nodes in the container with policy - 'REP 2 CBF 2 SELECT 4 FROM *'")
|
||||
def test_not_enough_nodes_in_container_rep_2(
|
||||
self,
|
||||
default_wallet: str,
|
||||
cluster_state_controller: ClusterStateController,
|
||||
simple_file: str,
|
||||
up_stop_nodes: None,
|
||||
):
|
||||
with allure.step("Creating a container with a full network map"):
|
||||
select = len(self.cluster.cluster_nodes)
|
||||
placement_rule = f"REP {select - 2} IN X CBF 2 SELECT {select} FROM * AS X"
|
||||
cid_1 = create_container(
|
||||
default_wallet,
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
rule=placement_rule,
|
||||
basic_acl=PUBLIC_ACL,
|
||||
)
|
||||
with allure.step("Put object"):
|
||||
oid = put_object(
|
||||
wallet=default_wallet,
|
||||
path=simple_file,
|
||||
cid=cid_1,
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
with allure.step("Search nodes with object"):
|
||||
object_nodes = get_object_nodes(
|
||||
cluster=self.cluster,
|
||||
wallet=default_wallet,
|
||||
cid=cid_1,
|
||||
oid=oid,
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
with allure.step("Turn off random node with object"):
|
||||
cluster_state_controller.stop_node_host(node=random.choice(object_nodes[1:]), mode="hard")
|
||||
with allure.step("Checking PUT operation"):
|
||||
oid_2 = put_object(
|
||||
wallet=default_wallet,
|
||||
path=simple_file,
|
||||
cid=cid_1,
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
with allure.step("Checking GET operation"):
|
||||
get_file = get_object(
|
||||
wallet=default_wallet,
|
||||
cid=cid_1,
|
||||
oid=oid_2,
|
||||
shell=self.shell,
|
||||
endpoint=object_nodes[0].storage_node.get_rpc_endpoint(),
|
||||
)
|
||||
os.remove(get_file)
|
||||
with allure.step("Checking creating container"):
|
||||
_ = create_container(
|
||||
default_wallet,
|
||||
shell=self.shell,
|
||||
endpoint=object_nodes[0].storage_node.get_rpc_endpoint(),
|
||||
rule=placement_rule,
|
||||
basic_acl=PUBLIC_ACL,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue