forked from TrueCloudLab/frostfs-testcases
Add metabase loss test
This commit is contained in:
parent
07e3ef3972
commit
61771ed534
2 changed files with 91 additions and 6 deletions
|
@ -1,5 +1,4 @@
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
from time import sleep
|
from time import sleep
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
|
@ -22,26 +21,43 @@ from frostfs_testlib.steps.node_management import (
|
||||||
)
|
)
|
||||||
from frostfs_testlib.steps.s3 import s3_helper
|
from frostfs_testlib.steps.s3 import s3_helper
|
||||||
from frostfs_testlib.storage.cluster import Cluster, StorageNode
|
from frostfs_testlib.storage.cluster import Cluster, StorageNode
|
||||||
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
|
from frostfs_testlib.storage.controllers import ClusterStateController
|
||||||
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
||||||
|
from frostfs_testlib.testing.test_control import expect_not_raises
|
||||||
from frostfs_testlib.utils import datetime_utils
|
from frostfs_testlib.utils import datetime_utils
|
||||||
from frostfs_testlib.utils.failover_utils import (
|
from frostfs_testlib.utils.failover_utils import (
|
||||||
wait_all_storage_nodes_returned,
|
wait_all_storage_nodes_returned,
|
||||||
wait_object_replication,
|
wait_object_replication,
|
||||||
)
|
)
|
||||||
|
from frostfs_testlib.utils.file_keeper import FileKeeper
|
||||||
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
|
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
|
||||||
|
|
||||||
logger = logging.getLogger("NeoLogger")
|
logger = logging.getLogger("NeoLogger")
|
||||||
stopped_nodes: list[StorageNode] = []
|
stopped_nodes: list[StorageNode] = []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="function")
|
||||||
|
@allure.title("Provide File Keeper")
|
||||||
|
def file_keeper():
|
||||||
|
keeper = FileKeeper()
|
||||||
|
yield keeper
|
||||||
|
keeper.restore_files()
|
||||||
|
|
||||||
|
|
||||||
@allure.step("Return all stopped hosts")
|
@allure.step("Return all stopped hosts")
|
||||||
@pytest.fixture(scope="function", autouse=True)
|
@pytest.fixture(scope="function", autouse=True)
|
||||||
def after_run_return_all_stopped_hosts(client_shell: Shell, cluster: Cluster):
|
def after_run_return_all_stopped_hosts(client_shell: Shell, cluster: Cluster) -> str:
|
||||||
yield
|
yield "After this test stopped services will be started automatically via fixture"
|
||||||
return_stopped_hosts(client_shell, cluster)
|
return_stopped_hosts(client_shell, cluster)
|
||||||
|
|
||||||
|
|
||||||
|
@allure.step("Return all stopped storage services after test")
|
||||||
|
@pytest.fixture(scope="function")
|
||||||
|
def after_run_return_all_stopped_services(cluster_state_controller: ClusterStateController):
|
||||||
|
yield
|
||||||
|
cluster_state_controller.start_stopped_storage_services()
|
||||||
|
|
||||||
|
|
||||||
def panic_reboot_host(host: Host) -> None:
|
def panic_reboot_host(host: Host) -> None:
|
||||||
shell = host.get_shell()
|
shell = host.get_shell()
|
||||||
shell.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"')
|
shell.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"')
|
||||||
|
@ -495,3 +511,72 @@ class TestEmptyMap(ClusterTestBase):
|
||||||
|
|
||||||
with allure.step("Delete bucket"):
|
with allure.step("Delete bucket"):
|
||||||
s3_client.delete_bucket(bucket)
|
s3_client.delete_bucket(bucket)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.failover
|
||||||
|
@pytest.mark.failover_data_loss
|
||||||
|
class TestStorageDataLoss(ClusterTestBase):
|
||||||
|
@allure.title(
|
||||||
|
"After metabase loss on all nodes operations on objects and buckets should be still available via S3"
|
||||||
|
)
|
||||||
|
@pytest.mark.metabase_loss
|
||||||
|
def test_metabase_loss(
|
||||||
|
self,
|
||||||
|
s3_client: S3ClientWrapper,
|
||||||
|
simple_object_size: int,
|
||||||
|
complex_object_size: int,
|
||||||
|
cluster_state_controller: ClusterStateController,
|
||||||
|
after_run_return_all_stopped_services: str,
|
||||||
|
file_keeper: FileKeeper,
|
||||||
|
):
|
||||||
|
allure.dynamic.description(after_run_return_all_stopped_services)
|
||||||
|
|
||||||
|
with allure.step("Create bucket"):
|
||||||
|
bucket = s3_client.create_bucket()
|
||||||
|
|
||||||
|
with allure.step("Put objects into bucket"):
|
||||||
|
simple_object_path = generate_file(simple_object_size)
|
||||||
|
simple_object_key = s3_helper.object_key_from_file_path(simple_object_path)
|
||||||
|
|
||||||
|
complex_object_path = generate_file(complex_object_size)
|
||||||
|
complex_object_key = s3_helper.object_key_from_file_path(complex_object_path)
|
||||||
|
|
||||||
|
s3_client.put_object(bucket, simple_object_path)
|
||||||
|
s3_client.put_object(bucket, complex_object_path)
|
||||||
|
|
||||||
|
with allure.step("Check objects are in bucket"):
|
||||||
|
s3_helper.check_objects_in_bucket(
|
||||||
|
s3_client, bucket, expected_objects=[simple_object_key, complex_object_key]
|
||||||
|
)
|
||||||
|
|
||||||
|
with allure.step("Stop storage services on all nodes"):
|
||||||
|
cluster_state_controller.stop_all_storage_services()
|
||||||
|
|
||||||
|
with allure.step("Delete metabase from all nodes"):
|
||||||
|
for node in cluster_state_controller.cluster.storage_nodes:
|
||||||
|
node.delete_metabase()
|
||||||
|
|
||||||
|
with allure.step("Enable resync_metabase option for storage services"):
|
||||||
|
for storage_node in cluster_state_controller.cluster.storage_nodes:
|
||||||
|
with allure.step(f"Enable resync_metabase option for {storage_node}"):
|
||||||
|
config_file_path, config = storage_node.get_config()
|
||||||
|
if not config["storage"]["shard"]["default"]["resync_metabase"]:
|
||||||
|
file_keeper.add(storage_node, config_file_path)
|
||||||
|
config["storage"]["shard"]["default"]["resync_metabase"] = True
|
||||||
|
storage_node.save_config(config)
|
||||||
|
|
||||||
|
with allure.step("Start storage services on all nodes"):
|
||||||
|
cluster_state_controller.start_stopped_storage_services()
|
||||||
|
|
||||||
|
with allure.step("Delete objects from bucket"):
|
||||||
|
with allure.step("Delete simple object from bucket"):
|
||||||
|
with expect_not_raises():
|
||||||
|
s3_client.delete_object(bucket, simple_object_key)
|
||||||
|
|
||||||
|
with allure.step("Delete complex object from bucket"):
|
||||||
|
with expect_not_raises():
|
||||||
|
s3_client.delete_object(bucket, complex_object_key)
|
||||||
|
|
||||||
|
with allure.step("Delete bucket"):
|
||||||
|
with expect_not_raises():
|
||||||
|
s3_client.delete_bucket(bucket)
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
allure-pytest==2.9.45
|
allure-pytest==2.13.2
|
||||||
allure-python-commons==2.9.45
|
allure-python-commons==2.13.2
|
||||||
base58==2.1.0
|
base58==2.1.0
|
||||||
boto3==1.16.33
|
boto3==1.16.33
|
||||||
botocore==1.19.33
|
botocore==1.19.33
|
||||||
|
|
Loading…
Reference in a new issue