Update pilorama loss and shards test cases
This commit is contained in:
parent
d327e8149b
commit
9e89dba03d
2 changed files with 21 additions and 120 deletions
|
|
@ -425,7 +425,8 @@ class TestEmptyMap(ClusterTestBase):
|
|||
|
||||
with reporter.step("Delete pilorama.db from all nodes"):
|
||||
for node in self.cluster.storage_nodes:
|
||||
node.delete_pilorama()
|
||||
for shard in node.get_shards():
|
||||
node.delete_file(shard.pilorama)
|
||||
|
||||
with reporter.step("Start all storage nodes"):
|
||||
cluster_state_controller.start_all_stopped_services()
|
||||
|
|
@ -445,17 +446,6 @@ class TestEmptyMap(ClusterTestBase):
|
|||
@pytest.mark.failover
|
||||
@pytest.mark.failover_data_loss
|
||||
class TestStorageDataLoss(ClusterTestBase):
|
||||
@reporter.step("Get list of all piloramas on node")
|
||||
def get_piloramas_list(self, node: StorageNode) -> list:
|
||||
data_directory_path = node.get_data_directory()
|
||||
|
||||
cmd = f"sudo ls -1 {data_directory_path}/meta*/pilorama*"
|
||||
shell = node.host.get_shell()
|
||||
stdout = shell.exec(cmd).stdout
|
||||
|
||||
piloramas = stdout.split("\n")
|
||||
return piloramas
|
||||
|
||||
@allure.title(
|
||||
"After metabase loss on all nodes operations on objects and buckets should be still available via S3 (s3_client={s3_client})"
|
||||
)
|
||||
|
|
@ -663,15 +653,22 @@ class TestStorageDataLoss(ClusterTestBase):
|
|||
object_versions.append(put_object)
|
||||
|
||||
node_to_check = self.cluster.storage_nodes[0]
|
||||
piloramas_list_before_removing = {}
|
||||
with reporter.step("Get list of all pilorama.db"):
|
||||
piloramas_list_before_removing = self.get_piloramas_list(node_to_check)
|
||||
|
||||
piloramas_list_before_removing = []
|
||||
with reporter.step("Get list of all pilorama.db on shards"):
|
||||
for shard in node_to_check.get_shards():
|
||||
piloramas_list_before_removing.append(shard.pilorama)
|
||||
|
||||
with reporter.step("Check that all pilorama.db files exist on node"):
|
||||
for pilorama in piloramas_list_before_removing:
|
||||
assert node_to_check.is_file_exist(pilorama), f"File {pilorama} does not exist"
|
||||
|
||||
with reporter.step("Stop all storage nodes"):
|
||||
cluster_state_controller.stop_services_of_type(StorageNode)
|
||||
|
||||
with reporter.step("Delete pilorama.db from one node"):
|
||||
node_to_check.delete_pilorama()
|
||||
for pilorama in piloramas_list_before_removing:
|
||||
node_to_check.delete_file(pilorama)
|
||||
|
||||
with reporter.step("Start all storage nodes"):
|
||||
cluster_state_controller.start_all_stopped_services()
|
||||
|
|
@ -680,10 +677,9 @@ class TestStorageDataLoss(ClusterTestBase):
|
|||
self.tick_epochs(1)
|
||||
sleep(120)
|
||||
|
||||
piloramas_list_afrer_removing = {}
|
||||
with reporter.step("Get list of all pilorama.db after sync"):
|
||||
piloramas_list_afrer_removing = self.get_piloramas_list(node_to_check)
|
||||
assert piloramas_list_afrer_removing == piloramas_list_before_removing, "List of pilorama.db is different"
|
||||
for pilorama in piloramas_list_before_removing:
|
||||
assert node_to_check.is_file_exist(pilorama), f"File {pilorama} does not exist"
|
||||
|
||||
with reporter.step("Check bucket versioning"):
|
||||
bucket_versioning = s3_client.get_bucket_versioning_status(bucket)
|
||||
|
|
|
|||
|
|
@ -1,108 +1,13 @@
|
|||
import json
|
||||
import pathlib
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from io import StringIO
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
import yaml
|
||||
from configobj import ConfigObj
|
||||
from frostfs_testlib.cli import FrostfsCli
|
||||
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
|
||||
from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
|
||||
from frostfs_testlib.storage.cluster import Cluster, StorageNode
|
||||
|
||||
SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_"
|
||||
BLOBSTOR_PREFIX = "_BLOBSTOR_"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Blobstor:
|
||||
path: str
|
||||
path_type: str
|
||||
|
||||
def __eq__(self, other) -> bool:
|
||||
if not isinstance(other, self.__class__):
|
||||
raise RuntimeError(f"Only two {self.__class__.__name__} instances can be compared")
|
||||
return self.path == other.path and self.path_type == other.path_type
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.path, self.path_type))
|
||||
|
||||
@staticmethod
|
||||
def from_config_object(section: ConfigObj, shard_id: str, blobstor_id: str):
|
||||
var_prefix = f"{SHARD_PREFIX}{shard_id}{BLOBSTOR_PREFIX}{blobstor_id}"
|
||||
return Blobstor(section.get(f"{var_prefix}_PATH"), section.get(f"{var_prefix}_TYPE"))
|
||||
|
||||
|
||||
@dataclass
|
||||
class Shard:
|
||||
blobstor: list[Blobstor]
|
||||
metabase: str
|
||||
writecache: str
|
||||
|
||||
def __eq__(self, other) -> bool:
|
||||
if not isinstance(other, self.__class__):
|
||||
raise RuntimeError(f"Only two {self.__class__.__name__} instances can be compared")
|
||||
return (
|
||||
set(self.blobstor) == set(other.blobstor)
|
||||
and self.metabase == other.metabase
|
||||
and self.writecache == other.writecache
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.metabase, self.writecache))
|
||||
|
||||
@staticmethod
|
||||
def _get_blobstor_count_from_section(config_object: ConfigObj, shard_id: int):
|
||||
pattern = f"{SHARD_PREFIX}{shard_id}{BLOBSTOR_PREFIX}"
|
||||
blobstors = {key[: len(pattern) + 2] for key in config_object.keys() if pattern in key}
|
||||
return len(blobstors)
|
||||
|
||||
@staticmethod
|
||||
def from_config_object(config_object: ConfigObj, shard_id: int):
|
||||
var_prefix = f"{SHARD_PREFIX}{shard_id}"
|
||||
|
||||
blobstor_count = Shard._get_blobstor_count_from_section(config_object, shard_id)
|
||||
blobstors = [
|
||||
Blobstor.from_config_object(config_object, shard_id, blobstor_id) for blobstor_id in range(blobstor_count)
|
||||
]
|
||||
|
||||
write_cache_enabled = config_object.as_bool(f"{var_prefix}_WRITECACHE_ENABLED")
|
||||
|
||||
return Shard(
|
||||
blobstors,
|
||||
config_object.get(f"{var_prefix}_METABASE_PATH"),
|
||||
config_object.get(f"{var_prefix}_WRITECACHE_PATH") if write_cache_enabled else "",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def from_object(shard):
|
||||
metabase = shard["metabase"]["path"] if "path" in shard["metabase"] else shard["metabase"]
|
||||
writecache = shard["writecache"]["path"] if "path" in shard["writecache"] else shard["writecache"]
|
||||
|
||||
return Shard(
|
||||
blobstor=[Blobstor(path=blobstor["path"], path_type=blobstor["type"]) for blobstor in shard["blobstor"]],
|
||||
metabase=metabase,
|
||||
writecache=writecache,
|
||||
)
|
||||
|
||||
|
||||
def shards_from_yaml(contents: str) -> list[Shard]:
|
||||
config = yaml.safe_load(contents)
|
||||
config["storage"]["shard"].pop("default")
|
||||
|
||||
return [Shard.from_object(shard) for shard in config["storage"]["shard"].values()]
|
||||
|
||||
|
||||
def shards_from_env(contents: str) -> list[Shard]:
|
||||
configObj = ConfigObj(StringIO(contents))
|
||||
|
||||
pattern = f"{SHARD_PREFIX}\d*"
|
||||
num_shards = len(set(re.findall(pattern, contents)))
|
||||
|
||||
return [Shard.from_config_object(configObj, shard_id) for shard_id in range(num_shards)]
|
||||
from frostfs_testlib.storage.dataclasses.shard import Shard
|
||||
|
||||
|
||||
@pytest.mark.shard
|
||||
|
|
@ -111,15 +16,14 @@ class TestControlShard:
|
|||
def get_shards_from_config(node: StorageNode) -> list[Shard]:
|
||||
config_file = node.get_shard_config_path()
|
||||
file_type = pathlib.Path(config_file).suffix
|
||||
contents = node.host.get_shell().exec(f"cat {config_file}").stdout
|
||||
|
||||
parser_method = {
|
||||
".env": shards_from_env,
|
||||
".yaml": shards_from_yaml,
|
||||
".yml": shards_from_yaml,
|
||||
".env": node.get_shards_from_env,
|
||||
".yaml": node.get_shards,
|
||||
".yml": node.get_shards,
|
||||
}
|
||||
|
||||
shards = parser_method[file_type](contents)
|
||||
shards = parser_method[file_type]()
|
||||
return shards
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -145,4 +49,5 @@ class TestControlShard:
|
|||
for storage_node in cluster.storage_nodes:
|
||||
shards_from_config = self.get_shards_from_config(storage_node)
|
||||
shards_from_cli = self.get_shards_from_cli(storage_node)
|
||||
|
||||
assert set(shards_from_config) == set(shards_from_cli)
|
||||
|
|
|
|||
Loading…
Reference in a new issue