|
|
|
@ -13,8 +13,8 @@ from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
|
|
|
|
|
from frostfs_testlib.shell import Shell
|
|
|
|
|
from frostfs_testlib.steps.epoch import tick_epoch, wait_for_epochs_align
|
|
|
|
|
from frostfs_testlib.storage.cluster import Cluster, StorageNode
|
|
|
|
|
from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate
|
|
|
|
|
from frostfs_testlib.utils import datetime_utils
|
|
|
|
|
from frostfs_testlib.hosting import Host
|
|
|
|
|
|
|
|
|
|
logger = logging.getLogger("NeoLogger")
|
|
|
|
|
|
|
|
|
@ -36,7 +36,7 @@ class HealthStatus:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@reporter.step("Get Locode from random storage node")
|
|
|
|
|
def get_locode_from_random_node(cluster: Cluster) -> str:
|
|
|
|
|
def get_locode_from_random_node( cluster: Cluster) -> str:
|
|
|
|
|
node = random.choice(cluster.services(StorageNode))
|
|
|
|
|
locode = node.get_un_locode()
|
|
|
|
|
logger.info(f"Chosen '{locode}' locode from node {node}")
|
|
|
|
@ -44,7 +44,7 @@ def get_locode_from_random_node(cluster: Cluster) -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@reporter.step("Healthcheck for storage node {node}")
|
|
|
|
|
def storage_node_healthcheck(node: StorageNode) -> HealthStatus:
|
|
|
|
|
def storage_node_healthcheck(self, node: StorageNode) -> HealthStatus:
|
|
|
|
|
"""
|
|
|
|
|
The function returns storage node's health status.
|
|
|
|
|
Args:
|
|
|
|
@ -52,9 +52,23 @@ def storage_node_healthcheck(node: StorageNode) -> HealthStatus:
|
|
|
|
|
Returns:
|
|
|
|
|
health status as HealthStatus object.
|
|
|
|
|
"""
|
|
|
|
|
command = "control healthcheck"
|
|
|
|
|
output = _run_control_command_with_retries(node, command)
|
|
|
|
|
return HealthStatus.from_stdout(output)
|
|
|
|
|
|
|
|
|
|
host = node.host
|
|
|
|
|
service_config = host.get_service_config(node.name)
|
|
|
|
|
wallet_path = service_config.attributes["wallet_path"]
|
|
|
|
|
wallet_password = service_config.attributes["wallet_password"]
|
|
|
|
|
|
|
|
|
|
shell = host.get_shell()
|
|
|
|
|
wallet_config_path = f"/tmp/{node.name}-config.yaml"
|
|
|
|
|
wallet_config = f'wallet: {wallet_path}\npassword: "{wallet_password}"'
|
|
|
|
|
shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
|
|
|
|
|
|
|
|
|
|
cli_config = host.get_cli_config("frostfs-cli")
|
|
|
|
|
|
|
|
|
|
cli = FrostfsCli(shell, cli_config.exec_path, wallet_config_path)
|
|
|
|
|
endpoint = node.get_control_endpoint()
|
|
|
|
|
output = cli.control.healthcheck(endpoint)
|
|
|
|
|
return HealthStatus.from_stdout(output.stdout)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@reporter.step("Set status for {node}")
|
|
|
|
@ -66,8 +80,19 @@ def storage_node_set_status(node: StorageNode, status: str, retries: int = 0) ->
|
|
|
|
|
status: online or offline.
|
|
|
|
|
retries (optional, int): number of retry attempts if it didn't work from the first time
|
|
|
|
|
"""
|
|
|
|
|
command = f"control set-status --status {status}"
|
|
|
|
|
_run_control_command_with_retries(node, command, retries)
|
|
|
|
|
host = node.host
|
|
|
|
|
shell = host.get_shell()
|
|
|
|
|
cli_config = host.get_cli_config("frostfs-cli")
|
|
|
|
|
wallet_config_path = f"/tmp/{node.name}-config.yaml"
|
|
|
|
|
service_config = host.get_service_config(node.name)
|
|
|
|
|
wallet_path = service_config.attributes["wallet_path"]
|
|
|
|
|
wallet_password = service_config.attributes["wallet_password"]
|
|
|
|
|
wallet_config = f'wallet: {wallet_path}\npassword: "{wallet_password}"'
|
|
|
|
|
shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
|
|
|
|
|
|
|
|
|
|
cli = FrostfsCli(shell, cli_config.exec_path, wallet_config_path)
|
|
|
|
|
endpoint = node.get_control_endpoint()
|
|
|
|
|
cli.control.set_status(endpoint, status)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@reporter.step("Get netmap snapshot")
|
|
|
|
@ -91,7 +116,7 @@ def get_netmap_snapshot(node: StorageNode, shell: Shell) -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@reporter.step("Get shard list for {node}")
|
|
|
|
|
def node_shard_list(node: StorageNode) -> list[str]:
|
|
|
|
|
def node_shard_list(node: StorageNode, json: Optional[bool] = None) -> list[str]:
|
|
|
|
|
"""
|
|
|
|
|
The function returns list of shards for specified storage node.
|
|
|
|
|
Args:
|
|
|
|
@ -99,20 +124,44 @@ def node_shard_list(node: StorageNode) -> list[str]:
|
|
|
|
|
Returns:
|
|
|
|
|
list of shards.
|
|
|
|
|
"""
|
|
|
|
|
command = "control shards list"
|
|
|
|
|
output = _run_control_command_with_retries(node, command)
|
|
|
|
|
host = node.host
|
|
|
|
|
shell = host.get_shell()
|
|
|
|
|
cli_config = host.get_cli_config("frostfs-cli")
|
|
|
|
|
wallet_config_path = f"/tmp/{node.name}-config.yaml"
|
|
|
|
|
wallet_password = host.get_service_config(node.name).attributes["wallet_password"]
|
|
|
|
|
wallet_config = f'password: "{wallet_password}"'
|
|
|
|
|
service_config = host.get_service_config(node.name)
|
|
|
|
|
wallet_path = service_config.attributes["wallet_path"]
|
|
|
|
|
wallet_password = service_config.attributes["wallet_password"]
|
|
|
|
|
wallet_config = f'wallet: {wallet_path}\npassword: "{wallet_password}"'
|
|
|
|
|
shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
|
|
|
|
|
|
|
|
|
|
cli = FrostfsCli(shell, cli_config.exec_path, wallet_config_path)
|
|
|
|
|
endpoint = node.get_control_endpoint()
|
|
|
|
|
output = cli.shards.list(endpoint=endpoint, json_mode=json)
|
|
|
|
|
return re.findall(r"Shard (.*):", output)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@reporter.step("Shard set for {node}")
|
|
|
|
|
def node_shard_set_mode(node: StorageNode, shard: str, mode: str) -> str:
|
|
|
|
|
def node_shard_set_mode(node: StorageNode, shard: list[str], mode: str) -> str:
|
|
|
|
|
"""
|
|
|
|
|
The function sets mode for specified shard.
|
|
|
|
|
Args:
|
|
|
|
|
node: node on which shard mode should be set.
|
|
|
|
|
"""
|
|
|
|
|
command = f"control shards set-mode --id {shard} --mode {mode}"
|
|
|
|
|
return _run_control_command_with_retries(node, command)
|
|
|
|
|
host = node.host
|
|
|
|
|
shell = host.get_shell()
|
|
|
|
|
cli_config = host.get_cli_config("frostfs-cli")
|
|
|
|
|
wallet_config_path = f"/tmp/{node.name}-config.yaml"
|
|
|
|
|
service_config = host.get_service_config(node.name)
|
|
|
|
|
wallet_path = service_config.attributes["wallet_path"]
|
|
|
|
|
wallet_password = service_config.attributes["wallet_password"]
|
|
|
|
|
wallet_config = f'wallet: {wallet_path}\npassword: "{wallet_password}"'
|
|
|
|
|
shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
|
|
|
|
|
|
|
|
|
|
cli = FrostfsCli(shell, cli_config.exec_path, wallet_config_path)
|
|
|
|
|
endpoint = node.get_control_endpoint()
|
|
|
|
|
return cli.shards.set_mode(endpoint=endpoint, id=shard, mode=mode)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@reporter.step("Drop object from {node}")
|
|
|
|
@ -120,10 +169,22 @@ def drop_object(node: StorageNode, cid: str, oid: str) -> str:
|
|
|
|
|
"""
|
|
|
|
|
The function drops object from specified node.
|
|
|
|
|
Args:
|
|
|
|
|
node_id str: node from which object should be dropped.
|
|
|
|
|
node: node from which object should be dropped.
|
|
|
|
|
"""
|
|
|
|
|
command = f"control drop-objects -o {cid}/{oid}"
|
|
|
|
|
return _run_control_command_with_retries(node, command)
|
|
|
|
|
host = node.host
|
|
|
|
|
shell = host.get_shell()
|
|
|
|
|
cli_config = host.get_cli_config("frostfs-cli")
|
|
|
|
|
wallet_config_path = f"/tmp/{node.name}-config.yaml"
|
|
|
|
|
service_config = host.get_service_config(node.name)
|
|
|
|
|
wallet_path = service_config.attributes["wallet_path"]
|
|
|
|
|
wallet_password = service_config.attributes["wallet_password"]
|
|
|
|
|
wallet_config = f'wallet: {wallet_path}\npassword: "{wallet_password}"'
|
|
|
|
|
shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
|
|
|
|
|
|
|
|
|
|
cli = FrostfsCli(shell, cli_config.exec_path, wallet_config_path)
|
|
|
|
|
endpoint = node.get_control_endpoint()
|
|
|
|
|
objects = f"{cid}/{oid}"
|
|
|
|
|
return cli.control.drop_objects(endpoint, objects)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@reporter.step("Delete data from host for node {node}")
|
|
|
|
@ -238,38 +299,3 @@ def remove_nodes_from_map_morph(
|
|
|
|
|
config_file=FROSTFS_ADM_CONFIG_PATH,
|
|
|
|
|
)
|
|
|
|
|
frostfsadm.morph.remove_nodes(node_netmap_keys)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _run_control_command_with_retries(node: StorageNode, command: str, retries: int = 0) -> str:
|
|
|
|
|
for attempt in range(1 + retries): # original attempt + specified retries
|
|
|
|
|
try:
|
|
|
|
|
return _run_control_command(node, command)
|
|
|
|
|
except AssertionError as err:
|
|
|
|
|
if attempt < retries:
|
|
|
|
|
logger.warning(f"Command {command} failed with error {err} and will be retried")
|
|
|
|
|
continue
|
|
|
|
|
raise AssertionError(f"Command {command} failed with error {err}") from err
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _run_control_command(node: StorageNode, command: str) -> None:
|
|
|
|
|
host = node.host
|
|
|
|
|
|
|
|
|
|
service_config = host.get_service_config(node.name)
|
|
|
|
|
wallet_path = service_config.attributes["wallet_path"]
|
|
|
|
|
wallet_password = service_config.attributes["wallet_password"]
|
|
|
|
|
control_endpoint = service_config.attributes["control_endpoint"]
|
|
|
|
|
|
|
|
|
|
shell = host.get_shell()
|
|
|
|
|
wallet_config_path = f"/tmp/{node.name}-config.yaml"
|
|
|
|
|
wallet_config = f'password: "{wallet_password}"'
|
|
|
|
|
shell.exec(f"echo '{wallet_config}' > {wallet_config_path}")
|
|
|
|
|
|
|
|
|
|
cli_config = host.get_cli_config("frostfs-cli")
|
|
|
|
|
|
|
|
|
|
# TODO: implement cli.control
|
|
|
|
|
# cli = FrostfsCli(shell, cli_config.exec_path, wallet_config_path)
|
|
|
|
|
result = shell.exec(
|
|
|
|
|
f"{cli_config.exec_path} {command} --endpoint {control_endpoint} "
|
|
|
|
|
f"--wallet {wallet_path} --config {wallet_config_path}"
|
|
|
|
|
)
|
|
|
|
|
return result.stdout
|
|
|
|
|