anurindm/frostfs-testlib
1
0
Fork 0
forked from TrueCloudLab/frostfs-testlib
Code Pull requests Activity

Add support test maintenance

Browse source 
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
This commit is contained in:
Dmitriy Zayakin 2023-11-20 13:54:47 +03:00 • committed by Dmitriy Zayakin
parent 22647c6d59
commit ed70dada96
8 changed files with 290 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

96
src/frostfs_testlib/cli/frostfs_adm/morph.py
View file

@ -27,11 +27,7 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph deposit-notary",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def dump_balances(
@ -56,11 +52,7 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph dump-balances",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def dump_config(self, rpc_endpoint: str) -> CommandResult:
@ -74,11 +66,25 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph dump-config",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def set_config(
self, set_key_value: str, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None
) -> CommandResult:
"""Add/update global config value in the FrostFS network.
Args:
set_key_value: key1=val1 [key2=val2 ...]
alphabet_wallets: Path to alphabet wallets dir
rpc_endpoint: N3 RPC node endpoint
Returns:
Command's result.
"""
return self._execute(
f"morph set-config {set_key_value}",
**{param: param_value for param, param_value in locals().items() if param not in ["self", "set_key_value"]},
)
def dump_containers(
@ -101,11 +107,7 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph dump-containers",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def dump_hashes(self, rpc_endpoint: str) -> CommandResult:
@ -119,11 +121,7 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph dump-hashes",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def force_new_epoch(
@ -140,11 +138,7 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph force-new-epoch",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def generate_alphabet(
@ -165,11 +159,7 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph generate-alphabet",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def generate_storage_wallet(
@ -192,11 +182,7 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph generate-storage-wallet",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def init(
@ -232,11 +218,7 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph init",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def refill_gas(
@ -259,11 +241,7 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph refill-gas",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def restore_containers(
@ -286,11 +264,7 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph restore-containers",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def set_policy(
@ -348,17 +322,13 @@ class FrostfsAdmMorph(CliCommand):
"""
return self._execute(
"morph update-contracts",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def remove_nodes(
self, node_netmap_keys: list[str], rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None
) -> CommandResult:
""" Move node to the Offline state in the candidates list
"""Move node to the Offline state in the candidates list
and tick an epoch to update the netmap using frostfs-adm
Args:
@ -371,7 +341,7 @@ class FrostfsAdmMorph(CliCommand):
"""
if not len(node_netmap_keys):
raise AttributeError("Got empty node_netmap_keys list")
return self._execute(
f"morph remove-nodes {' '.join(node_netmap_keys)}",
**{
@ -379,4 +349,4 @@ class FrostfsAdmMorph(CliCommand):
for param, param_value in locals().items()
if param not in ["self", "node_netmap_keys"]
},
)
)

3
src/frostfs_testlib/cli/frostfs_cli/cli.py
View file

@ -3,6 +3,7 @@ from typing import Optional
from frostfs_testlib.cli.frostfs_cli.accounting import FrostfsCliAccounting
from frostfs_testlib.cli.frostfs_cli.acl import FrostfsCliACL
from frostfs_testlib.cli.frostfs_cli.container import FrostfsCliContainer
from frostfs_testlib.cli.frostfs_cli.control import FrostfsCliControl
from frostfs_testlib.cli.frostfs_cli.netmap import FrostfsCliNetmap
from frostfs_testlib.cli.frostfs_cli.object import FrostfsCliObject
from frostfs_testlib.cli.frostfs_cli.session import FrostfsCliSession
@ -25,6 +26,7 @@ class FrostfsCli:
storagegroup: FrostfsCliStorageGroup
util: FrostfsCliUtil
version: FrostfsCliVersion
control: FrostfsCliControl
def __init__(self, shell: Shell, frostfs_cli_exec_path: str, config_file: Optional[str] = None):
self.accounting = FrostfsCliAccounting(shell, frostfs_cli_exec_path, config=config_file)
@ -38,3 +40,4 @@ class FrostfsCli:
self.util = FrostfsCliUtil(shell, frostfs_cli_exec_path, config=config_file)
self.version = FrostfsCliVersion(shell, frostfs_cli_exec_path, config=config_file)
self.tree = FrostfsCliTree(shell, frostfs_cli_exec_path, config=config_file)
self.control = FrostfsCliControl(shell, frostfs_cli_exec_path, config=config_file)

58
src/frostfs_testlib/cli/frostfs_cli/control.py Normal file
View file

@ -0,0 +1,58 @@
from typing import Optional
from frostfs_testlib.cli.cli_command import CliCommand
from frostfs_testlib.shell import CommandResult
class FrostfsCliControl(CliCommand):
def set_status(
self,
endpoint: str,
status: str,
wallet: Optional[str] = None,
force: Optional[bool] = None,
address: Optional[str] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""Set status of the storage node in FrostFS network map
Args:
wallet: Path to the wallet or binary key
address: Address of wallet account
endpoint: Remote node control address (as 'multiaddr' or '<host>:<port>')
force: Force turning to local maintenance
status: New netmap status keyword ('online', 'offline', 'maintenance')
timeout: Timeout for an operation (default 15s)
Returns:
Command`s result.
"""
return self._execute(
"control set-status",
**{param: value for param, value in locals().items() if param not in ["self"]},
)
def healthcheck(
self,
endpoint: str,
wallet: Optional[str] = None,
address: Optional[str] = None,
timeout: Optional[str] = None,
) -> CommandResult:
"""Set status of the storage node in FrostFS network map
Args:
wallet: Path to the wallet or binary key
address: Address of wallet account
endpoint: Remote node control address (as 'multiaddr' or '<host>:<port>')
force: Force turning to local maintenance
status: New netmap status keyword ('online', 'offline', 'maintenance')
timeout: Timeout for an operation (default 15s)
Returns:
Command`s result.
"""
return self._execute(
"control healthcheck",
**{param: value for param, value in locals().items() if param not in ["self"]},
)

86
src/frostfs_testlib/cli/netmap_parser.py Normal file
View file

@ -0,0 +1,86 @@
import re
from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.dataclasses.storage_object_info import NodeNetInfo, NodeNetmapInfo
class NetmapParser:
@staticmethod
def netinfo(output: str) -> NodeNetInfo:
regexes = {
"epoch": r"Epoch: (?P<epoch>\d+)",
"network_magic": r"Network magic: (?P<network_magic>.*$)",
"time_per_block": r"Time per block: (?P<time_per_block>\d+\w+)",
"container_fee": r"Container fee: (?P<container_fee>\d+)",
"epoch_duration": r"Epoch duration: (?P<epoch_duration>\d+)",
"inner_ring_candidate_fee": r"Inner Ring candidate fee: (?P<inner_ring_candidate_fee>\d+)",
"maximum_object_size": r"Maximum object size: (?P<maximum_object_size>\d+)",
"withdrawal_fee": r"Withdrawal fee: (?P<withdrawal_fee>\d+)",
"homomorphic_hashing_disabled": r"Homomorphic hashing disabled: (?P<homomorphic_hashing_disabled>true|false)",
"maintenance_mode_allowed": r"Maintenance mode allowed: (?P<maintenance_mode_allowed>true|false)",
"eigen_trust_alpha": r"EigenTrustAlpha: (?P<eigen_trust_alpha>\d+\w+$)",
"eigen_trust_iterations": r"EigenTrustIterations: (?P<eigen_trust_iterations>\d+)",
}
parse_result = {}
for key, regex in regexes.items():
search_result = re.search(regex, output, flags=re.MULTILINE)
if search_result == None:
parse_result[key] = None
continue
parse_result[key] = search_result[key].strip()
node_netinfo = NodeNetInfo(**parse_result)
return node_netinfo
@staticmethod
def snapshot_all_nodes(output: str) -> list[NodeNetmapInfo]:
"""The code will parse each line and return each node as dataclass."""
netmap_nodes = output.split("Node ")[1:]
dataclasses_netmap = []
result_netmap = {}
regexes = {
"node_id": r"\d+: (?P<node_id>\w+)",
"node_data_ips": r"(?P<node_data_ips>/ip4/.+?)$",
"node_status": r"(?P<node_status>ONLINE|OFFLINE)",
"cluster_name": r"ClusterName: (?P<cluster_name>\w+)",
"continent": r"Continent: (?P<continent>\w+)",
"country": r"Country: (?P<country>\w+)",
"country_code": r"CountryCode: (?P<country_code>\w+)",
"external_address": r"ExternalAddr: (?P<external_address>/ip[4].+?)$",
"location": r"Location: (?P<location>\w+.*)",
"node": r"Node: (?P<node>\d+\.\d+\.\d+\.\d+)",
"price": r"Price: (?P<price>\d+)",
"sub_div": r"SubDiv: (?P<sub_div>.*)",
"sub_div_code": r"SubDivCode: (?P<sub_div_code>\w+)",
"un_locode": r"UN-LOCODE: (?P<un_locode>\w+.*)",
"role": r"role: (?P<role>\w+)",
}
for node in netmap_nodes:
for key, regex in regexes.items():
search_result = re.search(regex, node, flags=re.MULTILINE)
if key == "node_data_ips":
result_netmap[key] = search_result[key].strip().split(" ")
continue
if key == "external_address":
result_netmap[key] = search_result[key].strip().split(",")
continue
if search_result == None:
result_netmap[key] = None
continue
result_netmap[key] = search_result[key].strip()
dataclasses_netmap.append(NodeNetmapInfo(**result_netmap))
return dataclasses_netmap
@staticmethod
def snapshot_one_node(output: str, cluster_node: ClusterNode) -> NodeNetmapInfo | None:
snapshot_nodes = NetmapParser.snapshot_all_nodes(output=output)
snapshot_node = [node for node in snapshot_nodes if node.node == cluster_node.host_ip]
if not snapshot_node:
return None
return snapshot_node[0]