[#330] Switch to new command netmap snapshot

1. Add netmap command to NeofsCli wrapper.
2. Update node_management steps to use netmap.snapshot method instead of
   deprecated "neofs-cli control netmap-snapshot" command.
3. Switch node's public key in netmap from base58-encoding to hex-encoding.

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
Vladimir Domnich 2022-09-28 13:53:08 +04:00 committed by Vladimir
parent 2a175b5824
commit 26032a67ec
4 changed files with 181 additions and 97 deletions

View file

@ -176,12 +176,9 @@ def test_nodes_management(prepare_tmp_dir):
random_node = choice(list(NEOFS_NETMAP_DICT)) random_node = choice(list(NEOFS_NETMAP_DICT))
alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != random_node]) alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != random_node])
# Calculate public key that identifies node in netmap (we need base58-formatted key # Calculate public key that identifies node in netmap
# because keys of storage nodes are base58-encoded in netmap)
random_node_wallet_path = NEOFS_NETMAP_DICT[random_node]["wallet_path"] random_node_wallet_path = NEOFS_NETMAP_DICT[random_node]["wallet_path"]
random_node_netmap_key = get_wallet_public_key( random_node_netmap_key = get_wallet_public_key(random_node_wallet_path, STORAGE_WALLET_PASS)
random_node_wallet_path, STORAGE_WALLET_PASS, format="base58"
)
with allure.step("Check node {random_node} is in netmap"): with allure.step("Check node {random_node} is in netmap"):
snapshot = get_netmap_snapshot(node_name=alive_node) snapshot = get_netmap_snapshot(node_name=alive_node)

View file

@ -1,5 +1,6 @@
from typing import Optional from typing import Optional
from cli_utils.cli.netmap import NeofsCliNetmap
from common import NEOFS_CLI_EXEC from common import NEOFS_CLI_EXEC
from .accounting import NeofsCliAccounting from .accounting import NeofsCliAccounting
@ -10,19 +11,23 @@ from .version import NeofsCliVersion
class NeofsCli: class NeofsCli:
neofs_cli_exec_path: Optional[str] = None accounting: NeofsCliAccounting
config: Optional[str] = None acl: NeofsCliACL
accounting: Optional[NeofsCliAccounting] = None container: NeofsCliContainer
acl: Optional[NeofsCliACL] = None netmap: NeofsCliNetmap
container: Optional[NeofsCliContainer] = None object: NeofsCliObject
object: Optional[NeofsCliObject] = None version: NeofsCliVersion
version: Optional[NeofsCliVersion] = None
def __init__(self, neofs_cli_exec_path: Optional[str] = None, config: Optional[str] = None, timeout: int = 30): def __init__(
self.config = config # config(str): config file (default is $HOME/.config/neofs-cli/config.yaml) self,
self.neofs_cli_exec_path = neofs_cli_exec_path or NEOFS_CLI_EXEC neofs_cli_exec_path: Optional[str] = None,
self.accounting = NeofsCliAccounting(self.neofs_cli_exec_path, timeout=timeout, config=config) config: Optional[str] = None,
self.acl = NeofsCliACL(self.neofs_cli_exec_path, timeout=timeout, config=config) timeout: int = 30,
self.container = NeofsCliContainer(self.neofs_cli_exec_path, timeout=timeout, config=config) ):
self.object = NeofsCliObject(self.neofs_cli_exec_path, timeout=timeout, config=config) neofs_cli_exec_path = neofs_cli_exec_path or NEOFS_CLI_EXEC
self.version = NeofsCliVersion(self.neofs_cli_exec_path, timeout=timeout, config=config) self.accounting = NeofsCliAccounting(neofs_cli_exec_path, timeout=timeout, config=config)
self.acl = NeofsCliACL(neofs_cli_exec_path, timeout=timeout, config=config)
self.container = NeofsCliContainer(neofs_cli_exec_path, timeout=timeout, config=config)
self.netmap = NeofsCliNetmap(neofs_cli_exec_path, timeout=timeout, config=config)
self.object = NeofsCliObject(neofs_cli_exec_path, timeout=timeout, config=config)
self.version = NeofsCliVersion(neofs_cli_exec_path, timeout=timeout, config=config)

View file

@ -0,0 +1,119 @@
from typing import Optional
from cli_utils.cli_command import NeofsCliCommand
class NeofsCliNetmap(NeofsCliCommand):
def epoch(
self,
rpc_endpoint: str,
wallet: str,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
) -> str:
"""
Get current epoch number.
Args:
address: address of wallet account
generate_key: generate new private key
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
ttl: TTL value in request meta header (default 2)
wallet: path to the wallet or binary key
xhdr: request X-Headers in form of Key=Value
Returns:
str: Raw command output
"""
return self._execute(
"netmap epoch",
**{param: value for param, value in locals().items() if param not in ["self"]},
)
def netinfo(
self,
rpc_endpoint: str,
wallet: str,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
) -> str:
"""
Get information about NeoFS network.
Args:
address: address of wallet account
generate_key: generate new private key
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
ttl: TTL value in request meta header (default 2)
wallet: path to the wallet or binary key
xhdr: request X-Headers in form of Key=Value
Returns:
str: Raw command output
"""
return self._execute(
"netmap netinfo",
**{param: value for param, value in locals().items() if param not in ["self"]},
)
def nodeinfo(
self,
rpc_endpoint: str,
wallet: str,
address: Optional[str] = None,
generate_key: bool = False,
json: bool = False,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
) -> str:
"""
Get target node info.
Args:
address: address of wallet account
generate_key: generate new private key
json: print node info in JSON format
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
ttl: TTL value in request meta header (default 2)
wallet: path to the wallet or binary key
xhdr: request X-Headers in form of Key=Value
Returns:
str: Raw command output
"""
return self._execute(
"netmap nodeinfo",
**{param: value for param, value in locals().items() if param not in ["self"]},
)
def snapshot(
self,
rpc_endpoint: str,
wallet: str,
address: Optional[str] = None,
generate_key: bool = False,
ttl: Optional[int] = None,
xhdr: Optional[dict] = None,
) -> str:
"""
Request current local snapshot of the network map.
Args:
address: address of wallet account
generate_key: generate new private key
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
ttl: TTL value in request meta header (default 2)
wallet: path to the wallet or binary key
xhdr: request X-Headers in form of Key=Value
Returns:
str: Raw command output
"""
return self._execute(
"netmap snapshot",
**{param: value for param, value in locals().items() if param not in ["self"]},
)

View file

@ -1,9 +1,5 @@
#!/usr/bin/python3.9 #!/usr/bin/python3.9
"""
This module contains keywords for tests that check management of storage nodes.
"""
import logging import logging
import random import random
import re import re
@ -12,7 +8,8 @@ from dataclasses import dataclass
from typing import Optional from typing import Optional
import allure import allure
from common import MORPH_BLOCK_TIME, NEOFS_NETMAP_DICT, STORAGE_WALLET_PASS from cli_utils.cli.cli import NeofsCli
from common import MORPH_BLOCK_TIME, NEOFS_NETMAP_DICT, STORAGE_WALLET_CONFIG, STORAGE_WALLET_PASS
from data_formatters import get_wallet_public_key from data_formatters import get_wallet_public_key
from epoch import tick_epoch from epoch import tick_epoch
from service_helper import get_storage_service_helper from service_helper import get_storage_service_helper
@ -23,8 +20,8 @@ logger = logging.getLogger("NeoLogger")
@dataclass @dataclass
class HealthStatus: class HealthStatus:
network_status: str = None network_status: Optional[str] = None
health_status: str = None health_status: Optional[str] = None
@staticmethod @staticmethod
def from_stdout(output: str) -> "HealthStatus": def from_stdout(output: str) -> "HealthStatus":
@ -37,16 +34,15 @@ class HealthStatus:
return HealthStatus(network, health) return HealthStatus(network, health)
@allure.step("Stop Nodes") @allure.step("Stop storage nodes")
def stop_nodes(number: int, nodes: list) -> list: def stop_nodes(number: int, nodes: list[str]) -> list[str]:
""" """
The function shuts down the given number of randomly Shuts down the given number of randomly selected storage nodes.
selected nodes in docker.
Args: Args:
number (int): the number of nodes to shut down number (int): the number of nodes to shut down
nodes (list): the list of nodes for possible shut down nodes (list): the list of nodes for possible shut down
Returns: Returns:
(list): the list of nodes which have been shut down (list): the list of nodes that were shut down
""" """
helper = get_storage_service_helper() helper = get_storage_service_helper()
nodes_to_stop = random.sample(nodes, number) nodes_to_stop = random.sample(nodes, number)
@ -55,62 +51,32 @@ def stop_nodes(number: int, nodes: list) -> list:
return nodes_to_stop return nodes_to_stop
@allure.step("Start Nodes") @allure.step("Start storage nodes")
def start_nodes(nodes: list) -> None: def start_nodes(nodes: list[str]) -> None:
""" """
The function raises the given nodes. The function starts specified storage nodes.
Args: Args:
nodes (list): the list of nodes to raise nodes (list): the list of nodes to start
Returns:
(void)
""" """
helper = get_storage_service_helper() helper = get_storage_service_helper()
for node in nodes: for node in nodes:
helper.start(node) helper.start(node)
@allure.step("Get control endpoint and wallet")
def get_control_endpoint_and_wallet(endpoint_number: str = ""):
"""
Gets control endpoint for a random or given node
Args:
endpoint_number (optional, str): the number of the node
in the form of 's01', 's02', etc.
given in NEOFS_NETMAP_DICT as keys
Returns:
(str): the number of the node
(str): endpoint control for the node
(str): the wallet of the respective node
"""
if endpoint_number == "":
endpoint_num = random.choice(list(NEOFS_NETMAP_DICT.keys()))
logger.info(f"Random node chosen: {endpoint_num}")
else:
endpoint_num = endpoint_number
endpoint_values = NEOFS_NETMAP_DICT[f"{endpoint_num}"]
endpoint_control = endpoint_values["control"]
wallet = endpoint_values["wallet_path"]
return endpoint_num, endpoint_control, wallet
@allure.step("Get Locode") @allure.step("Get Locode")
def get_locode(): def get_locode() -> str:
endpoint_values = random.choice(list(NEOFS_NETMAP_DICT.values())) endpoint_values = random.choice(list(NEOFS_NETMAP_DICT.values()))
locode = endpoint_values["UN-LOCODE"] locode = endpoint_values["UN-LOCODE"]
logger.info(f"Random locode chosen: {locode}") logger.info(f"Random locode chosen: {locode}")
return locode return locode
@allure.step("Healthcheck for node") @allure.step("Healthcheck for node {node_name}")
def node_healthcheck(node_name: str) -> HealthStatus: def node_healthcheck(node_name: str) -> HealthStatus:
""" """
The function returns node's health status. The function returns node's health status.
Args: Args:
node_name str: node name to use for netmap snapshot operation node_name str: node name for which health status should be retrieved.
Returns: Returns:
health status as HealthStatus object. health status as HealthStatus object.
""" """
@ -119,41 +85,42 @@ def node_healthcheck(node_name: str) -> HealthStatus:
return HealthStatus.from_stdout(output) return HealthStatus.from_stdout(output)
@allure.step("Set status for node") @allure.step("Set status for node {node_name}")
def node_set_status(node_name: str, status: str, retries: int = 0) -> None: def node_set_status(node_name: str, status: str, retries: int = 0) -> None:
""" """
The function sets particular status for given node. The function sets particular status for given node.
Args: Args:
node_name str: node name to use for netmap snapshot operation node_name str: node name for which status should be set.
status str: online or offline. status str: online or offline.
retries (optional, int): number of retry attempts if it didn't work from the first time retries (optional, int): number of retry attempts if it didn't work from the first time
Returns:
(void)
""" """
command = f"control set-status --status {status}" command = f"control set-status --status {status}"
_run_control_command(node_name, command, retries) _run_control_command(node_name, command, retries)
@allure.step("Get netmap snapshot") @allure.step("Get netmap snapshot")
def get_netmap_snapshot(node_name: Optional[str] = None) -> str: def get_netmap_snapshot(node_name: str) -> str:
""" """
The function returns string representation of netmap-snapshot. The function returns string representation of netmap snapshot.
Args: Args:
node_name str: node name to use for netmap snapshot operation node_name str: node name from which netmap snapshot should be requested.
Returns: Returns:
string representation of netmap-snapshot string representation of netmap
""" """
node_name = node_name or list(NEOFS_NETMAP_DICT)[0] node_info = NEOFS_NETMAP_DICT[node_name]
command = "control netmap-snapshot" cli = NeofsCli(config=STORAGE_WALLET_CONFIG)
return _run_control_command(node_name, command) return cli.netmap.snapshot(
rpc_endpoint=node_info["rpc"],
wallet=node_info["wallet_path"],
)
@allure.step("Shard list for node") @allure.step("Get shard list for node {node_name}")
def node_shard_list(node_name: str) -> list[str]: def node_shard_list(node_name: str) -> list[str]:
""" """
The function returns list of shards for particular node. The function returns list of shards for specified node.
Args: Args:
node_name str: node name to use for netmap snapshot operation node_name str: node name for which shards should be returned.
Returns: Returns:
list of shards. list of shards.
""" """
@ -162,27 +129,23 @@ def node_shard_list(node_name: str) -> list[str]:
return re.findall(r"Shard (.*):", output) return re.findall(r"Shard (.*):", output)
@allure.step("Shard list for node") @allure.step("Shard set for node {node_name}")
def node_shard_set_mode(node_name: str, shard: str, mode: str) -> str: def node_shard_set_mode(node_name: str, shard: str, mode: str) -> str:
""" """
The function sets mode for node's particular shard. The function sets mode for specified shard.
Args: Args:
node_name str: node name to use for netmap snapshot operation node_name str: node name on which shard mode should be set.
Returns:
health status as HealthStatus object.
""" """
command = f"control shards set-mode --id {shard} --mode {mode}" command = f"control shards set-mode --id {shard} --mode {mode}"
return _run_control_command(node_name, command) return _run_control_command(node_name, command)
@allure.step("Drop object from node {node_name}") @allure.step("Drop object from node {node_name}")
def drop_object(node_name: str, cid: str, oid: str) -> str: def drop_object(node_name: str, cid: str, oid: str) -> str:
""" """
The function drops object from particular node. The function drops object from specified node.
Args: Args:
node_name str: node name to use for netmap snapshot operation node_name str: node name from which object should be dropped.
Returns:
health status as HealthStatus object.
""" """
command = f"control drop-objects -o {cid}/{oid}" command = f"control drop-objects -o {cid}/{oid}"
return _run_control_command(node_name, command) return _run_control_command(node_name, command)
@ -197,9 +160,9 @@ def delete_node_data(node_name: str) -> None:
@allure.step("Exclude node {node_to_exclude} from network map") @allure.step("Exclude node {node_to_exclude} from network map")
def exclude_node_from_network_map(node_to_exclude, alive_node): def exclude_node_from_network_map(node_to_exclude: str, alive_node: str) -> None:
node_wallet_path = NEOFS_NETMAP_DICT[node_to_exclude]["wallet_path"] node_wallet_path = NEOFS_NETMAP_DICT[node_to_exclude]["wallet_path"]
node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS, format="base58") node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS)
node_set_status(node_to_exclude, status="offline") node_set_status(node_to_exclude, status="offline")
@ -223,10 +186,10 @@ def include_node_to_network_map(node_to_include: str, alive_node: str) -> None:
@allure.step("Check node {node_name} in network map") @allure.step("Check node {node_name} in network map")
def check_node_in_map(node_name: str, alive_node: str = None): def check_node_in_map(node_name: str, alive_node: Optional[str] = None) -> None:
alive_node = alive_node or node_name alive_node = alive_node or node_name
node_wallet_path = NEOFS_NETMAP_DICT[node_name]["wallet_path"] node_wallet_path = NEOFS_NETMAP_DICT[node_name]["wallet_path"]
node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS, format="base58") node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS)
logger.info(f"Node {node_name} netmap key: {node_netmap_key}") logger.info(f"Node {node_name} netmap key: {node_netmap_key}")