From 26032a67ec09b9affb75a10955f88f9644ca6877 Mon Sep 17 00:00:00 2001 From: Vladimir Domnich Date: Wed, 28 Sep 2022 13:53:08 +0400 Subject: [PATCH] [#330] Switch to new command netmap snapshot 1. Add netmap command to NeofsCli wrapper. 2. Update node_management steps to use netmap.snapshot method instead of deprecated "neofs-cli control netmap-snapshot" command. 3. Switch node's public key in netmap from base58-encoding to hex-encoding. Signed-off-by: Vladimir Domnich --- .../network/test_node_management.py | 7 +- .../lib/python_keywords/cli_utils/cli/cli.py | 35 +++--- .../python_keywords/cli_utils/cli/netmap.py | 119 ++++++++++++++++++ .../lib/python_keywords/node_management.py | 117 ++++++----------- 4 files changed, 181 insertions(+), 97 deletions(-) create mode 100644 robot/resources/lib/python_keywords/cli_utils/cli/netmap.py diff --git a/pytest_tests/testsuites/network/test_node_management.py b/pytest_tests/testsuites/network/test_node_management.py index 367d709..dbbee60 100644 --- a/pytest_tests/testsuites/network/test_node_management.py +++ b/pytest_tests/testsuites/network/test_node_management.py @@ -176,12 +176,9 @@ def test_nodes_management(prepare_tmp_dir): random_node = choice(list(NEOFS_NETMAP_DICT)) alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != random_node]) - # Calculate public key that identifies node in netmap (we need base58-formatted key - # because keys of storage nodes are base58-encoded in netmap) + # Calculate public key that identifies node in netmap random_node_wallet_path = NEOFS_NETMAP_DICT[random_node]["wallet_path"] - random_node_netmap_key = get_wallet_public_key( - random_node_wallet_path, STORAGE_WALLET_PASS, format="base58" - ) + random_node_netmap_key = get_wallet_public_key(random_node_wallet_path, STORAGE_WALLET_PASS) with allure.step("Check node {random_node} is in netmap"): snapshot = get_netmap_snapshot(node_name=alive_node) diff --git a/robot/resources/lib/python_keywords/cli_utils/cli/cli.py b/robot/resources/lib/python_keywords/cli_utils/cli/cli.py index 2bd7eb0..6b1fda5 100644 --- a/robot/resources/lib/python_keywords/cli_utils/cli/cli.py +++ b/robot/resources/lib/python_keywords/cli_utils/cli/cli.py @@ -1,5 +1,6 @@ from typing import Optional +from cli_utils.cli.netmap import NeofsCliNetmap from common import NEOFS_CLI_EXEC from .accounting import NeofsCliAccounting @@ -10,19 +11,23 @@ from .version import NeofsCliVersion class NeofsCli: - neofs_cli_exec_path: Optional[str] = None - config: Optional[str] = None - accounting: Optional[NeofsCliAccounting] = None - acl: Optional[NeofsCliACL] = None - container: Optional[NeofsCliContainer] = None - object: Optional[NeofsCliObject] = None - version: Optional[NeofsCliVersion] = None + accounting: NeofsCliAccounting + acl: NeofsCliACL + container: NeofsCliContainer + netmap: NeofsCliNetmap + object: NeofsCliObject + version: NeofsCliVersion - def __init__(self, neofs_cli_exec_path: Optional[str] = None, config: Optional[str] = None, timeout: int = 30): - self.config = config # config(str): config file (default is $HOME/.config/neofs-cli/config.yaml) - self.neofs_cli_exec_path = neofs_cli_exec_path or NEOFS_CLI_EXEC - self.accounting = NeofsCliAccounting(self.neofs_cli_exec_path, timeout=timeout, config=config) - self.acl = NeofsCliACL(self.neofs_cli_exec_path, timeout=timeout, config=config) - self.container = NeofsCliContainer(self.neofs_cli_exec_path, timeout=timeout, config=config) - self.object = NeofsCliObject(self.neofs_cli_exec_path, timeout=timeout, config=config) - self.version = NeofsCliVersion(self.neofs_cli_exec_path, timeout=timeout, config=config) + def __init__( + self, + neofs_cli_exec_path: Optional[str] = None, + config: Optional[str] = None, + timeout: int = 30, + ): + neofs_cli_exec_path = neofs_cli_exec_path or NEOFS_CLI_EXEC + self.accounting = NeofsCliAccounting(neofs_cli_exec_path, timeout=timeout, config=config) + self.acl = NeofsCliACL(neofs_cli_exec_path, timeout=timeout, config=config) + self.container = NeofsCliContainer(neofs_cli_exec_path, timeout=timeout, config=config) + self.netmap = NeofsCliNetmap(neofs_cli_exec_path, timeout=timeout, config=config) + self.object = NeofsCliObject(neofs_cli_exec_path, timeout=timeout, config=config) + self.version = NeofsCliVersion(neofs_cli_exec_path, timeout=timeout, config=config) diff --git a/robot/resources/lib/python_keywords/cli_utils/cli/netmap.py b/robot/resources/lib/python_keywords/cli_utils/cli/netmap.py new file mode 100644 index 0000000..98cb26c --- /dev/null +++ b/robot/resources/lib/python_keywords/cli_utils/cli/netmap.py @@ -0,0 +1,119 @@ +from typing import Optional + +from cli_utils.cli_command import NeofsCliCommand + + +class NeofsCliNetmap(NeofsCliCommand): + def epoch( + self, + rpc_endpoint: str, + wallet: str, + address: Optional[str] = None, + generate_key: bool = False, + ttl: Optional[int] = None, + xhdr: Optional[dict] = None, + ) -> str: + """ + Get current epoch number. + + Args: + address: address of wallet account + generate_key: generate new private key + rpc_endpoint: remote node address (as 'multiaddr' or ':') + ttl: TTL value in request meta header (default 2) + wallet: path to the wallet or binary key + xhdr: request X-Headers in form of Key=Value + + Returns: + str: Raw command output + """ + return self._execute( + "netmap epoch", + **{param: value for param, value in locals().items() if param not in ["self"]}, + ) + + def netinfo( + self, + rpc_endpoint: str, + wallet: str, + address: Optional[str] = None, + generate_key: bool = False, + ttl: Optional[int] = None, + xhdr: Optional[dict] = None, + ) -> str: + """ + Get information about NeoFS network. + + Args: + address: address of wallet account + generate_key: generate new private key + rpc_endpoint: remote node address (as 'multiaddr' or ':') + ttl: TTL value in request meta header (default 2) + wallet: path to the wallet or binary key + xhdr: request X-Headers in form of Key=Value + + Returns: + str: Raw command output + """ + return self._execute( + "netmap netinfo", + **{param: value for param, value in locals().items() if param not in ["self"]}, + ) + + def nodeinfo( + self, + rpc_endpoint: str, + wallet: str, + address: Optional[str] = None, + generate_key: bool = False, + json: bool = False, + ttl: Optional[int] = None, + xhdr: Optional[dict] = None, + ) -> str: + """ + Get target node info. + + Args: + address: address of wallet account + generate_key: generate new private key + json: print node info in JSON format + rpc_endpoint: remote node address (as 'multiaddr' or ':') + ttl: TTL value in request meta header (default 2) + wallet: path to the wallet or binary key + xhdr: request X-Headers in form of Key=Value + + Returns: + str: Raw command output + """ + return self._execute( + "netmap nodeinfo", + **{param: value for param, value in locals().items() if param not in ["self"]}, + ) + + def snapshot( + self, + rpc_endpoint: str, + wallet: str, + address: Optional[str] = None, + generate_key: bool = False, + ttl: Optional[int] = None, + xhdr: Optional[dict] = None, + ) -> str: + """ + Request current local snapshot of the network map. + + Args: + address: address of wallet account + generate_key: generate new private key + rpc_endpoint: remote node address (as 'multiaddr' or ':') + ttl: TTL value in request meta header (default 2) + wallet: path to the wallet or binary key + xhdr: request X-Headers in form of Key=Value + + Returns: + str: Raw command output + """ + return self._execute( + "netmap snapshot", + **{param: value for param, value in locals().items() if param not in ["self"]}, + ) diff --git a/robot/resources/lib/python_keywords/node_management.py b/robot/resources/lib/python_keywords/node_management.py index 2bd4935..aa0ee39 100644 --- a/robot/resources/lib/python_keywords/node_management.py +++ b/robot/resources/lib/python_keywords/node_management.py @@ -1,9 +1,5 @@ #!/usr/bin/python3.9 -""" - This module contains keywords for tests that check management of storage nodes. -""" - import logging import random import re @@ -12,7 +8,8 @@ from dataclasses import dataclass from typing import Optional import allure -from common import MORPH_BLOCK_TIME, NEOFS_NETMAP_DICT, STORAGE_WALLET_PASS +from cli_utils.cli.cli import NeofsCli +from common import MORPH_BLOCK_TIME, NEOFS_NETMAP_DICT, STORAGE_WALLET_CONFIG, STORAGE_WALLET_PASS from data_formatters import get_wallet_public_key from epoch import tick_epoch from service_helper import get_storage_service_helper @@ -23,8 +20,8 @@ logger = logging.getLogger("NeoLogger") @dataclass class HealthStatus: - network_status: str = None - health_status: str = None + network_status: Optional[str] = None + health_status: Optional[str] = None @staticmethod def from_stdout(output: str) -> "HealthStatus": @@ -37,16 +34,15 @@ class HealthStatus: return HealthStatus(network, health) -@allure.step("Stop Nodes") -def stop_nodes(number: int, nodes: list) -> list: +@allure.step("Stop storage nodes") +def stop_nodes(number: int, nodes: list[str]) -> list[str]: """ - The function shuts down the given number of randomly - selected nodes in docker. + Shuts down the given number of randomly selected storage nodes. Args: number (int): the number of nodes to shut down nodes (list): the list of nodes for possible shut down Returns: - (list): the list of nodes which have been shut down + (list): the list of nodes that were shut down """ helper = get_storage_service_helper() nodes_to_stop = random.sample(nodes, number) @@ -55,62 +51,32 @@ def stop_nodes(number: int, nodes: list) -> list: return nodes_to_stop -@allure.step("Start Nodes") -def start_nodes(nodes: list) -> None: +@allure.step("Start storage nodes") +def start_nodes(nodes: list[str]) -> None: """ - The function raises the given nodes. + The function starts specified storage nodes. Args: - nodes (list): the list of nodes to raise - Returns: - (void) + nodes (list): the list of nodes to start """ helper = get_storage_service_helper() for node in nodes: helper.start(node) -@allure.step("Get control endpoint and wallet") -def get_control_endpoint_and_wallet(endpoint_number: str = ""): - """ - Gets control endpoint for a random or given node - - Args: - endpoint_number (optional, str): the number of the node - in the form of 's01', 's02', etc. - given in NEOFS_NETMAP_DICT as keys - Returns: - (str): the number of the node - (str): endpoint control for the node - (str): the wallet of the respective node - """ - if endpoint_number == "": - endpoint_num = random.choice(list(NEOFS_NETMAP_DICT.keys())) - logger.info(f"Random node chosen: {endpoint_num}") - else: - endpoint_num = endpoint_number - - endpoint_values = NEOFS_NETMAP_DICT[f"{endpoint_num}"] - endpoint_control = endpoint_values["control"] - wallet = endpoint_values["wallet_path"] - - return endpoint_num, endpoint_control, wallet - - @allure.step("Get Locode") -def get_locode(): +def get_locode() -> str: endpoint_values = random.choice(list(NEOFS_NETMAP_DICT.values())) locode = endpoint_values["UN-LOCODE"] logger.info(f"Random locode chosen: {locode}") - return locode -@allure.step("Healthcheck for node") +@allure.step("Healthcheck for node {node_name}") def node_healthcheck(node_name: str) -> HealthStatus: """ The function returns node's health status. Args: - node_name str: node name to use for netmap snapshot operation + node_name str: node name for which health status should be retrieved. Returns: health status as HealthStatus object. """ @@ -119,41 +85,42 @@ def node_healthcheck(node_name: str) -> HealthStatus: return HealthStatus.from_stdout(output) -@allure.step("Set status for node") +@allure.step("Set status for node {node_name}") def node_set_status(node_name: str, status: str, retries: int = 0) -> None: """ The function sets particular status for given node. Args: - node_name str: node name to use for netmap snapshot operation + node_name str: node name for which status should be set. status str: online or offline. retries (optional, int): number of retry attempts if it didn't work from the first time - Returns: - (void) """ command = f"control set-status --status {status}" _run_control_command(node_name, command, retries) @allure.step("Get netmap snapshot") -def get_netmap_snapshot(node_name: Optional[str] = None) -> str: +def get_netmap_snapshot(node_name: str) -> str: """ - The function returns string representation of netmap-snapshot. + The function returns string representation of netmap snapshot. Args: - node_name str: node name to use for netmap snapshot operation + node_name str: node name from which netmap snapshot should be requested. Returns: - string representation of netmap-snapshot + string representation of netmap """ - node_name = node_name or list(NEOFS_NETMAP_DICT)[0] - command = "control netmap-snapshot" - return _run_control_command(node_name, command) + node_info = NEOFS_NETMAP_DICT[node_name] + cli = NeofsCli(config=STORAGE_WALLET_CONFIG) + return cli.netmap.snapshot( + rpc_endpoint=node_info["rpc"], + wallet=node_info["wallet_path"], + ) -@allure.step("Shard list for node") +@allure.step("Get shard list for node {node_name}") def node_shard_list(node_name: str) -> list[str]: """ - The function returns list of shards for particular node. + The function returns list of shards for specified node. Args: - node_name str: node name to use for netmap snapshot operation + node_name str: node name for which shards should be returned. Returns: list of shards. """ @@ -162,27 +129,23 @@ def node_shard_list(node_name: str) -> list[str]: return re.findall(r"Shard (.*):", output) -@allure.step("Shard list for node") +@allure.step("Shard set for node {node_name}") def node_shard_set_mode(node_name: str, shard: str, mode: str) -> str: """ - The function sets mode for node's particular shard. + The function sets mode for specified shard. Args: - node_name str: node name to use for netmap snapshot operation - Returns: - health status as HealthStatus object. + node_name str: node name on which shard mode should be set. """ - command = f"control shards set-mode --id {shard} --mode {mode}" + command = f"control shards set-mode --id {shard} --mode {mode}" return _run_control_command(node_name, command) @allure.step("Drop object from node {node_name}") def drop_object(node_name: str, cid: str, oid: str) -> str: """ - The function drops object from particular node. + The function drops object from specified node. Args: - node_name str: node name to use for netmap snapshot operation - Returns: - health status as HealthStatus object. + node_name str: node name from which object should be dropped. """ command = f"control drop-objects -o {cid}/{oid}" return _run_control_command(node_name, command) @@ -197,9 +160,9 @@ def delete_node_data(node_name: str) -> None: @allure.step("Exclude node {node_to_exclude} from network map") -def exclude_node_from_network_map(node_to_exclude, alive_node): +def exclude_node_from_network_map(node_to_exclude: str, alive_node: str) -> None: node_wallet_path = NEOFS_NETMAP_DICT[node_to_exclude]["wallet_path"] - node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS, format="base58") + node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS) node_set_status(node_to_exclude, status="offline") @@ -223,10 +186,10 @@ def include_node_to_network_map(node_to_include: str, alive_node: str) -> None: @allure.step("Check node {node_name} in network map") -def check_node_in_map(node_name: str, alive_node: str = None): +def check_node_in_map(node_name: str, alive_node: Optional[str] = None) -> None: alive_node = alive_node or node_name node_wallet_path = NEOFS_NETMAP_DICT[node_name]["wallet_path"] - node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS, format="base58") + node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS) logger.info(f"Node {node_name} netmap key: {node_netmap_key}")