From be39480adea9646df72049cda55a34d8c0b33d12 Mon Sep 17 00:00:00 2001 From: Vladimir Domnich Date: Thu, 7 Jul 2022 22:48:47 +0400 Subject: [PATCH] Fix node management tests Remote connection was created to the 1st storage node only. While in reality we want to create connection to a specific node. Signed-off-by: Vladimir Domnich --- neofs-keywords | 2 +- .../testsuites/container/test_container.py | 2 +- .../network/test_node_management.py | 60 ++++---- .../lib/python_keywords/node_management.py | 131 +++++++++--------- robot/variables/common.py | 3 +- 5 files changed, 99 insertions(+), 99 deletions(-) diff --git a/neofs-keywords b/neofs-keywords index 1d7ebe1d..58650099 160000 --- a/neofs-keywords +++ b/neofs-keywords @@ -1 +1 @@ -Subproject commit 1d7ebe1d6d4bb6c8ce62f36091347bddb54d333b +Subproject commit 58650099362f05b51b97f04e2244a4ba20f88351 diff --git a/pytest_tests/testsuites/container/test_container.py b/pytest_tests/testsuites/container/test_container.py index 79c64c58..9224ffaa 100644 --- a/pytest_tests/testsuites/container/test_container.py +++ b/pytest_tests/testsuites/container/test_container.py @@ -4,7 +4,7 @@ from time import sleep import allure import pytest -from contract_keywords import tick_epoch +from epoch import tick_epoch from python_keywords.container import create_container, get_container, list_containers, delete_container from utility import placement_policy_from_container diff --git a/pytest_tests/testsuites/network/test_node_management.py b/pytest_tests/testsuites/network/test_node_management.py index 593e03db..92fc97fc 100644 --- a/pytest_tests/testsuites/network/test_node_management.py +++ b/pytest_tests/testsuites/network/test_node_management.py @@ -6,7 +6,7 @@ import allure import pytest from common import (COMPLEX_OBJ_SIZE, MAINNET_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT, NEOFS_NETMAP_DICT, SHARD_0_GC_SLEEP) -from contract_keywords import tick_epoch +from epoch import tick_epoch from utility_keywords import generate_file from python_keywords.container import create_container, get_container from python_keywords.neofs_verbs import (delete_object, get_object, @@ -28,7 +28,7 @@ logger = logging.getLogger('NeoLogger') @pytest.fixture @allure.title('Create container and pick the node with data') -def crate_container_and_pick_node(create_remote_connection, prepare_wallet_and_deposit): +def crate_container_and_pick_node(prepare_wallet_and_deposit): wallet = prepare_wallet_and_deposit file_path = generate_file() placement_rule = 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X' @@ -44,63 +44,63 @@ def crate_container_and_pick_node(create_remote_connection, prepare_wallet_and_d yield cid, node_name - shards = node_shard_list(create_remote_connection, node_name) + shards = node_shard_list(node_name) assert shards for shard in shards: - node_shard_set_mode(create_remote_connection, node_name, shard, 'read-write') + node_shard_set_mode(node_name, shard, 'read-write') - node_shard_list(create_remote_connection, node_name) + node_shard_list(node_name) @pytest.fixture @pytest.mark.skip(reason="docker API works only for devenv") -def start_node_if_needed(create_remote_connection): +def start_node_if_needed(): yield try: - start_nodes_remote(create_remote_connection, list(NEOFS_NETMAP_DICT.keys())) + start_nodes_remote(list(NEOFS_NETMAP_DICT.keys())) except Exception as err: logger.error(f'Node start fails with error:\n{err}') @allure.title('Control Operations with storage nodes') @pytest.mark.node_mgmt -def test_nodes_management(prepare_tmp_dir, create_remote_connection): +def test_nodes_management(prepare_tmp_dir): """ This test checks base control operations with storage nodes (healthcheck, netmap-snapshot, set-status). """ random_node = choice(list(NEOFS_NETMAP_DICT)) alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != random_node]) - snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node) + snapshot = get_netmap_snapshot(node_name=alive_node) assert random_node in snapshot, f'Expected node {random_node} in netmap' with allure.step('Run health check for all storage nodes'): for node_name in NEOFS_NETMAP_DICT.keys(): - health_check = node_healthcheck(create_remote_connection, node_name) + health_check = node_healthcheck(node_name) assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE' with allure.step(f'Move node {random_node} to offline state'): - node_set_status(create_remote_connection, random_node, status='offline') + node_set_status(random_node, status='offline') sleep(robot_time_to_int(MAINNET_BLOCK_TIME)) tick_epoch() with allure.step(f'Check node {random_node} went to offline'): - health_check = node_healthcheck(create_remote_connection, random_node) + health_check = node_healthcheck(random_node) assert health_check.health_status == 'READY' and health_check.network_status == 'STATUS_UNDEFINED' - snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node) + snapshot = get_netmap_snapshot(node_name=alive_node) assert random_node not in snapshot, f'Expected node {random_node} not in netmap' with allure.step(f'Check node {random_node} went to online'): - node_set_status(create_remote_connection, random_node, status='online') + node_set_status(random_node, status='online') sleep(robot_time_to_int(MAINNET_BLOCK_TIME)) tick_epoch() with allure.step(f'Check node {random_node} went to online'): - health_check = node_healthcheck(create_remote_connection, random_node) + health_check = node_healthcheck(random_node) assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE' - snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node) + snapshot = get_netmap_snapshot(node_name=alive_node) assert random_node in snapshot, f'Expected node {random_node} in netmap' @@ -170,7 +170,7 @@ def test_placement_policy_negative(prepare_wallet_and_deposit, placement_rule, e @pytest.mark.node_mgmt @pytest.mark.skip(reason="docker API works only for devenv") @allure.title('NeoFS object replication on node failover') -def test_replication(prepare_wallet_and_deposit, create_remote_connection, start_node_if_needed): +def test_replication(prepare_wallet_and_deposit, start_node_if_needed): """ Test checks object replication on storage not failover and come back. """ @@ -185,22 +185,22 @@ def test_replication(prepare_wallet_and_deposit, create_remote_connection, start assert len(nodes) == expected_nodes_count, f'Expected {expected_nodes_count} copies, got {len(nodes)}' node_names = [name for name, config in NEOFS_NETMAP_DICT.items() if config.get('rpc') in nodes] - stopped_nodes = stop_nodes_remote(create_remote_connection, 1, node_names) + stopped_nodes = stop_nodes_remote(1, node_names) wait_for_expected_object_copies(wallet, cid, oid) - start_nodes_remote(create_remote_connection, stopped_nodes) + start_nodes_remote(stopped_nodes) tick_epoch() for node_name in node_names: - wait_for_node_go_online(create_remote_connection, node_name) + wait_for_node_go_online(node_name) wait_for_expected_object_copies(wallet, cid, oid) @pytest.mark.node_mgmt @allure.title('NeoFS object could be dropped using control command') -def test_drop_object(prepare_wallet_and_deposit, create_remote_connection): +def test_drop_object(prepare_wallet_and_deposit): """ Test checks object could be dropped using `neofs-cli control drop-objects` command. """ @@ -224,7 +224,7 @@ def test_drop_object(prepare_wallet_and_deposit, create_remote_connection): with allure.step(f'Drop object {oid}'): get_object(wallet, cid, oid) head_object(wallet, cid, oid) - drop_object(create_remote_connection, node_name, cid, oid) + drop_object(node_name, cid, oid) wait_for_obj_dropped(wallet, cid, oid, get_object) wait_for_obj_dropped(wallet, cid, oid, head_object) @@ -232,7 +232,7 @@ def test_drop_object(prepare_wallet_and_deposit, create_remote_connection): @pytest.mark.node_mgmt @pytest.mark.skip(reason='Need to clarify scenario') @allure.title('Control Operations with storage nodes') -def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_container_and_pick_node): +def test_shards(prepare_wallet_and_deposit, crate_container_and_pick_node): """ This test checks base control operations with storage nodes (healthcheck, netmap-snapshot, set-status). """ @@ -244,13 +244,13 @@ def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_cont # for mode in ('read-only', 'degraded'): for mode in ('degraded',): - shards = node_shard_list(create_remote_connection, node_name) + shards = node_shard_list(node_name) assert shards for shard in shards: - node_shard_set_mode(create_remote_connection, node_name, shard, mode) + node_shard_set_mode(node_name, shard, mode) - shards = node_shard_list(create_remote_connection, node_name) + shards = node_shard_list(node_name) assert shards with pytest.raises(RuntimeError): @@ -263,9 +263,9 @@ def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_cont get_object(wallet, cid, original_oid) for shard in shards: - node_shard_set_mode(create_remote_connection, node_name, shard, 'read-write') + node_shard_set_mode(node_name, shard, 'read-write') - shards = node_shard_list(create_remote_connection, node_name) + shards = node_shard_list(node_name) assert shards oid = put_object(wallet, file_path, cid) @@ -285,11 +285,11 @@ def validate_object_copies(wallet: str, placement_rule: str, file_path: str, exp @allure.step('Wait for node {node_name} goes online') -def wait_for_node_go_online(create_remote_connection, node_name: str): +def wait_for_node_go_online(node_name: str): timeout, attempts = 5, 20 for _ in range(attempts): try: - health_check = node_healthcheck(create_remote_connection, node_name) + health_check = node_healthcheck(node_name) assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE' return except Exception as err: diff --git a/robot/resources/lib/python_keywords/node_management.py b/robot/resources/lib/python_keywords/node_management.py index 24edc4dc..2c21a7fa 100644 --- a/robot/resources/lib/python_keywords/node_management.py +++ b/robot/resources/lib/python_keywords/node_management.py @@ -7,11 +7,12 @@ import random import re +from contextlib import contextmanager from dataclasses import dataclass from typing import List import docker -from common import NEOFS_NETMAP_DICT, STORAGE_NODE_BIN_PATH, STORAGE_NODE_CONFIG_PATH +from common import NEOFS_NETMAP_DICT, STORAGE_NODE_BIN_PATH, STORAGE_NODE_CONFIG_PATH, STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT, STORAGE_NODE_PWD, STORAGE_NODE_USER from robot.api import logger from robot.api.deco import keyword from ssh_helper import HostClient @@ -35,8 +36,23 @@ class HealthStatus: return HealthStatus(network, health) +@contextmanager +def create_ssh_client(node_name: str) -> HostClient: + if node_name not in NEOFS_NETMAP_DICT: + raise AssertionError(f'Node {node_name} is not found!') + + node_config = NEOFS_NETMAP_DICT.get(node_name) + host = node_config.get('control').split(':')[0] + ssh_client = HostClient(host, STORAGE_NODE_USER, STORAGE_NODE_PWD) + + try: + yield ssh_client + finally: + ssh_client.drop() + + @keyword('Stop Nodes') -def stop_nodes(number: int, nodes: list): +def stop_nodes(number: int, nodes: list) -> None: """ The function shuts down the given number of randomly selected nodes in docker. @@ -55,7 +71,7 @@ def stop_nodes(number: int, nodes: list): @keyword('Start Nodes') -def start_nodes(nodes: list): +def start_nodes(nodes: list) -> None: """ The function raises the given nodes. Args: @@ -106,12 +122,11 @@ def get_locode(): @keyword('Stop Nodes Remote') -def stop_nodes_remote(client: HostClient, number: int, nodes: list): +def stop_nodes_remote(number: int, nodes: list) -> None: """ The function shuts down the given number of randomly selected nodes in docker. Args: - client (HostClient): client that implements exec command number (int): the number of nodes to shut down nodes (list): the list of nodes for possible shut down Returns: @@ -120,137 +135,123 @@ def stop_nodes_remote(client: HostClient, number: int, nodes: list): nodes = random.sample(nodes, number) for node in nodes: node = node.split('.')[0] - client.exec(f'docker stop {node}') + with create_ssh_client(node) as ssh_client: + ssh_client.exec(f'docker stop {node}') return nodes @keyword('Start Nodes Remote') -def start_nodes_remote(client: HostClient, nodes: list): +def start_nodes_remote(nodes: list) -> None: """ The function starts nodes in docker. Args: - client (HostClient): client that implements exec command nodes (list): the list of nodes for possible shut down """ for node in nodes: node = node.split('.')[0] - client.exec(f'docker start {node}') + with create_ssh_client(node) as ssh_client: + ssh_client.exec(f'docker start {node}') @keyword('Healthcheck for node') -def node_healthcheck(client: HostClient, node_name: str) -> HealthStatus: +def node_healthcheck(node_name: str) -> HealthStatus: """ The function returns node's health status. Args: - client HostClient: client that implements exec command. node_name str: node name to use for netmap snapshot operation Returns: health status as HealthStatus object. """ - if node_name not in NEOFS_NETMAP_DICT: - raise AssertionError(f'Node {node_name} is not found!') - - node_config = NEOFS_NETMAP_DICT.get(node_name) - control_url = node_config.get('control') - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control healthcheck --endpoint {control_url} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH}' - output = client.exec_with_confirmation(cmd, ['']) - return HealthStatus.from_stdout(output.stdout) + with create_ssh_client(node_name) as ssh_client: + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control healthcheck ' \ + f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH}' + output = ssh_client.exec_with_confirmation(cmd, ['']) + return HealthStatus.from_stdout(output.stdout) @keyword('Set status for node') -def node_set_status(client: HostClient, node_name: str, status: str): +def node_set_status(node_name: str, status: str): """ The function sets particular status for given node. Args: - client HostClient: client that implements exec command. node_name str: node name to use for netmap snapshot operation status str: online or offline. Returns: (void) """ - if node_name not in NEOFS_NETMAP_DICT: - raise AssertionError(f'Node {node_name} is not found!') - - node_config = NEOFS_NETMAP_DICT.get(node_name) - control_url = node_config.get('control') - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control set-status --endpoint {control_url} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH} --status {status}' - client.exec_with_confirmation(cmd, ['']) + with create_ssh_client(node_name) as ssh_client: + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control set-status ' \ + f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH} --status {status}' + ssh_client.exec_with_confirmation(cmd, ['']) @keyword('Get netmap snapshot') -def get_netmap_snapshot(client: HostClient, node_name: str = None) -> str: +def get_netmap_snapshot(node_name: str = None) -> str: """ The function returns string representation of netmap-snapshot. Args: - client HostClient: client that implements exec command. node_name str: node name to use for netmap snapshot operation Returns: string representation of netmap-snapshot """ node_name = node_name or list(NEOFS_NETMAP_DICT)[0] - if node_name not in NEOFS_NETMAP_DICT: - raise AssertionError(f'Node {node_name} is not found!') - - node_config = NEOFS_NETMAP_DICT.get(node_name) - control_url = node_config.get('control') - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control netmap-snapshot --endpoint {control_url} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH}' - output = client.exec_with_confirmation(cmd, ['']) - return output.stdout + with create_ssh_client(node_name) as ssh_client: + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control netmap-snapshot ' \ + f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH}' + output = ssh_client.exec_with_confirmation(cmd, ['']) + return output.stdout @keyword('Shard list for node') -def node_shard_list(client: HostClient, node_name: str) -> List[str]: +def node_shard_list(node_name: str) -> List[str]: """ The function returns list of shards for particular node. Args: - client HostClient: client that implements exec command. node_name str: node name to use for netmap snapshot operation Returns: list of shards. """ - node_config = NEOFS_NETMAP_DICT.get(node_name) - control_url = node_config.get('control') - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards list --endpoint {control_url} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH}' - output = client.exec_with_confirmation(cmd, ['']) - return re.findall(r'Shard (.*):', output.stdout) + with create_ssh_client(node_name) as ssh_client: + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards list ' \ + f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH}' + output = ssh_client.exec_with_confirmation(cmd, ['']) + return re.findall(r'Shard (.*):', output.stdout) @keyword('Shard list for node') -def node_shard_set_mode(client: HostClient, node_name: str, shard: str, mode: str) -> str: +def node_shard_set_mode(node_name: str, shard: str, mode: str) -> str: """ The function sets mode for node's particular shard. Args: - client HostClient: client that implements exec command. node_name str: node name to use for netmap snapshot operation Returns: health status as HealthStatus object. """ - node_config = NEOFS_NETMAP_DICT.get(node_name) - control_url = node_config.get('control') - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards set-mode --endpoint {control_url} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH} --id {shard} --mode {mode}' - output = client.exec_with_confirmation(cmd, ['']) - return output.stdout + with create_ssh_client(node_name) as ssh_client: + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards set-mode ' \ + f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH} --id {shard} --mode {mode}' + output = ssh_client.exec_with_confirmation(cmd, ['']) + return output.stdout @keyword('Drop object from node {node_name}') -def drop_object(client: HostClient, node_name: str, cid: str, oid: str) -> str: +def drop_object(node_name: str, cid: str, oid: str) -> str: """ The function drops object from particular node. Args: - client HostClient: client that implements exec command. node_name str: node name to use for netmap snapshot operation Returns: health status as HealthStatus object. """ - node_config = NEOFS_NETMAP_DICT.get(node_name) - control_url = node_config.get('control') - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control drop-objects --endpoint {control_url} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH} -o {cid}/{oid}' - output = client.exec_with_confirmation(cmd, ['']) - return output.stdout + with create_ssh_client(node_name) as ssh_client: + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control drop-objects ' \ + f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH} -o {cid}/{oid}' + output = ssh_client.exec_with_confirmation(cmd, ['']) + return output.stdout diff --git a/robot/variables/common.py b/robot/variables/common.py index 483101e7..31183306 100644 --- a/robot/variables/common.py +++ b/robot/variables/common.py @@ -86,11 +86,10 @@ STORAGE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/storage/wallet01.json" S3_GATE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/s3_gate/wallet.json" S3_GATE_WALLET_PASS = 's3' -WALLET_PASS = f"{os.getcwd()}/wallet_pass.yml" - STORAGE_NODE_USER = os.getenv('STORAGE_NODE_USER', 'root') STORAGE_NODE_PWD = os.getenv('STORAGE_NODE_PWD') STORAGE_NODE_BIN_PATH = os.getenv('STORAGE_NODE_BIN_PATH', '/opt/dev-env/vendor/neofs-cli') STORAGE_NODE_CONFIG_PATH = os.getenv('STORAGE_NODE_CONFIG_PATH', '/opt/dev-env/services/storage/cli-cfg.yml') +STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT = os.getenv('STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT', 'localhost:8091') FREE_STORAGE = os.getenv('FREE_STORAGE', "false").lower() == "true"