From 68cbb7deea3071b5e6e47d4adc1687e835d9a6ad Mon Sep 17 00:00:00 2001 From: Vladimir Domnich Date: Tue, 5 Jul 2022 15:55:16 +0400 Subject: [PATCH] Update variables for node management tests New variables allow us: 1. To configure path of CLI binaries and config file on storage node. 2. Update variable names for storage node endpoints. Signed-off-by: Vladimir Domnich --- .../network/test_node_management.py | 2 + .../lib/python_keywords/node_management.py | 64 +++++----------- robot/variables/common.py | 73 +++++++++++-------- 3 files changed, 65 insertions(+), 74 deletions(-) diff --git a/pytest_tests/testsuites/network/test_node_management.py b/pytest_tests/testsuites/network/test_node_management.py index 707e188..593e03d 100644 --- a/pytest_tests/testsuites/network/test_node_management.py +++ b/pytest_tests/testsuites/network/test_node_management.py @@ -54,6 +54,7 @@ def crate_container_and_pick_node(create_remote_connection, prepare_wallet_and_d @pytest.fixture +@pytest.mark.skip(reason="docker API works only for devenv") def start_node_if_needed(create_remote_connection): yield try: @@ -167,6 +168,7 @@ def test_placement_policy_negative(prepare_wallet_and_deposit, placement_rule, e @pytest.mark.node_mgmt +@pytest.mark.skip(reason="docker API works only for devenv") @allure.title('NeoFS object replication on node failover') def test_replication(prepare_wallet_and_deposit, create_remote_connection, start_node_if_needed): """ diff --git a/robot/resources/lib/python_keywords/node_management.py b/robot/resources/lib/python_keywords/node_management.py index e7d986b..24edc4d 100644 --- a/robot/resources/lib/python_keywords/node_management.py +++ b/robot/resources/lib/python_keywords/node_management.py @@ -8,10 +8,10 @@ import random import re from dataclasses import dataclass -from typing import List, Tuple +from typing import List import docker -from common import DEPLOY_PATH, NEOFS_NETMAP_DICT +from common import NEOFS_NETMAP_DICT, STORAGE_NODE_BIN_PATH, STORAGE_NODE_CONFIG_PATH from robot.api import logger from robot.api.deco import keyword from ssh_helper import HostClient @@ -152,10 +152,8 @@ def node_healthcheck(client: HostClient, node_name: str) -> HealthStatus: node_config = NEOFS_NETMAP_DICT.get(node_name) control_url = node_config.get('control') - host, port = control_url.split(':') - cmd = f'{DEPLOY_PATH}/vendor/neofs-cli control healthcheck --endpoint {control_url} ' \ - f'--wallet {DEPLOY_PATH}/services/storage/wallet0{port[-1]}.json ' \ - f'--config {DEPLOY_PATH}/services/storage/cli-cfg.yml' + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control healthcheck --endpoint {control_url} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH}' output = client.exec_with_confirmation(cmd, ['']) return HealthStatus.from_stdout(output.stdout) @@ -176,10 +174,8 @@ def node_set_status(client: HostClient, node_name: str, status: str): node_config = NEOFS_NETMAP_DICT.get(node_name) control_url = node_config.get('control') - host, port = control_url.split(':') - cmd = f'{DEPLOY_PATH}/vendor/neofs-cli control set-status --endpoint {control_url} ' \ - f'--wallet {DEPLOY_PATH}/services/storage/wallet0{port[-1]}.json ' \ - f'--config {DEPLOY_PATH}/services/storage/cli-cfg.yml --status {status}' + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control set-status --endpoint {control_url} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH} --status {status}' client.exec_with_confirmation(cmd, ['']) @@ -200,10 +196,8 @@ def get_netmap_snapshot(client: HostClient, node_name: str = None) -> str: node_config = NEOFS_NETMAP_DICT.get(node_name) control_url = node_config.get('control') - host, port = control_url.split(':') - cmd = f'{DEPLOY_PATH}/vendor/neofs-cli control netmap-snapshot --endpoint {control_url} ' \ - f'--wallet {DEPLOY_PATH}/services/storage/wallet0{port[-1]}.json ' \ - f'--config {DEPLOY_PATH}/services/storage/cli-cfg.yml' + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control netmap-snapshot --endpoint {control_url} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH}' output = client.exec_with_confirmation(cmd, ['']) return output.stdout @@ -218,10 +212,10 @@ def node_shard_list(client: HostClient, node_name: str) -> List[str]: Returns: list of shards. """ - control_url, port = _url_port_for_node(node_name) - cmd = f'{DEPLOY_PATH}/vendor/neofs-cli control shards list --endpoint {control_url} ' \ - f'--wallet {DEPLOY_PATH}/services/storage/wallet0{port[-1]}.json ' \ - f'--config {DEPLOY_PATH}/services/storage/cli-cfg.yml' + node_config = NEOFS_NETMAP_DICT.get(node_name) + control_url = node_config.get('control') + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards list --endpoint {control_url} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH}' output = client.exec_with_confirmation(cmd, ['']) return re.findall(r'Shard (.*):', output.stdout) @@ -236,10 +230,10 @@ def node_shard_set_mode(client: HostClient, node_name: str, shard: str, mode: st Returns: health status as HealthStatus object. """ - control_url, port = _url_port_for_node(node_name) - cmd = f'{DEPLOY_PATH}/vendor/neofs-cli control shards set-mode --endpoint {control_url} ' \ - f'--wallet {DEPLOY_PATH}/services/storage/wallet0{port[-1]}.json ' \ - f'--config {DEPLOY_PATH}/services/storage/cli-cfg.yml --id {shard} --mode {mode}' + node_config = NEOFS_NETMAP_DICT.get(node_name) + control_url = node_config.get('control') + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards set-mode --endpoint {control_url} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH} --id {shard} --mode {mode}' output = client.exec_with_confirmation(cmd, ['']) return output.stdout @@ -254,27 +248,9 @@ def drop_object(client: HostClient, node_name: str, cid: str, oid: str) -> str: Returns: health status as HealthStatus object. """ - control_url, port = _url_port_for_node(node_name) - cmd = f'{DEPLOY_PATH}/vendor/neofs-cli control drop-objects --endpoint {control_url} ' \ - f'--wallet {DEPLOY_PATH}/services/storage/wallet0{port[-1]}.json ' \ - f'--config {DEPLOY_PATH}/services/storage/cli-cfg.yml -o {cid}/{oid}' - output = client.exec_with_confirmation(cmd, ['']) - return output.stdout - - -def _url_port_for_node(node_name: str) -> Tuple[str, str]: - """ - Returns control url and port for particular storage node. - Args: - node_name: str node bane from NEOFS_NETMAP_DICT - - Returns: - control url and port as a tuple. - """ - if node_name not in NEOFS_NETMAP_DICT: - raise AssertionError(f'Node {node_name} is not found!') - node_config = NEOFS_NETMAP_DICT.get(node_name) control_url = node_config.get('control') - port = control_url.split(':')[-1] - return control_url, port + cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control drop-objects --endpoint {control_url} ' \ + f'--config {STORAGE_NODE_CONFIG_PATH} -o {cid}/{oid}' + output = client.exec_with_confirmation(cmd, ['']) + return output.stdout diff --git a/robot/variables/common.py b/robot/variables/common.py index 2ed00ff..483101e 100644 --- a/robot/variables/common.py +++ b/robot/variables/common.py @@ -32,34 +32,44 @@ ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/") MORPH_MAGIC = os.getenv("MORPH_MAGIC") -STORAGE_NODE_1 = os.getenv('DATA_NODE_1', 's01.neofs.devenv:8080') -STORAGE_NODE_2 = os.getenv('DATA_NODE_2', 's02.neofs.devenv:8080') -STORAGE_NODE_3 = os.getenv('DATA_NODE_3', 's03.neofs.devenv:8080') -STORAGE_NODE_4 = os.getenv('DATA_NODE_4', 's04.neofs.devenv:8080') - -CONTROL_NODE_1 = os.getenv('NEOFS_CONTROL_NODE_1', 's01.neofs.devenv:8081') -CONTROL_NODE_2 = os.getenv('NEOFS_CONTROL_NODE_2', 's02.neofs.devenv:8081') -CONTROL_NODE_3 = os.getenv('NEOFS_CONTROL_NODE_3', 's03.neofs.devenv:8081') -CONTROL_NODE_4 = os.getenv('NEOFS_CONTROL_NODE_4', 's04.neofs.devenv:8081') - DEVENV_SERVICES_PATH = f"{os.getenv('DEVENV_PATH')}/services" -NEOFS_NETMAP_DICT = {'s01': {'rpc': STORAGE_NODE_1, - 'control': CONTROL_NODE_1, - 'wallet_path':f"{DEVENV_SERVICES_PATH}/storage/wallet01.json", - 'UN-LOCODE': 'RU MOW'}, - 's02': {'rpc': STORAGE_NODE_2, - 'control': CONTROL_NODE_2, - 'wallet_path': f"{DEVENV_SERVICES_PATH}/storage/wallet02.json", - 'UN-LOCODE': 'RU LED'}, - 's03': {'rpc': STORAGE_NODE_3, - 'control': CONTROL_NODE_3, - 'wallet_path': f"{DEVENV_SERVICES_PATH}/storage/wallet03.json", - 'UN-LOCODE': 'SE STO'}, - 's04': {'rpc': STORAGE_NODE_4, - 'control': CONTROL_NODE_4, - 'wallet_path': f"{DEVENV_SERVICES_PATH}/storage/wallet04.json", - 'UN-LOCODE': 'FI HEL'} - } + +STORAGE_RPC_ENDPOINT_1 = os.getenv('STORAGE_RPC_ENDPOINT_1', os.getenv('DATA_NODE_1', 's01.neofs.devenv:8080')) +STORAGE_RPC_ENDPOINT_2 = os.getenv('STORAGE_RPC_ENDPOINT_2', os.getenv('DATA_NODE_2', 's02.neofs.devenv:8080')) +STORAGE_RPC_ENDPOINT_3 = os.getenv('STORAGE_RPC_ENDPOINT_3', os.getenv('DATA_NODE_3', 's03.neofs.devenv:8080')) +STORAGE_RPC_ENDPOINT_4 = os.getenv('STORAGE_RPC_ENDPOINT_4', os.getenv('DATA_NODE_4', 's04.neofs.devenv:8080')) + +STORAGE_CONTROL_ENDPOINT_1 = os.getenv('STORAGE_CONTROL_ENDPOINT_1', os.getenv('NEOFS_CONTROL_NODE_1', 's01.neofs.devenv:8081')) +STORAGE_CONTROL_ENDPOINT_2 = os.getenv('STORAGE_CONTROL_ENDPOINT_2', os.getenv('NEOFS_CONTROL_NODE_2', 's02.neofs.devenv:8081')) +STORAGE_CONTROL_ENDPOINT_3 = os.getenv('STORAGE_CONTROL_ENDPOINT_3', os.getenv('NEOFS_CONTROL_NODE_3', 's03.neofs.devenv:8081')) +STORAGE_CONTROL_ENDPOINT_4 = os.getenv('STORAGE_CONTROL_ENDPOINT_4', os.getenv('NEOFS_CONTROL_NODE_4', 's04.neofs.devenv:8081')) + +NEOFS_NETMAP_DICT = { + 's01': { + 'rpc': STORAGE_RPC_ENDPOINT_1, + 'control': STORAGE_CONTROL_ENDPOINT_1, + 'wallet_path':f"{DEVENV_SERVICES_PATH}/storage/wallet01.json", + 'UN-LOCODE': 'RU MOW' + }, + 's02': { + 'rpc': STORAGE_RPC_ENDPOINT_2, + 'control': STORAGE_CONTROL_ENDPOINT_2, + 'wallet_path': f"{DEVENV_SERVICES_PATH}/storage/wallet02.json", + 'UN-LOCODE': 'RU LED' + }, + 's03': { + 'rpc': STORAGE_RPC_ENDPOINT_3, + 'control': STORAGE_CONTROL_ENDPOINT_3, + 'wallet_path': f"{DEVENV_SERVICES_PATH}/storage/wallet03.json", + 'UN-LOCODE': 'SE STO' + }, + 's04': { + 'rpc': STORAGE_RPC_ENDPOINT_4, + 'control': STORAGE_CONTROL_ENDPOINT_4, + 'wallet_path': f"{DEVENV_SERVICES_PATH}/storage/wallet04.json", + 'UN-LOCODE': 'FI HEL' + }, +} NEOFS_NETMAP = [i['rpc'] for i in NEOFS_NETMAP_DICT.values()] NEOGO_EXECUTABLE = os.getenv('NEOGO_EXECUTABLE', 'neo-go') NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli') @@ -76,8 +86,11 @@ STORAGE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/storage/wallet01.json" S3_GATE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/s3_gate/wallet.json" S3_GATE_WALLET_PASS = 's3' -CONTROL_NODE_USER = os.getenv('CONTROL_NODE_USER', 'root') -CONTROL_NODE_PWD = os.getenv('CONTROL_NODE_PWD') -DEPLOY_PATH = os.getenv('DEPLOY_PATH', '/opt/dev-env') +WALLET_PASS = f"{os.getcwd()}/wallet_pass.yml" + +STORAGE_NODE_USER = os.getenv('STORAGE_NODE_USER', 'root') +STORAGE_NODE_PWD = os.getenv('STORAGE_NODE_PWD') +STORAGE_NODE_BIN_PATH = os.getenv('STORAGE_NODE_BIN_PATH', '/opt/dev-env/vendor/neofs-cli') +STORAGE_NODE_CONFIG_PATH = os.getenv('STORAGE_NODE_CONFIG_PATH', '/opt/dev-env/services/storage/cli-cfg.yml') FREE_STORAGE = os.getenv('FREE_STORAGE', "false").lower() == "true"