From e89128673545d82c1c380accf77cc7609acd758c Mon Sep 17 00:00:00 2001 From: Vladimir Domnich Date: Mon, 18 Jul 2022 17:59:02 +0400 Subject: [PATCH] Fix call to control endpoint for remote devenv. --- Makefile | 24 ++-- .../lib/python_keywords/node_management.py | 115 ++++++++++-------- robot/variables/common.py | 6 +- 3 files changed, 76 insertions(+), 69 deletions(-) diff --git a/Makefile b/Makefile index ba8f005c..8014c043 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,7 @@ setup-for-remote-devenv: echo STORAGE_CONTROL_ENDPOINT_2=$(DEV_ENV_HOST_NAME):8182 >> .env echo STORAGE_CONTROL_ENDPOINT_3=$(DEV_ENV_HOST_NAME):8183 >> .env echo STORAGE_CONTROL_ENDPOINT_4=$(DEV_ENV_HOST_NAME):8184 >> .env + echo STORAGE_CONTROL_ENDPOINT_PRIVATE="true" >> .env echo STORAGE_RPC_ENDPOINT_1=$(DEV_ENV_HOST_NAME):8081 >> .env echo STORAGE_RPC_ENDPOINT_2=$(DEV_ENV_HOST_NAME):8082 >> .env echo STORAGE_RPC_ENDPOINT_3=$(DEV_ENV_HOST_NAME):8083 >> .env @@ -74,13 +75,10 @@ setup-for-remote-devenv: echo STORAGE_WALLET_PATH_2="/.setup/services/storage/wallet02.json" >> .env echo STORAGE_WALLET_PATH_3="/.setup/services/storage/wallet03.json" >> .env echo STORAGE_WALLET_PATH_4="/.setup/services/storage/wallet04.json" >> .env - # env: SSH connection to storage node + # env: SSH connection to storage node host and bin directory on that host + echo STORAGE_NODE_SSH_USER="root" >> .env echo STORAGE_NODE_SSH_PRIVATE_KEY_PATH="/root/.ssh/id_rsa" >> .env - # env: files inside storage node to make calls via SSH on that node echo STORAGE_NODE_BIN_PATH="$(DEV_ENV_DEPLOY_DIR)/vendor" >> .env - echo STORAGE_NODE_CONFIG_PATH="$(DEV_ENV_DEPLOY_DIR)/services/storage/cli-cfg.yml" >> .env - echo STORAGE_NODE_WALLET_PATH="$(DEV_ENV_DEPLOY_DIR)/services/storage/wallet01.json" >> .env - echo STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT="s01.neofs.devenv:8181" >> .env # env: s3 gateway public key ssh root@$(DEV_ENV_HOST_NAME) 'cd $(DEV_ENV_DEPLOY_DIR) && make env | grep NEOFS_IR_CONTRACTS_NEOFS' >> .env @@ -97,10 +95,11 @@ setup-for-sbercloud: echo MORPH_ENDPOINT=http://$(NODE1):40332 >> .env echo HTTP_GATE=http://$(NODE1):8888 >> .env echo S3_GATE=http://$(NODE1):8084 >> .env - echo STORAGE_CONTROL_ENDPOINT_1=$(NODE1):40332 >> .env - echo STORAGE_CONTROL_ENDPOINT_2=$(NODE2):40332 >> .env - echo STORAGE_CONTROL_ENDPOINT_3=$(NODE3):40332 >> .env - echo STORAGE_CONTROL_ENDPOINT_4=$(NODE4):40332 >> .env + echo STORAGE_CONTROL_ENDPOINT_1=localhost:8091 >> .env + echo STORAGE_CONTROL_ENDPOINT_2=localhost:8091 >> .env + echo STORAGE_CONTROL_ENDPOINT_3=localhost:8091 >> .env + echo STORAGE_CONTROL_ENDPOINT_4=localhost:8091 >> .env + echo STORAGE_CONTROL_ENDPOINT_PRIVATE="true" >> .env echo NEOFS_ENDPOINT=$(NODE1):8080 >> .env echo STORAGE_RPC_ENDPOINT_1=$(NODE1):8080 >> .env echo STORAGE_RPC_ENDPOINT_2=$(NODE2):8080 >> .env @@ -114,11 +113,10 @@ setup-for-sbercloud: echo FREE_STORAGE=true >> .env echo NEOFS_ADM_EXEC="neofs-adm" >> .env echo NEOFS_ADM_CONFIG_PATH="/.setup/config.yaml" >> .env - # env: files inside storage node to make calls via SSH on that node + # env: SSH connection to storage node host and bin directory on that host + echo STORAGE_NODE_SSH_USER="root" >> .env + echo STORAGE_NODE_SSH_PRIVATE_KEY_PATH="/root/.ssh/id_rsa" >> .env echo STORAGE_NODE_BIN_PATH="/usr/local/bin/" >> .env - echo STORAGE_NODE_CONFIG_PATH="/tmp/conf.yaml" >> .env - echo STORAGE_NODE_WALLET_PATH="/etc/neofs/storage/wallet.json" >> .env - echo STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT="localhost:8091" >> .env # env: s3 gateway public key echo S3_GATE_PUB_KEY=$(shell $(NEO_BIN_DIR)/neo-go wallet dump-keys -w $(CONFIG_DIR)/wallets/node1-s3.json | head -2 | tail -1) >> .env diff --git a/robot/resources/lib/python_keywords/node_management.py b/robot/resources/lib/python_keywords/node_management.py index a832c693..7a7ef0e4 100644 --- a/robot/resources/lib/python_keywords/node_management.py +++ b/robot/resources/lib/python_keywords/node_management.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python3.9 """ This module contains keywords for management test stand @@ -9,16 +9,16 @@ import random import re from contextlib import contextmanager from dataclasses import dataclass -from typing import List +from typing import Optional +from cli_helpers import _cmd_run import docker -from common import (NEOFS_NETMAP_DICT, STORAGE_NODE_BIN_PATH, STORAGE_NODE_CONFIG_PATH, - STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT, STORAGE_NODE_SSH_PASSWORD, - STORAGE_NODE_SSH_PRIVATE_KEY_PATH, STORAGE_NODE_SSH_USER, - STORAGE_NODE_WALLET_PATH) +from common import (NEOFS_CLI_EXEC, NEOFS_NETMAP_DICT, STORAGE_CONTROL_ENDPOINT_PRIVATE, + STORAGE_NODE_BIN_PATH, STORAGE_NODE_SSH_PASSWORD, + STORAGE_NODE_SSH_PRIVATE_KEY_PATH, STORAGE_NODE_SSH_USER, WALLET_CONFIG) from robot.api import logger from robot.api.deco import keyword -from ssh_helper import HostClient, HostIsNotAvailable +from ssh_helper import HostClient ROBOT_AUTO_KEYWORDS = False @@ -44,8 +44,10 @@ def create_ssh_client(node_name: str) -> HostClient: if node_name not in NEOFS_NETMAP_DICT: raise AssertionError(f'Node {node_name} is not found!') + # We use rpc endpoint to determine host address, because control endpoint + # (if it is private) will be a local address on the host machine node_config = NEOFS_NETMAP_DICT.get(node_name) - host = node_config.get('control').split(':')[0] + host = node_config.get('rpc').split(':')[0] ssh_client = HostClient( host, login=STORAGE_NODE_SSH_USER, @@ -115,9 +117,9 @@ def get_control_endpoint_and_wallet(endpoint_number: str = ''): endpoint_values = NEOFS_NETMAP_DICT[f'{endpoint_num}'] endpoint_control = endpoint_values['control'] - wlt = endpoint_values['wallet_path'] + wallet = endpoint_values['wallet_path'] - return endpoint_num, endpoint_control, wlt + return endpoint_num, endpoint_control, wallet @keyword('Get Locode') @@ -170,17 +172,13 @@ def node_healthcheck(node_name: str) -> HealthStatus: Returns: health status as HealthStatus object. """ - with create_ssh_client(node_name) as ssh_client: - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control healthcheck ' \ - f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ - f'{"-w " + STORAGE_NODE_WALLET_PATH if STORAGE_NODE_WALLET_PATH else ""} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH}' - output = ssh_client.exec_with_confirmation(cmd, ['']) - return HealthStatus.from_stdout(output.stdout) + command = "control healthcheck" + output = run_control_command(node_name, command) + return HealthStatus.from_stdout(output) @keyword('Set status for node') -def node_set_status(node_name: str, status: str): +def node_set_status(node_name: str, status: str) -> None: """ The function sets particular status for given node. Args: @@ -189,16 +187,12 @@ def node_set_status(node_name: str, status: str): Returns: (void) """ - with create_ssh_client(node_name) as ssh_client: - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control set-status ' \ - f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ - f'{"-w " + STORAGE_NODE_WALLET_PATH if STORAGE_NODE_WALLET_PATH else ""} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH} --status {status}' - ssh_client.exec_with_confirmation(cmd, ['']) + command = f"control set-status --status {status}" + run_control_command(node_name, command) @keyword('Get netmap snapshot') -def get_netmap_snapshot(node_name: str = None) -> str: +def get_netmap_snapshot(node_name: Optional[str] = None) -> str: """ The function returns string representation of netmap-snapshot. Args: @@ -207,18 +201,12 @@ def get_netmap_snapshot(node_name: str = None) -> str: string representation of netmap-snapshot """ node_name = node_name or list(NEOFS_NETMAP_DICT)[0] - - with create_ssh_client(node_name) as ssh_client: - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control netmap-snapshot ' \ - f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ - f'{"-w " + STORAGE_NODE_WALLET_PATH if STORAGE_NODE_WALLET_PATH else ""} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH}' - output = ssh_client.exec_with_confirmation(cmd, ['']) - return output.stdout + command = "control netmap-snapshot" + return run_control_command(node_name, command) @keyword('Shard list for node') -def node_shard_list(node_name: str) -> List[str]: +def node_shard_list(node_name: str) -> list[str]: """ The function returns list of shards for particular node. Args: @@ -226,13 +214,9 @@ def node_shard_list(node_name: str) -> List[str]: Returns: list of shards. """ - with create_ssh_client(node_name) as ssh_client: - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards list ' \ - f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ - f'{"-w " + STORAGE_NODE_WALLET_PATH if STORAGE_NODE_WALLET_PATH else ""} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH}' - output = ssh_client.exec_with_confirmation(cmd, ['']) - return re.findall(r'Shard (.*):', output.stdout) + command = "control shards list" + output = run_control_command(node_name, command) + return re.findall(r'Shard (.*):', output) @keyword('Shard list for node') @@ -244,13 +228,8 @@ def node_shard_set_mode(node_name: str, shard: str, mode: str) -> str: Returns: health status as HealthStatus object. """ - with create_ssh_client(node_name) as ssh_client: - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards set-mode ' \ - f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ - f'{"-w " + STORAGE_NODE_WALLET_PATH if STORAGE_NODE_WALLET_PATH else ""} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH} --id {shard} --mode {mode}' - output = ssh_client.exec_with_confirmation(cmd, ['']) - return output.stdout + command = f"control shards set-mode --id {shard} --mode {mode}" + return run_control_command(node_name, command) @keyword('Drop object from node {node_name}') @@ -262,10 +241,40 @@ def drop_object(node_name: str, cid: str, oid: str) -> str: Returns: health status as HealthStatus object. """ + command = f"control drop-objects -o {cid}/{oid}" + return run_control_command(node_name, command) + + +def run_control_command(node_name: str, command: str) -> str: + control_endpoint = NEOFS_NETMAP_DICT[node_name]["control"] + wallet_path = NEOFS_NETMAP_DICT[node_name]["wallet_path"] + + if not STORAGE_CONTROL_ENDPOINT_PRIVATE: + cmd = ( + f'{NEOFS_CLI_EXEC} {command} --rpc-endpoint {control_endpoint} ' + f'--wallet {wallet_path} --config {WALLET_CONFIG}' + ) + output = _cmd_run(cmd) + return output + + # Private control endpoint is accessible only from the host where storage node is running + # So, we connect to storage node host and run CLI command from there with create_ssh_client(node_name) as ssh_client: - cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control drop-objects ' \ - f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \ - f'{"-w " + STORAGE_NODE_WALLET_PATH if STORAGE_NODE_WALLET_PATH else ""} ' \ - f'--config {STORAGE_NODE_CONFIG_PATH} -o {cid}/{oid}' - output = ssh_client.exec_with_confirmation(cmd, ['']) + # Copy wallet content on storage node host + with open(wallet_path, "r") as file: + wallet = file.read() + remote_wallet_path = "/tmp/{node_name}-wallet.json" + ssh_client.exec_with_confirmation(f"echo '{wallet}' > {remote_wallet_path}", [""]) + + # Put config on storage node host + remote_config_path = "/tmp/{node_name}-config.yaml" + remote_config = 'password: ""' + ssh_client.exec_with_confirmation(f"echo '{remote_config}' > {remote_config_path}", [""]) + + # Execute command + cmd = ( + f'{STORAGE_NODE_BIN_PATH}/neofs-cli {command} --endpoint {control_endpoint} ' + f'--wallet {remote_wallet_path} --config {remote_config_path}' + ) + output = ssh_client.exec_with_confirmation(cmd, [""]) return output.stdout diff --git a/robot/variables/common.py b/robot/variables/common.py index 60b7d592..dd563cd8 100644 --- a/robot/variables/common.py +++ b/robot/variables/common.py @@ -43,6 +43,7 @@ STORAGE_CONTROL_ENDPOINT_1 = os.getenv("STORAGE_CONTROL_ENDPOINT_1", "s01.neofs. STORAGE_CONTROL_ENDPOINT_2 = os.getenv("STORAGE_CONTROL_ENDPOINT_2", "s02.neofs.devenv:8081") STORAGE_CONTROL_ENDPOINT_3 = os.getenv("STORAGE_CONTROL_ENDPOINT_3", "s03.neofs.devenv:8081") STORAGE_CONTROL_ENDPOINT_4 = os.getenv("STORAGE_CONTROL_ENDPOINT_4", "s04.neofs.devenv:8081") +STORAGE_CONTROL_ENDPOINT_PRIVATE = os.getenv("STORAGE_CONTROL_ENDPOINT_PRIVATE", "false").lower() == "true" STORAGE_WALLET_PATH_1 = os.getenv("STORAGE_WALLET_PATH_1", f"{DEVENV_PATH}/services/storage/wallet01.json") STORAGE_WALLET_PATH_2 = os.getenv("STORAGE_WALLET_PATH_2", f"{DEVENV_PATH}/services/storage/wallet02.json") @@ -93,7 +94,7 @@ IR_WALLET_CONFIG = os.getenv("IR_WALLET_CONFIG", f"{os.getcwd()}/neofs_cli_confi IR_WALLET_PASS = os.getenv("IR_WALLET_PASS", "one") # Parameters that control SSH connection to storage node -STORAGE_NODE_SSH_USER = os.getenv("STORAGE_NODE_SSH_USER", "root") +STORAGE_NODE_SSH_USER = os.getenv("STORAGE_NODE_SSH_USER") STORAGE_NODE_SSH_PASSWORD = os.getenv("STORAGE_NODE_SSH_PASSWORD") STORAGE_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("STORAGE_NODE_SSH_PRIVATE_KEY_PATH") @@ -103,10 +104,9 @@ STORAGE_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("STORAGE_NODE_SSH_PRIVATE_KEY_PATH STORAGE_NODE_BIN_PATH = os.getenv("STORAGE_NODE_BIN_PATH", f"{DEVENV_PATH}/vendor") STORAGE_NODE_CONFIG_PATH = os.getenv("STORAGE_NODE_CONFIG_PATH", f"{DEVENV_PATH}/services/storage/cli-cfg.yml") STORAGE_NODE_WALLET_PATH = os.getenv("STORAGE_NODE_WALLET_PATH") # allows to override path to wallet that was given in STORAGE_NODE_CONFIG_PATH, this is temp parameter, should be fixed in environment setup -STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT = os.getenv("STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT") # Path and config for neofs-adm utility. Optional if tests are running against devenv NEOFS_ADM_EXEC = os.getenv("NEOFS_ADM_EXEC") NEOFS_ADM_CONFIG_PATH = os.getenv("NEOFS_ADM_CONFIG_PATH") -FREE_STORAGE = os.getenv('FREE_STORAGE', "false").lower() == "true" +FREE_STORAGE = os.getenv("FREE_STORAGE", "false").lower() == "true"