Fix node management tests

Remote connection was created to the 1st storage node only. While in reality
we want to create connection to a specific node.

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>

neofs-keywords

@@ -1 +1 @@
-Subproject commit 1d7ebe1d6d4bb6c8ce62f36091347bddb54d333b
+Subproject commit 58650099362f05b51b97f04e2244a4ba20f88351

pytest_tests/testsuites/container/test_container.py

@@ -4,7 +4,7 @@ from time import sleep
 import allure
 import pytest
 
-from contract_keywords import tick_epoch
+from epoch import tick_epoch
 from python_keywords.container import create_container, get_container, list_containers, delete_container
 from utility import placement_policy_from_container
 

pytest_tests/testsuites/network/test_node_management.py

@@ -6,7 +6,7 @@ import allure
 import pytest
 from common import (COMPLEX_OBJ_SIZE, MAINNET_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT,
                     NEOFS_NETMAP_DICT, SHARD_0_GC_SLEEP)
-from contract_keywords import tick_epoch
+from epoch import tick_epoch
 from utility_keywords import generate_file
 from python_keywords.container import create_container, get_container
 from python_keywords.neofs_verbs import (delete_object, get_object,
@@ -28,7 +28,7 @@ logger = logging.getLogger('NeoLogger')
 
 @pytest.fixture
 @allure.title('Create container and pick the node with data')
-def crate_container_and_pick_node(create_remote_connection, prepare_wallet_and_deposit):
+def crate_container_and_pick_node(prepare_wallet_and_deposit):
     wallet = prepare_wallet_and_deposit
     file_path = generate_file()
     placement_rule = 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X'
@@ -44,63 +44,63 @@ def crate_container_and_pick_node(create_remote_connection, prepare_wallet_and_d
 
     yield cid, node_name
 
-    shards = node_shard_list(create_remote_connection, node_name)
+    shards = node_shard_list(node_name)
     assert shards
 
     for shard in shards:
-        node_shard_set_mode(create_remote_connection, node_name, shard, 'read-write')
+        node_shard_set_mode(node_name, shard, 'read-write')
 
-    node_shard_list(create_remote_connection, node_name)
+    node_shard_list(node_name)
 
 
 @pytest.fixture
 @pytest.mark.skip(reason="docker API works only for devenv")
-def start_node_if_needed(create_remote_connection):
+def start_node_if_needed():
     yield
     try:
-        start_nodes_remote(create_remote_connection, list(NEOFS_NETMAP_DICT.keys()))
+        start_nodes_remote(list(NEOFS_NETMAP_DICT.keys()))
     except Exception as err:
         logger.error(f'Node start fails with error:\n{err}')
 
 
 @allure.title('Control Operations with storage nodes')
 @pytest.mark.node_mgmt
-def test_nodes_management(prepare_tmp_dir, create_remote_connection):
+def test_nodes_management(prepare_tmp_dir):
     """
     This test checks base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
     """
     random_node = choice(list(NEOFS_NETMAP_DICT))
     alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != random_node])
-    snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node)
+    snapshot = get_netmap_snapshot(node_name=alive_node)
     assert random_node in snapshot, f'Expected node {random_node} in netmap'
 
     with allure.step('Run health check for all storage nodes'):
         for node_name in NEOFS_NETMAP_DICT.keys():
-            health_check = node_healthcheck(create_remote_connection, node_name)
+            health_check = node_healthcheck(node_name)
             assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE'
 
     with allure.step(f'Move node {random_node} to offline state'):
-        node_set_status(create_remote_connection, random_node, status='offline')
+        node_set_status(random_node, status='offline')
 
     sleep(robot_time_to_int(MAINNET_BLOCK_TIME))
     tick_epoch()
 
     with allure.step(f'Check node {random_node} went to offline'):
-        health_check = node_healthcheck(create_remote_connection, random_node)
+        health_check = node_healthcheck(random_node)
         assert health_check.health_status == 'READY' and health_check.network_status == 'STATUS_UNDEFINED'
-        snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node)
+        snapshot = get_netmap_snapshot(node_name=alive_node)
         assert random_node not in snapshot, f'Expected node {random_node} not in netmap'
 
     with allure.step(f'Check node {random_node} went to online'):
-        node_set_status(create_remote_connection, random_node, status='online')
+        node_set_status(random_node, status='online')
 
     sleep(robot_time_to_int(MAINNET_BLOCK_TIME))
     tick_epoch()
 
     with allure.step(f'Check node {random_node} went to online'):
-        health_check = node_healthcheck(create_remote_connection, random_node)
+        health_check = node_healthcheck(random_node)
         assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE'
-        snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node)
+        snapshot = get_netmap_snapshot(node_name=alive_node)
         assert random_node in snapshot, f'Expected node {random_node} in netmap'
 
 
@@ -170,7 +170,7 @@ def test_placement_policy_negative(prepare_wallet_and_deposit, placement_rule, e
 @pytest.mark.node_mgmt
 @pytest.mark.skip(reason="docker API works only for devenv")
 @allure.title('NeoFS object replication on node failover')
-def test_replication(prepare_wallet_and_deposit, create_remote_connection, start_node_if_needed):
+def test_replication(prepare_wallet_and_deposit, start_node_if_needed):
     """
     Test checks object replication on storage not failover and come back.
     """
@@ -185,22 +185,22 @@ def test_replication(prepare_wallet_and_deposit, create_remote_connection, start
     assert len(nodes) == expected_nodes_count, f'Expected {expected_nodes_count} copies, got {len(nodes)}'
 
     node_names = [name for name, config in NEOFS_NETMAP_DICT.items() if config.get('rpc') in nodes]
-    stopped_nodes = stop_nodes_remote(create_remote_connection, 1, node_names)
+    stopped_nodes = stop_nodes_remote(1, node_names)
 
     wait_for_expected_object_copies(wallet, cid, oid)
 
-    start_nodes_remote(create_remote_connection, stopped_nodes)
+    start_nodes_remote(stopped_nodes)
     tick_epoch()
 
     for node_name in node_names:
-        wait_for_node_go_online(create_remote_connection, node_name)
+        wait_for_node_go_online(node_name)
 
     wait_for_expected_object_copies(wallet, cid, oid)
 
 
 @pytest.mark.node_mgmt
 @allure.title('NeoFS object could be dropped using control command')
-def test_drop_object(prepare_wallet_and_deposit, create_remote_connection):
+def test_drop_object(prepare_wallet_and_deposit):
     """
     Test checks object could be dropped using `neofs-cli control drop-objects` command.
     """
@@ -224,7 +224,7 @@ def test_drop_object(prepare_wallet_and_deposit, create_remote_connection):
         with allure.step(f'Drop object {oid}'):
             get_object(wallet, cid, oid)
             head_object(wallet, cid, oid)
-            drop_object(create_remote_connection, node_name, cid, oid)
+            drop_object(node_name, cid, oid)
             wait_for_obj_dropped(wallet, cid, oid, get_object)
             wait_for_obj_dropped(wallet, cid, oid, head_object)
 
@@ -232,7 +232,7 @@ def test_drop_object(prepare_wallet_and_deposit, create_remote_connection):
 @pytest.mark.node_mgmt
 @pytest.mark.skip(reason='Need to clarify scenario')
 @allure.title('Control Operations with storage nodes')
-def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_container_and_pick_node):
+def test_shards(prepare_wallet_and_deposit, crate_container_and_pick_node):
     """
     This test checks base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
     """
@@ -244,13 +244,13 @@ def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_cont
 
     # for mode in ('read-only', 'degraded'):
     for mode in ('degraded',):
-        shards = node_shard_list(create_remote_connection, node_name)
+        shards = node_shard_list(node_name)
         assert shards
 
         for shard in shards:
-            node_shard_set_mode(create_remote_connection, node_name, shard, mode)
+            node_shard_set_mode(node_name, shard, mode)
 
-        shards = node_shard_list(create_remote_connection, node_name)
+        shards = node_shard_list(node_name)
         assert shards
 
         with pytest.raises(RuntimeError):
@@ -263,9 +263,9 @@ def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_cont
         get_object(wallet, cid, original_oid)
 
         for shard in shards:
-            node_shard_set_mode(create_remote_connection, node_name, shard, 'read-write')
+            node_shard_set_mode(node_name, shard, 'read-write')
 
-        shards = node_shard_list(create_remote_connection, node_name)
+        shards = node_shard_list(node_name)
         assert shards
 
         oid = put_object(wallet, file_path, cid)
@@ -285,11 +285,11 @@ def validate_object_copies(wallet: str, placement_rule: str, file_path: str, exp
 
 
 @allure.step('Wait for node {node_name} goes online')
-def wait_for_node_go_online(create_remote_connection, node_name: str):
+def wait_for_node_go_online(node_name: str):
     timeout, attempts = 5, 20
     for _ in range(attempts):
         try:
-            health_check = node_healthcheck(create_remote_connection, node_name)
+            health_check = node_healthcheck(node_name)
             assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE'
             return
         except Exception as err:

robot/resources/lib/python_keywords/node_management.py

@@ -7,11 +7,12 @@
 
 import random
 import re
+from contextlib import contextmanager
 from dataclasses import dataclass
 from typing import List
 
 import docker
-from common import NEOFS_NETMAP_DICT, STORAGE_NODE_BIN_PATH, STORAGE_NODE_CONFIG_PATH
+from common import NEOFS_NETMAP_DICT, STORAGE_NODE_BIN_PATH, STORAGE_NODE_CONFIG_PATH, STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT, STORAGE_NODE_PWD, STORAGE_NODE_USER
 from robot.api import logger
 from robot.api.deco import keyword
 from ssh_helper import HostClient
@@ -35,8 +36,23 @@ class HealthStatus:
         return HealthStatus(network, health)
 
 
+@contextmanager
+def create_ssh_client(node_name: str) -> HostClient:
+    if node_name not in NEOFS_NETMAP_DICT:
+        raise AssertionError(f'Node {node_name} is not found!')
+
+    node_config = NEOFS_NETMAP_DICT.get(node_name)
+    host = node_config.get('control').split(':')[0]
+    ssh_client = HostClient(host, STORAGE_NODE_USER, STORAGE_NODE_PWD)
+
+    try:
+        yield ssh_client
+    finally:
+        ssh_client.drop()
+
+
 @keyword('Stop Nodes')
-def stop_nodes(number: int, nodes: list):
+def stop_nodes(number: int, nodes: list) -> None:
     """
         The function shuts down the given number of randomly
         selected nodes in docker.
@@ -55,7 +71,7 @@ def stop_nodes(number: int, nodes: list):
 
 
 @keyword('Start Nodes')
-def start_nodes(nodes: list):
+def start_nodes(nodes: list) -> None:
     """
         The function raises the given nodes.
         Args:
@@ -106,12 +122,11 @@ def get_locode():
 
 
 @keyword('Stop Nodes Remote')
-def stop_nodes_remote(client: HostClient, number: int, nodes: list):
+def stop_nodes_remote(number: int, nodes: list) -> None:
     """
         The function shuts down the given number of randomly
         selected nodes in docker.
         Args:
-            client (HostClient): client that implements exec command
             number (int): the number of nodes to shut down
             nodes (list): the list of nodes for possible shut down
         Returns:
@@ -120,137 +135,123 @@ def stop_nodes_remote(client: HostClient, number: int, nodes: list):
     nodes = random.sample(nodes, number)
     for node in nodes:
         node = node.split('.')[0]
-        client.exec(f'docker stop {node}')
+        with create_ssh_client(node) as ssh_client:
+            ssh_client.exec(f'docker stop {node}')
     return nodes
 
 
 @keyword('Start Nodes Remote')
-def start_nodes_remote(client: HostClient, nodes: list):
+def start_nodes_remote(nodes: list) -> None:
     """
         The function starts nodes in docker.
         Args:
-           client (HostClient): client that implements exec command
            nodes (list): the list of nodes for possible shut down
     """
     for node in nodes:
         node = node.split('.')[0]
-        client.exec(f'docker start {node}')
+        with create_ssh_client(node) as ssh_client:
+            ssh_client.exec(f'docker start {node}')
 
 
 @keyword('Healthcheck for node')
-def node_healthcheck(client: HostClient, node_name: str) -> HealthStatus:
+def node_healthcheck(node_name: str) -> HealthStatus:
     """
         The function returns node's health status.
         Args:
-            client HostClient: client that implements exec command.
             node_name str: node name to use for netmap snapshot operation
         Returns:
             health status as HealthStatus object.
     """
-    if node_name not in NEOFS_NETMAP_DICT:
+    with create_ssh_client(node_name) as ssh_client:
-        raise AssertionError(f'Node {node_name} is not found!')
+        cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control healthcheck ' \
-
+            f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
-    node_config = NEOFS_NETMAP_DICT.get(node_name)
-    control_url = node_config.get('control')
-    cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control healthcheck --endpoint {control_url} ' \
             f'--config {STORAGE_NODE_CONFIG_PATH}'
-    output = client.exec_with_confirmation(cmd, [''])
+        output = ssh_client.exec_with_confirmation(cmd, [''])
         return HealthStatus.from_stdout(output.stdout)
 
 
 @keyword('Set status for node')
-def node_set_status(client: HostClient, node_name: str, status: str):
+def node_set_status(node_name: str, status: str):
     """
         The function sets particular status for given node.
         Args:
-            client HostClient: client that implements exec command.
             node_name str: node name to use for netmap snapshot operation
             status str: online or offline.
         Returns:
             (void)
     """
-    if node_name not in NEOFS_NETMAP_DICT:
+    with create_ssh_client(node_name) as ssh_client:
-        raise AssertionError(f'Node {node_name} is not found!')
+        cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control set-status ' \
-
+            f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
-    node_config = NEOFS_NETMAP_DICT.get(node_name)
-    control_url = node_config.get('control')
-    cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control set-status --endpoint {control_url} ' \
             f'--config {STORAGE_NODE_CONFIG_PATH} --status {status}'
-    client.exec_with_confirmation(cmd, [''])
+        ssh_client.exec_with_confirmation(cmd, [''])
 
 
 @keyword('Get netmap snapshot')
-def get_netmap_snapshot(client: HostClient, node_name: str = None) -> str:
+def get_netmap_snapshot(node_name: str = None) -> str:
     """
         The function returns string representation of netmap-snapshot.
         Args:
-            client HostClient: client that implements exec command.
             node_name str: node name to use for netmap snapshot operation
         Returns:
             string representation of netmap-snapshot
     """
     node_name = node_name or list(NEOFS_NETMAP_DICT)[0]
 
-    if node_name not in NEOFS_NETMAP_DICT:
+    with create_ssh_client(node_name) as ssh_client:
-        raise AssertionError(f'Node {node_name} is not found!')
+        cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control netmap-snapshot ' \
-
+            f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
-    node_config = NEOFS_NETMAP_DICT.get(node_name)
-    control_url = node_config.get('control')
-    cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control netmap-snapshot --endpoint {control_url} ' \
             f'--config {STORAGE_NODE_CONFIG_PATH}'
-    output = client.exec_with_confirmation(cmd, [''])
+        output = ssh_client.exec_with_confirmation(cmd, [''])
         return output.stdout
 
 
 @keyword('Shard list for node')
-def node_shard_list(client: HostClient, node_name: str) -> List[str]:
+def node_shard_list(node_name: str) -> List[str]:
     """
         The function returns list of shards for particular node.
         Args:
-            client HostClient: client that implements exec command.
             node_name str: node name to use for netmap snapshot operation
         Returns:
             list of shards.
     """
-    node_config = NEOFS_NETMAP_DICT.get(node_name)
+    with create_ssh_client(node_name) as ssh_client:
-    control_url = node_config.get('control')
+        cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards list ' \
-    cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards list --endpoint {control_url} ' \
+            f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
             f'--config {STORAGE_NODE_CONFIG_PATH}'
-    output = client.exec_with_confirmation(cmd, [''])
+        output = ssh_client.exec_with_confirmation(cmd, [''])
         return re.findall(r'Shard (.*):', output.stdout)
 
 
 @keyword('Shard list for node')
-def node_shard_set_mode(client: HostClient, node_name: str, shard: str, mode: str) -> str:
+def node_shard_set_mode(node_name: str, shard: str, mode: str) -> str:
     """
         The function sets mode for node's particular shard.
         Args:
-            client HostClient: client that implements exec command.
             node_name str: node name to use for netmap snapshot operation
         Returns:
             health status as HealthStatus object.
     """
-    node_config = NEOFS_NETMAP_DICT.get(node_name)
+    with create_ssh_client(node_name) as ssh_client:
-    control_url = node_config.get('control')
+        cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards set-mode ' \
-    cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards set-mode --endpoint {control_url} ' \
+            f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
             f'--config {STORAGE_NODE_CONFIG_PATH} --id {shard} --mode {mode}'
-    output = client.exec_with_confirmation(cmd, [''])
+        output = ssh_client.exec_with_confirmation(cmd, [''])
         return output.stdout
 
 
 @keyword('Drop object from node {node_name}')
-def drop_object(client: HostClient, node_name: str, cid: str, oid: str) -> str:
+def drop_object(node_name: str, cid: str, oid: str) -> str:
     """
         The function drops object from particular node.
         Args:
-            client HostClient: client that implements exec command.
             node_name str: node name to use for netmap snapshot operation
         Returns:
             health status as HealthStatus object.
     """
-    node_config = NEOFS_NETMAP_DICT.get(node_name)
+    with create_ssh_client(node_name) as ssh_client:
-    control_url = node_config.get('control')
+        cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control drop-objects ' \
-    cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control drop-objects --endpoint {control_url} ' \
+            f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
             f'--config {STORAGE_NODE_CONFIG_PATH} -o {cid}/{oid}'
-    output = client.exec_with_confirmation(cmd, [''])
+        output = ssh_client.exec_with_confirmation(cmd, [''])
         return output.stdout

robot/variables/common.py

@@ -86,11 +86,10 @@ STORAGE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/storage/wallet01.json"
 S3_GATE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/s3_gate/wallet.json"
 S3_GATE_WALLET_PASS = 's3'
 
-WALLET_PASS = f"{os.getcwd()}/wallet_pass.yml"
-
 STORAGE_NODE_USER = os.getenv('STORAGE_NODE_USER', 'root')
 STORAGE_NODE_PWD = os.getenv('STORAGE_NODE_PWD')
 STORAGE_NODE_BIN_PATH = os.getenv('STORAGE_NODE_BIN_PATH', '/opt/dev-env/vendor/neofs-cli')
 STORAGE_NODE_CONFIG_PATH = os.getenv('STORAGE_NODE_CONFIG_PATH', '/opt/dev-env/services/storage/cli-cfg.yml')
+STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT = os.getenv('STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT', 'localhost:8091')
 
 FREE_STORAGE = os.getenv('FREE_STORAGE', "false").lower() == "true"