anurindm/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Pull requests Activity

Fix node management tests.

Browse source 
Remote connection was created to the 1st storage node only.
In reality we wanted to create connection to specific node.
This commit is contained in:
Vladimir Domnich 2022-07-07 22:48:47 +04:00
parent 0daa262fdb
commit 7f0dcbfdca
3 changed files with 96 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
pytest_tests/testsuites/network/test_node_management.py
View file

@ -28,7 +28,7 @@ logger = logging.getLogger('NeoLogger')
@pytest.fixture
@allure.title('Create container and pick the node with data')
def crate_container_and_pick_node(create_remote_connection, prepare_wallet_and_deposit):
def crate_container_and_pick_node(prepare_wallet_and_deposit):
wallet = prepare_wallet_and_deposit
file_path = generate_file()
placement_rule = 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X'
@ -44,63 +44,63 @@ def crate_container_and_pick_node(create_remote_connection, prepare_wallet_and_d
yield cid, node_name
shards = node_shard_list(create_remote_connection, node_name)
shards = node_shard_list(node_name)
assert shards
for shard in shards:
node_shard_set_mode(create_remote_connection, node_name, shard, 'read-write')
node_shard_set_mode(node_name, shard, 'read-write')
node_shard_list(create_remote_connection, node_name)
node_shard_list(node_name)
@pytest.fixture
@pytest.mark.skip(reason="docker API works only for devenv")
def start_node_if_needed(create_remote_connection):
def start_node_if_needed():
yield
try:
start_nodes_remote(create_remote_connection, list(NEOFS_NETMAP_DICT.keys()))
start_nodes_remote(list(NEOFS_NETMAP_DICT.keys()))
except Exception as err:
logger.error(f'Node start fails with error:\n{err}')
@allure.title('Control Operations with storage nodes')
@pytest.mark.node_mgmt
def test_nodes_management(prepare_tmp_dir, create_remote_connection):
def test_nodes_management(prepare_tmp_dir):
"""
This test checks base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
"""
random_node = choice(list(NEOFS_NETMAP_DICT))
alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != random_node])
snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node)
snapshot = get_netmap_snapshot(node_name=alive_node)
assert random_node in snapshot, f'Expected node {random_node} in netmap'
with allure.step('Run health check for all storage nodes'):
for node_name in NEOFS_NETMAP_DICT.keys():
health_check = node_healthcheck(create_remote_connection, node_name)
health_check = node_healthcheck(node_name)
assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE'
with allure.step(f'Move node {random_node} to offline state'):
node_set_status(create_remote_connection, random_node, status='offline')
node_set_status(random_node, status='offline')
sleep(robot_time_to_int(MAINNET_BLOCK_TIME))
tick_epoch()
with allure.step(f'Check node {random_node} went to offline'):
health_check = node_healthcheck(create_remote_connection, random_node)
health_check = node_healthcheck(random_node)
assert health_check.health_status == 'READY' and health_check.network_status == 'STATUS_UNDEFINED'
snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node)
snapshot = get_netmap_snapshot(node_name=alive_node)
assert random_node not in snapshot, f'Expected node {random_node} not in netmap'
with allure.step(f'Check node {random_node} went to online'):
node_set_status(create_remote_connection, random_node, status='online')
node_set_status(random_node, status='online')
sleep(robot_time_to_int(MAINNET_BLOCK_TIME))
tick_epoch()
with allure.step(f'Check node {random_node} went to online'):
health_check = node_healthcheck(create_remote_connection, random_node)
health_check = node_healthcheck(random_node)
assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE'
snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node)
snapshot = get_netmap_snapshot(node_name=alive_node)
assert random_node in snapshot, f'Expected node {random_node} in netmap'
@ -170,7 +170,7 @@ def test_placement_policy_negative(prepare_wallet_and_deposit, placement_rule, e
@pytest.mark.node_mgmt
@pytest.mark.skip(reason="docker API works only for devenv")
@allure.title('NeoFS object replication on node failover')
def test_replication(prepare_wallet_and_deposit, create_remote_connection, start_node_if_needed):
def test_replication(prepare_wallet_and_deposit, start_node_if_needed):
"""
Test checks object replication on storage not failover and come back.
"""
@ -185,22 +185,22 @@ def test_replication(prepare_wallet_and_deposit, create_remote_connection, start
assert len(nodes) == expected_nodes_count, f'Expected {expected_nodes_count} copies, got {len(nodes)}'
node_names = [name for name, config in NEOFS_NETMAP_DICT.items() if config.get('rpc') in nodes]
stopped_nodes = stop_nodes_remote(create_remote_connection, 1, node_names)
stopped_nodes = stop_nodes_remote(1, node_names)
wait_for_expected_object_copies(wallet, cid, oid)
start_nodes_remote(create_remote_connection, stopped_nodes)
start_nodes_remote(stopped_nodes)
tick_epoch()
for node_name in node_names:
wait_for_node_go_online(create_remote_connection, node_name)
wait_for_node_go_online(node_name)
wait_for_expected_object_copies(wallet, cid, oid)
@pytest.mark.node_mgmt
@allure.title('NeoFS object could be dropped using control command')
def test_drop_object(prepare_wallet_and_deposit, create_remote_connection):
def test_drop_object(prepare_wallet_and_deposit):
"""
Test checks object could be dropped using `neofs-cli control drop-objects` command.
"""
@ -224,7 +224,7 @@ def test_drop_object(prepare_wallet_and_deposit, create_remote_connection):
with allure.step(f'Drop object {oid}'):
get_object(wallet, cid, oid)
head_object(wallet, cid, oid)
drop_object(create_remote_connection, node_name, cid, oid)
drop_object(node_name, cid, oid)
wait_for_obj_dropped(wallet, cid, oid, get_object)
wait_for_obj_dropped(wallet, cid, oid, head_object)
@ -232,7 +232,7 @@ def test_drop_object(prepare_wallet_and_deposit, create_remote_connection):
@pytest.mark.node_mgmt
@pytest.mark.skip(reason='Need to clarify scenario')
@allure.title('Control Operations with storage nodes')
def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_container_and_pick_node):
def test_shards(prepare_wallet_and_deposit, crate_container_and_pick_node):
"""
This test checks base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
"""
@ -244,13 +244,13 @@ def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_cont
# for mode in ('read-only', 'degraded'):
for mode in ('degraded',):
shards = node_shard_list(create_remote_connection, node_name)
shards = node_shard_list(node_name)
assert shards
for shard in shards:
node_shard_set_mode(create_remote_connection, node_name, shard, mode)
node_shard_set_mode(node_name, shard, mode)
shards = node_shard_list(create_remote_connection, node_name)
shards = node_shard_list(node_name)
assert shards
with pytest.raises(RuntimeError):
@ -263,9 +263,9 @@ def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_cont
get_object(wallet, cid, original_oid)
for shard in shards:
node_shard_set_mode(create_remote_connection, node_name, shard, 'read-write')
node_shard_set_mode(node_name, shard, 'read-write')
shards = node_shard_list(create_remote_connection, node_name)
shards = node_shard_list(node_name)
assert shards
oid = put_object(wallet, file_path, cid)
@ -285,11 +285,11 @@ def validate_object_copies(wallet: str, placement_rule: str, file_path: str, exp
@allure.step('Wait for node {node_name} goes online')
def wait_for_node_go_online(create_remote_connection, node_name: str):
def wait_for_node_go_online(node_name: str):
timeout, attempts = 5, 20
for _ in range(attempts):
try:
health_check = node_healthcheck(create_remote_connection, node_name)
health_check = node_healthcheck(node_name)
assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE'
return
except Exception as err:

131
robot/resources/lib/python_keywords/node_management.py
View file

@ -7,11 +7,12 @@
import random
import re
from contextlib import contextmanager
from dataclasses import dataclass
from typing import List
import docker
from common import NEOFS_NETMAP_DICT, STORAGE_NODE_BIN_PATH, STORAGE_NODE_CONFIG_PATH
from common import NEOFS_NETMAP_DICT, STORAGE_NODE_BIN_PATH, STORAGE_NODE_CONFIG_PATH, STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT, STORAGE_NODE_PWD, STORAGE_NODE_USER
from robot.api import logger
from robot.api.deco import keyword
from ssh_helper import HostClient
@ -35,8 +36,23 @@ class HealthStatus:
return HealthStatus(network, health)
@contextmanager
def create_ssh_client(node_name: str) -> HostClient:
if node_name not in NEOFS_NETMAP_DICT:
raise AssertionError(f'Node {node_name} is not found!')
node_config = NEOFS_NETMAP_DICT.get(node_name)
host = node_config.get('control').split(':')[0]
ssh_client = HostClient(host, STORAGE_NODE_USER, STORAGE_NODE_PWD)
try:
yield ssh_client
finally:
ssh_client.drop()
@keyword('Stop Nodes')
def stop_nodes(number: int, nodes: list):
def stop_nodes(number: int, nodes: list) -> None:
"""
The function shuts down the given number of randomly
selected nodes in docker.
@ -55,7 +71,7 @@ def stop_nodes(number: int, nodes: list):
@keyword('Start Nodes')
def start_nodes(nodes: list):
def start_nodes(nodes: list) -> None:
"""
The function raises the given nodes.
Args:
@ -106,12 +122,11 @@ def get_locode():
@keyword('Stop Nodes Remote')
def stop_nodes_remote(client: HostClient, number: int, nodes: list):
def stop_nodes_remote(number: int, nodes: list) -> None:
"""
The function shuts down the given number of randomly
selected nodes in docker.
Args:
client (HostClient): client that implements exec command
number (int): the number of nodes to shut down
nodes (list): the list of nodes for possible shut down
Returns:
@ -120,137 +135,123 @@ def stop_nodes_remote(client: HostClient, number: int, nodes: list):
nodes = random.sample(nodes, number)
for node in nodes:
node = node.split('.')[0]
client.exec(f'docker stop {node}')
with create_ssh_client(node) as ssh_client:
ssh_client.exec(f'docker stop {node}')
return nodes
@keyword('Start Nodes Remote')
def start_nodes_remote(client: HostClient, nodes: list):
def start_nodes_remote(nodes: list) -> None:
"""
The function starts nodes in docker.
Args:
client (HostClient): client that implements exec command
nodes (list): the list of nodes for possible shut down
"""
for node in nodes:
node = node.split('.')[0]
client.exec(f'docker start {node}')
with create_ssh_client(node) as ssh_client:
ssh_client.exec(f'docker start {node}')
@keyword('Healthcheck for node')
def node_healthcheck(client: HostClient, node_name: str) -> HealthStatus:
def node_healthcheck(node_name: str) -> HealthStatus:
"""
The function returns node's health status.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
Returns:
health status as HealthStatus object.
"""
if node_name not in NEOFS_NETMAP_DICT:
raise AssertionError(f'Node {node_name} is not found!')
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control healthcheck --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = client.exec_with_confirmation(cmd, [''])
return HealthStatus.from_stdout(output.stdout)
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control healthcheck ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = ssh_client.exec_with_confirmation(cmd, [''])
return HealthStatus.from_stdout(output.stdout)
@keyword('Set status for node')
def node_set_status(client: HostClient, node_name: str, status: str):
def node_set_status(node_name: str, status: str):
"""
The function sets particular status for given node.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
status str: online or offline.
Returns:
(void)
"""
if node_name not in NEOFS_NETMAP_DICT:
raise AssertionError(f'Node {node_name} is not found!')
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control set-status --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} --status {status}'
client.exec_with_confirmation(cmd, [''])
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control set-status ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} --status {status}'
ssh_client.exec_with_confirmation(cmd, [''])
@keyword('Get netmap snapshot')
def get_netmap_snapshot(client: HostClient, node_name: str = None) -> str:
def get_netmap_snapshot(node_name: str = None) -> str:
"""
The function returns string representation of netmap-snapshot.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
Returns:
string representation of netmap-snapshot
"""
node_name = node_name or list(NEOFS_NETMAP_DICT)[0]
if node_name not in NEOFS_NETMAP_DICT:
raise AssertionError(f'Node {node_name} is not found!')
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control netmap-snapshot --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = client.exec_with_confirmation(cmd, [''])
return output.stdout
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control netmap-snapshot ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = ssh_client.exec_with_confirmation(cmd, [''])
return output.stdout
@keyword('Shard list for node')
def node_shard_list(client: HostClient, node_name: str) -> List[str]:
def node_shard_list(node_name: str) -> List[str]:
"""
The function returns list of shards for particular node.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
Returns:
list of shards.
"""
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards list --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = client.exec_with_confirmation(cmd, [''])
return re.findall(r'Shard (.*):', output.stdout)
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards list ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = ssh_client.exec_with_confirmation(cmd, [''])
return re.findall(r'Shard (.*):', output.stdout)
@keyword('Shard list for node')
def node_shard_set_mode(client: HostClient, node_name: str, shard: str, mode: str) -> str:
def node_shard_set_mode(node_name: str, shard: str, mode: str) -> str:
"""
The function sets mode for node's particular shard.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
Returns:
health status as HealthStatus object.
"""
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards set-mode --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} --id {shard} --mode {mode}'
output = client.exec_with_confirmation(cmd, [''])
return output.stdout
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards set-mode ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} --id {shard} --mode {mode}'
output = ssh_client.exec_with_confirmation(cmd, [''])
return output.stdout
@keyword('Drop object from node {node_name}')
def drop_object(client: HostClient, node_name: str, cid: str, oid: str) -> str:
def drop_object(node_name: str, cid: str, oid: str) -> str:
"""
The function drops object from particular node.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
Returns:
health status as HealthStatus object.
"""
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control drop-objects --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} -o {cid}/{oid}'
output = client.exec_with_confirmation(cmd, [''])
return output.stdout
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control drop-objects ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} -o {cid}/{oid}'
output = ssh_client.exec_with_confirmation(cmd, [''])
return output.stdout

1
robot/variables/common.py
View file

@ -91,5 +91,6 @@ STORAGE_NODE_USER = os.getenv('STORAGE_NODE_USER', 'root')
STORAGE_NODE_PWD = os.getenv('STORAGE_NODE_PWD')
STORAGE_NODE_BIN_PATH = os.getenv('STORAGE_NODE_BIN_PATH', '/opt/dev-env/vendor/neofs-cli')
STORAGE_NODE_CONFIG_PATH = os.getenv('STORAGE_NODE_CONFIG_PATH', '/opt/dev-env/services/storage/cli-cfg.yml')
STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT = os.getenv('STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT', 'localhost:8091')
FREE_STORAGE = os.getenv('FREE_STORAGE', "false").lower() == "true"