aarifullin/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Pull requests Activity

Fix node management tests

Browse source 
Remote connection was created to the 1st storage node only. While in reality
we want to create connection to a specific node.

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
Vladimir Domnich 2022-07-07 22:48:47 +04:00
parent 68cbb7deea
commit be39480ade
5 changed files with 99 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
neofs-keywords

@ -1 +1 @@
Subproject commit 1d7ebe1d6d4bb6c8ce62f36091347bddb54d333b
Subproject commit 58650099362f05b51b97f04e2244a4ba20f88351

2
pytest_tests/testsuites/container/test_container.py
View file

@ -4,7 +4,7 @@ from time import sleep
import allure
import pytest
from contract_keywords import tick_epoch
from epoch import tick_epoch
from python_keywords.container import create_container, get_container, list_containers, delete_container
from utility import placement_policy_from_container

60
pytest_tests/testsuites/network/test_node_management.py
View file

@ -6,7 +6,7 @@ import allure
import pytest
from common import (COMPLEX_OBJ_SIZE, MAINNET_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT,
NEOFS_NETMAP_DICT, SHARD_0_GC_SLEEP)
from contract_keywords import tick_epoch
from epoch import tick_epoch
from utility_keywords import generate_file
from python_keywords.container import create_container, get_container
from python_keywords.neofs_verbs import (delete_object, get_object,
@ -28,7 +28,7 @@ logger = logging.getLogger('NeoLogger')
@pytest.fixture
@allure.title('Create container and pick the node with data')
def crate_container_and_pick_node(create_remote_connection, prepare_wallet_and_deposit):
def crate_container_and_pick_node(prepare_wallet_and_deposit):
wallet = prepare_wallet_and_deposit
file_path = generate_file()
placement_rule = 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X'
@ -44,63 +44,63 @@ def crate_container_and_pick_node(create_remote_connection, prepare_wallet_and_d
yield cid, node_name
shards = node_shard_list(create_remote_connection, node_name)
shards = node_shard_list(node_name)
assert shards
for shard in shards:
node_shard_set_mode(create_remote_connection, node_name, shard, 'read-write')
node_shard_set_mode(node_name, shard, 'read-write')
node_shard_list(create_remote_connection, node_name)
node_shard_list(node_name)
@pytest.fixture
@pytest.mark.skip(reason="docker API works only for devenv")
def start_node_if_needed(create_remote_connection):
def start_node_if_needed():
yield
try:
start_nodes_remote(create_remote_connection, list(NEOFS_NETMAP_DICT.keys()))
start_nodes_remote(list(NEOFS_NETMAP_DICT.keys()))
except Exception as err:
logger.error(f'Node start fails with error:\n{err}')
@allure.title('Control Operations with storage nodes')
@pytest.mark.node_mgmt
def test_nodes_management(prepare_tmp_dir, create_remote_connection):
def test_nodes_management(prepare_tmp_dir):
"""
This test checks base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
"""
random_node = choice(list(NEOFS_NETMAP_DICT))
alive_node = choice([node for node in NEOFS_NETMAP_DICT if node != random_node])
snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node)
snapshot = get_netmap_snapshot(node_name=alive_node)
assert random_node in snapshot, f'Expected node {random_node} in netmap'
with allure.step('Run health check for all storage nodes'):
for node_name in NEOFS_NETMAP_DICT.keys():
health_check = node_healthcheck(create_remote_connection, node_name)
health_check = node_healthcheck(node_name)
assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE'
with allure.step(f'Move node {random_node} to offline state'):
node_set_status(create_remote_connection, random_node, status='offline')
node_set_status(random_node, status='offline')
sleep(robot_time_to_int(MAINNET_BLOCK_TIME))
tick_epoch()
with allure.step(f'Check node {random_node} went to offline'):
health_check = node_healthcheck(create_remote_connection, random_node)
health_check = node_healthcheck(random_node)
assert health_check.health_status == 'READY' and health_check.network_status == 'STATUS_UNDEFINED'
snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node)
snapshot = get_netmap_snapshot(node_name=alive_node)
assert random_node not in snapshot, f'Expected node {random_node} not in netmap'
with allure.step(f'Check node {random_node} went to online'):
node_set_status(create_remote_connection, random_node, status='online')
node_set_status(random_node, status='online')
sleep(robot_time_to_int(MAINNET_BLOCK_TIME))
tick_epoch()
with allure.step(f'Check node {random_node} went to online'):
health_check = node_healthcheck(create_remote_connection, random_node)
health_check = node_healthcheck(random_node)
assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE'
snapshot = get_netmap_snapshot(create_remote_connection, node_name=alive_node)
snapshot = get_netmap_snapshot(node_name=alive_node)
assert random_node in snapshot, f'Expected node {random_node} in netmap'
@ -170,7 +170,7 @@ def test_placement_policy_negative(prepare_wallet_and_deposit, placement_rule, e
@pytest.mark.node_mgmt
@pytest.mark.skip(reason="docker API works only for devenv")
@allure.title('NeoFS object replication on node failover')
def test_replication(prepare_wallet_and_deposit, create_remote_connection, start_node_if_needed):
def test_replication(prepare_wallet_and_deposit, start_node_if_needed):
"""
Test checks object replication on storage not failover and come back.
"""
@ -185,22 +185,22 @@ def test_replication(prepare_wallet_and_deposit, create_remote_connection, start
assert len(nodes) == expected_nodes_count, f'Expected {expected_nodes_count} copies, got {len(nodes)}'
node_names = [name for name, config in NEOFS_NETMAP_DICT.items() if config.get('rpc') in nodes]
stopped_nodes = stop_nodes_remote(create_remote_connection, 1, node_names)
stopped_nodes = stop_nodes_remote(1, node_names)
wait_for_expected_object_copies(wallet, cid, oid)
start_nodes_remote(create_remote_connection, stopped_nodes)
start_nodes_remote(stopped_nodes)
tick_epoch()
for node_name in node_names:
wait_for_node_go_online(create_remote_connection, node_name)
wait_for_node_go_online(node_name)
wait_for_expected_object_copies(wallet, cid, oid)
@pytest.mark.node_mgmt
@allure.title('NeoFS object could be dropped using control command')
def test_drop_object(prepare_wallet_and_deposit, create_remote_connection):
def test_drop_object(prepare_wallet_and_deposit):
"""
Test checks object could be dropped using `neofs-cli control drop-objects` command.
"""
@ -224,7 +224,7 @@ def test_drop_object(prepare_wallet_and_deposit, create_remote_connection):
with allure.step(f'Drop object {oid}'):
get_object(wallet, cid, oid)
head_object(wallet, cid, oid)
drop_object(create_remote_connection, node_name, cid, oid)
drop_object(node_name, cid, oid)
wait_for_obj_dropped(wallet, cid, oid, get_object)
wait_for_obj_dropped(wallet, cid, oid, head_object)
@ -232,7 +232,7 @@ def test_drop_object(prepare_wallet_and_deposit, create_remote_connection):
@pytest.mark.node_mgmt
@pytest.mark.skip(reason='Need to clarify scenario')
@allure.title('Control Operations with storage nodes')
def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_container_and_pick_node):
def test_shards(prepare_wallet_and_deposit, crate_container_and_pick_node):
"""
This test checks base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
"""
@ -244,13 +244,13 @@ def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_cont
# for mode in ('read-only', 'degraded'):
for mode in ('degraded',):
shards = node_shard_list(create_remote_connection, node_name)
shards = node_shard_list(node_name)
assert shards
for shard in shards:
node_shard_set_mode(create_remote_connection, node_name, shard, mode)
node_shard_set_mode(node_name, shard, mode)
shards = node_shard_list(create_remote_connection, node_name)
shards = node_shard_list(node_name)
assert shards
with pytest.raises(RuntimeError):
@ -263,9 +263,9 @@ def test_shards(prepare_wallet_and_deposit, create_remote_connection, crate_cont
get_object(wallet, cid, original_oid)
for shard in shards:
node_shard_set_mode(create_remote_connection, node_name, shard, 'read-write')
node_shard_set_mode(node_name, shard, 'read-write')
shards = node_shard_list(create_remote_connection, node_name)
shards = node_shard_list(node_name)
assert shards
oid = put_object(wallet, file_path, cid)
@ -285,11 +285,11 @@ def validate_object_copies(wallet: str, placement_rule: str, file_path: str, exp
@allure.step('Wait for node {node_name} goes online')
def wait_for_node_go_online(create_remote_connection, node_name: str):
def wait_for_node_go_online(node_name: str):
timeout, attempts = 5, 20
for _ in range(attempts):
try:
health_check = node_healthcheck(create_remote_connection, node_name)
health_check = node_healthcheck(node_name)
assert health_check.health_status == 'READY' and health_check.network_status == 'ONLINE'
return
except Exception as err:

131
robot/resources/lib/python_keywords/node_management.py
View file

@ -7,11 +7,12 @@
import random
import re
from contextlib import contextmanager
from dataclasses import dataclass
from typing import List
import docker
from common import NEOFS_NETMAP_DICT, STORAGE_NODE_BIN_PATH, STORAGE_NODE_CONFIG_PATH
from common import NEOFS_NETMAP_DICT, STORAGE_NODE_BIN_PATH, STORAGE_NODE_CONFIG_PATH, STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT, STORAGE_NODE_PWD, STORAGE_NODE_USER
from robot.api import logger
from robot.api.deco import keyword
from ssh_helper import HostClient
@ -35,8 +36,23 @@ class HealthStatus:
return HealthStatus(network, health)
@contextmanager
def create_ssh_client(node_name: str) -> HostClient:
if node_name not in NEOFS_NETMAP_DICT:
raise AssertionError(f'Node {node_name} is not found!')
node_config = NEOFS_NETMAP_DICT.get(node_name)
host = node_config.get('control').split(':')[0]
ssh_client = HostClient(host, STORAGE_NODE_USER, STORAGE_NODE_PWD)
try:
yield ssh_client
finally:
ssh_client.drop()
@keyword('Stop Nodes')
def stop_nodes(number: int, nodes: list):
def stop_nodes(number: int, nodes: list) -> None:
"""
The function shuts down the given number of randomly
selected nodes in docker.
@ -55,7 +71,7 @@ def stop_nodes(number: int, nodes: list):
@keyword('Start Nodes')
def start_nodes(nodes: list):
def start_nodes(nodes: list) -> None:
"""
The function raises the given nodes.
Args:
@ -106,12 +122,11 @@ def get_locode():
@keyword('Stop Nodes Remote')
def stop_nodes_remote(client: HostClient, number: int, nodes: list):
def stop_nodes_remote(number: int, nodes: list) -> None:
"""
The function shuts down the given number of randomly
selected nodes in docker.
Args:
client (HostClient): client that implements exec command
number (int): the number of nodes to shut down
nodes (list): the list of nodes for possible shut down
Returns:
@ -120,137 +135,123 @@ def stop_nodes_remote(client: HostClient, number: int, nodes: list):
nodes = random.sample(nodes, number)
for node in nodes:
node = node.split('.')[0]
client.exec(f'docker stop {node}')
with create_ssh_client(node) as ssh_client:
ssh_client.exec(f'docker stop {node}')
return nodes
@keyword('Start Nodes Remote')
def start_nodes_remote(client: HostClient, nodes: list):
def start_nodes_remote(nodes: list) -> None:
"""
The function starts nodes in docker.
Args:
client (HostClient): client that implements exec command
nodes (list): the list of nodes for possible shut down
"""
for node in nodes:
node = node.split('.')[0]
client.exec(f'docker start {node}')
with create_ssh_client(node) as ssh_client:
ssh_client.exec(f'docker start {node}')
@keyword('Healthcheck for node')
def node_healthcheck(client: HostClient, node_name: str) -> HealthStatus:
def node_healthcheck(node_name: str) -> HealthStatus:
"""
The function returns node's health status.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
Returns:
health status as HealthStatus object.
"""
if node_name not in NEOFS_NETMAP_DICT:
raise AssertionError(f'Node {node_name} is not found!')
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control healthcheck --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = client.exec_with_confirmation(cmd, [''])
return HealthStatus.from_stdout(output.stdout)
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control healthcheck ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = ssh_client.exec_with_confirmation(cmd, [''])
return HealthStatus.from_stdout(output.stdout)
@keyword('Set status for node')
def node_set_status(client: HostClient, node_name: str, status: str):
def node_set_status(node_name: str, status: str):
"""
The function sets particular status for given node.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
status str: online or offline.
Returns:
(void)
"""
if node_name not in NEOFS_NETMAP_DICT:
raise AssertionError(f'Node {node_name} is not found!')
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control set-status --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} --status {status}'
client.exec_with_confirmation(cmd, [''])
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control set-status ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} --status {status}'
ssh_client.exec_with_confirmation(cmd, [''])
@keyword('Get netmap snapshot')
def get_netmap_snapshot(client: HostClient, node_name: str = None) -> str:
def get_netmap_snapshot(node_name: str = None) -> str:
"""
The function returns string representation of netmap-snapshot.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
Returns:
string representation of netmap-snapshot
"""
node_name = node_name or list(NEOFS_NETMAP_DICT)[0]
if node_name not in NEOFS_NETMAP_DICT:
raise AssertionError(f'Node {node_name} is not found!')
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control netmap-snapshot --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = client.exec_with_confirmation(cmd, [''])
return output.stdout
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control netmap-snapshot ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = ssh_client.exec_with_confirmation(cmd, [''])
return output.stdout
@keyword('Shard list for node')
def node_shard_list(client: HostClient, node_name: str) -> List[str]:
def node_shard_list(node_name: str) -> List[str]:
"""
The function returns list of shards for particular node.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
Returns:
list of shards.
"""
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards list --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = client.exec_with_confirmation(cmd, [''])
return re.findall(r'Shard (.*):', output.stdout)
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards list ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH}'
output = ssh_client.exec_with_confirmation(cmd, [''])
return re.findall(r'Shard (.*):', output.stdout)
@keyword('Shard list for node')
def node_shard_set_mode(client: HostClient, node_name: str, shard: str, mode: str) -> str:
def node_shard_set_mode(node_name: str, shard: str, mode: str) -> str:
"""
The function sets mode for node's particular shard.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
Returns:
health status as HealthStatus object.
"""
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards set-mode --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} --id {shard} --mode {mode}'
output = client.exec_with_confirmation(cmd, [''])
return output.stdout
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control shards set-mode ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} --id {shard} --mode {mode}'
output = ssh_client.exec_with_confirmation(cmd, [''])
return output.stdout
@keyword('Drop object from node {node_name}')
def drop_object(client: HostClient, node_name: str, cid: str, oid: str) -> str:
def drop_object(node_name: str, cid: str, oid: str) -> str:
"""
The function drops object from particular node.
Args:
client HostClient: client that implements exec command.
node_name str: node name to use for netmap snapshot operation
Returns:
health status as HealthStatus object.
"""
node_config = NEOFS_NETMAP_DICT.get(node_name)
control_url = node_config.get('control')
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control drop-objects --endpoint {control_url} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} -o {cid}/{oid}'
output = client.exec_with_confirmation(cmd, [''])
return output.stdout
with create_ssh_client(node_name) as ssh_client:
cmd = f'{STORAGE_NODE_BIN_PATH}/neofs-cli control drop-objects ' \
f'--endpoint {STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT} ' \
f'--config {STORAGE_NODE_CONFIG_PATH} -o {cid}/{oid}'
output = ssh_client.exec_with_confirmation(cmd, [''])
return output.stdout

3
robot/variables/common.py
View file

@ -86,11 +86,10 @@ STORAGE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/storage/wallet01.json"
S3_GATE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/s3_gate/wallet.json"
S3_GATE_WALLET_PASS = 's3'
WALLET_PASS = f"{os.getcwd()}/wallet_pass.yml"
STORAGE_NODE_USER = os.getenv('STORAGE_NODE_USER', 'root')
STORAGE_NODE_PWD = os.getenv('STORAGE_NODE_PWD')
STORAGE_NODE_BIN_PATH = os.getenv('STORAGE_NODE_BIN_PATH', '/opt/dev-env/vendor/neofs-cli')
STORAGE_NODE_CONFIG_PATH = os.getenv('STORAGE_NODE_CONFIG_PATH', '/opt/dev-env/services/storage/cli-cfg.yml')
STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT = os.getenv('STORAGE_NODE_PRIVATE_CONTROL_ENDPOINT', 'localhost:8091')
FREE_STORAGE = os.getenv('FREE_STORAGE', "false").lower() == "true"