From 48e53b3d863d866200e6ca1561e8b53474fc99e9 Mon Sep 17 00:00:00 2001 From: Vladimir Domnich Date: Thu, 13 Oct 2022 16:13:45 +0000 Subject: [PATCH] Switch failover test to hosting from testlib Signed-off-by: Vladimir Domnich --- pytest_tests/helpers/binary_version_helper.py | 25 +- pytest_tests/helpers/iptables_helper.py | 13 +- pytest_tests/helpers/remote_process.py | 4 + pytest_tests/helpers/ssh_helper.py | 250 ------------------ pytest_tests/requirements.txt | 2 +- pytest_tests/steps/s3_gate_base.py | 5 +- pytest_tests/testsuites/acl/test_acl.py | 1 - pytest_tests/testsuites/acl/test_eacl.py | 9 +- pytest_tests/testsuites/conftest.py | 29 +- .../failovers/test_failover_network.py | 106 ++++---- .../failovers/test_failover_storage.py | 104 ++++---- .../network/test_node_management.py | 72 +++-- .../testsuites/object/test_object_api.py | 2 +- .../testsuites/services/test_http_gate.py | 20 +- .../test_object_session_token.py | 2 +- .../lib/python_keywords/container.py | 9 +- .../lib/python_keywords/failover_utils.py | 4 +- .../lib/python_keywords/node_management.py | 12 +- .../lib/python_keywords/object_access.py | 8 +- robot/variables/common.py | 6 - 20 files changed, 242 insertions(+), 441 deletions(-) delete mode 100644 pytest_tests/helpers/ssh_helper.py diff --git a/pytest_tests/helpers/binary_version_helper.py b/pytest_tests/helpers/binary_version_helper.py index 4076b89..5fe334e 100644 --- a/pytest_tests/helpers/binary_version_helper.py +++ b/pytest_tests/helpers/binary_version_helper.py @@ -17,11 +17,11 @@ def get_local_binaries_versions(shell: Shell) -> dict[str, str]: versions[binary] = _parse_version(out) neofs_cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG) - versions["neofs-cli"] = _parse_version(neofs_cli.version.get()) + versions["neofs-cli"] = _parse_version(neofs_cli.version.get().stdout) try: neofs_adm = NeofsAdm(shell, NEOFS_ADM_EXEC) - versions["neofs-adm"] = _parse_version(neofs_adm.version.get()) + versions["neofs-adm"] = _parse_version(neofs_adm.version.get().stdout) except RuntimeError: logger.info(f"neofs-adm not installed") @@ -35,24 +35,23 @@ def get_local_binaries_versions(shell: Shell) -> dict[str, str]: def get_remote_binaries_versions(hosting: Hosting) -> dict[str, str]: versions_by_host = {} for host in hosting.hosts: - binaries = [] + binary_path_by_name = {} # Maps binary name to executable path for service_config in host.config.services: exec_path = service_config.attributes.get("exec_path") if exec_path: - binaries.append(exec_path) + binary_path_by_name[service_config.name] = exec_path for cli_config in host.config.clis: - binaries.append(cli_config.exec_path) + binary_path_by_name[cli_config.name] = cli_config.exec_path shell = host.get_shell() versions_at_host = {} - for binary in binaries: + for binary_name, binary_path in binary_path_by_name.items(): try: - result = shell.exec(f"{binary} --version") - versions_at_host[service_config.name] = _parse_version(result.stdout) + result = shell.exec(f"{binary_path} --version") + versions_at_host[binary_name] = _parse_version(result.stdout) except Exception as exc: - logger.error(f"Cannot get version for {exec_path} because of\n{exc}") - versions_at_host[service_config.name] = "Unknown" - continue + logger.error(f"Cannot get version for {binary_path} because of\n{exc}") + versions_at_host[binary_name] = "Unknown" versions_by_host[host.config.address] = versions_at_host # Consolidate versions across all hosts @@ -61,7 +60,9 @@ def get_remote_binaries_versions(hosting: Hosting) -> dict[str, str]: for name, version in binary_versions.items(): captured_version = versions.get(name) if captured_version: - assert captured_version == version, f"Binary {name} has inconsistent version on host {host}" + assert ( + captured_version == version + ), f"Binary {name} has inconsistent version on host {host}" else: versions[name] = version return versions diff --git a/pytest_tests/helpers/iptables_helper.py b/pytest_tests/helpers/iptables_helper.py index 338f034..50beb72 100644 --- a/pytest_tests/helpers/iptables_helper.py +++ b/pytest_tests/helpers/iptables_helper.py @@ -1,16 +1,13 @@ -from ssh_helper import HostClient +from neofs_testlib.shell import Shell -# TODO: convert to shell from hosting class IpTablesHelper: @staticmethod - def drop_input_traffic_to_port(client: HostClient, ports: list[str]): + def drop_input_traffic_to_port(shell: Shell, ports: list[str]) -> None: for port in ports: - cmd_output = client.exec(cmd=f"sudo iptables -A INPUT -p tcp --dport {port} -j DROP") - assert cmd_output.rc == 0 + shell.exec(f"sudo iptables -A INPUT -p tcp --dport {port} -j DROP") @staticmethod - def restore_input_traffic_to_port(client: HostClient, ports: list[str]): + def restore_input_traffic_to_port(shell: Shell, ports: list[str]) -> None: for port in ports: - cmd_output = client.exec(cmd=f"sudo iptables -D INPUT -p tcp --dport {port} -j DROP") - assert cmd_output.rc == 0 + shell.exec(f"sudo iptables -D INPUT -p tcp --dport {port} -j DROP") diff --git a/pytest_tests/helpers/remote_process.py b/pytest_tests/helpers/remote_process.py index 9caf8d0..b0e898f 100644 --- a/pytest_tests/helpers/remote_process.py +++ b/pytest_tests/helpers/remote_process.py @@ -4,6 +4,10 @@ import uuid from typing import Optional import allure + +# This file is broken, because tenacity is not registered in requirements.txt +# So, the file won't be fixed in scope of this PR, alipay will fix himself by +# switching RemoteProcess and K6 classes from HostClient to shell from hosting from tenacity import retry, stop_after_attempt, wait_fixed from pytest_tests.helpers.ssh_helper import HostClient diff --git a/pytest_tests/helpers/ssh_helper.py b/pytest_tests/helpers/ssh_helper.py deleted file mode 100644 index 5eea5eb..0000000 --- a/pytest_tests/helpers/ssh_helper.py +++ /dev/null @@ -1,250 +0,0 @@ -import logging -import socket -import tempfile -import textwrap -from contextlib import contextmanager -from dataclasses import dataclass -from datetime import datetime -from functools import wraps -from time import sleep -from typing import ClassVar, Optional - -import allure -from paramiko import AutoAddPolicy, RSAKey, SFTPClient, SSHClient, SSHException, ssh_exception -from paramiko.ssh_exception import AuthenticationException - - -class HostIsNotAvailable(Exception): - """Raises when host is not reachable.""" - - def __init__(self, ip: str = None, exc: Exception = None): - msg = f'Host is not available{f" by ip: {ip}" if ip else ""}' - if exc: - msg = f"{msg}. {exc}" - super().__init__(msg) - - -def log_command(func): - @wraps(func) - def wrapper(host: "HostClient", command: str, *args, **kwargs): - display_length = 60 - short = command.removeprefix("$ProgressPreference='SilentlyContinue'\n") - short = short[:display_length] - short += "..." if short != command else "" - with allure.step(f"SSH: {short}"): - logging.info(f'Execute command "{command}" on "{host.ip}"') - - start_time = datetime.utcnow() - cmd_result = func(host, command, *args, **kwargs) - end_time = datetime.utcnow() - - log_message = ( - f"HOST: {host.ip}\n" - f'COMMAND:\n{textwrap.indent(command, " ")}\n' - f"RC:\n {cmd_result.rc}\n" - f'STDOUT:\n{textwrap.indent(cmd_result.stdout, " ")}\n' - f'STDERR:\n{textwrap.indent(cmd_result.stderr, " ")}\n' - f"Start / End / Elapsed\t {start_time.time()} / {end_time.time()} / {end_time - start_time}" - ) - - logging.info(log_message) - allure.attach(log_message, "SSH command", allure.attachment_type.TEXT) - return cmd_result - - return wrapper - - -@dataclass -class SSHCommand: - stdout: str - stderr: str - rc: int - - -class HostClient: - ssh_client: SSHClient - SSH_CONNECTION_ATTEMPTS: ClassVar[int] = 3 - CONNECTION_TIMEOUT = 90 - - TIMEOUT_RESTORE_CONNECTION = 10, 24 - - def __init__( - self, - ip: str, - login: str, - password: Optional[str] = None, - private_key_path: Optional[str] = None, - private_key_passphrase: Optional[str] = None, - init_ssh_client=True, - ) -> None: - self.ip = ip - self.login = login - self.password = password - self.private_key_path = private_key_path - self.private_key_passphrase = private_key_passphrase - if init_ssh_client: - self.create_connection(self.SSH_CONNECTION_ATTEMPTS) - - def exec(self, cmd: str, verify=True, timeout=90) -> SSHCommand: - cmd_result = self._inner_exec(cmd, timeout) - if verify: - assert cmd_result.rc == 0, f'Non zero rc from command: "{cmd}"' - return cmd_result - - @log_command - def exec_with_confirmation( - self, cmd: str, confirmation: list, verify=True, timeout=90 - ) -> SSHCommand: - ssh_stdin, ssh_stdout, ssh_stderr = self.ssh_client.exec_command(cmd, timeout=timeout) - for line in confirmation: - if not line.endswith("\n"): - line = f"{line}\n" - try: - ssh_stdin.write(line) - except OSError as err: - logging.error(f"Got error {err} executing command {cmd}") - ssh_stdin.close() - output = SSHCommand( - stdout=ssh_stdout.read().decode(errors="ignore"), - stderr=ssh_stderr.read().decode(errors="ignore"), - rc=ssh_stdout.channel.recv_exit_status(), - ) - if verify: - debug_info = f"\nSTDOUT: {output.stdout}\nSTDERR: {output.stderr}\nRC: {output.rc}" - assert output.rc == 0, f'Non zero rc from command: "{cmd}"{debug_info}' - return output - - @contextmanager - def as_user(self, user: str, password: str): - keep_user, keep_password = self.login, self.password - self.login, self.password = user, password - self.create_connection() - yield - self.login, self.password = keep_user, keep_password - self.create_connection() - - @contextmanager - def create_ssh_connection(self) -> "SSHClient": - if not self.ssh_client: - self.create_connection() - try: - yield self.ssh_client - finally: - self.drop() - - @allure.step("Restore connection") - def restore_ssh_connection(self): - retry_time, retry_count = self.TIMEOUT_RESTORE_CONNECTION - for _ in range(retry_count): - try: - self.create_connection() - except AssertionError: - logging.warning(f"Host: Cant reach host: {self.ip}.") - sleep(retry_time) - else: - logging.info(f"Host: Cant reach host: {self.ip}.") - return - raise AssertionError(f"Host: Cant reach host: {self.ip} after 240 seconds..") - - @allure.step("Copy file {host_path_to_file} to local file {path_to_file}") - def copy_file_from_host(self, host_path_to_file: str, path_to_file: str): - with self._sftp_client() as sftp_client: - sftp_client.get(host_path_to_file, path_to_file) - - def copy_file_to_host(self, path_to_file: str, host_path_to_file: str): - with allure.step( - f"Copy local file {path_to_file} to remote file {host_path_to_file} on host {self.ip}" - ): - with self._sftp_client() as sftp_client: - sftp_client.put(path_to_file, host_path_to_file) - - @allure.step("Save string to remote file {host_path_to_file}") - def copy_str_to_host_file(self, string: str, host_path_to_file: str): - with tempfile.NamedTemporaryFile(mode="r+") as temp: - temp.writelines(string) - temp.flush() - with self._sftp_client() as client: - client.put(temp.name, host_path_to_file) - self.exec(f"cat {host_path_to_file}", verify=False) - - def create_connection(self, attempts=SSH_CONNECTION_ATTEMPTS): - exc_err = None - for attempt in range(attempts): - self.ssh_client = SSHClient() - self.ssh_client.set_missing_host_key_policy(AutoAddPolicy()) - try: - if self.private_key_path: - logging.info( - f"Trying to connect to host {self.ip} as {self.login} using SSH key " - f"{self.private_key_path} (attempt {attempt})" - ) - self.ssh_client.connect( - hostname=self.ip, - username=self.login, - pkey=RSAKey.from_private_key_file( - self.private_key_path, self.private_key_passphrase - ), - timeout=self.CONNECTION_TIMEOUT, - ) - else: - logging.info( - f"Trying to connect to host {self.ip} as {self.login} using password " - f"(attempt {attempt})" - ) - self.ssh_client.connect( - hostname=self.ip, - username=self.login, - password=self.password, - timeout=self.CONNECTION_TIMEOUT, - ) - return True - - except AuthenticationException as auth_err: - logging.error(f"Host: {self.ip}. {auth_err}") - self.drop() - raise auth_err - - except ( - SSHException, - ssh_exception.NoValidConnectionsError, - AttributeError, - socket.timeout, - OSError, - ) as ssh_err: - exc_err = ssh_err - self.drop() - logging.error(f"Host: {self.ip}, connection error. {exc_err}") - - raise HostIsNotAvailable(self.ip, exc_err) - - def drop(self): - self.ssh_client.close() - - @log_command - def _inner_exec(self, cmd: str, timeout: int) -> SSHCommand: - if not self.ssh_client: - self.create_connection() - for _ in range(self.SSH_CONNECTION_ATTEMPTS): - try: - _, stdout, stderr = self.ssh_client.exec_command(cmd, timeout=timeout) - return SSHCommand( - stdout=stdout.read().decode(errors="ignore"), - stderr=stderr.read().decode(errors="ignore"), - rc=stdout.channel.recv_exit_status(), - ) - except ( - SSHException, - TimeoutError, - ssh_exception.NoValidConnectionsError, - ConnectionResetError, - AttributeError, - socket.timeout, - ) as ssh_err: - logging.error(f"Host: {self.ip}, exec command error {ssh_err}") - self.create_connection() - raise HostIsNotAvailable(f"Host: {self.ip} is not reachable.") - - @contextmanager - def _sftp_client(self) -> SFTPClient: - with self.ssh_client.open_sftp() as sftp: - yield sftp diff --git a/pytest_tests/requirements.txt b/pytest_tests/requirements.txt index fe42f50..5c1e9f2 100644 --- a/pytest_tests/requirements.txt +++ b/pytest_tests/requirements.txt @@ -56,7 +56,7 @@ pyparsing==3.0.9 pyrsistent==0.18.1 pytest==7.1.2 python-dateutil==2.8.2 -requests==2.27.1 +requests==2.28.0 robotframework==4.1.2 s3transfer==0.3.7 six==1.16.0 diff --git a/pytest_tests/steps/s3_gate_base.py b/pytest_tests/steps/s3_gate_base.py index 874db61..001a3e2 100644 --- a/pytest_tests/steps/s3_gate_base.py +++ b/pytest_tests/steps/s3_gate_base.py @@ -20,6 +20,7 @@ from common import ( S3_GATE_WALLET_PATH, ) from data_formatters import get_wallet_public_key +from neofs_testlib.shell import Shell from python_keywords.container import list_containers from steps.aws_cli_client import AwsCliClient @@ -42,7 +43,7 @@ class TestS3GateBase: @pytest.fixture(scope="class", autouse=True) @allure.title("[Class/Autouse]: Create S3 client") - def s3_client(self, prepare_wallet_and_deposit, request): + def s3_client(self, prepare_wallet_and_deposit, client_shell: Shell, request): wallet = prepare_wallet_and_deposit s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json" @@ -53,7 +54,7 @@ class TestS3GateBase: secret_access_key, owner_private_key, ) = init_s3_credentials(wallet, s3_bearer_rules_file=s3_bearer_rules_file) - containers_list = list_containers(wallet) + containers_list = list_containers(wallet, shell=client_shell) assert cid in containers_list, f"Expected cid {cid} in {containers_list}" if request.param == "aws cli": diff --git a/pytest_tests/testsuites/acl/test_acl.py b/pytest_tests/testsuites/acl/test_acl.py index b0853b0..33b0681 100644 --- a/pytest_tests/testsuites/acl/test_acl.py +++ b/pytest_tests/testsuites/acl/test_acl.py @@ -1,6 +1,5 @@ import allure import pytest - from python_keywords.acl import EACLRole from python_keywords.container import create_container from python_keywords.container_access import ( diff --git a/pytest_tests/testsuites/acl/test_eacl.py b/pytest_tests/testsuites/acl/test_eacl.py index 27c32d0..3b99ea2 100644 --- a/pytest_tests/testsuites/acl/test_eacl.py +++ b/pytest_tests/testsuites/acl/test_eacl.py @@ -3,6 +3,7 @@ import pytest from common import NEOFS_NETMAP_DICT from failover_utils import wait_object_replication_on_nodes from neofs_testlib.hosting import Hosting +from neofs_testlib.shell import Shell from python_keywords.acl import ( EACLAccess, EACLOperation, @@ -28,7 +29,6 @@ from python_keywords.object_access import ( can_put_object, can_search_object, ) -from neofs_testlib.shell import Shell from wellknown_acl import PUBLIC_ACL @@ -195,7 +195,12 @@ class TestEACLContainer: @allure.title("Testcase to validate NeoFS replication with eACL deny rules.") def test_extended_acl_deny_replication( - self, wallets, client_shell, hosting: Hosting, eacl_full_placement_container_with_object, file_path + self, + wallets, + client_shell, + hosting: Hosting, + eacl_full_placement_container_with_object, + file_path, ): user_wallet = wallets.get_wallet() cid, oid, file_path = eacl_full_placement_container_with_object diff --git a/pytest_tests/testsuites/conftest.py b/pytest_tests/testsuites/conftest.py index 477b486..52cbac4 100644 --- a/pytest_tests/testsuites/conftest.py +++ b/pytest_tests/testsuites/conftest.py @@ -17,6 +17,7 @@ from common import ( ) from env_properties import save_env_properties from neofs_testlib.hosting import Hosting +from neofs_testlib.reporter import AllureHandler, get_reporter from neofs_testlib.shell import LocalShell, Shell from payment_neogo import neofs_deposit, transfer_mainnet_gas from python_keywords.node_management import node_healthcheck @@ -25,10 +26,26 @@ logger = logging.getLogger("NeoLogger") @pytest.fixture(scope="session") -def client_shell() -> Shell: +def configure_testlib(): + get_reporter().register_handler(AllureHandler()) + yield + + +@pytest.fixture(scope="session") +def client_shell(configure_testlib) -> Shell: yield LocalShell() +@pytest.fixture(scope="session") +def hosting(configure_testlib) -> Hosting: + with open(HOSTING_CONFIG_FILE, "r") as file: + hosting_config = yaml.full_load(file) + + hosting_instance = Hosting() + hosting_instance.configure(hosting_config) + yield hosting_instance + + @pytest.fixture(scope="session") def require_multiple_hosts(hosting: Hosting): """Fixture that is applied to tests that require that environment has multiple hosts.""" @@ -37,16 +54,6 @@ def require_multiple_hosts(hosting: Hosting): yield -@pytest.fixture(scope="session") -def hosting() -> Hosting: - with open(HOSTING_CONFIG_FILE, "r") as file: - hosting_config = yaml.full_load(file) - - hosting_instance = Hosting() - hosting_instance.configure(hosting_config) - yield hosting_instance - - @pytest.fixture(scope="session", autouse=True) @allure.title("Check binary versions") def check_binary_versions(request, hosting: Hosting, client_shell: Shell): diff --git a/pytest_tests/testsuites/failovers/test_failover_network.py b/pytest_tests/testsuites/failovers/test_failover_network.py index 9a33542..5b02a57 100644 --- a/pytest_tests/testsuites/failovers/test_failover_network.py +++ b/pytest_tests/testsuites/failovers/test_failover_network.py @@ -4,18 +4,12 @@ from time import sleep import allure import pytest -from common import ( - STORAGE_NODE_SSH_PASSWORD, - STORAGE_NODE_SSH_PRIVATE_KEY_PATH, - STORAGE_NODE_SSH_USER, -) from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes from file_helper import generate_file, get_file_hash from iptables_helper import IpTablesHelper from neofs_testlib.hosting import Hosting from python_keywords.container import create_container from python_keywords.neofs_verbs import get_object, put_object -from ssh_helper import HostClient from wellknown_acl import PUBLIC_ACL logger = logging.getLogger("NeoLogger") @@ -31,16 +25,10 @@ def restore_network(hosting: Hosting): yield not_empty = len(blocked_hosts) != 0 - for host in list(blocked_hosts): - with allure.step(f"Start storage node {host}"): - client = HostClient( - ip=host, - login=STORAGE_NODE_SSH_USER, - password=STORAGE_NODE_SSH_PASSWORD, - private_key_path=STORAGE_NODE_SSH_PRIVATE_KEY_PATH, - ) - with client.create_ssh_connection(): - IpTablesHelper.restore_input_traffic_to_port(client, PORTS_TO_BLOCK) + for host_address in list(blocked_hosts): + with allure.step(f"Restore network at host {host_address}"): + host = hosting.get_host_by_address(host_address) + IpTablesHelper.restore_input_traffic_to_port(host.get_shell(), PORTS_TO_BLOCK) blocked_hosts.remove(host) if not_empty: wait_all_storage_node_returned(hosting) @@ -49,67 +37,65 @@ def restore_network(hosting: Hosting): @allure.title("Block Storage node traffic") @pytest.mark.failover @pytest.mark.failover_net -def test_block_storage_node_traffic(prepare_wallet_and_deposit, client_shell, require_multiple_hosts): +def test_block_storage_node_traffic( + prepare_wallet_and_deposit, client_shell, require_multiple_hosts, hosting: Hosting +): """ Block storage nodes traffic using iptables and wait for replication for objects. """ wallet = prepare_wallet_and_deposit placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" - excluded_nodes = [] wakeup_node_timeout = 10 # timeout to let nodes detect that traffic has blocked nodes_to_block_count = 2 source_file_path = generate_file() cid = create_container(wallet, shell=client_shell, rule=placement_rule, basic_acl=PUBLIC_ACL) oid = put_object(wallet, source_file_path, cid, shell=client_shell) - nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell) - logger.info(f"Nodes are {nodes}") - random_nodes = [(node, node.split(":")[0]) for node in nodes] - if nodes_to_block_count > len(nodes): - random_nodes = [(node, node.split(":")[0]) for node in choices(nodes, k=2)] + # TODO: we need to refactor wait_object_replication_on_nodes so that it returns + # storage node names rather than endpoints + node_endpoints = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell) - for random_node, random_node_ip in random_nodes: - client = HostClient( - ip=random_node_ip, - login=STORAGE_NODE_SSH_USER, - password=STORAGE_NODE_SSH_PASSWORD, - private_key_path=STORAGE_NODE_SSH_PRIVATE_KEY_PATH, - ) + logger.info(f"Nodes are {node_endpoints}") + node_endpoints_to_block = node_endpoints + if nodes_to_block_count > len(node_endpoints): + # TODO: the intent of this logic is not clear, need to revisit + node_endpoints_to_block = choices(node_endpoints, k=2) - with allure.step(f"Block incoming traffic for node {random_node} on port {PORTS_TO_BLOCK}"): - with client.create_ssh_connection(): - IpTablesHelper.drop_input_traffic_to_port(client, PORTS_TO_BLOCK) - blocked_hosts.append(random_node_ip) - excluded_nodes.append(random_node) + excluded_nodes = [] + for node_endpoint in node_endpoints_to_block: + host_address = node_endpoint.split(":")[0] + host = hosting.get_host_by_address(host_address) + + with allure.step(f"Block incoming traffic at host {host_address} on port {PORTS_TO_BLOCK}"): + blocked_hosts.append(host_address) + excluded_nodes.append(node_endpoint) + IpTablesHelper.drop_input_traffic_to_port(host.get_shell(), PORTS_TO_BLOCK) sleep(wakeup_node_timeout) - new_nodes = wait_object_replication_on_nodes( - wallet, cid, oid, 2, excluded_nodes=excluded_nodes - ) + with allure.step(f"Check object is not stored on node {node_endpoint}"): + new_nodes = wait_object_replication_on_nodes( + wallet, cid, oid, 2, shell=client_shell, excluded_nodes=excluded_nodes + ) + assert node_endpoint not in new_nodes - assert random_node not in new_nodes + with allure.step(f"Check object data is not corrupted"): + got_file_path = get_object(wallet, cid, oid, endpoint=new_nodes[0], shell=client_shell) + assert get_file_hash(source_file_path) == get_file_hash(got_file_path) + + for node_endpoint in node_endpoints_to_block: + host_address = node_endpoint.split(":")[0] + host = hosting.get_host_by_address(host_address) + + with allure.step( + f"Unblock incoming traffic at host {host_address} on port {PORTS_TO_BLOCK}" + ): + IpTablesHelper.restore_input_traffic_to_port(host.get_shell(), PORTS_TO_BLOCK) + blocked_hosts.remove(host_address) + sleep(wakeup_node_timeout) + + with allure.step(f"Check object data is not corrupted"): + new_nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell) got_file_path = get_object(wallet, cid, oid, shell=client_shell, endpoint=new_nodes[0]) assert get_file_hash(source_file_path) == get_file_hash(got_file_path) - - for random_node, random_node_ip in random_nodes: - client = HostClient( - ip=random_node_ip, - login=STORAGE_NODE_SSH_USER, - password=STORAGE_NODE_SSH_PASSWORD, - private_key_path=STORAGE_NODE_SSH_PRIVATE_KEY_PATH, - ) - - with allure.step( - f"Unblock incoming traffic for node {random_node} on port {PORTS_TO_BLOCK}" - ): - with client.create_ssh_connection(): - IpTablesHelper.restore_input_traffic_to_port(client, PORTS_TO_BLOCK) - blocked_hosts.remove(random_node_ip) - sleep(wakeup_node_timeout) - - new_nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell) - - got_file_path = get_object(wallet, cid, oid, shell=client_shell, endpoint=new_nodes[0]) - assert get_file_hash(source_file_path) == get_file_hash(got_file_path) diff --git a/pytest_tests/testsuites/failovers/test_failover_storage.py b/pytest_tests/testsuites/failovers/test_failover_storage.py index a478bf0..1a0334c 100644 --- a/pytest_tests/testsuites/failovers/test_failover_storage.py +++ b/pytest_tests/testsuites/failovers/test_failover_storage.py @@ -2,17 +2,12 @@ import logging import allure import pytest -from common import ( - STORAGE_NODE_SSH_PASSWORD, - STORAGE_NODE_SSH_PRIVATE_KEY_PATH, - STORAGE_NODE_SSH_USER, -) from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes from file_helper import generate_file, get_file_hash -from neofs_testlib.hosting import Hosting +from neofs_testlib.hosting import Host, Hosting +from neofs_testlib.shell import CommandOptions from python_keywords.container import create_container from python_keywords.neofs_verbs import get_object, put_object -from ssh_helper import HostClient from wellknown_acl import PUBLIC_ACL logger = logging.getLogger("NeoLogger") @@ -20,27 +15,21 @@ stopped_hosts = [] @pytest.fixture(autouse=True) -@allure.step("Return all storage nodes") -def return_all_storage_nodes_fixture(hosting: Hosting): +@allure.step("Return all stopped hosts") +def after_run_return_all_stopped_hosts(hosting: Hosting): yield - return_all_storage_nodes(hosting) + return_stopped_hosts(hosting) -def panic_reboot_host(ip: str = None): - ssh = HostClient( - ip=ip, - login=STORAGE_NODE_SSH_USER, - password=STORAGE_NODE_SSH_PASSWORD, - private_key_path=STORAGE_NODE_SSH_PRIVATE_KEY_PATH, - ) - ssh.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"') - ssh_stdin, _, _ = ssh.ssh_client.exec_command( - 'sudo sh -c "echo b > /proc/sysrq-trigger"', timeout=1 - ) - ssh_stdin.close() +def panic_reboot_host(host: Host) -> None: + shell = host.get_shell() + shell.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"') + + options = CommandOptions(close_stdin=True, timeout=1, check=False) + shell.exec('sudo sh -c "echo b > /proc/sysrq-trigger"', options) -def return_all_storage_nodes(hosting: Hosting) -> None: +def return_stopped_hosts(hosting: Hosting) -> None: for host_address in list(stopped_hosts): with allure.step(f"Start host {host_address}"): host = hosting.get_host_by_address(host_address) @@ -50,10 +39,10 @@ def return_all_storage_nodes(hosting: Hosting) -> None: wait_all_storage_node_returned(hosting) -@allure.title("Lost and returned nodes") +@allure.title("Lose and return storage node's host") @pytest.mark.parametrize("hard_reboot", [True, False]) @pytest.mark.failover -def test_lost_storage_node( +def test_lose_storage_node_host( prepare_wallet_and_deposit, client_shell, hosting: Hosting, @@ -65,35 +54,44 @@ def test_lost_storage_node( source_file_path = generate_file() cid = create_container(wallet, shell=client_shell, rule=placement_rule, basic_acl=PUBLIC_ACL) oid = put_object(wallet, source_file_path, cid, shell=client_shell) - nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell) + node_endpoints = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell) - new_nodes = [] - for node in nodes: - host = hosting.get_host_by_service(node) + for node_endpoint in node_endpoints: + host_address = node_endpoint.split(":")[0] + host = hosting.get_host_by_address(host_address) stopped_hosts.append(host.config.address) - with allure.step(f"Stop storage node {node}"): + + with allure.step(f"Stop host {host_address}"): host.stop_host("hard" if hard_reboot else "soft") - new_nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell, excluded_nodes=[node]) - assert not [node for node in nodes if node in new_nodes] - got_file_path = get_object(wallet, cid, oid, shell=client_shell, endpoint=new_nodes[0]) - assert get_file_hash(source_file_path) == get_file_hash(got_file_path) + new_nodes = wait_object_replication_on_nodes( + wallet, cid, oid, 2, shell=client_shell, excluded_nodes=[node_endpoint] + ) + assert all(old_node not in new_nodes for old_node in node_endpoints) - with allure.step(f"Return storage nodes"): - return_all_storage_nodes(hosting) + with allure.step("Check object data is not corrupted"): + got_file_path = get_object(wallet, cid, oid, endpoint=new_nodes[0]) + assert get_file_hash(source_file_path) == get_file_hash(got_file_path) - new_nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell) + with allure.step(f"Return all hosts"): + return_stopped_hosts(hosting) - got_file_path = get_object(wallet, cid, oid, shell=client_shell, endpoint=new_nodes[0]) - assert get_file_hash(source_file_path) == get_file_hash(got_file_path) + with allure.step("Check object data is not corrupted"): + new_nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell) + got_file_path = get_object(wallet, cid, oid, shell=client_shell, endpoint=new_nodes[0]) + assert get_file_hash(source_file_path) == get_file_hash(got_file_path) -@allure.title("Panic storage node(s)") +@allure.title("Panic storage node's host") @pytest.mark.parametrize("sequence", [True, False]) @pytest.mark.failover_panic @pytest.mark.failover -def test_panic_storage_node( - prepare_wallet_and_deposit, client_shell, require_multiple_hosts, sequence: bool +def test_panic_storage_node_host( + prepare_wallet_and_deposit, + client_shell, + hosting: Hosting, + require_multiple_hosts, + sequence: bool, ): wallet = prepare_wallet_and_deposit placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" @@ -101,16 +99,24 @@ def test_panic_storage_node( cid = create_container(wallet, shell=client_shell, rule=placement_rule, basic_acl=PUBLIC_ACL) oid = put_object(wallet, source_file_path, cid, shell=client_shell) - nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell) + node_endpoints = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell) + allure.attach( + "\n".join(node_endpoints), + "Current nodes with object", + allure.attachment_type.TEXT, + ) + new_nodes: list[str] = [] - allure.attach("\n".join(nodes), "Current nodes with object", allure.attachment_type.TEXT) - for node in nodes: - with allure.step(f"Hard reboot host {node} via magic SysRq option"): - panic_reboot_host(ip=node.split(":")[-2]) + for node_endpoint in node_endpoints: + host_address = node_endpoint.split(":")[0] + + with allure.step(f"Hard reboot host {node_endpoint} via magic SysRq option"): + host = hosting.get_host_by_address(host_address) + panic_reboot_host(host) if sequence: try: new_nodes = wait_object_replication_on_nodes( - wallet, cid, oid, 2, shell=client_shell, excluded_nodes=[node] + wallet, cid, oid, 2, shell=client_shell, excluded_nodes=[node_endpoint] ) except AssertionError: new_nodes = wait_object_replication_on_nodes( @@ -119,7 +125,7 @@ def test_panic_storage_node( allure.attach( "\n".join(new_nodes), - f"Nodes with object after {node} fail", + f"Nodes with object after {node_endpoint} fail", allure.attachment_type.TEXT, ) diff --git a/pytest_tests/testsuites/network/test_node_management.py b/pytest_tests/testsuites/network/test_node_management.py index 727438c..469ae4f 100644 --- a/pytest_tests/testsuites/network/test_node_management.py +++ b/pytest_tests/testsuites/network/test_node_management.py @@ -5,7 +5,6 @@ from typing import Optional import allure import pytest -from neofs_testlib.shell import Shell from common import ( COMPLEX_OBJ_SIZE, MORPH_BLOCK_TIME, @@ -19,6 +18,7 @@ from epoch import tick_epoch from file_helper import generate_file from grpc_responses import OBJECT_NOT_FOUND, error_matches_status from neofs_testlib.hosting import Hosting +from neofs_testlib.shell import Shell from python_keywords.container import create_container, get_container from python_keywords.failover_utils import wait_object_replication_on_nodes from python_keywords.neofs_verbs import delete_object, get_object, head_object, put_object @@ -121,7 +121,11 @@ def return_nodes(shell: Shell, hosting: Hosting, alive_node: Optional[str] = Non @pytest.mark.add_nodes @pytest.mark.node_mgmt def test_add_nodes( - prepare_tmp_dir, client_shell, prepare_wallet_and_deposit, return_nodes_after_test_run, hosting: Hosting + prepare_tmp_dir, + client_shell, + prepare_wallet_and_deposit, + return_nodes_after_test_run, + hosting: Hosting, ): wallet = prepare_wallet_and_deposit placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X" @@ -141,14 +145,18 @@ def test_add_nodes( # Add node to recovery list before messing with it check_nodes.append(additional_node) - exclude_node_from_network_map(hosting, additional_node, alive_node) + exclude_node_from_network_map(hosting, additional_node, alive_node, shell=client_shell) delete_node_data(hosting, additional_node) - cid = create_container(wallet, rule=placement_rule_3, basic_acl=PUBLIC_ACL) + cid = create_container(wallet, rule=placement_rule_3, basic_acl=PUBLIC_ACL, shell=client_shell) oid = put_object( - wallet, source_file_path, cid, endpoint=NEOFS_NETMAP_DICT[alive_node].get("rpc") + wallet, + source_file_path, + cid, + endpoint=NEOFS_NETMAP_DICT[alive_node].get("rpc"), + shell=client_shell, ) - wait_object_replication_on_nodes(wallet, cid, oid, 3) + wait_object_replication_on_nodes(wallet, cid, oid, 3, shell=client_shell) return_nodes(shell=client_shell, hosting=hosting, alive_node=alive_node) @@ -156,16 +164,24 @@ def test_add_nodes( random_node = choice( [node for node in NEOFS_NETMAP_DICT if node not in (additional_node, alive_node)] ) - exclude_node_from_network_map(hosting, random_node, alive_node) + exclude_node_from_network_map(hosting, random_node, alive_node, shell=client_shell) - wait_object_replication_on_nodes(wallet, cid, oid, 3, excluded_nodes=[random_node]) + wait_object_replication_on_nodes( + wallet, cid, oid, 3, excluded_nodes=[random_node], shell=client_shell + ) include_node_to_network_map(hosting, random_node, alive_node, shell=client_shell) - wait_object_replication_on_nodes(wallet, cid, oid, 3) + wait_object_replication_on_nodes(wallet, cid, oid, 3, shell=client_shell) with allure.step("Check container could be created with new node"): - cid = create_container(wallet, rule=placement_rule_4, basic_acl=PUBLIC_ACL) + cid = create_container( + wallet, rule=placement_rule_4, basic_acl=PUBLIC_ACL, shell=client_shell + ) oid = put_object( - wallet, source_file_path, cid, endpoint=NEOFS_NETMAP_DICT[alive_node].get("rpc") + wallet, + source_file_path, + cid, + endpoint=NEOFS_NETMAP_DICT[alive_node].get("rpc"), + shell=client_shell, ) wait_object_replication_on_nodes(wallet, cid, oid, 4, shell=client_shell) @@ -231,13 +247,15 @@ def test_nodes_management(prepare_tmp_dir, client_shell, hosting: Hosting): ) @pytest.mark.node_mgmt @allure.title("Test object copies based on placement policy") -def test_placement_policy(prepare_wallet_and_deposit, placement_rule, expected_copies): +def test_placement_policy( + prepare_wallet_and_deposit, placement_rule, expected_copies, client_shell: Shell +): """ This test checks object's copies based on container's placement policy. """ wallet = prepare_wallet_and_deposit file_path = generate_file() - validate_object_copies(wallet, placement_rule, file_path, expected_copies) + validate_object_copies(wallet, placement_rule, file_path, expected_copies, shell=client_shell) @pytest.mark.parametrize( @@ -288,7 +306,7 @@ def test_placement_policy(prepare_wallet_and_deposit, placement_rule, expected_c @pytest.mark.node_mgmt @allure.title("Test object copies and storage nodes based on placement policy") def test_placement_policy_with_nodes( - prepare_wallet_and_deposit, placement_rule, expected_copies, nodes + prepare_wallet_and_deposit, placement_rule, expected_copies, nodes, client_shell: Shell ): """ Based on container's placement policy check that storage nodes are piked correctly and object has @@ -297,7 +315,7 @@ def test_placement_policy_with_nodes( wallet = prepare_wallet_and_deposit file_path = generate_file() cid, oid, found_nodes = validate_object_copies( - wallet, placement_rule, file_path, expected_copies + wallet, placement_rule, file_path, expected_copies, shell=client_shell ) expected_nodes = [NEOFS_NETMAP_DICT[node_name].get("rpc") for node_name in nodes] assert set(found_nodes) == set( @@ -313,17 +331,21 @@ def test_placement_policy_with_nodes( ) @pytest.mark.node_mgmt @allure.title("Negative cases for placement policy") -def test_placement_policy_negative(prepare_wallet_and_deposit, placement_rule, expected_copies): +def test_placement_policy_negative( + prepare_wallet_and_deposit, placement_rule, expected_copies, client_shell: Shell +): """ Negative test for placement policy. """ wallet = prepare_wallet_and_deposit file_path = generate_file() with pytest.raises(RuntimeError, match=".*not enough nodes to SELECT from.*"): - validate_object_copies(wallet, placement_rule, file_path, expected_copies) + validate_object_copies( + wallet, placement_rule, file_path, expected_copies, shell=client_shell + ) -@pytest.mark.skip(reason="We cover this scenario for Sbercloud in failover tests") +@pytest.mark.skip(reason="We cover this scenario in failover tests") @pytest.mark.node_mgmt @allure.title("NeoFS object replication on node failover") def test_replication(prepare_wallet_and_deposit, after_run_start_all_nodes, hosting: Hosting): @@ -430,14 +452,18 @@ def test_shards(prepare_wallet_and_deposit, create_container_and_pick_node, host @allure.step("Validate object has {expected_copies} copies") -def validate_object_copies(wallet: str, placement_rule: str, file_path: str, expected_copies: int): - cid = create_container(wallet, rule=placement_rule, basic_acl=PUBLIC_ACL) - got_policy = placement_policy_from_container(get_container(wallet, cid, json_mode=False)) +def validate_object_copies( + wallet: str, placement_rule: str, file_path: str, expected_copies: int, shell: Shell +): + cid = create_container(wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=shell) + got_policy = placement_policy_from_container( + get_container(wallet, cid, json_mode=False, shell=shell) + ) assert got_policy == placement_rule.replace( "'", "" ), f"Expected \n{placement_rule} and got policy \n{got_policy} are the same" - oid = put_object(wallet, file_path, cid) - nodes = get_nodes_with_object(wallet, cid, oid) + oid = put_object(wallet, file_path, cid, shell=shell) + nodes = get_nodes_with_object(wallet, cid, oid, shell=shell) assert len(nodes) == expected_copies, f"Expected {expected_copies} copies, got {len(nodes)}" return cid, oid, nodes diff --git a/pytest_tests/testsuites/object/test_object_api.py b/pytest_tests/testsuites/object/test_object_api.py index fe8be9e..163b6e0 100644 --- a/pytest_tests/testsuites/object/test_object_api.py +++ b/pytest_tests/testsuites/object/test_object_api.py @@ -8,6 +8,7 @@ from container import create_container from epoch import get_epoch, tick_epoch from file_helper import generate_file, get_file_content, get_file_hash from grpc_responses import OBJECT_ALREADY_REMOVED, OBJECT_NOT_FOUND, error_matches_status +from neofs_testlib.shell import Shell from python_keywords.neofs_verbs import ( delete_object, get_object, @@ -18,7 +19,6 @@ from python_keywords.neofs_verbs import ( search_object, ) from python_keywords.storage_policy import get_complex_object_copies, get_simple_object_copies -from neofs_testlib.shell import Shell from tombstone import verify_head_tombstone from utility import wait_for_gc_pass_on_storage_nodes diff --git a/pytest_tests/testsuites/services/test_http_gate.py b/pytest_tests/testsuites/services/test_http_gate.py index 757c14c..ce64f09 100644 --- a/pytest_tests/testsuites/services/test_http_gate.py +++ b/pytest_tests/testsuites/services/test_http_gate.py @@ -9,6 +9,7 @@ from common import COMPLEX_OBJ_SIZE from container import create_container from epoch import get_epoch, tick_epoch from file_helper import generate_file, get_file_hash +from neofs_testlib.shell import Shell from python_keywords.http_gate import ( get_via_http_curl, get_via_http_gate, @@ -17,7 +18,6 @@ from python_keywords.http_gate import ( upload_via_http_gate, upload_via_http_gate_curl, ) -from neofs_testlib.shell import Shell from python_keywords.neofs_verbs import get_object, put_object from python_keywords.storage_policy import get_nodes_without_object from utility import wait_for_gc_pass_on_storage_nodes @@ -225,7 +225,14 @@ class TestHttpGate: oid_curl = upload_via_http_gate_curl(cid=cid, filepath=file_path, large_object=True) self.get_object_and_verify_hashes(oid_gate, file_path, self.wallet, cid, shell=client_shell) - self.get_object_and_verify_hashes(oid_curl, file_path, self.wallet, cid, shell=client_shell, object_getter=get_via_http_curl) + self.get_object_and_verify_hashes( + oid_curl, + file_path, + self.wallet, + cid, + shell=client_shell, + object_getter=get_via_http_curl, + ) @pytest.mark.curl @allure.title("Test Put/Get over HTTP using Curl utility") @@ -243,7 +250,14 @@ class TestHttpGate: oid_large = upload_via_http_gate_curl(cid=cid, filepath=file_path_large) for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): - self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid, shell=client_shell, object_getter=get_via_http_curl) + self.get_object_and_verify_hashes( + oid, + file_path, + self.wallet, + cid, + shell=client_shell, + object_getter=get_via_http_curl, + ) @staticmethod @allure.step("Try to get object and expect error") diff --git a/pytest_tests/testsuites/session_token/test_object_session_token.py b/pytest_tests/testsuites/session_token/test_object_session_token.py index eec422c..1691af6 100644 --- a/pytest_tests/testsuites/session_token/test_object_session_token.py +++ b/pytest_tests/testsuites/session_token/test_object_session_token.py @@ -2,10 +2,10 @@ import random import allure import pytest -from neofs_testlib.shell import Shell from common import COMPLEX_OBJ_SIZE, NEOFS_NETMAP_DICT, SIMPLE_OBJ_SIZE from file_helper import generate_file from grpc_responses import SESSION_NOT_FOUND +from neofs_testlib.shell import Shell from payment_neogo import _address_from_wallet from python_keywords.container import create_container from python_keywords.neofs_verbs import ( diff --git a/robot/resources/lib/python_keywords/container.py b/robot/resources/lib/python_keywords/container.py index c79e5e0..6dc6cdc 100644 --- a/robot/resources/lib/python_keywords/container.py +++ b/robot/resources/lib/python_keywords/container.py @@ -129,7 +129,10 @@ def list_containers(wallet: str, shell: Shell) -> list[str]: @allure.step("Get Container") def get_container( - wallet: str, cid: str, shell: Shell, json_mode: bool = True, + wallet: str, + cid: str, + shell: Shell, + json_mode: bool = True, ) -> Union[dict, str]: """ A wrapper for `neofs-cli container get` call. It extracts container's @@ -162,9 +165,7 @@ def get_container( @allure.step("Delete Container") # TODO: make the error message about a non-found container more user-friendly # https://github.com/nspcc-dev/neofs-contract/issues/121 -def delete_container( - wallet: str, cid: str, shell: Shell, force: bool = False -) -> None: +def delete_container(wallet: str, cid: str, shell: Shell, force: bool = False) -> None: """ A wrapper for `neofs-cli container delete` call. Args: diff --git a/robot/resources/lib/python_keywords/failover_utils.py b/robot/resources/lib/python_keywords/failover_utils.py index 1ea4aeb..861e280 100644 --- a/robot/resources/lib/python_keywords/failover_utils.py +++ b/robot/resources/lib/python_keywords/failover_utils.py @@ -4,8 +4,8 @@ from typing import Optional import allure from common import NEOFS_NETMAP_DICT -from neofs_testlib.shell import Shell from neofs_testlib.hosting import Hosting +from neofs_testlib.shell import Shell from python_keywords.node_management import node_healthcheck from storage_policy import get_nodes_with_object @@ -22,7 +22,7 @@ def wait_object_replication_on_nodes( excluded_nodes: Optional[list[str]] = None, ) -> list[str]: excluded_nodes = excluded_nodes or [] - sleep_interval, attempts = 10, 18 + sleep_interval, attempts = 15, 20 nodes = [] for __attempt in range(attempts): nodes = get_nodes_with_object(wallet, cid, oid, shell=shell, skip_nodes=excluded_nodes) diff --git a/robot/resources/lib/python_keywords/node_management.py b/robot/resources/lib/python_keywords/node_management.py index 3f67309..bb379ff 100644 --- a/robot/resources/lib/python_keywords/node_management.py +++ b/robot/resources/lib/python_keywords/node_management.py @@ -16,8 +16,8 @@ from common import ( from data_formatters import get_wallet_public_key from epoch import tick_epoch from neofs_testlib.cli import NeofsCli -from neofs_testlib.shell import Shell from neofs_testlib.hosting import Hosting +from neofs_testlib.shell import Shell from utility import parse_time logger = logging.getLogger("NeoLogger") @@ -165,7 +165,9 @@ def delete_node_data(hosting: Hosting, node_name: str) -> None: @allure.step("Exclude node {node_to_exclude} from network map") -def exclude_node_from_network_map(hosting: Hosting, node_to_exclude: str, alive_node: str, shell: Shell) -> None: +def exclude_node_from_network_map( + hosting: Hosting, node_to_exclude: str, alive_node: str, shell: Shell +) -> None: node_wallet_path = NEOFS_NETMAP_DICT[node_to_exclude]["wallet_path"] node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS) @@ -181,13 +183,15 @@ def exclude_node_from_network_map(hosting: Hosting, node_to_exclude: str, alive_ @allure.step("Include node {node_to_include} into network map") -def include_node_to_network_map(hosting: Hosting, node_to_include: str, alive_node: str, shell: Shell) -> None: +def include_node_to_network_map( + hosting: Hosting, node_to_include: str, alive_node: str, shell: Shell +) -> None: node_set_status(hosting, node_to_include, status="online") time.sleep(parse_time(MORPH_BLOCK_TIME)) tick_epoch() - check_node_in_map(node_to_include, alive_node, shell=shell) + check_node_in_map(node_to_include, shell, alive_node) @allure.step("Check node {node_name} in network map") diff --git a/robot/resources/lib/python_keywords/object_access.py b/robot/resources/lib/python_keywords/object_access.py index 8c633ee..308a7e1 100644 --- a/robot/resources/lib/python_keywords/object_access.py +++ b/robot/resources/lib/python_keywords/object_access.py @@ -30,7 +30,13 @@ def can_get_object( with allure.step("Try get object from container"): try: got_file_path = get_object( - wallet, cid, oid, bearer_token=bearer, wallet_config=wallet_config, xhdr=xhdr, shell=shell, + wallet, + cid, + oid, + bearer_token=bearer, + wallet_config=wallet_config, + xhdr=xhdr, + shell=shell, ) except OPERATION_ERROR_TYPE as err: assert error_matches_status( diff --git a/robot/variables/common.py b/robot/variables/common.py index 79423a0..d753104 100644 --- a/robot/variables/common.py +++ b/robot/variables/common.py @@ -89,12 +89,6 @@ NEOFS_NETMAP_DICT = { } NEOFS_NETMAP = [node["rpc"] for node in NEOFS_NETMAP_DICT.values()] -# Parameters that control SSH connection to storage node -# TODO: we should use hosting instead -STORAGE_NODE_SSH_USER = os.getenv("STORAGE_NODE_SSH_USER") -STORAGE_NODE_SSH_PASSWORD = os.getenv("STORAGE_NODE_SSH_PASSWORD") -STORAGE_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("STORAGE_NODE_SSH_PRIVATE_KEY_PATH") - # Paths to CLI executables on machine that runs tests NEOGO_EXECUTABLE = os.getenv("NEOGO_EXECUTABLE", "neo-go") NEOFS_CLI_EXEC = os.getenv("NEOFS_CLI_EXEC", "neofs-cli")