Switch failover test to hosting from testlib
Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
parent
92c034c10b
commit
48e53b3d86
20 changed files with 242 additions and 441 deletions
|
@ -17,11 +17,11 @@ def get_local_binaries_versions(shell: Shell) -> dict[str, str]:
|
||||||
versions[binary] = _parse_version(out)
|
versions[binary] = _parse_version(out)
|
||||||
|
|
||||||
neofs_cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
|
neofs_cli = NeofsCli(shell, NEOFS_CLI_EXEC, WALLET_CONFIG)
|
||||||
versions["neofs-cli"] = _parse_version(neofs_cli.version.get())
|
versions["neofs-cli"] = _parse_version(neofs_cli.version.get().stdout)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
neofs_adm = NeofsAdm(shell, NEOFS_ADM_EXEC)
|
neofs_adm = NeofsAdm(shell, NEOFS_ADM_EXEC)
|
||||||
versions["neofs-adm"] = _parse_version(neofs_adm.version.get())
|
versions["neofs-adm"] = _parse_version(neofs_adm.version.get().stdout)
|
||||||
except RuntimeError:
|
except RuntimeError:
|
||||||
logger.info(f"neofs-adm not installed")
|
logger.info(f"neofs-adm not installed")
|
||||||
|
|
||||||
|
@ -35,24 +35,23 @@ def get_local_binaries_versions(shell: Shell) -> dict[str, str]:
|
||||||
def get_remote_binaries_versions(hosting: Hosting) -> dict[str, str]:
|
def get_remote_binaries_versions(hosting: Hosting) -> dict[str, str]:
|
||||||
versions_by_host = {}
|
versions_by_host = {}
|
||||||
for host in hosting.hosts:
|
for host in hosting.hosts:
|
||||||
binaries = []
|
binary_path_by_name = {} # Maps binary name to executable path
|
||||||
for service_config in host.config.services:
|
for service_config in host.config.services:
|
||||||
exec_path = service_config.attributes.get("exec_path")
|
exec_path = service_config.attributes.get("exec_path")
|
||||||
if exec_path:
|
if exec_path:
|
||||||
binaries.append(exec_path)
|
binary_path_by_name[service_config.name] = exec_path
|
||||||
for cli_config in host.config.clis:
|
for cli_config in host.config.clis:
|
||||||
binaries.append(cli_config.exec_path)
|
binary_path_by_name[cli_config.name] = cli_config.exec_path
|
||||||
|
|
||||||
shell = host.get_shell()
|
shell = host.get_shell()
|
||||||
versions_at_host = {}
|
versions_at_host = {}
|
||||||
for binary in binaries:
|
for binary_name, binary_path in binary_path_by_name.items():
|
||||||
try:
|
try:
|
||||||
result = shell.exec(f"{binary} --version")
|
result = shell.exec(f"{binary_path} --version")
|
||||||
versions_at_host[service_config.name] = _parse_version(result.stdout)
|
versions_at_host[binary_name] = _parse_version(result.stdout)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.error(f"Cannot get version for {exec_path} because of\n{exc}")
|
logger.error(f"Cannot get version for {binary_path} because of\n{exc}")
|
||||||
versions_at_host[service_config.name] = "Unknown"
|
versions_at_host[binary_name] = "Unknown"
|
||||||
continue
|
|
||||||
versions_by_host[host.config.address] = versions_at_host
|
versions_by_host[host.config.address] = versions_at_host
|
||||||
|
|
||||||
# Consolidate versions across all hosts
|
# Consolidate versions across all hosts
|
||||||
|
@ -61,7 +60,9 @@ def get_remote_binaries_versions(hosting: Hosting) -> dict[str, str]:
|
||||||
for name, version in binary_versions.items():
|
for name, version in binary_versions.items():
|
||||||
captured_version = versions.get(name)
|
captured_version = versions.get(name)
|
||||||
if captured_version:
|
if captured_version:
|
||||||
assert captured_version == version, f"Binary {name} has inconsistent version on host {host}"
|
assert (
|
||||||
|
captured_version == version
|
||||||
|
), f"Binary {name} has inconsistent version on host {host}"
|
||||||
else:
|
else:
|
||||||
versions[name] = version
|
versions[name] = version
|
||||||
return versions
|
return versions
|
||||||
|
|
|
@ -1,16 +1,13 @@
|
||||||
from ssh_helper import HostClient
|
from neofs_testlib.shell import Shell
|
||||||
|
|
||||||
|
|
||||||
# TODO: convert to shell from hosting
|
|
||||||
class IpTablesHelper:
|
class IpTablesHelper:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def drop_input_traffic_to_port(client: HostClient, ports: list[str]):
|
def drop_input_traffic_to_port(shell: Shell, ports: list[str]) -> None:
|
||||||
for port in ports:
|
for port in ports:
|
||||||
cmd_output = client.exec(cmd=f"sudo iptables -A INPUT -p tcp --dport {port} -j DROP")
|
shell.exec(f"sudo iptables -A INPUT -p tcp --dport {port} -j DROP")
|
||||||
assert cmd_output.rc == 0
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def restore_input_traffic_to_port(client: HostClient, ports: list[str]):
|
def restore_input_traffic_to_port(shell: Shell, ports: list[str]) -> None:
|
||||||
for port in ports:
|
for port in ports:
|
||||||
cmd_output = client.exec(cmd=f"sudo iptables -D INPUT -p tcp --dport {port} -j DROP")
|
shell.exec(f"sudo iptables -D INPUT -p tcp --dport {port} -j DROP")
|
||||||
assert cmd_output.rc == 0
|
|
||||||
|
|
|
@ -4,6 +4,10 @@ import uuid
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
|
|
||||||
|
# This file is broken, because tenacity is not registered in requirements.txt
|
||||||
|
# So, the file won't be fixed in scope of this PR, alipay will fix himself by
|
||||||
|
# switching RemoteProcess and K6 classes from HostClient to shell from hosting
|
||||||
from tenacity import retry, stop_after_attempt, wait_fixed
|
from tenacity import retry, stop_after_attempt, wait_fixed
|
||||||
|
|
||||||
from pytest_tests.helpers.ssh_helper import HostClient
|
from pytest_tests.helpers.ssh_helper import HostClient
|
||||||
|
|
|
@ -1,250 +0,0 @@
|
||||||
import logging
|
|
||||||
import socket
|
|
||||||
import tempfile
|
|
||||||
import textwrap
|
|
||||||
from contextlib import contextmanager
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import datetime
|
|
||||||
from functools import wraps
|
|
||||||
from time import sleep
|
|
||||||
from typing import ClassVar, Optional
|
|
||||||
|
|
||||||
import allure
|
|
||||||
from paramiko import AutoAddPolicy, RSAKey, SFTPClient, SSHClient, SSHException, ssh_exception
|
|
||||||
from paramiko.ssh_exception import AuthenticationException
|
|
||||||
|
|
||||||
|
|
||||||
class HostIsNotAvailable(Exception):
|
|
||||||
"""Raises when host is not reachable."""
|
|
||||||
|
|
||||||
def __init__(self, ip: str = None, exc: Exception = None):
|
|
||||||
msg = f'Host is not available{f" by ip: {ip}" if ip else ""}'
|
|
||||||
if exc:
|
|
||||||
msg = f"{msg}. {exc}"
|
|
||||||
super().__init__(msg)
|
|
||||||
|
|
||||||
|
|
||||||
def log_command(func):
|
|
||||||
@wraps(func)
|
|
||||||
def wrapper(host: "HostClient", command: str, *args, **kwargs):
|
|
||||||
display_length = 60
|
|
||||||
short = command.removeprefix("$ProgressPreference='SilentlyContinue'\n")
|
|
||||||
short = short[:display_length]
|
|
||||||
short += "..." if short != command else ""
|
|
||||||
with allure.step(f"SSH: {short}"):
|
|
||||||
logging.info(f'Execute command "{command}" on "{host.ip}"')
|
|
||||||
|
|
||||||
start_time = datetime.utcnow()
|
|
||||||
cmd_result = func(host, command, *args, **kwargs)
|
|
||||||
end_time = datetime.utcnow()
|
|
||||||
|
|
||||||
log_message = (
|
|
||||||
f"HOST: {host.ip}\n"
|
|
||||||
f'COMMAND:\n{textwrap.indent(command, " ")}\n'
|
|
||||||
f"RC:\n {cmd_result.rc}\n"
|
|
||||||
f'STDOUT:\n{textwrap.indent(cmd_result.stdout, " ")}\n'
|
|
||||||
f'STDERR:\n{textwrap.indent(cmd_result.stderr, " ")}\n'
|
|
||||||
f"Start / End / Elapsed\t {start_time.time()} / {end_time.time()} / {end_time - start_time}"
|
|
||||||
)
|
|
||||||
|
|
||||||
logging.info(log_message)
|
|
||||||
allure.attach(log_message, "SSH command", allure.attachment_type.TEXT)
|
|
||||||
return cmd_result
|
|
||||||
|
|
||||||
return wrapper
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class SSHCommand:
|
|
||||||
stdout: str
|
|
||||||
stderr: str
|
|
||||||
rc: int
|
|
||||||
|
|
||||||
|
|
||||||
class HostClient:
|
|
||||||
ssh_client: SSHClient
|
|
||||||
SSH_CONNECTION_ATTEMPTS: ClassVar[int] = 3
|
|
||||||
CONNECTION_TIMEOUT = 90
|
|
||||||
|
|
||||||
TIMEOUT_RESTORE_CONNECTION = 10, 24
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
ip: str,
|
|
||||||
login: str,
|
|
||||||
password: Optional[str] = None,
|
|
||||||
private_key_path: Optional[str] = None,
|
|
||||||
private_key_passphrase: Optional[str] = None,
|
|
||||||
init_ssh_client=True,
|
|
||||||
) -> None:
|
|
||||||
self.ip = ip
|
|
||||||
self.login = login
|
|
||||||
self.password = password
|
|
||||||
self.private_key_path = private_key_path
|
|
||||||
self.private_key_passphrase = private_key_passphrase
|
|
||||||
if init_ssh_client:
|
|
||||||
self.create_connection(self.SSH_CONNECTION_ATTEMPTS)
|
|
||||||
|
|
||||||
def exec(self, cmd: str, verify=True, timeout=90) -> SSHCommand:
|
|
||||||
cmd_result = self._inner_exec(cmd, timeout)
|
|
||||||
if verify:
|
|
||||||
assert cmd_result.rc == 0, f'Non zero rc from command: "{cmd}"'
|
|
||||||
return cmd_result
|
|
||||||
|
|
||||||
@log_command
|
|
||||||
def exec_with_confirmation(
|
|
||||||
self, cmd: str, confirmation: list, verify=True, timeout=90
|
|
||||||
) -> SSHCommand:
|
|
||||||
ssh_stdin, ssh_stdout, ssh_stderr = self.ssh_client.exec_command(cmd, timeout=timeout)
|
|
||||||
for line in confirmation:
|
|
||||||
if not line.endswith("\n"):
|
|
||||||
line = f"{line}\n"
|
|
||||||
try:
|
|
||||||
ssh_stdin.write(line)
|
|
||||||
except OSError as err:
|
|
||||||
logging.error(f"Got error {err} executing command {cmd}")
|
|
||||||
ssh_stdin.close()
|
|
||||||
output = SSHCommand(
|
|
||||||
stdout=ssh_stdout.read().decode(errors="ignore"),
|
|
||||||
stderr=ssh_stderr.read().decode(errors="ignore"),
|
|
||||||
rc=ssh_stdout.channel.recv_exit_status(),
|
|
||||||
)
|
|
||||||
if verify:
|
|
||||||
debug_info = f"\nSTDOUT: {output.stdout}\nSTDERR: {output.stderr}\nRC: {output.rc}"
|
|
||||||
assert output.rc == 0, f'Non zero rc from command: "{cmd}"{debug_info}'
|
|
||||||
return output
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def as_user(self, user: str, password: str):
|
|
||||||
keep_user, keep_password = self.login, self.password
|
|
||||||
self.login, self.password = user, password
|
|
||||||
self.create_connection()
|
|
||||||
yield
|
|
||||||
self.login, self.password = keep_user, keep_password
|
|
||||||
self.create_connection()
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def create_ssh_connection(self) -> "SSHClient":
|
|
||||||
if not self.ssh_client:
|
|
||||||
self.create_connection()
|
|
||||||
try:
|
|
||||||
yield self.ssh_client
|
|
||||||
finally:
|
|
||||||
self.drop()
|
|
||||||
|
|
||||||
@allure.step("Restore connection")
|
|
||||||
def restore_ssh_connection(self):
|
|
||||||
retry_time, retry_count = self.TIMEOUT_RESTORE_CONNECTION
|
|
||||||
for _ in range(retry_count):
|
|
||||||
try:
|
|
||||||
self.create_connection()
|
|
||||||
except AssertionError:
|
|
||||||
logging.warning(f"Host: Cant reach host: {self.ip}.")
|
|
||||||
sleep(retry_time)
|
|
||||||
else:
|
|
||||||
logging.info(f"Host: Cant reach host: {self.ip}.")
|
|
||||||
return
|
|
||||||
raise AssertionError(f"Host: Cant reach host: {self.ip} after 240 seconds..")
|
|
||||||
|
|
||||||
@allure.step("Copy file {host_path_to_file} to local file {path_to_file}")
|
|
||||||
def copy_file_from_host(self, host_path_to_file: str, path_to_file: str):
|
|
||||||
with self._sftp_client() as sftp_client:
|
|
||||||
sftp_client.get(host_path_to_file, path_to_file)
|
|
||||||
|
|
||||||
def copy_file_to_host(self, path_to_file: str, host_path_to_file: str):
|
|
||||||
with allure.step(
|
|
||||||
f"Copy local file {path_to_file} to remote file {host_path_to_file} on host {self.ip}"
|
|
||||||
):
|
|
||||||
with self._sftp_client() as sftp_client:
|
|
||||||
sftp_client.put(path_to_file, host_path_to_file)
|
|
||||||
|
|
||||||
@allure.step("Save string to remote file {host_path_to_file}")
|
|
||||||
def copy_str_to_host_file(self, string: str, host_path_to_file: str):
|
|
||||||
with tempfile.NamedTemporaryFile(mode="r+") as temp:
|
|
||||||
temp.writelines(string)
|
|
||||||
temp.flush()
|
|
||||||
with self._sftp_client() as client:
|
|
||||||
client.put(temp.name, host_path_to_file)
|
|
||||||
self.exec(f"cat {host_path_to_file}", verify=False)
|
|
||||||
|
|
||||||
def create_connection(self, attempts=SSH_CONNECTION_ATTEMPTS):
|
|
||||||
exc_err = None
|
|
||||||
for attempt in range(attempts):
|
|
||||||
self.ssh_client = SSHClient()
|
|
||||||
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
|
|
||||||
try:
|
|
||||||
if self.private_key_path:
|
|
||||||
logging.info(
|
|
||||||
f"Trying to connect to host {self.ip} as {self.login} using SSH key "
|
|
||||||
f"{self.private_key_path} (attempt {attempt})"
|
|
||||||
)
|
|
||||||
self.ssh_client.connect(
|
|
||||||
hostname=self.ip,
|
|
||||||
username=self.login,
|
|
||||||
pkey=RSAKey.from_private_key_file(
|
|
||||||
self.private_key_path, self.private_key_passphrase
|
|
||||||
),
|
|
||||||
timeout=self.CONNECTION_TIMEOUT,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logging.info(
|
|
||||||
f"Trying to connect to host {self.ip} as {self.login} using password "
|
|
||||||
f"(attempt {attempt})"
|
|
||||||
)
|
|
||||||
self.ssh_client.connect(
|
|
||||||
hostname=self.ip,
|
|
||||||
username=self.login,
|
|
||||||
password=self.password,
|
|
||||||
timeout=self.CONNECTION_TIMEOUT,
|
|
||||||
)
|
|
||||||
return True
|
|
||||||
|
|
||||||
except AuthenticationException as auth_err:
|
|
||||||
logging.error(f"Host: {self.ip}. {auth_err}")
|
|
||||||
self.drop()
|
|
||||||
raise auth_err
|
|
||||||
|
|
||||||
except (
|
|
||||||
SSHException,
|
|
||||||
ssh_exception.NoValidConnectionsError,
|
|
||||||
AttributeError,
|
|
||||||
socket.timeout,
|
|
||||||
OSError,
|
|
||||||
) as ssh_err:
|
|
||||||
exc_err = ssh_err
|
|
||||||
self.drop()
|
|
||||||
logging.error(f"Host: {self.ip}, connection error. {exc_err}")
|
|
||||||
|
|
||||||
raise HostIsNotAvailable(self.ip, exc_err)
|
|
||||||
|
|
||||||
def drop(self):
|
|
||||||
self.ssh_client.close()
|
|
||||||
|
|
||||||
@log_command
|
|
||||||
def _inner_exec(self, cmd: str, timeout: int) -> SSHCommand:
|
|
||||||
if not self.ssh_client:
|
|
||||||
self.create_connection()
|
|
||||||
for _ in range(self.SSH_CONNECTION_ATTEMPTS):
|
|
||||||
try:
|
|
||||||
_, stdout, stderr = self.ssh_client.exec_command(cmd, timeout=timeout)
|
|
||||||
return SSHCommand(
|
|
||||||
stdout=stdout.read().decode(errors="ignore"),
|
|
||||||
stderr=stderr.read().decode(errors="ignore"),
|
|
||||||
rc=stdout.channel.recv_exit_status(),
|
|
||||||
)
|
|
||||||
except (
|
|
||||||
SSHException,
|
|
||||||
TimeoutError,
|
|
||||||
ssh_exception.NoValidConnectionsError,
|
|
||||||
ConnectionResetError,
|
|
||||||
AttributeError,
|
|
||||||
socket.timeout,
|
|
||||||
) as ssh_err:
|
|
||||||
logging.error(f"Host: {self.ip}, exec command error {ssh_err}")
|
|
||||||
self.create_connection()
|
|
||||||
raise HostIsNotAvailable(f"Host: {self.ip} is not reachable.")
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def _sftp_client(self) -> SFTPClient:
|
|
||||||
with self.ssh_client.open_sftp() as sftp:
|
|
||||||
yield sftp
|
|
|
@ -56,7 +56,7 @@ pyparsing==3.0.9
|
||||||
pyrsistent==0.18.1
|
pyrsistent==0.18.1
|
||||||
pytest==7.1.2
|
pytest==7.1.2
|
||||||
python-dateutil==2.8.2
|
python-dateutil==2.8.2
|
||||||
requests==2.27.1
|
requests==2.28.0
|
||||||
robotframework==4.1.2
|
robotframework==4.1.2
|
||||||
s3transfer==0.3.7
|
s3transfer==0.3.7
|
||||||
six==1.16.0
|
six==1.16.0
|
||||||
|
|
|
@ -20,6 +20,7 @@ from common import (
|
||||||
S3_GATE_WALLET_PATH,
|
S3_GATE_WALLET_PATH,
|
||||||
)
|
)
|
||||||
from data_formatters import get_wallet_public_key
|
from data_formatters import get_wallet_public_key
|
||||||
|
from neofs_testlib.shell import Shell
|
||||||
from python_keywords.container import list_containers
|
from python_keywords.container import list_containers
|
||||||
|
|
||||||
from steps.aws_cli_client import AwsCliClient
|
from steps.aws_cli_client import AwsCliClient
|
||||||
|
@ -42,7 +43,7 @@ class TestS3GateBase:
|
||||||
|
|
||||||
@pytest.fixture(scope="class", autouse=True)
|
@pytest.fixture(scope="class", autouse=True)
|
||||||
@allure.title("[Class/Autouse]: Create S3 client")
|
@allure.title("[Class/Autouse]: Create S3 client")
|
||||||
def s3_client(self, prepare_wallet_and_deposit, request):
|
def s3_client(self, prepare_wallet_and_deposit, client_shell: Shell, request):
|
||||||
wallet = prepare_wallet_and_deposit
|
wallet = prepare_wallet_and_deposit
|
||||||
s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json"
|
s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json"
|
||||||
|
|
||||||
|
@ -53,7 +54,7 @@ class TestS3GateBase:
|
||||||
secret_access_key,
|
secret_access_key,
|
||||||
owner_private_key,
|
owner_private_key,
|
||||||
) = init_s3_credentials(wallet, s3_bearer_rules_file=s3_bearer_rules_file)
|
) = init_s3_credentials(wallet, s3_bearer_rules_file=s3_bearer_rules_file)
|
||||||
containers_list = list_containers(wallet)
|
containers_list = list_containers(wallet, shell=client_shell)
|
||||||
assert cid in containers_list, f"Expected cid {cid} in {containers_list}"
|
assert cid in containers_list, f"Expected cid {cid} in {containers_list}"
|
||||||
|
|
||||||
if request.param == "aws cli":
|
if request.param == "aws cli":
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
import allure
|
import allure
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from python_keywords.acl import EACLRole
|
from python_keywords.acl import EACLRole
|
||||||
from python_keywords.container import create_container
|
from python_keywords.container import create_container
|
||||||
from python_keywords.container_access import (
|
from python_keywords.container_access import (
|
||||||
|
|
|
@ -3,6 +3,7 @@ import pytest
|
||||||
from common import NEOFS_NETMAP_DICT
|
from common import NEOFS_NETMAP_DICT
|
||||||
from failover_utils import wait_object_replication_on_nodes
|
from failover_utils import wait_object_replication_on_nodes
|
||||||
from neofs_testlib.hosting import Hosting
|
from neofs_testlib.hosting import Hosting
|
||||||
|
from neofs_testlib.shell import Shell
|
||||||
from python_keywords.acl import (
|
from python_keywords.acl import (
|
||||||
EACLAccess,
|
EACLAccess,
|
||||||
EACLOperation,
|
EACLOperation,
|
||||||
|
@ -28,7 +29,6 @@ from python_keywords.object_access import (
|
||||||
can_put_object,
|
can_put_object,
|
||||||
can_search_object,
|
can_search_object,
|
||||||
)
|
)
|
||||||
from neofs_testlib.shell import Shell
|
|
||||||
from wellknown_acl import PUBLIC_ACL
|
from wellknown_acl import PUBLIC_ACL
|
||||||
|
|
||||||
|
|
||||||
|
@ -195,7 +195,12 @@ class TestEACLContainer:
|
||||||
|
|
||||||
@allure.title("Testcase to validate NeoFS replication with eACL deny rules.")
|
@allure.title("Testcase to validate NeoFS replication with eACL deny rules.")
|
||||||
def test_extended_acl_deny_replication(
|
def test_extended_acl_deny_replication(
|
||||||
self, wallets, client_shell, hosting: Hosting, eacl_full_placement_container_with_object, file_path
|
self,
|
||||||
|
wallets,
|
||||||
|
client_shell,
|
||||||
|
hosting: Hosting,
|
||||||
|
eacl_full_placement_container_with_object,
|
||||||
|
file_path,
|
||||||
):
|
):
|
||||||
user_wallet = wallets.get_wallet()
|
user_wallet = wallets.get_wallet()
|
||||||
cid, oid, file_path = eacl_full_placement_container_with_object
|
cid, oid, file_path = eacl_full_placement_container_with_object
|
||||||
|
|
|
@ -17,6 +17,7 @@ from common import (
|
||||||
)
|
)
|
||||||
from env_properties import save_env_properties
|
from env_properties import save_env_properties
|
||||||
from neofs_testlib.hosting import Hosting
|
from neofs_testlib.hosting import Hosting
|
||||||
|
from neofs_testlib.reporter import AllureHandler, get_reporter
|
||||||
from neofs_testlib.shell import LocalShell, Shell
|
from neofs_testlib.shell import LocalShell, Shell
|
||||||
from payment_neogo import neofs_deposit, transfer_mainnet_gas
|
from payment_neogo import neofs_deposit, transfer_mainnet_gas
|
||||||
from python_keywords.node_management import node_healthcheck
|
from python_keywords.node_management import node_healthcheck
|
||||||
|
@ -25,10 +26,26 @@ logger = logging.getLogger("NeoLogger")
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
@pytest.fixture(scope="session")
|
||||||
def client_shell() -> Shell:
|
def configure_testlib():
|
||||||
|
get_reporter().register_handler(AllureHandler())
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def client_shell(configure_testlib) -> Shell:
|
||||||
yield LocalShell()
|
yield LocalShell()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="session")
|
||||||
|
def hosting(configure_testlib) -> Hosting:
|
||||||
|
with open(HOSTING_CONFIG_FILE, "r") as file:
|
||||||
|
hosting_config = yaml.full_load(file)
|
||||||
|
|
||||||
|
hosting_instance = Hosting()
|
||||||
|
hosting_instance.configure(hosting_config)
|
||||||
|
yield hosting_instance
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
@pytest.fixture(scope="session")
|
||||||
def require_multiple_hosts(hosting: Hosting):
|
def require_multiple_hosts(hosting: Hosting):
|
||||||
"""Fixture that is applied to tests that require that environment has multiple hosts."""
|
"""Fixture that is applied to tests that require that environment has multiple hosts."""
|
||||||
|
@ -37,16 +54,6 @@ def require_multiple_hosts(hosting: Hosting):
|
||||||
yield
|
yield
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
|
||||||
def hosting() -> Hosting:
|
|
||||||
with open(HOSTING_CONFIG_FILE, "r") as file:
|
|
||||||
hosting_config = yaml.full_load(file)
|
|
||||||
|
|
||||||
hosting_instance = Hosting()
|
|
||||||
hosting_instance.configure(hosting_config)
|
|
||||||
yield hosting_instance
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session", autouse=True)
|
@pytest.fixture(scope="session", autouse=True)
|
||||||
@allure.title("Check binary versions")
|
@allure.title("Check binary versions")
|
||||||
def check_binary_versions(request, hosting: Hosting, client_shell: Shell):
|
def check_binary_versions(request, hosting: Hosting, client_shell: Shell):
|
||||||
|
|
|
@ -4,18 +4,12 @@ from time import sleep
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
import pytest
|
import pytest
|
||||||
from common import (
|
|
||||||
STORAGE_NODE_SSH_PASSWORD,
|
|
||||||
STORAGE_NODE_SSH_PRIVATE_KEY_PATH,
|
|
||||||
STORAGE_NODE_SSH_USER,
|
|
||||||
)
|
|
||||||
from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes
|
from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes
|
||||||
from file_helper import generate_file, get_file_hash
|
from file_helper import generate_file, get_file_hash
|
||||||
from iptables_helper import IpTablesHelper
|
from iptables_helper import IpTablesHelper
|
||||||
from neofs_testlib.hosting import Hosting
|
from neofs_testlib.hosting import Hosting
|
||||||
from python_keywords.container import create_container
|
from python_keywords.container import create_container
|
||||||
from python_keywords.neofs_verbs import get_object, put_object
|
from python_keywords.neofs_verbs import get_object, put_object
|
||||||
from ssh_helper import HostClient
|
|
||||||
from wellknown_acl import PUBLIC_ACL
|
from wellknown_acl import PUBLIC_ACL
|
||||||
|
|
||||||
logger = logging.getLogger("NeoLogger")
|
logger = logging.getLogger("NeoLogger")
|
||||||
|
@ -31,16 +25,10 @@ def restore_network(hosting: Hosting):
|
||||||
yield
|
yield
|
||||||
|
|
||||||
not_empty = len(blocked_hosts) != 0
|
not_empty = len(blocked_hosts) != 0
|
||||||
for host in list(blocked_hosts):
|
for host_address in list(blocked_hosts):
|
||||||
with allure.step(f"Start storage node {host}"):
|
with allure.step(f"Restore network at host {host_address}"):
|
||||||
client = HostClient(
|
host = hosting.get_host_by_address(host_address)
|
||||||
ip=host,
|
IpTablesHelper.restore_input_traffic_to_port(host.get_shell(), PORTS_TO_BLOCK)
|
||||||
login=STORAGE_NODE_SSH_USER,
|
|
||||||
password=STORAGE_NODE_SSH_PASSWORD,
|
|
||||||
private_key_path=STORAGE_NODE_SSH_PRIVATE_KEY_PATH,
|
|
||||||
)
|
|
||||||
with client.create_ssh_connection():
|
|
||||||
IpTablesHelper.restore_input_traffic_to_port(client, PORTS_TO_BLOCK)
|
|
||||||
blocked_hosts.remove(host)
|
blocked_hosts.remove(host)
|
||||||
if not_empty:
|
if not_empty:
|
||||||
wait_all_storage_node_returned(hosting)
|
wait_all_storage_node_returned(hosting)
|
||||||
|
@ -49,67 +37,65 @@ def restore_network(hosting: Hosting):
|
||||||
@allure.title("Block Storage node traffic")
|
@allure.title("Block Storage node traffic")
|
||||||
@pytest.mark.failover
|
@pytest.mark.failover
|
||||||
@pytest.mark.failover_net
|
@pytest.mark.failover_net
|
||||||
def test_block_storage_node_traffic(prepare_wallet_and_deposit, client_shell, require_multiple_hosts):
|
def test_block_storage_node_traffic(
|
||||||
|
prepare_wallet_and_deposit, client_shell, require_multiple_hosts, hosting: Hosting
|
||||||
|
):
|
||||||
"""
|
"""
|
||||||
Block storage nodes traffic using iptables and wait for replication for objects.
|
Block storage nodes traffic using iptables and wait for replication for objects.
|
||||||
"""
|
"""
|
||||||
wallet = prepare_wallet_and_deposit
|
wallet = prepare_wallet_and_deposit
|
||||||
placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
|
placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
|
||||||
excluded_nodes = []
|
|
||||||
wakeup_node_timeout = 10 # timeout to let nodes detect that traffic has blocked
|
wakeup_node_timeout = 10 # timeout to let nodes detect that traffic has blocked
|
||||||
nodes_to_block_count = 2
|
nodes_to_block_count = 2
|
||||||
|
|
||||||
source_file_path = generate_file()
|
source_file_path = generate_file()
|
||||||
cid = create_container(wallet, shell=client_shell, rule=placement_rule, basic_acl=PUBLIC_ACL)
|
cid = create_container(wallet, shell=client_shell, rule=placement_rule, basic_acl=PUBLIC_ACL)
|
||||||
oid = put_object(wallet, source_file_path, cid, shell=client_shell)
|
oid = put_object(wallet, source_file_path, cid, shell=client_shell)
|
||||||
nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell)
|
|
||||||
|
|
||||||
logger.info(f"Nodes are {nodes}")
|
# TODO: we need to refactor wait_object_replication_on_nodes so that it returns
|
||||||
random_nodes = [(node, node.split(":")[0]) for node in nodes]
|
# storage node names rather than endpoints
|
||||||
if nodes_to_block_count > len(nodes):
|
node_endpoints = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell)
|
||||||
random_nodes = [(node, node.split(":")[0]) for node in choices(nodes, k=2)]
|
|
||||||
|
|
||||||
for random_node, random_node_ip in random_nodes:
|
logger.info(f"Nodes are {node_endpoints}")
|
||||||
client = HostClient(
|
node_endpoints_to_block = node_endpoints
|
||||||
ip=random_node_ip,
|
if nodes_to_block_count > len(node_endpoints):
|
||||||
login=STORAGE_NODE_SSH_USER,
|
# TODO: the intent of this logic is not clear, need to revisit
|
||||||
password=STORAGE_NODE_SSH_PASSWORD,
|
node_endpoints_to_block = choices(node_endpoints, k=2)
|
||||||
private_key_path=STORAGE_NODE_SSH_PRIVATE_KEY_PATH,
|
|
||||||
)
|
|
||||||
|
|
||||||
with allure.step(f"Block incoming traffic for node {random_node} on port {PORTS_TO_BLOCK}"):
|
excluded_nodes = []
|
||||||
with client.create_ssh_connection():
|
for node_endpoint in node_endpoints_to_block:
|
||||||
IpTablesHelper.drop_input_traffic_to_port(client, PORTS_TO_BLOCK)
|
host_address = node_endpoint.split(":")[0]
|
||||||
blocked_hosts.append(random_node_ip)
|
host = hosting.get_host_by_address(host_address)
|
||||||
excluded_nodes.append(random_node)
|
|
||||||
|
with allure.step(f"Block incoming traffic at host {host_address} on port {PORTS_TO_BLOCK}"):
|
||||||
|
blocked_hosts.append(host_address)
|
||||||
|
excluded_nodes.append(node_endpoint)
|
||||||
|
IpTablesHelper.drop_input_traffic_to_port(host.get_shell(), PORTS_TO_BLOCK)
|
||||||
sleep(wakeup_node_timeout)
|
sleep(wakeup_node_timeout)
|
||||||
|
|
||||||
new_nodes = wait_object_replication_on_nodes(
|
with allure.step(f"Check object is not stored on node {node_endpoint}"):
|
||||||
wallet, cid, oid, 2, excluded_nodes=excluded_nodes
|
new_nodes = wait_object_replication_on_nodes(
|
||||||
)
|
wallet, cid, oid, 2, shell=client_shell, excluded_nodes=excluded_nodes
|
||||||
|
)
|
||||||
|
assert node_endpoint not in new_nodes
|
||||||
|
|
||||||
assert random_node not in new_nodes
|
with allure.step(f"Check object data is not corrupted"):
|
||||||
|
got_file_path = get_object(wallet, cid, oid, endpoint=new_nodes[0], shell=client_shell)
|
||||||
|
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
|
||||||
|
|
||||||
|
for node_endpoint in node_endpoints_to_block:
|
||||||
|
host_address = node_endpoint.split(":")[0]
|
||||||
|
host = hosting.get_host_by_address(host_address)
|
||||||
|
|
||||||
|
with allure.step(
|
||||||
|
f"Unblock incoming traffic at host {host_address} on port {PORTS_TO_BLOCK}"
|
||||||
|
):
|
||||||
|
IpTablesHelper.restore_input_traffic_to_port(host.get_shell(), PORTS_TO_BLOCK)
|
||||||
|
blocked_hosts.remove(host_address)
|
||||||
|
sleep(wakeup_node_timeout)
|
||||||
|
|
||||||
|
with allure.step(f"Check object data is not corrupted"):
|
||||||
|
new_nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell)
|
||||||
|
|
||||||
got_file_path = get_object(wallet, cid, oid, shell=client_shell, endpoint=new_nodes[0])
|
got_file_path = get_object(wallet, cid, oid, shell=client_shell, endpoint=new_nodes[0])
|
||||||
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
|
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
|
||||||
|
|
||||||
for random_node, random_node_ip in random_nodes:
|
|
||||||
client = HostClient(
|
|
||||||
ip=random_node_ip,
|
|
||||||
login=STORAGE_NODE_SSH_USER,
|
|
||||||
password=STORAGE_NODE_SSH_PASSWORD,
|
|
||||||
private_key_path=STORAGE_NODE_SSH_PRIVATE_KEY_PATH,
|
|
||||||
)
|
|
||||||
|
|
||||||
with allure.step(
|
|
||||||
f"Unblock incoming traffic for node {random_node} on port {PORTS_TO_BLOCK}"
|
|
||||||
):
|
|
||||||
with client.create_ssh_connection():
|
|
||||||
IpTablesHelper.restore_input_traffic_to_port(client, PORTS_TO_BLOCK)
|
|
||||||
blocked_hosts.remove(random_node_ip)
|
|
||||||
sleep(wakeup_node_timeout)
|
|
||||||
|
|
||||||
new_nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell)
|
|
||||||
|
|
||||||
got_file_path = get_object(wallet, cid, oid, shell=client_shell, endpoint=new_nodes[0])
|
|
||||||
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
|
|
||||||
|
|
|
@ -2,17 +2,12 @@ import logging
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
import pytest
|
import pytest
|
||||||
from common import (
|
|
||||||
STORAGE_NODE_SSH_PASSWORD,
|
|
||||||
STORAGE_NODE_SSH_PRIVATE_KEY_PATH,
|
|
||||||
STORAGE_NODE_SSH_USER,
|
|
||||||
)
|
|
||||||
from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes
|
from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes
|
||||||
from file_helper import generate_file, get_file_hash
|
from file_helper import generate_file, get_file_hash
|
||||||
from neofs_testlib.hosting import Hosting
|
from neofs_testlib.hosting import Host, Hosting
|
||||||
|
from neofs_testlib.shell import CommandOptions
|
||||||
from python_keywords.container import create_container
|
from python_keywords.container import create_container
|
||||||
from python_keywords.neofs_verbs import get_object, put_object
|
from python_keywords.neofs_verbs import get_object, put_object
|
||||||
from ssh_helper import HostClient
|
|
||||||
from wellknown_acl import PUBLIC_ACL
|
from wellknown_acl import PUBLIC_ACL
|
||||||
|
|
||||||
logger = logging.getLogger("NeoLogger")
|
logger = logging.getLogger("NeoLogger")
|
||||||
|
@ -20,27 +15,21 @@ stopped_hosts = []
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
@allure.step("Return all storage nodes")
|
@allure.step("Return all stopped hosts")
|
||||||
def return_all_storage_nodes_fixture(hosting: Hosting):
|
def after_run_return_all_stopped_hosts(hosting: Hosting):
|
||||||
yield
|
yield
|
||||||
return_all_storage_nodes(hosting)
|
return_stopped_hosts(hosting)
|
||||||
|
|
||||||
|
|
||||||
def panic_reboot_host(ip: str = None):
|
def panic_reboot_host(host: Host) -> None:
|
||||||
ssh = HostClient(
|
shell = host.get_shell()
|
||||||
ip=ip,
|
shell.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"')
|
||||||
login=STORAGE_NODE_SSH_USER,
|
|
||||||
password=STORAGE_NODE_SSH_PASSWORD,
|
options = CommandOptions(close_stdin=True, timeout=1, check=False)
|
||||||
private_key_path=STORAGE_NODE_SSH_PRIVATE_KEY_PATH,
|
shell.exec('sudo sh -c "echo b > /proc/sysrq-trigger"', options)
|
||||||
)
|
|
||||||
ssh.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"')
|
|
||||||
ssh_stdin, _, _ = ssh.ssh_client.exec_command(
|
|
||||||
'sudo sh -c "echo b > /proc/sysrq-trigger"', timeout=1
|
|
||||||
)
|
|
||||||
ssh_stdin.close()
|
|
||||||
|
|
||||||
|
|
||||||
def return_all_storage_nodes(hosting: Hosting) -> None:
|
def return_stopped_hosts(hosting: Hosting) -> None:
|
||||||
for host_address in list(stopped_hosts):
|
for host_address in list(stopped_hosts):
|
||||||
with allure.step(f"Start host {host_address}"):
|
with allure.step(f"Start host {host_address}"):
|
||||||
host = hosting.get_host_by_address(host_address)
|
host = hosting.get_host_by_address(host_address)
|
||||||
|
@ -50,10 +39,10 @@ def return_all_storage_nodes(hosting: Hosting) -> None:
|
||||||
wait_all_storage_node_returned(hosting)
|
wait_all_storage_node_returned(hosting)
|
||||||
|
|
||||||
|
|
||||||
@allure.title("Lost and returned nodes")
|
@allure.title("Lose and return storage node's host")
|
||||||
@pytest.mark.parametrize("hard_reboot", [True, False])
|
@pytest.mark.parametrize("hard_reboot", [True, False])
|
||||||
@pytest.mark.failover
|
@pytest.mark.failover
|
||||||
def test_lost_storage_node(
|
def test_lose_storage_node_host(
|
||||||
prepare_wallet_and_deposit,
|
prepare_wallet_and_deposit,
|
||||||
client_shell,
|
client_shell,
|
||||||
hosting: Hosting,
|
hosting: Hosting,
|
||||||
|
@ -65,35 +54,44 @@ def test_lost_storage_node(
|
||||||
source_file_path = generate_file()
|
source_file_path = generate_file()
|
||||||
cid = create_container(wallet, shell=client_shell, rule=placement_rule, basic_acl=PUBLIC_ACL)
|
cid = create_container(wallet, shell=client_shell, rule=placement_rule, basic_acl=PUBLIC_ACL)
|
||||||
oid = put_object(wallet, source_file_path, cid, shell=client_shell)
|
oid = put_object(wallet, source_file_path, cid, shell=client_shell)
|
||||||
nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell)
|
node_endpoints = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell)
|
||||||
|
|
||||||
new_nodes = []
|
for node_endpoint in node_endpoints:
|
||||||
for node in nodes:
|
host_address = node_endpoint.split(":")[0]
|
||||||
host = hosting.get_host_by_service(node)
|
host = hosting.get_host_by_address(host_address)
|
||||||
stopped_hosts.append(host.config.address)
|
stopped_hosts.append(host.config.address)
|
||||||
with allure.step(f"Stop storage node {node}"):
|
|
||||||
|
with allure.step(f"Stop host {host_address}"):
|
||||||
host.stop_host("hard" if hard_reboot else "soft")
|
host.stop_host("hard" if hard_reboot else "soft")
|
||||||
new_nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell, excluded_nodes=[node])
|
|
||||||
|
|
||||||
assert not [node for node in nodes if node in new_nodes]
|
new_nodes = wait_object_replication_on_nodes(
|
||||||
got_file_path = get_object(wallet, cid, oid, shell=client_shell, endpoint=new_nodes[0])
|
wallet, cid, oid, 2, shell=client_shell, excluded_nodes=[node_endpoint]
|
||||||
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
|
)
|
||||||
|
assert all(old_node not in new_nodes for old_node in node_endpoints)
|
||||||
|
|
||||||
with allure.step(f"Return storage nodes"):
|
with allure.step("Check object data is not corrupted"):
|
||||||
return_all_storage_nodes(hosting)
|
got_file_path = get_object(wallet, cid, oid, endpoint=new_nodes[0])
|
||||||
|
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
|
||||||
|
|
||||||
new_nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell)
|
with allure.step(f"Return all hosts"):
|
||||||
|
return_stopped_hosts(hosting)
|
||||||
|
|
||||||
got_file_path = get_object(wallet, cid, oid, shell=client_shell, endpoint=new_nodes[0])
|
with allure.step("Check object data is not corrupted"):
|
||||||
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
|
new_nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell)
|
||||||
|
got_file_path = get_object(wallet, cid, oid, shell=client_shell, endpoint=new_nodes[0])
|
||||||
|
assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
|
||||||
|
|
||||||
|
|
||||||
@allure.title("Panic storage node(s)")
|
@allure.title("Panic storage node's host")
|
||||||
@pytest.mark.parametrize("sequence", [True, False])
|
@pytest.mark.parametrize("sequence", [True, False])
|
||||||
@pytest.mark.failover_panic
|
@pytest.mark.failover_panic
|
||||||
@pytest.mark.failover
|
@pytest.mark.failover
|
||||||
def test_panic_storage_node(
|
def test_panic_storage_node_host(
|
||||||
prepare_wallet_and_deposit, client_shell, require_multiple_hosts, sequence: bool
|
prepare_wallet_and_deposit,
|
||||||
|
client_shell,
|
||||||
|
hosting: Hosting,
|
||||||
|
require_multiple_hosts,
|
||||||
|
sequence: bool,
|
||||||
):
|
):
|
||||||
wallet = prepare_wallet_and_deposit
|
wallet = prepare_wallet_and_deposit
|
||||||
placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
|
placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
|
||||||
|
@ -101,16 +99,24 @@ def test_panic_storage_node(
|
||||||
cid = create_container(wallet, shell=client_shell, rule=placement_rule, basic_acl=PUBLIC_ACL)
|
cid = create_container(wallet, shell=client_shell, rule=placement_rule, basic_acl=PUBLIC_ACL)
|
||||||
oid = put_object(wallet, source_file_path, cid, shell=client_shell)
|
oid = put_object(wallet, source_file_path, cid, shell=client_shell)
|
||||||
|
|
||||||
nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell)
|
node_endpoints = wait_object_replication_on_nodes(wallet, cid, oid, 2, shell=client_shell)
|
||||||
|
allure.attach(
|
||||||
|
"\n".join(node_endpoints),
|
||||||
|
"Current nodes with object",
|
||||||
|
allure.attachment_type.TEXT,
|
||||||
|
)
|
||||||
|
|
||||||
new_nodes: list[str] = []
|
new_nodes: list[str] = []
|
||||||
allure.attach("\n".join(nodes), "Current nodes with object", allure.attachment_type.TEXT)
|
for node_endpoint in node_endpoints:
|
||||||
for node in nodes:
|
host_address = node_endpoint.split(":")[0]
|
||||||
with allure.step(f"Hard reboot host {node} via magic SysRq option"):
|
|
||||||
panic_reboot_host(ip=node.split(":")[-2])
|
with allure.step(f"Hard reboot host {node_endpoint} via magic SysRq option"):
|
||||||
|
host = hosting.get_host_by_address(host_address)
|
||||||
|
panic_reboot_host(host)
|
||||||
if sequence:
|
if sequence:
|
||||||
try:
|
try:
|
||||||
new_nodes = wait_object_replication_on_nodes(
|
new_nodes = wait_object_replication_on_nodes(
|
||||||
wallet, cid, oid, 2, shell=client_shell, excluded_nodes=[node]
|
wallet, cid, oid, 2, shell=client_shell, excluded_nodes=[node_endpoint]
|
||||||
)
|
)
|
||||||
except AssertionError:
|
except AssertionError:
|
||||||
new_nodes = wait_object_replication_on_nodes(
|
new_nodes = wait_object_replication_on_nodes(
|
||||||
|
@ -119,7 +125,7 @@ def test_panic_storage_node(
|
||||||
|
|
||||||
allure.attach(
|
allure.attach(
|
||||||
"\n".join(new_nodes),
|
"\n".join(new_nodes),
|
||||||
f"Nodes with object after {node} fail",
|
f"Nodes with object after {node_endpoint} fail",
|
||||||
allure.attachment_type.TEXT,
|
allure.attachment_type.TEXT,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -5,7 +5,6 @@ from typing import Optional
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
import pytest
|
import pytest
|
||||||
from neofs_testlib.shell import Shell
|
|
||||||
from common import (
|
from common import (
|
||||||
COMPLEX_OBJ_SIZE,
|
COMPLEX_OBJ_SIZE,
|
||||||
MORPH_BLOCK_TIME,
|
MORPH_BLOCK_TIME,
|
||||||
|
@ -19,6 +18,7 @@ from epoch import tick_epoch
|
||||||
from file_helper import generate_file
|
from file_helper import generate_file
|
||||||
from grpc_responses import OBJECT_NOT_FOUND, error_matches_status
|
from grpc_responses import OBJECT_NOT_FOUND, error_matches_status
|
||||||
from neofs_testlib.hosting import Hosting
|
from neofs_testlib.hosting import Hosting
|
||||||
|
from neofs_testlib.shell import Shell
|
||||||
from python_keywords.container import create_container, get_container
|
from python_keywords.container import create_container, get_container
|
||||||
from python_keywords.failover_utils import wait_object_replication_on_nodes
|
from python_keywords.failover_utils import wait_object_replication_on_nodes
|
||||||
from python_keywords.neofs_verbs import delete_object, get_object, head_object, put_object
|
from python_keywords.neofs_verbs import delete_object, get_object, head_object, put_object
|
||||||
|
@ -121,7 +121,11 @@ def return_nodes(shell: Shell, hosting: Hosting, alive_node: Optional[str] = Non
|
||||||
@pytest.mark.add_nodes
|
@pytest.mark.add_nodes
|
||||||
@pytest.mark.node_mgmt
|
@pytest.mark.node_mgmt
|
||||||
def test_add_nodes(
|
def test_add_nodes(
|
||||||
prepare_tmp_dir, client_shell, prepare_wallet_and_deposit, return_nodes_after_test_run, hosting: Hosting
|
prepare_tmp_dir,
|
||||||
|
client_shell,
|
||||||
|
prepare_wallet_and_deposit,
|
||||||
|
return_nodes_after_test_run,
|
||||||
|
hosting: Hosting,
|
||||||
):
|
):
|
||||||
wallet = prepare_wallet_and_deposit
|
wallet = prepare_wallet_and_deposit
|
||||||
placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X"
|
placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X"
|
||||||
|
@ -141,14 +145,18 @@ def test_add_nodes(
|
||||||
|
|
||||||
# Add node to recovery list before messing with it
|
# Add node to recovery list before messing with it
|
||||||
check_nodes.append(additional_node)
|
check_nodes.append(additional_node)
|
||||||
exclude_node_from_network_map(hosting, additional_node, alive_node)
|
exclude_node_from_network_map(hosting, additional_node, alive_node, shell=client_shell)
|
||||||
delete_node_data(hosting, additional_node)
|
delete_node_data(hosting, additional_node)
|
||||||
|
|
||||||
cid = create_container(wallet, rule=placement_rule_3, basic_acl=PUBLIC_ACL)
|
cid = create_container(wallet, rule=placement_rule_3, basic_acl=PUBLIC_ACL, shell=client_shell)
|
||||||
oid = put_object(
|
oid = put_object(
|
||||||
wallet, source_file_path, cid, endpoint=NEOFS_NETMAP_DICT[alive_node].get("rpc")
|
wallet,
|
||||||
|
source_file_path,
|
||||||
|
cid,
|
||||||
|
endpoint=NEOFS_NETMAP_DICT[alive_node].get("rpc"),
|
||||||
|
shell=client_shell,
|
||||||
)
|
)
|
||||||
wait_object_replication_on_nodes(wallet, cid, oid, 3)
|
wait_object_replication_on_nodes(wallet, cid, oid, 3, shell=client_shell)
|
||||||
|
|
||||||
return_nodes(shell=client_shell, hosting=hosting, alive_node=alive_node)
|
return_nodes(shell=client_shell, hosting=hosting, alive_node=alive_node)
|
||||||
|
|
||||||
|
@ -156,16 +164,24 @@ def test_add_nodes(
|
||||||
random_node = choice(
|
random_node = choice(
|
||||||
[node for node in NEOFS_NETMAP_DICT if node not in (additional_node, alive_node)]
|
[node for node in NEOFS_NETMAP_DICT if node not in (additional_node, alive_node)]
|
||||||
)
|
)
|
||||||
exclude_node_from_network_map(hosting, random_node, alive_node)
|
exclude_node_from_network_map(hosting, random_node, alive_node, shell=client_shell)
|
||||||
|
|
||||||
wait_object_replication_on_nodes(wallet, cid, oid, 3, excluded_nodes=[random_node])
|
wait_object_replication_on_nodes(
|
||||||
|
wallet, cid, oid, 3, excluded_nodes=[random_node], shell=client_shell
|
||||||
|
)
|
||||||
include_node_to_network_map(hosting, random_node, alive_node, shell=client_shell)
|
include_node_to_network_map(hosting, random_node, alive_node, shell=client_shell)
|
||||||
wait_object_replication_on_nodes(wallet, cid, oid, 3)
|
wait_object_replication_on_nodes(wallet, cid, oid, 3, shell=client_shell)
|
||||||
|
|
||||||
with allure.step("Check container could be created with new node"):
|
with allure.step("Check container could be created with new node"):
|
||||||
cid = create_container(wallet, rule=placement_rule_4, basic_acl=PUBLIC_ACL)
|
cid = create_container(
|
||||||
|
wallet, rule=placement_rule_4, basic_acl=PUBLIC_ACL, shell=client_shell
|
||||||
|
)
|
||||||
oid = put_object(
|
oid = put_object(
|
||||||
wallet, source_file_path, cid, endpoint=NEOFS_NETMAP_DICT[alive_node].get("rpc")
|
wallet,
|
||||||
|
source_file_path,
|
||||||
|
cid,
|
||||||
|
endpoint=NEOFS_NETMAP_DICT[alive_node].get("rpc"),
|
||||||
|
shell=client_shell,
|
||||||
)
|
)
|
||||||
wait_object_replication_on_nodes(wallet, cid, oid, 4, shell=client_shell)
|
wait_object_replication_on_nodes(wallet, cid, oid, 4, shell=client_shell)
|
||||||
|
|
||||||
|
@ -231,13 +247,15 @@ def test_nodes_management(prepare_tmp_dir, client_shell, hosting: Hosting):
|
||||||
)
|
)
|
||||||
@pytest.mark.node_mgmt
|
@pytest.mark.node_mgmt
|
||||||
@allure.title("Test object copies based on placement policy")
|
@allure.title("Test object copies based on placement policy")
|
||||||
def test_placement_policy(prepare_wallet_and_deposit, placement_rule, expected_copies):
|
def test_placement_policy(
|
||||||
|
prepare_wallet_and_deposit, placement_rule, expected_copies, client_shell: Shell
|
||||||
|
):
|
||||||
"""
|
"""
|
||||||
This test checks object's copies based on container's placement policy.
|
This test checks object's copies based on container's placement policy.
|
||||||
"""
|
"""
|
||||||
wallet = prepare_wallet_and_deposit
|
wallet = prepare_wallet_and_deposit
|
||||||
file_path = generate_file()
|
file_path = generate_file()
|
||||||
validate_object_copies(wallet, placement_rule, file_path, expected_copies)
|
validate_object_copies(wallet, placement_rule, file_path, expected_copies, shell=client_shell)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
|
@ -288,7 +306,7 @@ def test_placement_policy(prepare_wallet_and_deposit, placement_rule, expected_c
|
||||||
@pytest.mark.node_mgmt
|
@pytest.mark.node_mgmt
|
||||||
@allure.title("Test object copies and storage nodes based on placement policy")
|
@allure.title("Test object copies and storage nodes based on placement policy")
|
||||||
def test_placement_policy_with_nodes(
|
def test_placement_policy_with_nodes(
|
||||||
prepare_wallet_and_deposit, placement_rule, expected_copies, nodes
|
prepare_wallet_and_deposit, placement_rule, expected_copies, nodes, client_shell: Shell
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Based on container's placement policy check that storage nodes are piked correctly and object has
|
Based on container's placement policy check that storage nodes are piked correctly and object has
|
||||||
|
@ -297,7 +315,7 @@ def test_placement_policy_with_nodes(
|
||||||
wallet = prepare_wallet_and_deposit
|
wallet = prepare_wallet_and_deposit
|
||||||
file_path = generate_file()
|
file_path = generate_file()
|
||||||
cid, oid, found_nodes = validate_object_copies(
|
cid, oid, found_nodes = validate_object_copies(
|
||||||
wallet, placement_rule, file_path, expected_copies
|
wallet, placement_rule, file_path, expected_copies, shell=client_shell
|
||||||
)
|
)
|
||||||
expected_nodes = [NEOFS_NETMAP_DICT[node_name].get("rpc") for node_name in nodes]
|
expected_nodes = [NEOFS_NETMAP_DICT[node_name].get("rpc") for node_name in nodes]
|
||||||
assert set(found_nodes) == set(
|
assert set(found_nodes) == set(
|
||||||
|
@ -313,17 +331,21 @@ def test_placement_policy_with_nodes(
|
||||||
)
|
)
|
||||||
@pytest.mark.node_mgmt
|
@pytest.mark.node_mgmt
|
||||||
@allure.title("Negative cases for placement policy")
|
@allure.title("Negative cases for placement policy")
|
||||||
def test_placement_policy_negative(prepare_wallet_and_deposit, placement_rule, expected_copies):
|
def test_placement_policy_negative(
|
||||||
|
prepare_wallet_and_deposit, placement_rule, expected_copies, client_shell: Shell
|
||||||
|
):
|
||||||
"""
|
"""
|
||||||
Negative test for placement policy.
|
Negative test for placement policy.
|
||||||
"""
|
"""
|
||||||
wallet = prepare_wallet_and_deposit
|
wallet = prepare_wallet_and_deposit
|
||||||
file_path = generate_file()
|
file_path = generate_file()
|
||||||
with pytest.raises(RuntimeError, match=".*not enough nodes to SELECT from.*"):
|
with pytest.raises(RuntimeError, match=".*not enough nodes to SELECT from.*"):
|
||||||
validate_object_copies(wallet, placement_rule, file_path, expected_copies)
|
validate_object_copies(
|
||||||
|
wallet, placement_rule, file_path, expected_copies, shell=client_shell
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip(reason="We cover this scenario for Sbercloud in failover tests")
|
@pytest.mark.skip(reason="We cover this scenario in failover tests")
|
||||||
@pytest.mark.node_mgmt
|
@pytest.mark.node_mgmt
|
||||||
@allure.title("NeoFS object replication on node failover")
|
@allure.title("NeoFS object replication on node failover")
|
||||||
def test_replication(prepare_wallet_and_deposit, after_run_start_all_nodes, hosting: Hosting):
|
def test_replication(prepare_wallet_and_deposit, after_run_start_all_nodes, hosting: Hosting):
|
||||||
|
@ -430,14 +452,18 @@ def test_shards(prepare_wallet_and_deposit, create_container_and_pick_node, host
|
||||||
|
|
||||||
|
|
||||||
@allure.step("Validate object has {expected_copies} copies")
|
@allure.step("Validate object has {expected_copies} copies")
|
||||||
def validate_object_copies(wallet: str, placement_rule: str, file_path: str, expected_copies: int):
|
def validate_object_copies(
|
||||||
cid = create_container(wallet, rule=placement_rule, basic_acl=PUBLIC_ACL)
|
wallet: str, placement_rule: str, file_path: str, expected_copies: int, shell: Shell
|
||||||
got_policy = placement_policy_from_container(get_container(wallet, cid, json_mode=False))
|
):
|
||||||
|
cid = create_container(wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=shell)
|
||||||
|
got_policy = placement_policy_from_container(
|
||||||
|
get_container(wallet, cid, json_mode=False, shell=shell)
|
||||||
|
)
|
||||||
assert got_policy == placement_rule.replace(
|
assert got_policy == placement_rule.replace(
|
||||||
"'", ""
|
"'", ""
|
||||||
), f"Expected \n{placement_rule} and got policy \n{got_policy} are the same"
|
), f"Expected \n{placement_rule} and got policy \n{got_policy} are the same"
|
||||||
oid = put_object(wallet, file_path, cid)
|
oid = put_object(wallet, file_path, cid, shell=shell)
|
||||||
nodes = get_nodes_with_object(wallet, cid, oid)
|
nodes = get_nodes_with_object(wallet, cid, oid, shell=shell)
|
||||||
assert len(nodes) == expected_copies, f"Expected {expected_copies} copies, got {len(nodes)}"
|
assert len(nodes) == expected_copies, f"Expected {expected_copies} copies, got {len(nodes)}"
|
||||||
return cid, oid, nodes
|
return cid, oid, nodes
|
||||||
|
|
||||||
|
|
|
@ -8,6 +8,7 @@ from container import create_container
|
||||||
from epoch import get_epoch, tick_epoch
|
from epoch import get_epoch, tick_epoch
|
||||||
from file_helper import generate_file, get_file_content, get_file_hash
|
from file_helper import generate_file, get_file_content, get_file_hash
|
||||||
from grpc_responses import OBJECT_ALREADY_REMOVED, OBJECT_NOT_FOUND, error_matches_status
|
from grpc_responses import OBJECT_ALREADY_REMOVED, OBJECT_NOT_FOUND, error_matches_status
|
||||||
|
from neofs_testlib.shell import Shell
|
||||||
from python_keywords.neofs_verbs import (
|
from python_keywords.neofs_verbs import (
|
||||||
delete_object,
|
delete_object,
|
||||||
get_object,
|
get_object,
|
||||||
|
@ -18,7 +19,6 @@ from python_keywords.neofs_verbs import (
|
||||||
search_object,
|
search_object,
|
||||||
)
|
)
|
||||||
from python_keywords.storage_policy import get_complex_object_copies, get_simple_object_copies
|
from python_keywords.storage_policy import get_complex_object_copies, get_simple_object_copies
|
||||||
from neofs_testlib.shell import Shell
|
|
||||||
from tombstone import verify_head_tombstone
|
from tombstone import verify_head_tombstone
|
||||||
from utility import wait_for_gc_pass_on_storage_nodes
|
from utility import wait_for_gc_pass_on_storage_nodes
|
||||||
|
|
||||||
|
|
|
@ -9,6 +9,7 @@ from common import COMPLEX_OBJ_SIZE
|
||||||
from container import create_container
|
from container import create_container
|
||||||
from epoch import get_epoch, tick_epoch
|
from epoch import get_epoch, tick_epoch
|
||||||
from file_helper import generate_file, get_file_hash
|
from file_helper import generate_file, get_file_hash
|
||||||
|
from neofs_testlib.shell import Shell
|
||||||
from python_keywords.http_gate import (
|
from python_keywords.http_gate import (
|
||||||
get_via_http_curl,
|
get_via_http_curl,
|
||||||
get_via_http_gate,
|
get_via_http_gate,
|
||||||
|
@ -17,7 +18,6 @@ from python_keywords.http_gate import (
|
||||||
upload_via_http_gate,
|
upload_via_http_gate,
|
||||||
upload_via_http_gate_curl,
|
upload_via_http_gate_curl,
|
||||||
)
|
)
|
||||||
from neofs_testlib.shell import Shell
|
|
||||||
from python_keywords.neofs_verbs import get_object, put_object
|
from python_keywords.neofs_verbs import get_object, put_object
|
||||||
from python_keywords.storage_policy import get_nodes_without_object
|
from python_keywords.storage_policy import get_nodes_without_object
|
||||||
from utility import wait_for_gc_pass_on_storage_nodes
|
from utility import wait_for_gc_pass_on_storage_nodes
|
||||||
|
@ -225,7 +225,14 @@ class TestHttpGate:
|
||||||
oid_curl = upload_via_http_gate_curl(cid=cid, filepath=file_path, large_object=True)
|
oid_curl = upload_via_http_gate_curl(cid=cid, filepath=file_path, large_object=True)
|
||||||
|
|
||||||
self.get_object_and_verify_hashes(oid_gate, file_path, self.wallet, cid, shell=client_shell)
|
self.get_object_and_verify_hashes(oid_gate, file_path, self.wallet, cid, shell=client_shell)
|
||||||
self.get_object_and_verify_hashes(oid_curl, file_path, self.wallet, cid, shell=client_shell, object_getter=get_via_http_curl)
|
self.get_object_and_verify_hashes(
|
||||||
|
oid_curl,
|
||||||
|
file_path,
|
||||||
|
self.wallet,
|
||||||
|
cid,
|
||||||
|
shell=client_shell,
|
||||||
|
object_getter=get_via_http_curl,
|
||||||
|
)
|
||||||
|
|
||||||
@pytest.mark.curl
|
@pytest.mark.curl
|
||||||
@allure.title("Test Put/Get over HTTP using Curl utility")
|
@allure.title("Test Put/Get over HTTP using Curl utility")
|
||||||
|
@ -243,7 +250,14 @@ class TestHttpGate:
|
||||||
oid_large = upload_via_http_gate_curl(cid=cid, filepath=file_path_large)
|
oid_large = upload_via_http_gate_curl(cid=cid, filepath=file_path_large)
|
||||||
|
|
||||||
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
|
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
|
||||||
self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid, shell=client_shell, object_getter=get_via_http_curl)
|
self.get_object_and_verify_hashes(
|
||||||
|
oid,
|
||||||
|
file_path,
|
||||||
|
self.wallet,
|
||||||
|
cid,
|
||||||
|
shell=client_shell,
|
||||||
|
object_getter=get_via_http_curl,
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@allure.step("Try to get object and expect error")
|
@allure.step("Try to get object and expect error")
|
||||||
|
|
|
@ -2,10 +2,10 @@ import random
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
import pytest
|
import pytest
|
||||||
from neofs_testlib.shell import Shell
|
|
||||||
from common import COMPLEX_OBJ_SIZE, NEOFS_NETMAP_DICT, SIMPLE_OBJ_SIZE
|
from common import COMPLEX_OBJ_SIZE, NEOFS_NETMAP_DICT, SIMPLE_OBJ_SIZE
|
||||||
from file_helper import generate_file
|
from file_helper import generate_file
|
||||||
from grpc_responses import SESSION_NOT_FOUND
|
from grpc_responses import SESSION_NOT_FOUND
|
||||||
|
from neofs_testlib.shell import Shell
|
||||||
from payment_neogo import _address_from_wallet
|
from payment_neogo import _address_from_wallet
|
||||||
from python_keywords.container import create_container
|
from python_keywords.container import create_container
|
||||||
from python_keywords.neofs_verbs import (
|
from python_keywords.neofs_verbs import (
|
||||||
|
|
|
@ -129,7 +129,10 @@ def list_containers(wallet: str, shell: Shell) -> list[str]:
|
||||||
|
|
||||||
@allure.step("Get Container")
|
@allure.step("Get Container")
|
||||||
def get_container(
|
def get_container(
|
||||||
wallet: str, cid: str, shell: Shell, json_mode: bool = True,
|
wallet: str,
|
||||||
|
cid: str,
|
||||||
|
shell: Shell,
|
||||||
|
json_mode: bool = True,
|
||||||
) -> Union[dict, str]:
|
) -> Union[dict, str]:
|
||||||
"""
|
"""
|
||||||
A wrapper for `neofs-cli container get` call. It extracts container's
|
A wrapper for `neofs-cli container get` call. It extracts container's
|
||||||
|
@ -162,9 +165,7 @@ def get_container(
|
||||||
@allure.step("Delete Container")
|
@allure.step("Delete Container")
|
||||||
# TODO: make the error message about a non-found container more user-friendly
|
# TODO: make the error message about a non-found container more user-friendly
|
||||||
# https://github.com/nspcc-dev/neofs-contract/issues/121
|
# https://github.com/nspcc-dev/neofs-contract/issues/121
|
||||||
def delete_container(
|
def delete_container(wallet: str, cid: str, shell: Shell, force: bool = False) -> None:
|
||||||
wallet: str, cid: str, shell: Shell, force: bool = False
|
|
||||||
) -> None:
|
|
||||||
"""
|
"""
|
||||||
A wrapper for `neofs-cli container delete` call.
|
A wrapper for `neofs-cli container delete` call.
|
||||||
Args:
|
Args:
|
||||||
|
|
|
@ -4,8 +4,8 @@ from typing import Optional
|
||||||
|
|
||||||
import allure
|
import allure
|
||||||
from common import NEOFS_NETMAP_DICT
|
from common import NEOFS_NETMAP_DICT
|
||||||
from neofs_testlib.shell import Shell
|
|
||||||
from neofs_testlib.hosting import Hosting
|
from neofs_testlib.hosting import Hosting
|
||||||
|
from neofs_testlib.shell import Shell
|
||||||
from python_keywords.node_management import node_healthcheck
|
from python_keywords.node_management import node_healthcheck
|
||||||
from storage_policy import get_nodes_with_object
|
from storage_policy import get_nodes_with_object
|
||||||
|
|
||||||
|
@ -22,7 +22,7 @@ def wait_object_replication_on_nodes(
|
||||||
excluded_nodes: Optional[list[str]] = None,
|
excluded_nodes: Optional[list[str]] = None,
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
excluded_nodes = excluded_nodes or []
|
excluded_nodes = excluded_nodes or []
|
||||||
sleep_interval, attempts = 10, 18
|
sleep_interval, attempts = 15, 20
|
||||||
nodes = []
|
nodes = []
|
||||||
for __attempt in range(attempts):
|
for __attempt in range(attempts):
|
||||||
nodes = get_nodes_with_object(wallet, cid, oid, shell=shell, skip_nodes=excluded_nodes)
|
nodes = get_nodes_with_object(wallet, cid, oid, shell=shell, skip_nodes=excluded_nodes)
|
||||||
|
|
|
@ -16,8 +16,8 @@ from common import (
|
||||||
from data_formatters import get_wallet_public_key
|
from data_formatters import get_wallet_public_key
|
||||||
from epoch import tick_epoch
|
from epoch import tick_epoch
|
||||||
from neofs_testlib.cli import NeofsCli
|
from neofs_testlib.cli import NeofsCli
|
||||||
from neofs_testlib.shell import Shell
|
|
||||||
from neofs_testlib.hosting import Hosting
|
from neofs_testlib.hosting import Hosting
|
||||||
|
from neofs_testlib.shell import Shell
|
||||||
from utility import parse_time
|
from utility import parse_time
|
||||||
|
|
||||||
logger = logging.getLogger("NeoLogger")
|
logger = logging.getLogger("NeoLogger")
|
||||||
|
@ -165,7 +165,9 @@ def delete_node_data(hosting: Hosting, node_name: str) -> None:
|
||||||
|
|
||||||
|
|
||||||
@allure.step("Exclude node {node_to_exclude} from network map")
|
@allure.step("Exclude node {node_to_exclude} from network map")
|
||||||
def exclude_node_from_network_map(hosting: Hosting, node_to_exclude: str, alive_node: str, shell: Shell) -> None:
|
def exclude_node_from_network_map(
|
||||||
|
hosting: Hosting, node_to_exclude: str, alive_node: str, shell: Shell
|
||||||
|
) -> None:
|
||||||
node_wallet_path = NEOFS_NETMAP_DICT[node_to_exclude]["wallet_path"]
|
node_wallet_path = NEOFS_NETMAP_DICT[node_to_exclude]["wallet_path"]
|
||||||
node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS)
|
node_netmap_key = get_wallet_public_key(node_wallet_path, STORAGE_WALLET_PASS)
|
||||||
|
|
||||||
|
@ -181,13 +183,15 @@ def exclude_node_from_network_map(hosting: Hosting, node_to_exclude: str, alive_
|
||||||
|
|
||||||
|
|
||||||
@allure.step("Include node {node_to_include} into network map")
|
@allure.step("Include node {node_to_include} into network map")
|
||||||
def include_node_to_network_map(hosting: Hosting, node_to_include: str, alive_node: str, shell: Shell) -> None:
|
def include_node_to_network_map(
|
||||||
|
hosting: Hosting, node_to_include: str, alive_node: str, shell: Shell
|
||||||
|
) -> None:
|
||||||
node_set_status(hosting, node_to_include, status="online")
|
node_set_status(hosting, node_to_include, status="online")
|
||||||
|
|
||||||
time.sleep(parse_time(MORPH_BLOCK_TIME))
|
time.sleep(parse_time(MORPH_BLOCK_TIME))
|
||||||
tick_epoch()
|
tick_epoch()
|
||||||
|
|
||||||
check_node_in_map(node_to_include, alive_node, shell=shell)
|
check_node_in_map(node_to_include, shell, alive_node)
|
||||||
|
|
||||||
|
|
||||||
@allure.step("Check node {node_name} in network map")
|
@allure.step("Check node {node_name} in network map")
|
||||||
|
|
|
@ -30,7 +30,13 @@ def can_get_object(
|
||||||
with allure.step("Try get object from container"):
|
with allure.step("Try get object from container"):
|
||||||
try:
|
try:
|
||||||
got_file_path = get_object(
|
got_file_path = get_object(
|
||||||
wallet, cid, oid, bearer_token=bearer, wallet_config=wallet_config, xhdr=xhdr, shell=shell,
|
wallet,
|
||||||
|
cid,
|
||||||
|
oid,
|
||||||
|
bearer_token=bearer,
|
||||||
|
wallet_config=wallet_config,
|
||||||
|
xhdr=xhdr,
|
||||||
|
shell=shell,
|
||||||
)
|
)
|
||||||
except OPERATION_ERROR_TYPE as err:
|
except OPERATION_ERROR_TYPE as err:
|
||||||
assert error_matches_status(
|
assert error_matches_status(
|
||||||
|
|
|
@ -89,12 +89,6 @@ NEOFS_NETMAP_DICT = {
|
||||||
}
|
}
|
||||||
NEOFS_NETMAP = [node["rpc"] for node in NEOFS_NETMAP_DICT.values()]
|
NEOFS_NETMAP = [node["rpc"] for node in NEOFS_NETMAP_DICT.values()]
|
||||||
|
|
||||||
# Parameters that control SSH connection to storage node
|
|
||||||
# TODO: we should use hosting instead
|
|
||||||
STORAGE_NODE_SSH_USER = os.getenv("STORAGE_NODE_SSH_USER")
|
|
||||||
STORAGE_NODE_SSH_PASSWORD = os.getenv("STORAGE_NODE_SSH_PASSWORD")
|
|
||||||
STORAGE_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("STORAGE_NODE_SSH_PRIVATE_KEY_PATH")
|
|
||||||
|
|
||||||
# Paths to CLI executables on machine that runs tests
|
# Paths to CLI executables on machine that runs tests
|
||||||
NEOGO_EXECUTABLE = os.getenv("NEOGO_EXECUTABLE", "neo-go")
|
NEOGO_EXECUTABLE = os.getenv("NEOGO_EXECUTABLE", "neo-go")
|
||||||
NEOFS_CLI_EXEC = os.getenv("NEOFS_CLI_EXEC", "neofs-cli")
|
NEOFS_CLI_EXEC = os.getenv("NEOFS_CLI_EXEC", "neofs-cli")
|
||||||
|
|
Loading…
Reference in a new issue