Remove dockerfile and removed docker run command from Makefile to keep testcases environment agnostic.

Add logic to retrieve mainnet wallet file from control node as WIF for this wallet is not available in config anymore.

Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
This commit is contained in:
a.y.volkov 2022-07-01 20:30:07 +03:00 committed by Anastasia Prasolova
parent 6b1e1ab28d
commit 8acf738147
7 changed files with 252 additions and 107 deletions

View file

@ -7,13 +7,6 @@ SHELL = bash
OUTPUT_DIR = artifacts/ OUTPUT_DIR = artifacts/
KEYWORDS_REPO = git@github.com:nspcc-dev/neofs-keywords.git KEYWORDS_REPO = git@github.com:nspcc-dev/neofs-keywords.git
VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a) VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a)
ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
DEV_IMAGE_PY ?= registry.spb.yadro.com/tools/pytest-neofs-x86_64:4
ifeq ($(shell uname -s),Darwin)
DOCKER_NETWORK = --network bridge -p 389:389 -p 636:636
endif
.PHONY: all .PHONY: all
all: venvs all: venvs
@ -39,32 +32,7 @@ clean:
pytest-local: pytest-local:
@echo "⇒ Run Pytest" @echo "⇒ Run Pytest"
export PYTHONPATH=$(ROOT_DIR)/neofs-keywords/lib:$(ROOT_DIR)/neofs-keywords/robot:$(ROOT_DIR)/robot/resources/lib:$(ROOT_DIR)/robot/resources/lib/python_keywords:$(ROOT_DIR)/robot/variables && \
python -m pytest pytest_tests/testsuites/ python -m pytest pytest_tests/testsuites/
help: help:
@echo "⇒ run Run testcases ${R}" @echo "⇒ run Run testcases ${R}"
.PHONY: pytest-docker
pytest-docker:
-docker ps
-docker rm neofs_tests_py
-docker pull $(DEV_IMAGE_PY)
docker run -t --rm \
-w /tests \
--name neofs_tests_py \
-e PYTHONPATH="/tests/neofs-keywords/lib:/tests/neofs-keywords/robot:/tests/robot/resources/lib:/tests/robot/resources/lib/python_keywords:/tests/robot/variables:/tests/pytest_tests/helpers" \
-v $(CURDIR):/tests \
-v /var/run/docker.sock:/var/run/docker.sock \
-v $(NEO_BIN_DIR):/neofs \
--privileged \
$(DOCKER_NETWORK) \
--env-file $(CURDIR)/.env \
$(DEV_IMAGE_PY) \
-v \
-m "$(CI_MARKERS)" \
--color=no \
--junitxml=/tests/xunit_results.xml \
--alluredir=/tests/allure_results \
--setup-show \
/tests/pytest_tests/testsuites

View file

@ -1,40 +0,0 @@
ARG ARCH
FROM python:3.9-slim
ARG ARCH
RUN apt-get -y update && apt-get -y install \
gcc \
make \
git \
curl \
wget \
openssh-client \
iputils-ping \
unzip \
vim \
dbus \
lsof \
tini \
libssl-dev \
expect \
runc \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get clean
COPY requirements.txt ./
# install aws cli utility v2 from pip
RUN python3 -m pip install awscliv2
RUN ln /usr/local/bin/awscliv2 /usr/bin/aws
RUN awsv2 --install
RUN aws --version
RUN pip install --no-cache-dir -r requirements.txt
ENV PATH=$PATH:/root:/neofs
WORKDIR /root
ENTRYPOINT ["tini", "--", "pytest"]

View file

@ -0,0 +1,208 @@
import logging
import socket
import tempfile
import textwrap
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import datetime
from functools import wraps
from time import sleep
from typing import ClassVar
import os
import allure
from paramiko import (AutoAddPolicy, SFTPClient, SSHClient, SSHException,
ssh_exception, RSAKey)
from paramiko.ssh_exception import AuthenticationException
class HostIsNotAvailable(Exception):
"""Raises when host is not reachable."""
def __init__(self, ip: str = None, exc: Exception = None):
msg = f'Host is not available{f" by ip: {ip}" if ip else ""}'
if exc:
msg = f'{msg}. {exc}'
super().__init__(msg)
def log_command(func):
@wraps(func)
def wrapper(host: 'HostClient', command: str, *args, **kwargs):
display_length = 60
short = command.removeprefix("$ProgressPreference='SilentlyContinue'\n")
short = short[:display_length]
short += '...' if short != command else ''
with allure.step(f'SSH: {short}'):
logging.info(f'Execute command "{command}" on "{host.ip}"')
start_time = datetime.now()
cmd_result = func(host, command, *args, **kwargs)
end_time = datetime.now()
log_message = f'HOST: {host.ip}\n' \
f'COMMAND:\n{textwrap.indent(command, " ")}\n' \
f'RC:\n {cmd_result.rc}\n' \
f'STDOUT:\n{textwrap.indent(cmd_result.stdout, " ")}\n' \
f'STDERR:\n{textwrap.indent(cmd_result.stderr, " ")}\n' \
f'Start / End / Elapsed\t {start_time.time()} / {end_time.time()} / {end_time - start_time}'
logging.info(log_message)
allure.attach(log_message, 'SSH command', allure.attachment_type.TEXT)
return cmd_result
return wrapper
@dataclass
class SSHCommand:
stdout: str
stderr: str
rc: int
class HostClient:
ssh_client: SSHClient
SSH_CONNECTION_ATTEMPTS: ClassVar[int] = 3
CONNECTION_TIMEOUT = 30
TIMEOUT_RESTORE_CONNECTION = 10, 24
def __init__(self, ip: str, login: str, password: str, init_ssh_client=True):
self.ip = ip
self.login = login
self.password = password
self.pk = os.getenv('SSH_PK_PATH', '/root/.ssh/id_rsa')
if init_ssh_client:
self.create_connection(self.SSH_CONNECTION_ATTEMPTS)
def exec(self, cmd: str, verify=True, timeout=30) -> SSHCommand:
cmd_result = self._inner_exec(cmd, timeout)
if verify:
assert cmd_result.rc == 0, f'Non zero rc from command: "{cmd}"'
return cmd_result
@log_command
def exec_with_confirmation(self, cmd: str, confirmation: list, verify=True, timeout=10) -> SSHCommand:
ssh_stdin, ssh_stdout, ssh_stderr = self.ssh_client.exec_command(cmd, timeout=timeout)
for line in confirmation:
if not line.endswith('\n'):
line = f'{line}\n'
try:
ssh_stdin.write(line)
except OSError as err:
logging.error(f'Got error {err} executing command {cmd}')
ssh_stdin.close()
output = SSHCommand(stdout=ssh_stdout.read().decode(errors='ignore'),
stderr=ssh_stderr.read().decode(errors='ignore'),
rc=ssh_stdout.channel.recv_exit_status())
if verify:
debug_info = f'\nSTDOUT: {output.stdout}\nSTDERR: {output.stderr}\nRC: {output.rc}'
assert output.rc == 0, f'Non zero rc from command: "{cmd}"{debug_info}'
return output
@contextmanager
def as_user(self, user: str, password: str):
keep_user, keep_password = self.login, self.password
self.login, self.password = user, password
self.create_connection()
yield
self.login, self.password = keep_user, keep_password
self.create_connection()
@allure.step('Restore connection')
def restore_ssh_connection(self):
retry_time, retry_count = self.TIMEOUT_RESTORE_CONNECTION
for _ in range(retry_count):
try:
self.create_connection()
except AssertionError:
logging.warning(f'Host: Cant reach host: {self.ip}.')
sleep(retry_time)
else:
logging.info(f'Host: Cant reach host: {self.ip}.')
return
raise AssertionError(f'Host: Cant reach host: {self.ip} after 240 seconds..')
@allure.step('Copy file {host_path_to_file} to local file {path_to_file}')
def copy_file_from_host(self, host_path_to_file: str, path_to_file: str):
with self._sftp_client() as sftp_client:
sftp_client.get(host_path_to_file, path_to_file)
def copy_file_to_host(self, path_to_file: str, host_path_to_file: str):
with allure.step(f'Copy local file {path_to_file} to remote file {host_path_to_file} on host {self.ip}'):
with self._sftp_client() as sftp_client:
sftp_client.put(path_to_file, host_path_to_file)
@allure.step('Save string to remote file {host_path_to_file}')
def copy_str_to_host_file(self, string: str, host_path_to_file: str):
with tempfile.NamedTemporaryFile(mode='r+') as temp:
temp.writelines(string)
temp.flush()
with self._sftp_client() as client:
client.put(temp.name, host_path_to_file)
self.exec(f'cat {host_path_to_file}', verify=False)
def create_connection(self, attempts=SSH_CONNECTION_ATTEMPTS):
exc_err = None
for attempt in range(attempts):
logging.info(f'Try to establish connection {attempt + 1} time to Host: {self.ip} under {self.login} '
f'user with {self.password} password..')
self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
try:
if self.password:
self.ssh_client.connect(hostname=str(self.ip), username=self.login, password=str(self.password),
timeout=self.CONNECTION_TIMEOUT)
else:
self.ssh_client.connect(hostname=str(self.ip), pkey=RSAKey.from_private_key_file(self.pk),
timeout=self.CONNECTION_TIMEOUT)
return True
except AuthenticationException as auth_err:
logging.error(f'Host: {self.ip}. {auth_err}')
raise auth_err
except (
SSHException,
ssh_exception.NoValidConnectionsError,
AttributeError,
socket.timeout,
OSError
) as ssh_err:
exc_err = ssh_err
logging.error(f'Host: {self.ip}, connection error. {exc_err}')
raise HostIsNotAvailable(self.ip, exc_err)
def drop(self):
self.ssh_client.close()
@log_command
def _inner_exec(self, cmd: str, timeout: int) -> SSHCommand:
if not self.ssh_client:
self.create_connection()
for _ in range(self.SSH_CONNECTION_ATTEMPTS):
try:
_, stdout, stderr = self.ssh_client.exec_command(cmd, timeout=timeout)
return SSHCommand(
stdout=stdout.read().decode(errors='ignore'),
stderr=stderr.read().decode(errors='ignore'),
rc=stdout.channel.recv_exit_status()
)
except (
SSHException,
TimeoutError,
ssh_exception.NoValidConnectionsError,
ConnectionResetError,
AttributeError,
socket.timeout,
) as ssh_err:
logging.error(f'Host: {self.ip}, exec command error {ssh_err}')
self.create_connection()
raise HostIsNotAvailable(f'Host: {self.ip} is not reachable.')
@contextmanager
def _sftp_client(self) -> SFTPClient:
with self.ssh_client.open_sftp() as sftp:
yield sftp

View file

@ -6,21 +6,23 @@ from time import sleep
import allure import allure
import pytest import pytest
from robot.api import deco
import rpc_client import rpc_client
import wallet import wallet
from cli_helpers import _cmd_run from cli_helpers import _cmd_run
from common import (ASSETS_DIR, COMMON_PLACEMENT_RULE, COMPLEX_OBJ_SIZE, from common import (ASSETS_DIR, COMPLEX_OBJ_SIZE, COMMON_PLACEMENT_RULE,
MAINNET_WALLET_WIF, NEO_MAINNET_ENDPOINT, SIMPLE_OBJ_SIZE) MAINNET_WALLET_PATH, NEO_MAINNET_ENDPOINT, SIMPLE_OBJ_SIZE, REMOTE_HOST, CONTROL_NODE_USER,
CONTROL_NODE_PWD)
from payment_neogo import neofs_deposit, transfer_mainnet_gas
from python_keywords.container import create_container from python_keywords.container import create_container
from python_keywords.payment_neogo import get_balance from python_keywords.payment_neogo import get_balance
from python_keywords.utility_keywords import generate_file_and_file_hash from python_keywords.utility_keywords import generate_file_and_file_hash
from robot.api import deco from ssh_helper import HostClient
from wallet_keywords import neofs_deposit, transfer_mainnet_gas
from wellknown_acl import PUBLIC_ACL from wellknown_acl import PUBLIC_ACL
deco.keyword = allure.step deco.keyword = allure.step
logger = logging.getLogger('NeoLogger') logger = logging.getLogger('NeoLogger')
@ -57,39 +59,43 @@ def init_wallet_with_address():
@allure.title('Prepare wallet and deposit') @allure.title('Prepare wallet and deposit')
def prepare_wallet_and_deposit(init_wallet_with_address): def prepare_wallet_and_deposit(init_wallet_with_address):
deposit = 30 deposit = 30
wallet, addr, wif = init_wallet_with_address local_wallet_path = None
logger.info(f'Init wallet: {wallet},\naddr: {addr},\nwif: {wif}') wallet, addr, _ = init_wallet_with_address
logger.info(f'Init wallet: {wallet},\naddr: {addr}')
txid = transfer_mainnet_gas(MAINNET_WALLET_WIF, addr, deposit + 1) if REMOTE_HOST:
wait_unitl_transaction_accepted_in_block(txid) ssh_client = HostClient(REMOTE_HOST, CONTROL_NODE_USER, CONTROL_NODE_PWD)
deposit_tx = neofs_deposit(wif, deposit) local_wallet_path = os.path.join(ASSETS_DIR, os.path.basename(MAINNET_WALLET_PATH))
wait_unitl_transaction_accepted_in_block(deposit_tx) ssh_client.copy_file_from_host(MAINNET_WALLET_PATH, local_wallet_path)
return wallet, wif transfer_mainnet_gas(wallet, deposit + 1, wallet_path=local_wallet_path or MAINNET_WALLET_PATH)
neofs_deposit(wallet, deposit)
return wallet
@pytest.fixture() @pytest.fixture()
@allure.title('Create Container') @allure.title('Create Container')
def prepare_container(prepare_wallet_and_deposit): def prepare_container(prepare_wallet_and_deposit):
wallet, wif = prepare_wallet_and_deposit wallet = prepare_wallet_and_deposit
return prepare_container_impl(wallet, wif) return prepare_container_impl(wallet)
@pytest.fixture(scope='module') @pytest.fixture(scope='module')
@allure.title('Create Public Container') @allure.title('Create Public Container')
def prepare_public_container(prepare_wallet_and_deposit): def prepare_public_container(prepare_wallet_and_deposit):
placement_rule = 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X' placement_rule = 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X'
wallet, wif = prepare_wallet_and_deposit wallet = prepare_wallet_and_deposit
return prepare_container_impl(wallet, wif, rule=placement_rule, basic_acl=PUBLIC_ACL) return prepare_container_impl(wallet, rule=placement_rule, basic_acl=PUBLIC_ACL)
def prepare_container_impl(wallet: str, wif: str, rule=COMMON_PLACEMENT_RULE, basic_acl: str = ''): def prepare_container_impl(wallet: str, rule=COMMON_PLACEMENT_RULE, basic_acl: str = ''):
balance = get_balance(wif) balance = get_balance(wallet)
assert balance > 0, f'Expected balance is greater than 0. Got {balance}' assert balance > 0, f'Expected balance is greater than 0. Got {balance}'
cid = create_container(wallet, rule=rule, basic_acl=basic_acl) cid = create_container(wallet, rule=rule, basic_acl=basic_acl)
new_balance = get_balance(wif) new_balance = get_balance(wallet)
assert new_balance < balance, 'Expected some fee has charged' assert new_balance < balance, 'Expected some fee has charged'
return cid, wallet return cid, wallet

View file

@ -29,7 +29,7 @@ class TestS3Gate:
@pytest.fixture(scope='class', autouse=True) @pytest.fixture(scope='class', autouse=True)
@allure.title('[Class/Autouse]: Create S3 client') @allure.title('[Class/Autouse]: Create S3 client')
def s3_client(self, prepare_wallet_and_deposit, request): def s3_client(self, prepare_wallet_and_deposit, request):
wallet, wif = prepare_wallet_and_deposit wallet = prepare_wallet_and_deposit
s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json" s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json"
cid, bucket, access_key_id, secret_access_key, owner_private_key = \ cid, bucket, access_key_id, secret_access_key, owner_private_key = \

View file

@ -3,6 +3,10 @@
import re import re
import time import time
from neo3 import wallet
from robot.api import logger
from robot.api.deco import keyword
import contract import contract
import converters import converters
import rpc_client import rpc_client
@ -10,9 +14,6 @@ from common import (GAS_HASH, MAINNET_SINGLE_ADDR, MAINNET_WALLET_PATH,
MORPH_ENDPOINT, NEO_MAINNET_ENDPOINT, NEOFS_CONTRACT, MORPH_ENDPOINT, NEO_MAINNET_ENDPOINT, NEOFS_CONTRACT,
NEOGO_CLI_EXEC) NEOGO_CLI_EXEC)
from converters import load_wallet from converters import load_wallet
from neo3 import wallet
from robot.api import logger
from robot.api.deco import keyword
from wallet import nep17_transfer from wallet import nep17_transfer
from wrappers import run_sh_with_passwd_contract from wrappers import run_sh_with_passwd_contract
@ -101,9 +102,9 @@ def get_balance(wallet_path: str):
@keyword('Transfer Mainnet Gas') @keyword('Transfer Mainnet Gas')
def transfer_mainnet_gas(wallet_to: str, amount: int, def transfer_mainnet_gas(wallet_to: str, amount: int, wallet_password: str = EMPTY_PASSWORD,
wallet_password: str = EMPTY_PASSWORD): wallet_path: str = MAINNET_WALLET_PATH):
''' """
This function transfer GAS in main chain from mainnet wallet to This function transfer GAS in main chain from mainnet wallet to
the provided wallet. If the wallet contains more than one address, the provided wallet. If the wallet contains more than one address,
the assets will be transferred to the last one. the assets will be transferred to the last one.
@ -112,15 +113,14 @@ def transfer_mainnet_gas(wallet_to: str, amount: int,
amount (int): amount of gas to transfer amount (int): amount of gas to transfer
wallet_password (optional, str): password of the wallet; it is wallet_password (optional, str): password of the wallet; it is
required to decode the wallet and extract its addresses required to decode the wallet and extract its addresses
wallet_path (str): path to chain node wallet
Returns: Returns:
(void) (void)
''' """
address_to = _address_from_wallet(wallet_to, wallet_password) address_to = _address_from_wallet(wallet_to, wallet_password)
txid = nep17_transfer(MAINNET_WALLET_PATH, address_to, amount, txid = nep17_transfer(wallet_path, address_to, amount, NEO_MAINNET_ENDPOINT,
NEO_MAINNET_ENDPOINT, wallet_pass=MAINNET_WALLET_PASS, addr_from=MAINNET_SINGLE_ADDR)
wallet_pass=MAINNET_WALLET_PASS,
addr_from=MAINNET_SINGLE_ADDR)
if not transaction_accepted(txid): if not transaction_accepted(txid):
raise AssertionError(f"TX {txid} hasn't been processed") raise AssertionError(f"TX {txid} hasn't been processed")
@ -137,14 +137,13 @@ def neofs_deposit(wallet_to: str, amount: int,
address_to = _address_from_wallet(wallet_to, wallet_password) address_to = _address_from_wallet(wallet_to, wallet_password)
txid = nep17_transfer(wallet_to, deposit_addr, amount, txid = nep17_transfer(wallet_to, deposit_addr, amount, NEO_MAINNET_ENDPOINT,
NEO_MAINNET_ENDPOINT, wallet_pass=wallet_password, wallet_pass=wallet_password, addr_from=address_to)
addr_from=address_to)
if not transaction_accepted(txid): if not transaction_accepted(txid):
raise AssertionError(f"TX {txid} hasn't been processed") raise AssertionError(f"TX {txid} hasn't been processed")
def _address_from_wallet(wlt: str, wallet_password: str): def _address_from_wallet(wlt: str, wallet_password: str):
""" """
Extracting the address from the given wallet. Extracting the address from the given wallet.
Args: Args:

View file

@ -71,3 +71,7 @@ IR_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/ir/wallet01.json"
IR_WALLET_CONFIG = f"{os.getcwd()}/neofs_cli_configs/one_wallet_password.yml" IR_WALLET_CONFIG = f"{os.getcwd()}/neofs_cli_configs/one_wallet_password.yml"
IR_WALLET_PASS = 'one' IR_WALLET_PASS = 'one'
STORAGE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/storage/wallet01.json" STORAGE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/storage/wallet01.json"
CONTROL_NODE_USER = os.getenv('CONTROL_NODE_USER', 'root')
CONTROL_NODE_PWD = os.getenv('CONTROL_NODE_PWD')
REMOTE_HOST = os.getenv('REMOTE_HOST')