mmalygina/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Pull requests Activity

Remove dockerfile and removed docker run command from Makefile to keep testcases environment agnostic.

Browse source 
Add logic to retrieve mainnet wallet file from control node as WIF for this wallet is not available in config anymore.

Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
This commit is contained in:
a.y.volkov 2022-07-01 20:30:07 +03:00 committed by Anastasia Prasolova
parent 6b1e1ab28d
commit 8acf738147
7 changed files with 252 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
Makefile
View file

@ -7,13 +7,6 @@ SHELL = bash
OUTPUT_DIR = artifacts/
KEYWORDS_REPO = git@github.com:nspcc-dev/neofs-keywords.git
VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a)
ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
DEV_IMAGE_PY ?= registry.spb.yadro.com/tools/pytest-neofs-x86_64:4
ifeq ($(shell uname -s),Darwin)
DOCKER_NETWORK = --network bridge -p 389:389 -p 636:636
endif
.PHONY: all
all: venvs
@ -39,32 +32,7 @@ clean:
pytest-local:
@echo "⇒ Run Pytest"
export PYTHONPATH=$(ROOT_DIR)/neofs-keywords/lib:$(ROOT_DIR)/neofs-keywords/robot:$(ROOT_DIR)/robot/resources/lib:$(ROOT_DIR)/robot/resources/lib/python_keywords:$(ROOT_DIR)/robot/variables && \
python -m pytest pytest_tests/testsuites/
help:
@echo "⇒ run Run testcases ${R}"
.PHONY: pytest-docker
pytest-docker:
-docker ps
-docker rm neofs_tests_py
-docker pull $(DEV_IMAGE_PY)
docker run -t --rm \
-w /tests \
--name neofs_tests_py \
-e PYTHONPATH="/tests/neofs-keywords/lib:/tests/neofs-keywords/robot:/tests/robot/resources/lib:/tests/robot/resources/lib/python_keywords:/tests/robot/variables:/tests/pytest_tests/helpers" \
-v $(CURDIR):/tests \
-v /var/run/docker.sock:/var/run/docker.sock \
-v $(NEO_BIN_DIR):/neofs \
--privileged \
$(DOCKER_NETWORK) \
--env-file $(CURDIR)/.env \
$(DEV_IMAGE_PY) \
-v \
-m "$(CI_MARKERS)" \
--color=no \
--junitxml=/tests/xunit_results.xml \
--alluredir=/tests/allure_results \
--setup-show \
/tests/pytest_tests/testsuites

40
pytest_tests/Dockerfile
View file

@ -1,40 +0,0 @@
ARG ARCH
FROM python:3.9-slim
ARG ARCH
RUN apt-get -y update && apt-get -y install \
gcc \
make \
git \
curl \
wget \
openssh-client \
iputils-ping \
unzip \
vim \
dbus \
lsof \
tini \
libssl-dev \
expect \
runc \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get clean
COPY requirements.txt ./
# install aws cli utility v2 from pip
RUN python3 -m pip install awscliv2
RUN ln /usr/local/bin/awscliv2 /usr/bin/aws
RUN awsv2 --install
RUN aws --version
RUN pip install --no-cache-dir -r requirements.txt
ENV PATH=$PATH:/root:/neofs
WORKDIR /root
ENTRYPOINT ["tini", "--", "pytest"]

208
pytest_tests/helpers/ssh_helper.py Normal file
View file

@ -0,0 +1,208 @@
import logging
import socket
import tempfile
import textwrap
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import datetime
from functools import wraps
from time import sleep
from typing import ClassVar
import os
import allure
from paramiko import (AutoAddPolicy, SFTPClient, SSHClient, SSHException,
ssh_exception, RSAKey)
from paramiko.ssh_exception import AuthenticationException
class HostIsNotAvailable(Exception):
"""Raises when host is not reachable."""
def __init__(self, ip: str = None, exc: Exception = None):
msg = f'Host is not available{f" by ip: {ip}" if ip else ""}'
if exc:
msg = f'{msg}. {exc}'
super().__init__(msg)
def log_command(func):
@wraps(func)
def wrapper(host: 'HostClient', command: str, *args, **kwargs):
display_length = 60
short = command.removeprefix("$ProgressPreference='SilentlyContinue'\n")
short = short[:display_length]
short += '...' if short != command else ''
with allure.step(f'SSH: {short}'):
logging.info(f'Execute command "{command}" on "{host.ip}"')
start_time = datetime.now()
cmd_result = func(host, command, *args, **kwargs)
end_time = datetime.now()
log_message = f'HOST: {host.ip}\n' \
f'COMMAND:\n{textwrap.indent(command, " ")}\n' \
f'RC:\n {cmd_result.rc}\n' \
f'STDOUT:\n{textwrap.indent(cmd_result.stdout, " ")}\n' \
f'STDERR:\n{textwrap.indent(cmd_result.stderr, " ")}\n' \
f'Start / End / Elapsed\t {start_time.time()} / {end_time.time()} / {end_time - start_time}'
logging.info(log_message)
allure.attach(log_message, 'SSH command', allure.attachment_type.TEXT)
return cmd_result
return wrapper
@dataclass
class SSHCommand:
stdout: str
stderr: str
rc: int
class HostClient:
ssh_client: SSHClient
SSH_CONNECTION_ATTEMPTS: ClassVar[int] = 3
CONNECTION_TIMEOUT = 30
TIMEOUT_RESTORE_CONNECTION = 10, 24
def __init__(self, ip: str, login: str, password: str, init_ssh_client=True):
self.ip = ip
self.login = login
self.password = password
self.pk = os.getenv('SSH_PK_PATH', '/root/.ssh/id_rsa')
if init_ssh_client:
self.create_connection(self.SSH_CONNECTION_ATTEMPTS)
def exec(self, cmd: str, verify=True, timeout=30) -> SSHCommand:
cmd_result = self._inner_exec(cmd, timeout)
if verify:
assert cmd_result.rc == 0, f'Non zero rc from command: "{cmd}"'
return cmd_result
@log_command
def exec_with_confirmation(self, cmd: str, confirmation: list, verify=True, timeout=10) -> SSHCommand:
ssh_stdin, ssh_stdout, ssh_stderr = self.ssh_client.exec_command(cmd, timeout=timeout)
for line in confirmation:
if not line.endswith('\n'):
line = f'{line}\n'
try:
ssh_stdin.write(line)
except OSError as err:
logging.error(f'Got error {err} executing command {cmd}')
ssh_stdin.close()
output = SSHCommand(stdout=ssh_stdout.read().decode(errors='ignore'),
stderr=ssh_stderr.read().decode(errors='ignore'),
rc=ssh_stdout.channel.recv_exit_status())
if verify:
debug_info = f'\nSTDOUT: {output.stdout}\nSTDERR: {output.stderr}\nRC: {output.rc}'
assert output.rc == 0, f'Non zero rc from command: "{cmd}"{debug_info}'
return output
@contextmanager
def as_user(self, user: str, password: str):
keep_user, keep_password = self.login, self.password
self.login, self.password = user, password
self.create_connection()
yield
self.login, self.password = keep_user, keep_password
self.create_connection()
@allure.step('Restore connection')
def restore_ssh_connection(self):
retry_time, retry_count = self.TIMEOUT_RESTORE_CONNECTION
for _ in range(retry_count):
try:
self.create_connection()
except AssertionError:
logging.warning(f'Host: Cant reach host: {self.ip}.')
sleep(retry_time)
else:
logging.info(f'Host: Cant reach host: {self.ip}.')
return
raise AssertionError(f'Host: Cant reach host: {self.ip} after 240 seconds..')
@allure.step('Copy file {host_path_to_file} to local file {path_to_file}')
def copy_file_from_host(self, host_path_to_file: str, path_to_file: str):
with self._sftp_client() as sftp_client:
sftp_client.get(host_path_to_file, path_to_file)
def copy_file_to_host(self, path_to_file: str, host_path_to_file: str):
with allure.step(f'Copy local file {path_to_file} to remote file {host_path_to_file} on host {self.ip}'):
with self._sftp_client() as sftp_client:
sftp_client.put(path_to_file, host_path_to_file)
@allure.step('Save string to remote file {host_path_to_file}')
def copy_str_to_host_file(self, string: str, host_path_to_file: str):
with tempfile.NamedTemporaryFile(mode='r+') as temp:
temp.writelines(string)
temp.flush()
with self._sftp_client() as client:
client.put(temp.name, host_path_to_file)
self.exec(f'cat {host_path_to_file}', verify=False)
def create_connection(self, attempts=SSH_CONNECTION_ATTEMPTS):
exc_err = None
for attempt in range(attempts):
logging.info(f'Try to establish connection {attempt + 1} time to Host: {self.ip} under {self.login} '
f'user with {self.password} password..')
self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
try:
if self.password:
self.ssh_client.connect(hostname=str(self.ip), username=self.login, password=str(self.password),
timeout=self.CONNECTION_TIMEOUT)
else:
self.ssh_client.connect(hostname=str(self.ip), pkey=RSAKey.from_private_key_file(self.pk),
timeout=self.CONNECTION_TIMEOUT)
return True
except AuthenticationException as auth_err:
logging.error(f'Host: {self.ip}. {auth_err}')
raise auth_err
except (
SSHException,
ssh_exception.NoValidConnectionsError,
AttributeError,
socket.timeout,
OSError
) as ssh_err:
exc_err = ssh_err
logging.error(f'Host: {self.ip}, connection error. {exc_err}')
raise HostIsNotAvailable(self.ip, exc_err)
def drop(self):
self.ssh_client.close()
@log_command
def _inner_exec(self, cmd: str, timeout: int) -> SSHCommand:
if not self.ssh_client:
self.create_connection()
for _ in range(self.SSH_CONNECTION_ATTEMPTS):
try:
_, stdout, stderr = self.ssh_client.exec_command(cmd, timeout=timeout)
return SSHCommand(
stdout=stdout.read().decode(errors='ignore'),
stderr=stderr.read().decode(errors='ignore'),
rc=stdout.channel.recv_exit_status()
)
except (
SSHException,
TimeoutError,
ssh_exception.NoValidConnectionsError,
ConnectionResetError,
AttributeError,
socket.timeout,
) as ssh_err:
logging.error(f'Host: {self.ip}, exec command error {ssh_err}')
self.create_connection()
raise HostIsNotAvailable(f'Host: {self.ip} is not reachable.')
@contextmanager
def _sftp_client(self) -> SFTPClient:
with self.ssh_client.open_sftp() as sftp:
yield sftp

44
pytest_tests/testsuites/conftest.py
View file

@ -6,21 +6,23 @@ from time import sleep
import allure
import pytest
from robot.api import deco
import rpc_client
import wallet
from cli_helpers import _cmd_run
from common import (ASSETS_DIR, COMMON_PLACEMENT_RULE, COMPLEX_OBJ_SIZE,
MAINNET_WALLET_WIF, NEO_MAINNET_ENDPOINT, SIMPLE_OBJ_SIZE)
from common import (ASSETS_DIR, COMPLEX_OBJ_SIZE, COMMON_PLACEMENT_RULE,
MAINNET_WALLET_PATH, NEO_MAINNET_ENDPOINT, SIMPLE_OBJ_SIZE, REMOTE_HOST, CONTROL_NODE_USER,
CONTROL_NODE_PWD)
from payment_neogo import neofs_deposit, transfer_mainnet_gas
from python_keywords.container import create_container
from python_keywords.payment_neogo import get_balance
from python_keywords.utility_keywords import generate_file_and_file_hash
from robot.api import deco
from wallet_keywords import neofs_deposit, transfer_mainnet_gas
from ssh_helper import HostClient
from wellknown_acl import PUBLIC_ACL
deco.keyword = allure.step
logger = logging.getLogger('NeoLogger')
@ -57,39 +59,43 @@ def init_wallet_with_address():
@allure.title('Prepare wallet and deposit')
def prepare_wallet_and_deposit(init_wallet_with_address):
deposit = 30
wallet, addr, wif = init_wallet_with_address
logger.info(f'Init wallet: {wallet},\naddr: {addr},\nwif: {wif}')
local_wallet_path = None
wallet, addr, _ = init_wallet_with_address
logger.info(f'Init wallet: {wallet},\naddr: {addr}')
txid = transfer_mainnet_gas(MAINNET_WALLET_WIF, addr, deposit + 1)
wait_unitl_transaction_accepted_in_block(txid)
deposit_tx = neofs_deposit(wif, deposit)
wait_unitl_transaction_accepted_in_block(deposit_tx)
if REMOTE_HOST:
ssh_client = HostClient(REMOTE_HOST, CONTROL_NODE_USER, CONTROL_NODE_PWD)
local_wallet_path = os.path.join(ASSETS_DIR, os.path.basename(MAINNET_WALLET_PATH))
ssh_client.copy_file_from_host(MAINNET_WALLET_PATH, local_wallet_path)
return wallet, wif
transfer_mainnet_gas(wallet, deposit + 1, wallet_path=local_wallet_path or MAINNET_WALLET_PATH)
neofs_deposit(wallet, deposit)
return wallet
@pytest.fixture()
@allure.title('Create Container')
def prepare_container(prepare_wallet_and_deposit):
wallet, wif = prepare_wallet_and_deposit
return prepare_container_impl(wallet, wif)
wallet = prepare_wallet_and_deposit
return prepare_container_impl(wallet)
@pytest.fixture(scope='module')
@allure.title('Create Public Container')
def prepare_public_container(prepare_wallet_and_deposit):
placement_rule = 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X'
wallet, wif = prepare_wallet_and_deposit
return prepare_container_impl(wallet, wif, rule=placement_rule, basic_acl=PUBLIC_ACL)
wallet = prepare_wallet_and_deposit
return prepare_container_impl(wallet, rule=placement_rule, basic_acl=PUBLIC_ACL)
def prepare_container_impl(wallet: str, wif: str, rule=COMMON_PLACEMENT_RULE, basic_acl: str = ''):
balance = get_balance(wif)
def prepare_container_impl(wallet: str, rule=COMMON_PLACEMENT_RULE, basic_acl: str = ''):
balance = get_balance(wallet)
assert balance > 0, f'Expected balance is greater than 0. Got {balance}'
cid = create_container(wallet, rule=rule, basic_acl=basic_acl)
new_balance = get_balance(wif)
new_balance = get_balance(wallet)
assert new_balance < balance, 'Expected some fee has charged'
return cid, wallet

2
pytest_tests/testsuites/services/test_s3_gate.py
View file

@ -29,7 +29,7 @@ class TestS3Gate:
@pytest.fixture(scope='class', autouse=True)
@allure.title('[Class/Autouse]: Create S3 client')
def s3_client(self, prepare_wallet_and_deposit, request):
wallet, wif = prepare_wallet_and_deposit
wallet = prepare_wallet_and_deposit
s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json"
cid, bucket, access_key_id, secret_access_key, owner_private_key = \

29
robot/resources/lib/python_keywords/payment_neogo.py
View file

@ -3,6 +3,10 @@
import re
import time
from neo3 import wallet
from robot.api import logger
from robot.api.deco import keyword
import contract
import converters
import rpc_client
@ -10,9 +14,6 @@ from common import (GAS_HASH, MAINNET_SINGLE_ADDR, MAINNET_WALLET_PATH,
MORPH_ENDPOINT, NEO_MAINNET_ENDPOINT, NEOFS_CONTRACT,
NEOGO_CLI_EXEC)
from converters import load_wallet
from neo3 import wallet
from robot.api import logger
from robot.api.deco import keyword
from wallet import nep17_transfer
from wrappers import run_sh_with_passwd_contract
@ -101,9 +102,9 @@ def get_balance(wallet_path: str):
@keyword('Transfer Mainnet Gas')
def transfer_mainnet_gas(wallet_to: str, amount: int,
wallet_password: str = EMPTY_PASSWORD):
'''
def transfer_mainnet_gas(wallet_to: str, amount: int, wallet_password: str = EMPTY_PASSWORD,
wallet_path: str = MAINNET_WALLET_PATH):
"""
This function transfer GAS in main chain from mainnet wallet to
the provided wallet. If the wallet contains more than one address,
the assets will be transferred to the last one.
@ -112,15 +113,14 @@ def transfer_mainnet_gas(wallet_to: str, amount: int,
amount (int): amount of gas to transfer
wallet_password (optional, str): password of the wallet; it is
required to decode the wallet and extract its addresses
wallet_path (str): path to chain node wallet
Returns:
(void)
'''
"""
address_to = _address_from_wallet(wallet_to, wallet_password)
txid = nep17_transfer(MAINNET_WALLET_PATH, address_to, amount,
NEO_MAINNET_ENDPOINT,
wallet_pass=MAINNET_WALLET_PASS,
addr_from=MAINNET_SINGLE_ADDR)
txid = nep17_transfer(wallet_path, address_to, amount, NEO_MAINNET_ENDPOINT,
wallet_pass=MAINNET_WALLET_PASS, addr_from=MAINNET_SINGLE_ADDR)
if not transaction_accepted(txid):
raise AssertionError(f"TX {txid} hasn't been processed")
@ -137,14 +137,13 @@ def neofs_deposit(wallet_to: str, amount: int,
address_to = _address_from_wallet(wallet_to, wallet_password)
txid = nep17_transfer(wallet_to, deposit_addr, amount,
NEO_MAINNET_ENDPOINT, wallet_pass=wallet_password,
addr_from=address_to)
txid = nep17_transfer(wallet_to, deposit_addr, amount, NEO_MAINNET_ENDPOINT,
wallet_pass=wallet_password, addr_from=address_to)
if not transaction_accepted(txid):
raise AssertionError(f"TX {txid} hasn't been processed")
def _address_from_wallet(wlt: str, wallet_password: str):
def _address_from_wallet(wlt: str, wallet_password: str):
"""
Extracting the address from the given wallet.
Args:

4
robot/variables/common.py
View file

@ -71,3 +71,7 @@ IR_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/ir/wallet01.json"
IR_WALLET_CONFIG = f"{os.getcwd()}/neofs_cli_configs/one_wallet_password.yml"
IR_WALLET_PASS = 'one'
STORAGE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/storage/wallet01.json"
CONTROL_NODE_USER = os.getenv('CONTROL_NODE_USER', 'root')
CONTROL_NODE_PWD = os.getenv('CONTROL_NODE_PWD')
REMOTE_HOST = os.getenv('REMOTE_HOST')