Replace prepare_container*** fixtures with a function.

The change is motivated by variety of standard ACLs that will be hard to manage with set of fixtures.

Remove logic that initializes wallet from remote devenv host.
This setup action should be handled outside tests.

Add ability to establish SSH connection using SSH key instead of password.

Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
Vladimir Domnich 2022-07-12 13:59:19 +04:00 committed by Anastasia Prasolova
parent ffa40112a1
commit 84230d12e3
11 changed files with 142 additions and 199 deletions

View file

@ -7,12 +7,10 @@ from dataclasses import dataclass
from datetime import datetime from datetime import datetime
from functools import wraps from functools import wraps
from time import sleep from time import sleep
from typing import ClassVar from typing import ClassVar, Optional
import os
import allure import allure
from paramiko import (AutoAddPolicy, SFTPClient, SSHClient, SSHException, from paramiko import AutoAddPolicy, SFTPClient, SSHClient, SSHException, ssh_exception, RSAKey
ssh_exception, RSAKey)
from paramiko.ssh_exception import AuthenticationException from paramiko.ssh_exception import AuthenticationException
@ -36,9 +34,9 @@ def log_command(func):
with allure.step(f'SSH: {short}'): with allure.step(f'SSH: {short}'):
logging.info(f'Execute command "{command}" on "{host.ip}"') logging.info(f'Execute command "{command}" on "{host.ip}"')
start_time = datetime.now() start_time = datetime.utcnow()
cmd_result = func(host, command, *args, **kwargs) cmd_result = func(host, command, *args, **kwargs)
end_time = datetime.now() end_time = datetime.utcnow()
log_message = f'HOST: {host.ip}\n' \ log_message = f'HOST: {host.ip}\n' \
f'COMMAND:\n{textwrap.indent(command, " ")}\n' \ f'COMMAND:\n{textwrap.indent(command, " ")}\n' \
@ -67,11 +65,12 @@ class HostClient:
TIMEOUT_RESTORE_CONNECTION = 10, 24 TIMEOUT_RESTORE_CONNECTION = 10, 24
def __init__(self, ip: str, login: str, password: str, init_ssh_client=True): def __init__(self, ip: str, login: str, password: Optional[str],
private_key_path: Optional[str] = None, init_ssh_client=True) -> None:
self.ip = ip self.ip = ip
self.login = login self.login = login
self.password = password self.password = password
self.pk = os.getenv('SSH_PK_PATH') self.private_key_path = private_key_path
if init_ssh_client: if init_ssh_client:
self.create_connection(self.SSH_CONNECTION_ATTEMPTS) self.create_connection(self.SSH_CONNECTION_ATTEMPTS)
@ -145,17 +144,30 @@ class HostClient:
def create_connection(self, attempts=SSH_CONNECTION_ATTEMPTS): def create_connection(self, attempts=SSH_CONNECTION_ATTEMPTS):
exc_err = None exc_err = None
for attempt in range(attempts): for attempt in range(attempts):
logging.info(f'Try to establish connection {attempt + 1} time to Host: {self.ip} under {self.login} '
f'user with {self.password} password..')
self.ssh_client = SSHClient() self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy()) self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
try: try:
if self.password: if self.private_key_path:
self.ssh_client.connect(hostname=str(self.ip), username=self.login, password=str(self.password), logging.info(
timeout=self.CONNECTION_TIMEOUT) f"Trying to connect to host {self.ip} using SSH key "
f"{self.private_key_path} (attempt {attempt})"
)
self.ssh_client.connect(
hostname=self.ip,
pkey=RSAKey.from_private_key_file(self.private_key_path, self.password),
timeout=self.CONNECTION_TIMEOUT
)
else: else:
self.ssh_client.connect(hostname=str(self.ip), pkey=RSAKey.from_private_key_file(self.pk), logging.info(
timeout=self.CONNECTION_TIMEOUT) f"Trying to connect to host {self.ip} as {self.login} using password "
f"{self.password[:2] + '***' if self.password else ''} (attempt {attempt})"
)
self.ssh_client.connect(
hostname=self.ip,
username=self.login,
password=self.password,
timeout=self.CONNECTION_TIMEOUT
)
return True return True
except AuthenticationException as auth_err: except AuthenticationException as auth_err:

View file

@ -1,6 +1,5 @@
import os import os
import uuid import uuid
from typing import List
from common import ASSETS_DIR, SIMPLE_OBJ_SIZE from common import ASSETS_DIR, SIMPLE_OBJ_SIZE
@ -30,7 +29,7 @@ def get_file_content(file_path: str) -> str:
return content return content
def split_file(file_path: str, parts: int) -> List[str]: def split_file(file_path: str, parts: int) -> list[str]:
files = [] files = []
with open(file_path, 'rb') as in_file: with open(file_path, 'rb') as in_file:
data = in_file.read() data = in_file.read()

View file

@ -2,23 +2,19 @@ import logging
import os import os
import shutil import shutil
from re import search from re import search
from time import sleep
import allure import allure
import pytest import pytest
from robot.api import deco from robot.api import deco
import rpc_client
import wallet import wallet
from cli_helpers import _cmd_run from cli_helpers import _cmd_run
from common import (ASSETS_DIR, COMMON_PLACEMENT_RULE, CONTROL_NODE_USER, CONTROL_NODE_PWD, from common import ASSETS_DIR, FREE_STORAGE, MAINNET_WALLET_PATH
FREE_STORAGE, MAINNET_WALLET_PATH, NEO_MAINNET_ENDPOINT, REMOTE_HOST)
from payment_neogo import neofs_deposit, transfer_mainnet_gas from payment_neogo import neofs_deposit, transfer_mainnet_gas
from python_keywords.container import create_container
from ssh_helper import HostClient
from wellknown_acl import PUBLIC_ACL
deco.keyword = allure.step def robot_keyword_adapter(name=None, tags=(), types=()):
return allure.step(name)
deco.keyword = robot_keyword_adapter
logger = logging.getLogger('NeoLogger') logger = logging.getLogger('NeoLogger')
@ -55,65 +51,12 @@ def init_wallet_with_address():
@pytest.fixture(scope='session') @pytest.fixture(scope='session')
@allure.title('Prepare wallet and deposit') @allure.title('Prepare wallet and deposit')
def prepare_wallet_and_deposit(init_wallet_with_address): def prepare_wallet_and_deposit(init_wallet_with_address):
local_wallet_path = None
wallet, addr, _ = init_wallet_with_address wallet, addr, _ = init_wallet_with_address
logger.info(f'Init wallet: {wallet},\naddr: {addr}') logger.info(f'Init wallet: {wallet},\naddr: {addr}')
if REMOTE_HOST:
ssh_client = HostClient(REMOTE_HOST, CONTROL_NODE_USER, CONTROL_NODE_PWD)
local_wallet_path = os.path.join(ASSETS_DIR, os.path.basename(MAINNET_WALLET_PATH))
ssh_client.copy_file_from_host(MAINNET_WALLET_PATH, local_wallet_path)
if not FREE_STORAGE: if not FREE_STORAGE:
deposit = 30 deposit = 30
transfer_mainnet_gas(wallet, deposit + 1, wallet_path=local_wallet_path or MAINNET_WALLET_PATH) transfer_mainnet_gas(wallet, deposit + 1, wallet_path=MAINNET_WALLET_PATH)
neofs_deposit(wallet, deposit) neofs_deposit(wallet, deposit)
return wallet return wallet
@pytest.fixture()
@allure.title('Create Container')
def prepare_container(prepare_wallet_and_deposit):
wallet = prepare_wallet_and_deposit
return prepare_container_impl(wallet)
@pytest.fixture(scope='module')
@allure.title('Create Public Container')
def prepare_public_container(prepare_wallet_and_deposit):
placement_rule = 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X'
wallet = prepare_wallet_and_deposit
return prepare_container_impl(wallet, rule=placement_rule, basic_acl=PUBLIC_ACL)
def prepare_container_impl(wallet: str, rule=COMMON_PLACEMENT_RULE, basic_acl: str = ''):
cid = create_container(wallet, rule=rule, basic_acl=basic_acl)
return cid, wallet
@allure.step('Wait until transaction accepted in block')
def wait_until_transaction_accepted_in_block(tx_id: str):
"""
This function return True in case of accepted TX.
Parameters:
:param tx_id: transaction ID
"""
mainnet_rpc_cli = rpc_client.RPCClient(NEO_MAINNET_ENDPOINT)
if isinstance(tx_id, bytes):
tx_id = tx_id.decode()
sleep_interval, attempts = 5, 10
for __attempt in range(attempts):
try:
resp = mainnet_rpc_cli.get_transaction_height(tx_id)
if resp is not None:
logger.info(f"got block height: {resp}")
return True
except Exception as e:
logger.info(f"request failed with error: {e}")
raise e
sleep(sleep_interval)
raise TimeoutError(f'Timeout {sleep_interval * attempts} sec. reached on waiting for transaction accepted')

View file

@ -3,8 +3,9 @@ from time import sleep
import allure import allure
import pytest import pytest
from container import create_container
from epoch import tick_epoch from epoch import tick_epoch
from python_keywords.neofs import verify_head_tombstone from tombstone import verify_head_tombstone
from python_keywords.neofs_verbs import (delete_object, get_object, get_range, from python_keywords.neofs_verbs import (delete_object, get_object, get_range,
get_range_hash, head_object, get_range_hash, head_object,
put_object, search_object) put_object, search_object)
@ -19,8 +20,9 @@ CLEANUP_TIMEOUT = 10
@allure.title('Test native object API') @allure.title('Test native object API')
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.grpc_api @pytest.mark.grpc_api
def test_object_api(prepare_container): def test_object_api(prepare_wallet_and_deposit):
cid, wallet = prepare_container wallet = prepare_wallet_and_deposit
cid = create_container(wallet)
wallet_cid = {'wallet': wallet, 'cid': cid} wallet_cid = {'wallet': wallet, 'cid': cid}
file_usr_header = {'key1': 1, 'key2': 'abc'} file_usr_header = {'key1': 1, 'key2': 'abc'}
file_usr_header_oth = {'key1': 2} file_usr_header_oth = {'key1': 2}

View file

@ -4,17 +4,17 @@ from random import choice
from time import sleep from time import sleep
import allure import allure
from common import COMPLEX_OBJ_SIZE
import pytest import pytest
from common import COMPLEX_OBJ_SIZE
from container import create_container
from epoch import get_epoch, tick_epoch from epoch import get_epoch, tick_epoch
from python_keywords.http_gate import (get_via_http_curl, get_via_http_gate, from python_keywords.http_gate import (get_via_http_curl, get_via_http_gate,
get_via_http_gate_by_attribute, get_via_http_gate_by_attribute, get_via_zip_http_gate,
get_via_zip_http_gate, upload_via_http_gate, upload_via_http_gate_curl)
upload_via_http_gate,
upload_via_http_gate_curl)
from python_keywords.neofs_verbs import get_object, put_object from python_keywords.neofs_verbs import get_object, put_object
from python_keywords.storage_policy import get_nodes_without_object from python_keywords.storage_policy import get_nodes_without_object
from python_keywords.utility_keywords import generate_file, get_file_hash from python_keywords.utility_keywords import generate_file, get_file_hash
from wellknown_acl import PUBLIC_ACL
logger = logging.getLogger('NeoLogger') logger = logging.getLogger('NeoLogger')
@ -26,9 +26,15 @@ CLEANUP_TIMEOUT = 10
@allure.link('https://github.com/nspcc-dev/neofs-http-gw#downloading', name='downloading') @allure.link('https://github.com/nspcc-dev/neofs-http-gw#downloading', name='downloading')
@pytest.mark.http_gate @pytest.mark.http_gate
class TestHttpGate: class TestHttpGate:
PLACEMENT_RULE = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
@pytest.fixture(scope="class", autouse=True)
@allure.title('[Class/Autouse]: Prepare wallet and deposit')
def prepare_wallet(self, prepare_wallet_and_deposit):
TestHttpGate.wallet = prepare_wallet_and_deposit
@allure.title('Test Put over gRPC, Get over HTTP') @allure.title('Test Put over gRPC, Get over HTTP')
def test_put_grpc_get_http(self, prepare_public_container): def test_put_grpc_get_http(self):
""" """
Test that object can be put using gRPC interface and get using HTTP. Test that object can be put using gRPC interface and get using HTTP.
@ -43,21 +49,21 @@ class TestHttpGate:
Expected result: Expected result:
Hashes must be the same. Hashes must be the same.
""" """
cid, wallet = prepare_public_container cid = create_container(self.wallet, rule=self.PLACEMENT_RULE, basic_acl=PUBLIC_ACL)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
with allure.step('Put objects using gRPC'): with allure.step('Put objects using gRPC'):
oid_simple = put_object(wallet=wallet, path=file_path_simple, cid=cid) oid_simple = put_object(wallet=self.wallet, path=file_path_simple, cid=cid)
oid_large = put_object(wallet=wallet, path=file_path_large, cid=cid) oid_large = put_object(wallet=self.wallet, path=file_path_large, cid=cid)
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
self.get_object_and_verify_hashes(oid, file_path, wallet, cid) self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid)
@allure.link('https://github.com/nspcc-dev/neofs-http-gw#uploading', name='uploading') @allure.link('https://github.com/nspcc-dev/neofs-http-gw#uploading', name='uploading')
@allure.link('https://github.com/nspcc-dev/neofs-http-gw#downloading', name='downloading') @allure.link('https://github.com/nspcc-dev/neofs-http-gw#downloading', name='downloading')
@pytest.mark.sanity @pytest.mark.sanity
@allure.title('Test Put over HTTP, Get over HTTP') @allure.title('Test Put over HTTP, Get over HTTP')
def test_put_http_get_http(self, prepare_public_container): def test_put_http_get_http(self):
""" """
Test that object can be put and get using HTTP interface. Test that object can be put and get using HTTP interface.
@ -70,7 +76,7 @@ class TestHttpGate:
Expected result: Expected result:
Hashes must be the same. Hashes must be the same.
""" """
cid, wallet = prepare_public_container cid = create_container(self.wallet, rule=self.PLACEMENT_RULE, basic_acl=PUBLIC_ACL)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
with allure.step('Put objects using HTTP'): with allure.step('Put objects using HTTP'):
@ -78,18 +84,20 @@ class TestHttpGate:
oid_large = upload_via_http_gate(cid=cid, path=file_path_large) oid_large = upload_via_http_gate(cid=cid, path=file_path_large)
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
self.get_object_and_verify_hashes(oid, file_path, wallet, cid) self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid)
@allure.link('https://github.com/nspcc-dev/neofs-http-gw#by-attributes', name='download by attributes') @allure.link('https://github.com/nspcc-dev/neofs-http-gw#by-attributes', name='download by attributes')
@allure.title('Test Put over HTTP, Get over HTTP with headers') @allure.title('Test Put over HTTP, Get over HTTP with headers')
@pytest.mark.parametrize('attributes', @pytest.mark.parametrize(
'attributes',
[ [
{'fileName': 'simple_obj_filename'}, {'fileName': 'simple_obj_filename'},
{'file-Name': 'simple obj filename'}, {'file-Name': 'simple obj filename'},
{'cat%jpeg': 'cat%jpeg'} {'cat%jpeg': 'cat%jpeg'}
], ids=['simple', 'hyphen', 'percent'] ],
ids=['simple', 'hyphen', 'percent']
) )
def test_put_http_get_http_with_headers(self, prepare_public_container, attributes): def test_put_http_get_http_with_headers(self, attributes: dict):
""" """
Test that object can be downloaded using different attributes in HTTP header. Test that object can be downloaded using different attributes in HTTP header.
@ -102,17 +110,18 @@ class TestHttpGate:
Expected result: Expected result:
Hashes must be the same. Hashes must be the same.
""" """
cid, wallet = prepare_public_container cid = create_container(self.wallet, rule=self.PLACEMENT_RULE, basic_acl=PUBLIC_ACL)
file_path = generate_file() file_path = generate_file()
with allure.step('Put objects using HTTP with attribute'): with allure.step('Put objects using HTTP with attribute'):
oid_simple = upload_via_http_gate(cid=cid, path=file_path, headers=self._attr_into_header(attributes)) headers = self._attr_into_header(attributes)
oid = upload_via_http_gate(cid=cid, path=file_path, headers=headers)
self.get_object_by_attr_and_verify_hashes(oid_simple, file_path, cid, attributes) self.get_object_by_attr_and_verify_hashes(oid, file_path, cid, attributes)
@allure.title('Test Expiration-Epoch in HTTP header') @allure.title('Test Expiration-Epoch in HTTP header')
def test_expiration_epoch_in_http(self, prepare_public_container): def test_expiration_epoch_in_http(self):
cid, wallet = prepare_public_container cid = create_container(self.wallet, rule=self.PLACEMENT_RULE, basic_acl=PUBLIC_ACL)
file_path = generate_file() file_path = generate_file()
object_not_found_err = 'object not found' object_not_found_err = 'object not found'
oids = [] oids = []
@ -137,15 +146,19 @@ class TestHttpGate:
sleep(CLEANUP_TIMEOUT) sleep(CLEANUP_TIMEOUT)
for oid in expired_objects: for oid in expired_objects:
self.try_to_get_object_and_expect_error(cid=cid, oid=oid, expected_err=object_not_found_err) self.try_to_get_object_and_expect_error(
cid=cid,
oid=oid,
expected_err=object_not_found_err
)
with allure.step('Other objects can be get'): with allure.step('Other objects can be get'):
for oid in not_expired_objects: for oid in not_expired_objects:
get_via_http_gate(cid=cid, oid=oid) get_via_http_gate(cid=cid, oid=oid)
@allure.title('Test Zip in HTTP header') @allure.title('Test Zip in HTTP header')
def test_zip_in_http(self, prepare_public_container): def test_zip_in_http(self):
cid, wallet = prepare_public_container cid = create_container(self.wallet, rule=self.PLACEMENT_RULE, basic_acl=PUBLIC_ACL)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
common_prefix = 'my_files' common_prefix = 'my_files'
@ -164,29 +177,29 @@ class TestHttpGate:
@pytest.mark.curl @pytest.mark.curl
@pytest.mark.long @pytest.mark.long
@allure.title('Test Put over HTTP/Curl, Get over HTTP/Curl for large object') @allure.title('Test Put over HTTP/Curl, Get over HTTP/Curl for large object')
def test_put_http_get_http_large_file(self, prepare_public_container): def test_put_http_get_http_large_file(self):
""" """
This test checks upload and download using curl with 'large' object. Large is object with size up to 20Mb. This test checks upload and download using curl with 'large' object. Large is object with size up to 20Mb.
""" """
cid, wallet = prepare_public_container cid = create_container(self.wallet, rule=self.PLACEMENT_RULE, basic_acl=PUBLIC_ACL)
obj_size = int(os.getenv('BIG_OBJ_SIZE', COMPLEX_OBJ_SIZE)) obj_size = int(os.getenv('BIG_OBJ_SIZE', COMPLEX_OBJ_SIZE))
file_path = generate_file(obj_size) file_path = generate_file(obj_size)
with allure.step('Put objects using HTTP'): with allure.step('Put objects using HTTP'):
oid_simple = upload_via_http_gate(cid=cid, path=file_path) oid_gate = upload_via_http_gate(cid=cid, path=file_path)
oid_curl = upload_via_http_gate_curl(cid=cid, filepath=file_path, large_object=True) oid_curl = upload_via_http_gate_curl(cid=cid, filepath=file_path, large_object=True)
self.get_object_and_verify_hashes(oid_simple, file_path, wallet, cid) self.get_object_and_verify_hashes(oid_gate, file_path, self.wallet, cid)
self.get_object_and_verify_hashes(oid_curl, file_path, wallet, cid, object_getter=get_via_http_curl) self.get_object_and_verify_hashes(oid_curl, file_path, self.wallet, cid, get_via_http_curl)
@pytest.mark.curl @pytest.mark.curl
@allure.title('Test Put/Get over HTTP using Curl utility') @allure.title('Test Put/Get over HTTP using Curl utility')
def test_put_http_get_http_curl(self, prepare_public_container): def test_put_http_get_http_curl(self):
""" """
Test checks upload and download over HTTP using curl utility. Test checks upload and download over HTTP using curl utility.
""" """
cid, wallet = prepare_public_container cid = create_container(self.wallet, rule=self.PLACEMENT_RULE, basic_acl=PUBLIC_ACL)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
with allure.step('Put objects using curl utility'): with allure.step('Put objects using curl utility'):
@ -194,7 +207,7 @@ class TestHttpGate:
oid_large = upload_via_http_gate_curl(cid=cid, filepath=file_path_large) oid_large = upload_via_http_gate_curl(cid=cid, filepath=file_path_large)
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
self.get_object_and_verify_hashes(oid, file_path, wallet, cid, object_getter=get_via_http_curl) self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid, get_via_http_curl)
@staticmethod @staticmethod
@allure.step('Try to get object and expect error') @allure.step('Try to get object and expect error')

View file

@ -1,16 +1,15 @@
#!/usr/bin/python3.8 #!/usr/bin/python3.9
""" """
Helper functions to use with `neofs-cli`, `neo-go` Helper functions to use with `neofs-cli`, `neo-go` and other CLIs.
and other CLIs.
""" """
from typing import Union import json
import subprocess import subprocess
import sys import sys
from contextlib import suppress from contextlib import suppress
from datetime import datetime from datetime import datetime
from json import dumps
from textwrap import shorten from textwrap import shorten
from typing import Union
import allure import allure
import pexpect import pexpect
@ -19,21 +18,21 @@ from robot.api import logger
ROBOT_AUTO_KEYWORDS = False ROBOT_AUTO_KEYWORDS = False
def _cmd_run(cmd, timeout=30): def _cmd_run(cmd: str, timeout: int = 30) -> str:
""" """
Runs given shell command <cmd>, in case of success returns its stdout, Runs given shell command <cmd>, in case of success returns its stdout,
in case of failure returns error message. in case of failure returns error message.
""" """
try: try:
logger.info(f"Executing command: {cmd}") logger.info(f"Executing command: {cmd}")
start_time = datetime.now() start_time = datetime.utcnow()
compl_proc = subprocess.run(cmd, check=True, universal_newlines=True, compl_proc = subprocess.run(cmd, check=True, universal_newlines=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
timeout=timeout, timeout=timeout,
shell=True) shell=True)
output = compl_proc.stdout output = compl_proc.stdout
return_code = compl_proc.returncode return_code = compl_proc.returncode
end_time = datetime.now() end_time = datetime.utcnow()
logger.info(f"Output: {output}") logger.info(f"Output: {output}")
_attach_allure_log(cmd, output, return_code, start_time, end_time) _attach_allure_log(cmd, output, return_code, start_time, end_time)
@ -50,7 +49,7 @@ def _cmd_run(cmd, timeout=30):
raise raise
def _run_with_passwd(cmd): def _run_with_passwd(cmd: str) -> str:
child = pexpect.spawn(cmd) child = pexpect.spawn(cmd)
child.delaybeforesend = 1 child.delaybeforesend = 1
child.expect(".*") child.expect(".*")
@ -64,7 +63,7 @@ def _run_with_passwd(cmd):
return cmd.decode() return cmd.decode()
def _configure_aws_cli(cmd, key_id, access_key, out_format='json'): def _configure_aws_cli(cmd: str, key_id: str, access_key: str, out_format: str = "json") -> str:
child = pexpect.spawn(cmd) child = pexpect.spawn(cmd)
child.delaybeforesend = 1 child.delaybeforesend = 1
@ -87,7 +86,8 @@ def _configure_aws_cli(cmd, key_id, access_key, out_format='json'):
return cmd.decode() return cmd.decode()
def _attach_allure_log(cmd: str, output: str, return_code: int, start_time: datetime, end_time: datetime): def _attach_allure_log(cmd: str, output: str, return_code: int, start_time: datetime,
end_time: datetime) -> None:
if 'allure' in sys.modules: if 'allure' in sys.modules:
command_attachment = ( command_attachment = (
f"COMMAND: '{cmd}'\n" f"COMMAND: '{cmd}'\n"
@ -99,11 +99,11 @@ def _attach_allure_log(cmd: str, output: str, return_code: int, start_time: date
allure.attach(command_attachment, 'Command execution', allure.attachment_type.TEXT) allure.attach(command_attachment, 'Command execution', allure.attachment_type.TEXT)
def log_command_execution(cmd: str, output: Union[str, dict]): def log_command_execution(cmd: str, output: Union[str, dict]) -> None:
logger.info(f'{cmd}: {output}') logger.info(f'{cmd}: {output}')
if 'allure' in sys.modules: if 'allure' in sys.modules:
with suppress(Exception): with suppress(Exception):
json_output = dumps(output, indent=4, sort_keys=True) json_output = json.dumps(output, indent=4, sort_keys=True)
output = json_output output = json_output
command_attachment = ( command_attachment = (
f"COMMAND: '{cmd}'\n" f"COMMAND: '{cmd}'\n"

View file

@ -1,29 +1,28 @@
#!/usr/bin/python3 #!/usr/bin/python3.9
""" """
This module contains keywords which utilize `neofs-cli container` This module contains keywords that utilize `neofs-cli container` commands.
commands.
""" """
import json import json
import time import time
from typing import Optional
from common import NEOFS_ENDPOINT, COMMON_PLACEMENT_RULE, NEOFS_CLI_EXEC, WALLET_CONFIG
from cli_helpers import _cmd_run
from data_formatters import dict_to_attrs
import json_transformers import json_transformers
from data_formatters import dict_to_attrs
from cli_helpers import _cmd_run
from common import NEOFS_ENDPOINT, NEOFS_CLI_EXEC, WALLET_CONFIG
from robot.api import logger from robot.api import logger
from robot.api.deco import keyword from robot.api.deco import keyword
ROBOT_AUTO_KEYWORDS = False ROBOT_AUTO_KEYWORDS = False
DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
@keyword('Create Container') @keyword('Create Container')
def create_container(wallet: str, rule: str = COMMON_PLACEMENT_RULE, basic_acl: str = '', def create_container(wallet: str, rule: str = DEFAULT_PLACEMENT_RULE, basic_acl: str = '',
attributes: dict = {}, session_token: str = '', session_wallet: str = '', attributes: Optional[dict] = None, session_token: str = '',
options: str = ''): session_wallet: str = '', options: str = '') -> str:
""" """
A wrapper for `neofs-cli container create` call. A wrapper for `neofs-cli container create` call.
@ -74,7 +73,7 @@ def create_container(wallet: str, rule: str = COMMON_PLACEMENT_RULE, basic_acl:
@keyword('List Containers') @keyword('List Containers')
def list_containers(wallet: str): def list_containers(wallet: str) -> list[str]:
""" """
A wrapper for `neofs-cli container list` call. It returns all the A wrapper for `neofs-cli container list` call. It returns all the
available containers for the given wallet. available containers for the given wallet.
@ -92,12 +91,12 @@ def list_containers(wallet: str):
@keyword('Get Container') @keyword('Get Container')
def get_container(wallet: str, cid: str): def get_container(wallet: str, cid: str) -> dict:
""" """
A wrapper for `neofs-cli container get` call. It extracts A wrapper for `neofs-cli container get` call. It extracts container's
container attributes and rearranges them to more compact view. attributes and rearranges them into a more compact view.
Args: Args:
wallet (str): a wallet on whose behalf we get the container wallet (str): path to a wallet on whose behalf we get the container
cid (str): ID of the container to get cid (str): ID of the container to get
Returns: Returns:
(dict): dict of container attributes (dict): dict of container attributes
@ -112,19 +111,18 @@ def get_container(wallet: str, cid: str):
for attr in container_info['attributes']: for attr in container_info['attributes']:
attributes[attr['key']] = attr['value'] attributes[attr['key']] = attr['value']
container_info['attributes'] = attributes container_info['attributes'] = attributes
container_info['ownerID'] = json_transformers.json_reencode( container_info['ownerID'] = json_transformers.json_reencode(container_info['ownerID']['value'])
container_info['ownerID']['value'])
return container_info return container_info
@keyword('Delete Container') @keyword('Delete Container')
# TODO: make the error message about a non-found container more user-friendly # TODO: make the error message about a non-found container more user-friendly
# https://github.com/nspcc-dev/neofs-contract/issues/121 # https://github.com/nspcc-dev/neofs-contract/issues/121
def delete_container(wallet: str, cid: str): def delete_container(wallet: str, cid: str) -> None:
""" """
A wrapper for `neofs-cli container delete` call. A wrapper for `neofs-cli container delete` call.
Args: Args:
wallet (str): a wallet on whose behalf we delete the container wallet (str): path to a wallet on whose behalf we delete the container
cid (str): ID of the container to delete cid (str): ID of the container to delete
This function doesn't return anything. This function doesn't return anything.
""" """
@ -136,27 +134,26 @@ def delete_container(wallet: str, cid: str):
_cmd_run(cmd) _cmd_run(cmd)
def _parse_cid(ouptut: str): def _parse_cid(output: str) -> str:
""" """
This function parses CID from given CLI output. The input string we Parses container ID from a given CLI output. The input string we expect:
expect:
container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN
awaiting... awaiting...
container has been persisted on sidechain container has been persisted on sidechain
We want to take 'container ID' value from the string. We want to take 'container ID' value from the string.
Args: Args:
ouptut (str): a command run output output (str): CLI output to parse
Returns: Returns:
(str): extracted CID (str): extracted CID
""" """
try: try:
# taking first string from command output # taking first line from command's output
fst_str = ouptut.split('\n')[0] first_line = output.split('\n')[0]
except Exception: except Exception:
logger.error(f"Got empty output: {ouptut}") logger.error(f"Got empty output: {output}")
splitted = fst_str.split(": ") splitted = first_line.split(": ")
if len(splitted) != 2: if len(splitted) != 2:
raise ValueError(f"no CID was parsed from command output: \t{fst_str}") raise ValueError(f"no CID was parsed from command output: \t{first_line}")
return splitted[1] return splitted[1]

View file

@ -1,16 +1,8 @@
"""
A bunch of functions which might rearrange some data or
change their representation.
"""
from functools import reduce def dict_to_attrs(attrs: dict) -> str:
def dict_to_attrs(attrs: dict):
""" """
This function takes dictionary of object attributes and converts them This function takes a dictionary of object's attributes and converts them
into the string. The string is passed to `--attributes` key of the into string. The string is passed to `--attributes` key of neofs-cli.
neofs-cli.
Args: Args:
attrs (dict): object attributes in {"a": "b", "c": "d"} format. attrs (dict): object attributes in {"a": "b", "c": "d"} format.
@ -18,4 +10,4 @@ def dict_to_attrs(attrs: dict):
Returns: Returns:
(str): string in "a=b,c=d" format. (str): string in "a=b,c=d" format.
""" """
return reduce(lambda a, b: f"{a},{b}", map(lambda i: f"{i}={attrs[i]}", attrs)) return ",".join(f"{key}={value}" for key, value in attrs.items())

View file

@ -1,32 +1,20 @@
#!/usr/bin/python3.9 #!/usr/bin/python3.9
import contract import contract
import sys
from robot.api import logger from robot.api import logger
from robot.api.deco import keyword from robot.api.deco import keyword
from robot.libraries.BuiltIn import BuiltIn
from common import IR_WALLET_PATH, IR_WALLET_PASS, MORPH_ENDPOINT
ROBOT_AUTO_KEYWORDS = False ROBOT_AUTO_KEYWORDS = False
if "pytest" in sys.modules:
import os
IR_WALLET_PATH = os.getenv("IR_WALLET_PATH")
IR_WALLET_PASS = os.getenv("IR_WALLET_PASS")
SIDECHAIN_EP = os.getenv("MORPH_ENDPOINT")
else:
IR_WALLET_PATH = BuiltIn().get_variable_value("${IR_WALLET_PATH}")
IR_WALLET_PASS = BuiltIn().get_variable_value("${IR_WALLET_PASS}")
SIDECHAIN_EP = BuiltIn().get_variable_value("${MORPH_ENDPOINT}")
@keyword('Get Epoch') @keyword('Get Epoch')
def get_epoch(): def get_epoch():
epoch = int(contract.testinvoke_contract( epoch = int(contract.testinvoke_contract(
contract.get_netmap_contract_hash(SIDECHAIN_EP), contract.get_netmap_contract_hash(MORPH_ENDPOINT),
'epoch', "epoch",
SIDECHAIN_EP) MORPH_ENDPOINT)
) )
logger.info(f"Got epoch {epoch}") logger.info(f"Got epoch {epoch}")
return epoch return epoch
@ -36,6 +24,6 @@ def get_epoch():
def tick_epoch(): def tick_epoch():
cur_epoch = get_epoch() cur_epoch = get_epoch()
return contract.invoke_contract_multisig( return contract.invoke_contract_multisig(
contract.get_netmap_contract_hash(SIDECHAIN_EP), contract.get_netmap_contract_hash(MORPH_ENDPOINT),
f"newEpoch int:{cur_epoch+1}", f"newEpoch int:{cur_epoch+1}",
IR_WALLET_PATH, IR_WALLET_PASS, SIDECHAIN_EP) IR_WALLET_PATH, IR_WALLET_PASS, MORPH_ENDPOINT)

View file

@ -1,9 +1,9 @@
#!/usr/bin/python3 #!/usr/bin/python3.9
import os import os
import uuid import uuid
from enum import Enum from enum import Enum
from typing import Optional, List from typing import Optional
import urllib3 import urllib3
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
@ -196,7 +196,7 @@ def create_multipart_upload_s3(s3_client, bucket_name: str, object_key: str) ->
@keyword('List multipart uploads S3') @keyword('List multipart uploads S3')
def list_multipart_uploads_s3(s3_client, bucket_name: str) -> Optional[List[dict]]: def list_multipart_uploads_s3(s3_client, bucket_name: str) -> Optional[list[dict]]:
try: try:
response = s3_client.list_multipart_uploads(Bucket=bucket_name) response = s3_client.list_multipart_uploads(Bucket=bucket_name)
log_command_execution('S3 List multipart upload', response) log_command_execution('S3 List multipart upload', response)
@ -240,7 +240,7 @@ def upload_part_s3(s3_client, bucket_name: str, object_key: str, upload_id: str,
@keyword('List parts S3') @keyword('List parts S3')
def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> List[dict]: def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> list[dict]:
try: try:
response = s3_client.list_parts(UploadId=upload_id, Bucket=bucket_name, Key=object_key) response = s3_client.list_parts(UploadId=upload_id, Bucket=bucket_name, Key=object_key)
log_command_execution('S3 List part', response) log_command_execution('S3 List part', response)

View file

@ -28,8 +28,6 @@ GAS_HASH = '0xd2a4cff31913016155e38e474a2c06d08be276cf'
NEOFS_CONTRACT = os.getenv("NEOFS_IR_CONTRACTS_NEOFS") NEOFS_CONTRACT = os.getenv("NEOFS_IR_CONTRACTS_NEOFS")
COMMON_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/") ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/")
MORPH_MAGIC = os.getenv("MORPH_MAGIC") MORPH_MAGIC = os.getenv("MORPH_MAGIC")
@ -74,6 +72,5 @@ STORAGE_WALLET_PATH = f"{DEVENV_SERVICES_PATH}/storage/wallet01.json"
CONTROL_NODE_USER = os.getenv('CONTROL_NODE_USER', 'root') CONTROL_NODE_USER = os.getenv('CONTROL_NODE_USER', 'root')
CONTROL_NODE_PWD = os.getenv('CONTROL_NODE_PWD') CONTROL_NODE_PWD = os.getenv('CONTROL_NODE_PWD')
REMOTE_HOST = os.getenv('REMOTE_HOST')
FREE_STORAGE = os.getenv('FREE_STORAGE', "false").lower() == "true" FREE_STORAGE = os.getenv('FREE_STORAGE', "false").lower() == "true"