diff --git a/pytest_tests/testsuites/container/test_container.py b/pytest_tests/testsuites/container/test_container.py index 8f96bc8..484b363 100644 --- a/pytest_tests/testsuites/container/test_container.py +++ b/pytest_tests/testsuites/container/test_container.py @@ -1,13 +1,11 @@ import json -from time import sleep import allure import pytest - from epoch import tick_epoch from grpc_responses import CONTAINER_NOT_FOUND, error_matches_status -from python_keywords.container import (create_container, delete_container, get_container, - list_containers) +from python_keywords.container import (create_container, delete_container, get_container, list_containers, + wait_for_container_creation, wait_for_container_deletion) from utility import placement_policy_from_container from wellknown_acl import PRIVATE_ACL_F @@ -24,13 +22,12 @@ def test_container_creation(prepare_wallet_and_deposit, name): json_wallet = json.load(file) placement_rule = 'REP 2 IN X CBF 1 SELECT 2 FROM * AS X' - options = f"--name {name}" if name else "" - cid = create_container(wallet, rule=placement_rule, options=options) + cid = create_container(wallet, rule=placement_rule, name=name) containers = list_containers(wallet) assert cid in containers, f'Expected container {cid} in containers: {containers}' - container_info: str = get_container(wallet, cid, flag='') + container_info: str = get_container(wallet, cid, json_mode=False) container_info = container_info.casefold() # To ignore case when comparing with expected values info_to_check = { @@ -58,17 +55,25 @@ def test_container_creation(prepare_wallet_and_deposit, name): wait_for_container_deletion(wallet, cid) -@allure.step('Wait for container deletion') -def wait_for_container_deletion(wallet: str, cid: str) -> None: - attempts, sleep_interval = 10, 5 - for _ in range(attempts): - try: - get_container(wallet, cid) - sleep(sleep_interval) - continue - except Exception as err: - if error_matches_status(err, CONTAINER_NOT_FOUND): - return - raise AssertionError(f'Expected "{CONTAINER_NOT_FOUND}" error, got\n{err}') +@allure.title('Parallel container creation and deletion') +@pytest.mark.sanity +@pytest.mark.container +def test_container_creation_deletion_parallel(prepare_wallet_and_deposit): + containers_count = 3 + wallet = prepare_wallet_and_deposit + placement_rule = 'REP 2 IN X CBF 1 SELECT 2 FROM * AS X' - raise AssertionError(f'Container was not deleted within {attempts * sleep_interval} sec') + cids: list[str] = [] + with allure.step(f'Create {containers_count} containers'): + for _ in range(containers_count): + cids.append(create_container(wallet, rule=placement_rule, await_mode=False, wait_for_creation=False)) + + with allure.step(f'Wait for containers occur in container list'): + for cid in cids: + wait_for_container_creation(wallet, cid, sleep_interval=containers_count) + + with allure.step('Delete containers and check they were deleted'): + for cid in cids: + delete_container(wallet, cid) + tick_epoch() + wait_for_container_deletion(wallet, cid) diff --git a/pytest_tests/testsuites/failovers/test_failover_network.py b/pytest_tests/testsuites/failovers/test_failover_network.py index 4f8abcf..b8efebf 100644 --- a/pytest_tests/testsuites/failovers/test_failover_network.py +++ b/pytest_tests/testsuites/failovers/test_failover_network.py @@ -4,9 +4,8 @@ from time import sleep import allure import pytest - -from common import (STORAGE_NODE_SSH_PRIVATE_KEY_PATH, STORAGE_NODE_SSH_USER, - STORAGE_NODE_SSH_PASSWORD) +from common import (STORAGE_NODE_SSH_PASSWORD, STORAGE_NODE_SSH_PRIVATE_KEY_PATH, + STORAGE_NODE_SSH_USER) from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes from iptables_helper import IpTablesHelper from python_keywords.container import create_container diff --git a/pytest_tests/testsuites/failovers/test_failover_storage.py b/pytest_tests/testsuites/failovers/test_failover_storage.py index 2694530..e07492f 100644 --- a/pytest_tests/testsuites/failovers/test_failover_storage.py +++ b/pytest_tests/testsuites/failovers/test_failover_storage.py @@ -2,8 +2,7 @@ import logging import allure import pytest -from common import (STORAGE_NODE_SSH_PRIVATE_KEY_PATH, STORAGE_NODE_SSH_USER, - STORAGE_NODE_SSH_PASSWORD) +from common import STORAGE_NODE_SSH_PASSWORD, STORAGE_NODE_SSH_PRIVATE_KEY_PATH, STORAGE_NODE_SSH_USER from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes from python_keywords.container import create_container from python_keywords.neofs_verbs import get_object, put_object @@ -12,7 +11,6 @@ from sbercloud_helper import SberCloud, SberCloudConfig from ssh_helper import HostClient from wellknown_acl import PUBLIC_ACL - logger = logging.getLogger('NeoLogger') stopped_hosts = [] @@ -52,11 +50,11 @@ def return_all_storage_nodes(sbercloud_client: SberCloud) -> None: wait_all_storage_node_returned() -@allure.title('Lost and return nodes') +@allure.title('Lost and returned nodes') @pytest.mark.parametrize('hard_reboot', [True, False]) @pytest.mark.failover -def test_lost_storage_node(prepare_wallet_and_deposit, sbercloud_client: SberCloud, - cloud_infrastructure_check, hard_reboot: bool): +def test_lost_storage_node(prepare_wallet_and_deposit, sbercloud_client: SberCloud, cloud_infrastructure_check, + hard_reboot: bool): wallet = prepare_wallet_and_deposit placement_rule = 'REP 2 IN X CBF 2 SELECT 2 FROM * AS X' source_file_path = generate_file() @@ -97,6 +95,7 @@ def test_panic_storage_node(prepare_wallet_and_deposit, cloud_infrastructure_che oid = put_object(wallet, source_file_path, cid) nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2) + new_nodes: list[str] = [] allure.attach('\n'.join(nodes), 'Current nodes with object', allure.attachment_type.TEXT) for node in nodes: with allure.step(f'Hard reboot host {node} via magic SysRq option'): diff --git a/pytest_tests/testsuites/network/test_node_management.py b/pytest_tests/testsuites/network/test_node_management.py index 08a5e47..c1e9de8 100644 --- a/pytest_tests/testsuites/network/test_node_management.py +++ b/pytest_tests/testsuites/network/test_node_management.py @@ -360,7 +360,7 @@ def test_shards(prepare_wallet_and_deposit, create_container_and_pick_node): @allure.step('Validate object has {expected_copies} copies') def validate_object_copies(wallet: str, placement_rule: str, file_path: str, expected_copies: int): cid = create_container(wallet, rule=placement_rule, basic_acl=PUBLIC_ACL) - got_policy = placement_policy_from_container(get_container(wallet, cid, flag='')) + got_policy = placement_policy_from_container(get_container(wallet, cid, json_mode=False)) assert got_policy == placement_rule.replace('\'', ''), \ f'Expected \n{placement_rule} and got policy \n{got_policy} are the same' oid = put_object(wallet, file_path, cid) diff --git a/pytest_tests/testsuites/object/test_object_api.py b/pytest_tests/testsuites/object/test_object_api.py index 62ca523..407b83d 100644 --- a/pytest_tests/testsuites/object/test_object_api.py +++ b/pytest_tests/testsuites/object/test_object_api.py @@ -3,14 +3,12 @@ from time import sleep import allure import pytest - -from common import SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE +from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE from container import create_container from epoch import get_epoch, tick_epoch from grpc_responses import OBJECT_ALREADY_REMOVED, OBJECT_NOT_FOUND, error_matches_status -from python_keywords.neofs_verbs import (delete_object, get_object, get_range, - get_range_hash, head_object, - put_object, search_object) +from python_keywords.neofs_verbs import (delete_object, get_object, get_range, get_range_hash, head_object, put_object, + search_object) from python_keywords.storage_policy import get_simple_object_copies from python_keywords.utility_keywords import generate_file, get_file_hash from tombstone import verify_head_tombstone @@ -116,7 +114,7 @@ def test_object_api_lifetime(prepare_wallet_and_deposit, request, object_size): file_hash = get_file_hash(file_path) epoch = get_epoch() - oid = put_object(wallet, file_path, cid, options=f'--expire-at {epoch + 1}') + oid = put_object(wallet, file_path, cid, expire_at=epoch + 1) got_file = get_object(wallet, cid, oid) assert get_file_hash(got_file) == file_hash diff --git a/robot/resources/lib/python_keywords/cli/__init__.py b/robot/resources/lib/python_keywords/cli/__init__.py new file mode 100644 index 0000000..8627b23 --- /dev/null +++ b/robot/resources/lib/python_keywords/cli/__init__.py @@ -0,0 +1 @@ +from .cli import NeofsCli diff --git a/robot/resources/lib/python_keywords/cli/accounting.py b/robot/resources/lib/python_keywords/cli/accounting.py new file mode 100644 index 0000000..84cf53b --- /dev/null +++ b/robot/resources/lib/python_keywords/cli/accounting.py @@ -0,0 +1,25 @@ +from typing import Optional + +from .cli_command import NeofsCliCommandBase + + +class NeofsCliAccounting(NeofsCliCommandBase): + def balance(self, wallet: str, rpc_endpoint: str, address: Optional[str] = None, + owner: Optional[str] = None) -> str: + """Get internal balance of NeoFS account + + Args: + address: address of wallet account + owner: owner of balance account (omit to use owner from private key) + rpc_endpoint: remote node address (as 'multiaddr' or ':') + wallet: WIF (NEP-2) string or path to the wallet or binary key + + + Returns: + str: Command string + + """ + return self._execute( + 'accounting balance', + **{param: param_value for param, param_value in locals().items() if param not in ['self']} + ) diff --git a/robot/resources/lib/python_keywords/cli/cli.py b/robot/resources/lib/python_keywords/cli/cli.py new file mode 100644 index 0000000..2cb4292 --- /dev/null +++ b/robot/resources/lib/python_keywords/cli/cli.py @@ -0,0 +1,32 @@ +from typing import Optional + +from common import NEOFS_CLI_EXEC + +from .accounting import NeofsCliAccounting +from .cli_command import NeofsCliCommandBase +from .container import NeofsCliContainer +from .object import NeofsCliObject + + +class NeofsCli: + neofs_cli_exec_path: Optional[str] = None + config: Optional[str] = None + accounting: Optional[NeofsCliAccounting] = None + container: Optional[NeofsCliContainer] = None + object: Optional[NeofsCliObject] = None + + def __init__(self, neofs_cli_exec_path: Optional[str] = None, config: Optional[str] = None, timeout: int = 30): + self.config = config # config(str): config file (default is $HOME/.config/neofs-cli/config.yaml) + self.neofs_cli_exec_path = neofs_cli_exec_path or NEOFS_CLI_EXEC + self.accounting = NeofsCliAccounting(self.neofs_cli_exec_path, timeout=timeout, config=config) + self.container = NeofsCliContainer(self.neofs_cli_exec_path, timeout=timeout, config=config) + self.object = NeofsCliObject(self.neofs_cli_exec_path, timeout=timeout, config=config) + + def version(self) -> str: + """Application version and NeoFS API compatibility + + Returns: + str: Command string + + """ + return NeofsCliCommandBase(self.neofs_cli_exec_path, config=self.config)._execute(command=None, version=True) diff --git a/robot/resources/lib/python_keywords/cli/cli_command.py b/robot/resources/lib/python_keywords/cli/cli_command.py new file mode 100644 index 0000000..6066ee7 --- /dev/null +++ b/robot/resources/lib/python_keywords/cli/cli_command.py @@ -0,0 +1,39 @@ +from typing import Optional + +from cli_helpers import _cmd_run + + +class NeofsCliCommandBase: + neofs_cli_exec: Optional[str] = None + timeout: Optional[int] = None + __base_params: Optional[str] = None + map_params = {'json_mode': 'json', 'await_mode': 'await', 'hash_type': 'hash'} + + def __init__(self, neofs_cli_exec: str, timeout: int = 30, **base_params): + self.neofs_cli_exec = neofs_cli_exec + self.timeout = timeout + self.__base_params = ' '.join([f'--{param} {value}' for param, value in base_params.items() if value]) + + def _format_command(self, command: str, **params) -> str: + param_str = [] + for param, value in params.items(): + if param in self.map_params.keys(): + param = self.map_params[param] + param = param.replace('_', '-') + if not value: + continue + if isinstance(value, bool): + param_str.append(f'--{param}') + elif isinstance(value, list): + param_str.append(f'--{param} \'{",".join(value)}\'') + elif isinstance(value, dict): + param_str.append(f'--{param} \'{",".join(f"{key}={val}" for key, val in value.items())}\'') + else: + value_str = str(value).replace("'", "\\'") + param_str.append(f"--{param} '{value_str}'") + param_str = ' '.join(param_str) + + return f'{self.neofs_cli_exec} {self.__base_params} {command or ""} {param_str}' + + def _execute(self, command: Optional[str], **params) -> str: + return _cmd_run(self._format_command(command, **params), timeout=self.timeout) diff --git a/robot/resources/lib/python_keywords/cli/container.py b/robot/resources/lib/python_keywords/cli/container.py new file mode 100644 index 0000000..61b358f --- /dev/null +++ b/robot/resources/lib/python_keywords/cli/container.py @@ -0,0 +1,184 @@ +from typing import Optional + +from .cli_command import NeofsCliCommandBase + + +class NeofsCliContainer(NeofsCliCommandBase): + def create(self, rpc_endpoint: str, wallet: str, address: Optional[str] = None, attributes: Optional[dict] = None, + basic_acl: Optional[str] = None, await_mode: bool = False, disable_timestamp: bool = False, + name: Optional[str] = None, nonce: Optional[str] = None, policy: Optional[str] = None, + session: Optional[str] = None, subnet: Optional[str] = None, ttl: Optional[int] = None, + xhdr: Optional[list] = None) -> str: + """Create a new container and register it in the NeoFS. + It will be stored in the sidechain when the Inner Ring accepts it. + + Args: + address: address of wallet account + attributes: comma separated pairs of container attributes in form of Key1=Value1,Key2=Value2 + await_mode: block execution until container is persisted + basic_acl: hex encoded basic ACL value or keywords like 'public-read-write', 'private', + 'eacl-public-read' (default "private") + disable_timestamp: disable timestamp container attribute + name: container name attribute + nonce: UUIDv4 nonce value for container + policy: QL-encoded or JSON-encoded placement policy or path to file with it + rpc_endpoint: remote node address (as 'multiaddr' or ':') + session: path to a JSON-encoded container session token + subnet: string representation of container subnetwork + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + Returns: + str: Command string + + """ + return self._execute( + 'container create', + **{param: param_value for param, param_value in locals().items() if param not in ['self']} + ) + + def delete(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None, await_mode: bool = False, + session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None) -> str: + """Delete an existing container. + Only the owner of the container has permission to remove the container. + + Args: + address: address of wallet account + await_mode: block execution until container is removed + cid: container ID + rpc_endpoint: remote node address (as 'multiaddr' or ':') + session: path to a JSON-encoded container session token + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + + Returns: + str: Command string + """ + + return self._execute( + 'container delete', + **{param: param_value for param, param_value in locals().items() if param not in ['self']} + ) + + def get(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None, await_mode: bool = False, + to: Optional[str] = None, json_mode: bool = False, ttl: Optional[int] = None, + xhdr: Optional[dict] = None) -> str: + """Get container field info + + Args: + address: address of wallet account + await_mode: block execution until container is removed + cid: container ID + json_mode: print or dump container in JSON format + rpc_endpoint: remote node address (as 'multiaddr' or ':') + to: path to dump encoded container + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + Returns: + str: Command string + + """ + + return self._execute( + 'container get', + **{param: param_value for param, param_value in locals().items() if param not in ['self']} + ) + + def get_eacl(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None, + await_mode: bool = False, to: Optional[str] = None, session: Optional[str] = None, + ttl: Optional[int] = None, xhdr: Optional[dict] = None) -> str: + """Get extended ACL talbe of container + + Args: + address: address of wallet account + await_mode: block execution until container is removed + cid: container ID + rpc_endpoint: remote node address (as 'multiaddr' or ':') + to: path to dump encoded container + session: path to a JSON-encoded container session token + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + Returns: + str: Command string + + """ + return self._execute( + 'container get-eacl', + **{param: param_value for param, param_value in locals().items() if param not in ['self']} + ) + + def list(self, rpc_endpoint: str, wallet: str, address: Optional[str] = None, + owner: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, **params) -> str: + """List all created containers + + Args: + address: address of wallet account + owner: owner of containers (omit to use owner from private key) + rpc_endpoint: remote node address (as 'multiaddr' or ':') + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + Returns: + str: Command string + + """ + return self._execute( + 'container list', + **{param: param_value for param, param_value in locals().items() if param not in ['self']} + ) + + def list_objects(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None, + ttl: Optional[int] = None, xhdr: Optional[dict] = None) -> str: + """List existing objects in container + + Args: + address: address of wallet account + cid: container ID + rpc_endpoint: remote node address (as 'multiaddr' or ':') + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + Returns: + str: Command string + + """ + + return self._execute( + 'container list-objects', + **{param: param_value for param, param_value in locals().items() if param not in ['self']} + ) + + def set_eacl(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None, + await_mode: bool = False, table: Optional[str] = None, session: Optional[str] = None, + ttl: Optional[int] = None, xhdr: Optional[dict] = None) -> str: + """Set a new extended ACL table for the container. + Container ID in the EACL table will be substituted with the ID from the CLI. + + Args: + address: address of wallet account + await_mode: block execution until container is removed + cid: container ID + rpc_endpoint: remote node address (as 'multiaddr' or ':') + session: path to a JSON-encoded container session token + table: path to file with JSON or binary encoded EACL table + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + Returns: + str: Command string + + """ + return self._execute( + 'container set-eacl', + **{param: param_value for param, param_value in locals().items() if param not in ['self']} + ) diff --git a/robot/resources/lib/python_keywords/cli/object.py b/robot/resources/lib/python_keywords/cli/object.py new file mode 100644 index 0000000..59eefca --- /dev/null +++ b/robot/resources/lib/python_keywords/cli/object.py @@ -0,0 +1,240 @@ +from typing import Optional + +from .cli_command import NeofsCliCommandBase + + +class NeofsCliObject(NeofsCliCommandBase): + def delete(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None, + bearer: Optional[str] = None, session: Optional[str] = None, ttl: Optional[int] = None, + xhdr: Optional[list] = None, **params) -> str: + """Delete object from NeoFS + + Args: + address: address of wallet account + bearer: File with signed JSON or binary encoded bearer token + cid: Container ID + oid: Object ID + rpc_endpoint: remote node address (as 'multiaddr' or ':') + session: path to a JSON-encoded container session token + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + Returns: + str: Command string + + """ + return self._execute( + 'object delete', + **{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']} + ) + + def get(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None, + bearer: Optional[str] = None, file: Optional[str] = None, + header: Optional[str] = None, no_progress: bool = False, raw: bool = False, + session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str: + """Get object from NeoFS + + Args: + address: address of wallet account + bearer: File with signed JSON or binary encoded bearer token + cid: Container ID + file: File to write object payload to. Default: stdout. + header: File to write header to. Default: stdout. + no_progress: Do not show progress bar + oid: Object ID + raw: Set raw request option + rpc_endpoint: remote node address (as 'multiaddr' or ':') + session: path to a JSON-encoded container session token + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + Returns: + str: Command string + + """ + return self._execute( + 'object get', + **{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']} + ) + + def hash(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None, + bearer: Optional[str] = None, range: Optional[str] = None, salt: Optional[str] = None, + ttl: Optional[int] = None, hash_type: Optional[str] = None, xhdr: Optional[list] = None, + **params) -> str: + """Get object hash + + Args: + address: address of wallet account + bearer: File with signed JSON or binary encoded bearer token + cid: Container ID + oid: Object ID + range: Range to take hash from in the form offset1:length1,... + rpc_endpoint: remote node address (as 'multiaddr' or ':') + salt: Salt in hex format + ttl: TTL value in request meta header (default 2) + hash_type: Hash type. Either 'sha256' or 'tz' (default "sha256") + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + Returns: + str: Command string + + """ + return self._execute( + 'object hash', + **{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']} + ) + + def head(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None, + bearer: Optional[str] = None, file: Optional[str] = None, + json_mode: bool = False, main_only: bool = False, proto: bool = False, raw: bool = False, + session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str: + """Get object header + + Args: + address: address of wallet account + bearer: File with signed JSON or binary encoded bearer token + cid: Container ID + file: File to write object payload to. Default: stdout. + json_mode: Marshal output in JSON + main_only: Return only main fields + oid: Object ID + proto: Marshal output in Protobuf + raw: Set raw request option + rpc_endpoint: remote node address (as 'multiaddr' or ':') + session: path to a JSON-encoded container session token + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + + Returns: + str: Command string + + """ + return self._execute( + 'object head', + **{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']} + ) + + def lock(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, lifetime: int, address: Optional[str] = None, + bearer: Optional[str] = None, session: Optional[str] = None, + ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str: + """Lock object in container + + Args: + address: address of wallet account + bearer: File with signed JSON or binary encoded bearer token + cid: Container ID + oid: Object ID + lifetime: Object lifetime + rpc_endpoint: remote node address (as 'multiaddr' or ':') + session: path to a JSON-encoded container session token + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + + Returns: + str: Command string + + """ + return self._execute( + 'object lock', + **{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']} + ) + + def put(self, rpc_endpoint: str, wallet: str, cid: str, file: str, address: Optional[str] = None, + attributes: Optional[dict] = None, bearer: Optional[str] = None, disable_filename: bool = False, + disable_timestamp: bool = False, expire_at: Optional[int] = None, no_progress: bool = False, + notify: Optional[str] = None, session: Optional[str] = None, ttl: Optional[int] = None, + xhdr: Optional[list] = None, **params) -> str: + """Put object to NeoFS + + Args: + address: address of wallet account + attributes: User attributes in form of Key1=Value1,Key2=Value2 + bearer: File with signed JSON or binary encoded bearer token + cid: Container ID + disable_filename: Do not set well-known filename attribute + disable_timestamp: Do not set well-known timestamp attribute + expire_at: Last epoch in the life of the object + file: File with object payload + no_progress: Do not show progress bar + notify: Object notification in the form of *epoch*:*topic*; '-' topic means using default + rpc_endpoint: remote node address (as 'multiaddr' or ':') + session: path to a JSON-encoded container session token + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + Returns: + str: Command string + + """ + return self._execute( + 'object put', + **{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']} + ) + + def range(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, range: str, address: Optional[str] = None, + bearer: Optional[str] = None, file: Optional[str] = None, json_mode: bool = False, raw: bool = False, + session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str: + """Get payload range data of an object + + Args: + address: address of wallet account + bearer: File with signed JSON or binary encoded bearer token + cid: Container ID + file: File to write object payload to. Default: stdout. + json_mode: Marshal output in JSON + oid: Object ID + range: Range to take data from in the form offset:length + raw: Set raw request option + rpc_endpoint: remote node address (as 'multiaddr' or ':') + session: path to a JSON-encoded container session token + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + + Returns: + str: Command string + + """ + return self._execute( + 'object range', + **{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']} + ) + + def search(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None, + bearer: Optional[str] = None, filters: Optional[list] = None, oid: Optional[str] = None, + phy: bool = False, root: bool = False, session: Optional[str] = None, ttl: Optional[int] = None, + xhdr: Optional[list] = None, **params) -> str: + """Search object + + Args: + address: address of wallet account + bearer: File with signed JSON or binary encoded bearer token + cid: Container ID + filters: Repeated filter expressions or files with protobuf JSON + oid: Object ID + phy: Search physically stored objects + root: Search for user objects + rpc_endpoint: remote node address (as 'multiaddr' or ':') + session: path to a JSON-encoded container session token + ttl: TTL value in request meta header (default 2) + wallet: WIF (NEP-2) string or path to the wallet or binary key + xhdr: Request X-Headers in form of Key=Value + + + Returns: + str: Command string + + """ + return self._execute( + 'object search', + **{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']} + ) diff --git a/robot/resources/lib/python_keywords/cli_helpers.py b/robot/resources/lib/python_keywords/cli_helpers.py index 919e251..ea1ed2e 100644 --- a/robot/resources/lib/python_keywords/cli_helpers.py +++ b/robot/resources/lib/python_keywords/cli_helpers.py @@ -42,22 +42,26 @@ def _cmd_run(cmd: str, timeout: int = 30) -> str: return output except subprocess.CalledProcessError as exc: - logger.info(f"Error:\nreturn code: {exc.returncode} " + logger.info(f"Command: {cmd}\n" + f"Error:\nreturn code: {exc.returncode} " f"\nOutput: {exc.output}") end_time = datetime.now() return_code, cmd_output = subprocess.getstatusoutput(cmd) _attach_allure_log(cmd, cmd_output, return_code, start_time, end_time) - raise RuntimeError(f"Error:\nreturn code: {exc.returncode} " - f"\nOutput: {exc.output}") from exc + raise RuntimeError(f"Command: {cmd}\n" + f"Error:\nreturn code: {exc.returncode}\n" + f"Output: {exc.output}") from exc except OSError as exc: - raise RuntimeError(f"Output: {exc.strerror}") from exc + raise RuntimeError(f"Command: {cmd}\n" + f"Output: {exc.strerror}") from exc except Exception as exc: return_code, cmd_output = subprocess.getstatusoutput(cmd) end_time = datetime.now() _attach_allure_log(cmd, cmd_output, return_code, start_time, end_time) - logger.info(f"Error:\nreturn code: {return_code}\nOutput: " - f"{exc.output.decode('utf-8') if type(exc.output) is bytes else exc.output}") + logger.info(f"Command: {cmd}\n" + f"Error:\nreturn code: {return_code}\n" + f"Output: {exc.output.decode('utf-8') if type(exc.output) is bytes else exc.output}") raise diff --git a/robot/resources/lib/python_keywords/container.py b/robot/resources/lib/python_keywords/container.py index 5d53a31..16d527f 100644 --- a/robot/resources/lib/python_keywords/container.py +++ b/robot/resources/lib/python_keywords/container.py @@ -5,24 +5,24 @@ """ import json -import time -from typing import Optional +from time import sleep +from typing import Optional, Union import json_transformers -from data_formatters import dict_to_attrs -from cli_helpers import _cmd_run -from common import NEOFS_ENDPOINT, NEOFS_CLI_EXEC, WALLET_CONFIG - +from cli import NeofsCli +from common import NEOFS_ENDPOINT, WALLET_CONFIG from robot.api import logger from robot.api.deco import keyword ROBOT_AUTO_KEYWORDS = False DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" + @keyword('Create Container') def create_container(wallet: str, rule: str = DEFAULT_PLACEMENT_RULE, basic_acl: str = '', attributes: Optional[dict] = None, session_token: str = '', - session_wallet: str = '', options: str = '') -> str: + session_wallet: str = '', name: str = None, options: dict = None, + await_mode: bool = True, wait_for_creation: bool = True) -> str: """ A wrapper for `neofs-cli container create` call. @@ -37,39 +37,51 @@ def create_container(wallet: str, rule: str = DEFAULT_PLACEMENT_RULE, basic_acl: session_wallet(optional, str): a path to the wallet which signed the session token; this parameter makes sense when paired with `session_token` - options (optional, str): any other options to pass to the call + options (optional, dict): any other options to pass to the call + name (optional, str): container name attribute + await_mode (bool): block execution until container is persisted + wait_for_creation (): Wait for container shows in container list Returns: (str): CID of the created container """ - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} container create ' - f'--wallet {session_wallet if session_wallet else wallet} ' - f'--config {WALLET_CONFIG} --policy "{rule}" ' - f'{"--basic-acl " + basic_acl if basic_acl else ""} ' - f'{"--attributes " + dict_to_attrs(attributes) if attributes else ""} ' - f'{"--session " + session_token if session_token else ""} ' - f'{options} --await' - ) - output = _cmd_run(cmd, timeout=60) + cli = NeofsCli(config=WALLET_CONFIG, timeout=60) + output = cli.container.create(rpc_endpoint=NEOFS_ENDPOINT, wallet=session_wallet if session_wallet else wallet, + policy=rule, basic_acl=basic_acl, attributes=attributes, name=name, + session=session_token, await_mode=await_mode, **options or {}) + cid = _parse_cid(output) - logger.info("Container created; waiting until it is persisted in sidechain") + logger.info("Container created; waiting until it is persisted in the sidechain") - deadline_to_persist = 15 # seconds - for i in range(0, deadline_to_persist): - time.sleep(1) + if wait_for_creation: + wait_for_container_creation(wallet, cid) + + return cid + + +def wait_for_container_creation(wallet: str, cid: str, attempts: int = 15, sleep_interval: int = 1): + for _ in range(attempts): containers = list_containers(wallet) if cid in containers: - break - logger.info(f"There is no {cid} in {containers} yet; continue") - if i + 1 == deadline_to_persist: - raise RuntimeError( - f"After {deadline_to_persist} seconds the container " - f"{cid} hasn't been persisted; exiting" - ) - return cid + return + logger.info(f"There is no {cid} in {containers} yet; sleep {sleep_interval} and continue") + sleep(sleep_interval) + raise RuntimeError(f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting") + + +def wait_for_container_deletion(wallet: str, cid: str, attempts: int = 30, sleep_interval: int = 1): + for _ in range(attempts): + try: + get_container(wallet, cid) + sleep(sleep_interval) + continue + except Exception as err: + if 'container not found' not in str(err): + raise AssertionError(f'Expected "container not found" in error, got\n{err}') + return + raise AssertionError(f'Expected container deleted during {attempts * sleep_interval} sec.') @keyword('List Containers') @@ -82,33 +94,30 @@ def list_containers(wallet: str) -> list[str]: Returns: (list): list of containers """ - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} ' - f'--config {WALLET_CONFIG} container list' - ) - output = _cmd_run(cmd) + cli = NeofsCli(config=WALLET_CONFIG) + output = cli.container.list(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet) + logger.info(f"Containers: \n{output}") return output.split() @keyword('Get Container') -def get_container(wallet: str, cid: str, flag: str = '--json') -> dict: +def get_container(wallet: str, cid: str, json_mode: bool = True) -> Union[dict, str]: """ A wrapper for `neofs-cli container get` call. It extracts container's attributes and rearranges them into a more compact view. Args: wallet (str): path to a wallet on whose behalf we get the container cid (str): ID of the container to get - flag (str): output as json or plain text + json_mode (bool): return container in JSON format Returns: (dict, str): dict of container attributes """ - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} ' - f'--config {WALLET_CONFIG} --cid {cid} container get {flag}' - ) - output = _cmd_run(cmd) - if flag != '--json': + cli = NeofsCli(config=WALLET_CONFIG) + output = cli.container.get(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, json_mode=json_mode) + + if not json_mode: return output + container_info = json.loads(output) attributes = dict() for attr in container_info['attributes']: @@ -130,11 +139,8 @@ def delete_container(wallet: str, cid: str) -> None: This function doesn't return anything. """ - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} ' - f'--config {WALLET_CONFIG} container delete --cid {cid}' - ) - _cmd_run(cmd) + cli = NeofsCli(config=WALLET_CONFIG) + cli.container.delete(wallet=wallet, cid=cid, rpc_endpoint=NEOFS_ENDPOINT) def _parse_cid(output: str) -> str: diff --git a/robot/resources/lib/python_keywords/neofs_verbs.py b/robot/resources/lib/python_keywords/neofs_verbs.py index 978c8db..20b2d9e 100644 --- a/robot/resources/lib/python_keywords/neofs_verbs.py +++ b/robot/resources/lib/python_keywords/neofs_verbs.py @@ -1,32 +1,28 @@ #!/usr/bin/python3 -''' +""" This module contains wrappers for NeoFS verbs executed via neofs-cli. -''' +""" import json -import os import random import re import uuid +from typing import Optional import json_transformers -from cli_helpers import _cmd_run +from cli import NeofsCli from common import ASSETS_DIR, NEOFS_ENDPOINT, NEOFS_NETMAP, WALLET_CONFIG -from data_formatters import dict_to_attrs from robot.api import logger from robot.api.deco import keyword ROBOT_AUTO_KEYWORDS = False -# path to neofs-cli executable -NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli') - @keyword('Get object') -def get_object(wallet: str, cid: str, oid: str, bearer_token: str = "", - write_object: str = "", endpoint: str = "", options: str = "", - wallet_config: str = WALLET_CONFIG): +def get_object(wallet: str, cid: str, oid: str, bearer_token: Optional[str] = None, write_object: str = "", + endpoint: str = "", options: Optional[dict] = None, wallet_config: str = WALLET_CONFIG, + no_progress: bool = True): """ GET from NeoFS. @@ -38,6 +34,7 @@ def get_object(wallet: str, cid: str, oid: str, bearer_token: str = "", write_object (optional, str): path to downloaded file, appends to `--file` key endpoint (optional, str): NeoFS endpoint to send request to, appends to `--rpc-endpoint` key wallet_config(optional, str): path to the wallet config + no_progress(optional, bool): do not show progress bar options (optional, str): any options which `neofs-cli object get` accepts Returns: (str): path to downloaded file @@ -50,20 +47,17 @@ def get_object(wallet: str, cid: str, oid: str, bearer_token: str = "", if not endpoint: endpoint = random.sample(NEOFS_NETMAP, 1)[0] - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint} --wallet {wallet} ' - f'object get --cid {cid} --oid {oid} --file {file_path} --config {wallet_config} ' - f'{"--bearer " + bearer_token if bearer_token else ""} ' - f'{options}' - ) - _cmd_run(cmd) + cli = NeofsCli(config=wallet_config) + cli.object.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, oid=oid, file=file_path, + bearer=bearer_token, no_progress=no_progress, **options or {}) + return file_path # TODO: make `bearer_token` optional @keyword('Get Range Hash') def get_range_hash(wallet: str, cid: str, oid: str, bearer_token: str, range_cut: str, - wallet_config: str = WALLET_CONFIG, options: str = ""): + wallet_config: str = WALLET_CONFIG, options: Optional[dict] = None): """ GETRANGEHASH of given Object. @@ -79,20 +73,19 @@ def get_range_hash(wallet: str, cid: str, oid: str, bearer_token: str, range_cut Returns: None """ - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} ' - f'object hash --cid {cid} --oid {oid} --range {range_cut} --config {wallet_config} ' - f'{"--bearer " + bearer_token if bearer_token else ""} ' - f'{options}' - ) - output = _cmd_run(cmd) + + cli = NeofsCli(config=wallet_config) + output = cli.object.hash(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid, range=range_cut, + bearer=bearer_token, **options or {}) + # cutting off output about range offset and length return output.split(':')[1].strip() @keyword('Put object') -def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers: dict = {}, - endpoint: str = "", wallet_config: str = WALLET_CONFIG, options: str = ""): +def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers: Optional[dict] = None, + endpoint: str = "", wallet_config: str = WALLET_CONFIG, expire_at: Optional[int] = None, + no_progress: bool = True, options: Optional[dict] = None): """ PUT of given file. @@ -104,7 +97,9 @@ def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers: user_headers (optional, dict): Object attributes, append to `--attributes` key endpoint(optional, str): NeoFS endpoint to send request to wallet_config(optional, str): path to the wallet config + no_progress(optional, bool): do not show progress bar options (optional, str): any options which `neofs-cli object put` accepts + expire_at (optional, int): Last epoch in the life of the object Returns: (str): ID of uploaded Object """ @@ -112,13 +107,12 @@ def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers: endpoint = random.sample(NEOFS_NETMAP, 1)[0] if not endpoint: logger.info(f'---DEB:\n{NEOFS_NETMAP}') - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint} --wallet {wallet} ' - f'object put --file {path} --cid {cid} {options} --config {wallet_config} ' - f'{"--bearer " + bearer if bearer else ""} ' - f'{"--attributes " + dict_to_attrs(user_headers) if user_headers else ""}' - ) - output = _cmd_run(cmd) + + cli = NeofsCli(config=wallet_config) + output = cli.object.put(rpc_endpoint=endpoint, wallet=wallet, file=path, cid=cid, bearer=bearer, + expire_at=expire_at, no_progress=no_progress, + attributes=user_headers or {}, **options or {}) + # splitting CLI output to lines and taking the penultimate line id_str = output.strip().split('\n')[-2] oid = id_str.split(':')[1] @@ -127,7 +121,7 @@ def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers: @keyword('Delete object') def delete_object(wallet: str, cid: str, oid: str, bearer: str = "", wallet_config: str = WALLET_CONFIG, - options: str = ""): + options: Optional[dict] = None): """ DELETE an Object. @@ -137,16 +131,15 @@ def delete_object(wallet: str, cid: str, oid: str, bearer: str = "", wallet_conf oid (str): ID of Object we are going to delete bearer (optional, str): path to Bearer Token file, appends to `--bearer` key wallet_config(optional, str): path to the wallet config - options (optional, str): any options which `neofs-cli object delete` accepts + options (optional, dict): any options which `neofs-cli object delete` accepts Returns: (str): Tombstone ID """ - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} ' - f'object delete --cid {cid} --oid {oid} {options} --config {wallet_config} ' - f'{"--bearer " + bearer if bearer else ""}' - ) - output = _cmd_run(cmd) + + cli = NeofsCli(config=wallet_config) + output = cli.object.delete(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid, bearer=bearer, + **options or {}) + id_str = output.split('\n')[1] tombstone = id_str.split(':')[1] return tombstone.strip() @@ -154,7 +147,7 @@ def delete_object(wallet: str, cid: str, oid: str, bearer: str = "", wallet_conf @keyword('Get Range') def get_range(wallet: str, cid: str, oid: str, range_cut: str, wallet_config: str = WALLET_CONFIG, - bearer: str = "", options: str = ""): + bearer: str = "", options: Optional[dict] = None): """ GETRANGE an Object. @@ -165,35 +158,31 @@ def get_range(wallet: str, cid: str, oid: str, range_cut: str, wallet_config: st range_cut (str): range to take data from in the form offset:length bearer (optional, str): path to Bearer Token file, appends to `--bearer` key wallet_config(optional, str): path to the wallet config - options (optional, str): any options which `neofs-cli object range` accepts + options (optional, dict): any options which `neofs-cli object range` accepts Returns: (str, bytes) - path to the file with range content and content of this file as bytes """ range_file = f"{ASSETS_DIR}/{uuid.uuid4()}" - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} ' - f'object range --cid {cid} --oid {oid} --range {range_cut} --config {wallet_config} ' - f'{options} --file {range_file} ' - f'{"--bearer " + bearer if bearer else ""} ' - ) - _cmd_run(cmd) - content = '' + + cli = NeofsCli(config=wallet_config) + cli.object.range(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid, range=range_cut, file=range_file, + bearer=bearer, **options or {}) + with open(range_file, 'rb') as fout: content = fout.read() return range_file, content @keyword('Search object') -def search_object(wallet: str, cid: str, keys: str = "", bearer: str = "", filters: dict = {}, - expected_objects_list=[], wallet_config: str = WALLET_CONFIG, options: str = ""): +def search_object(wallet: str, cid: str, bearer: str = "", filters: Optional[dict] = None, + expected_objects_list: Optional[list] = None, wallet_config: str = WALLET_CONFIG, + options: Optional[dict] = None): """ SEARCH an Object. Args: wallet (str): wallet on whose behalf SEARCH is done cid (str): ID of Container where we get the Object from - keys(optional, str): any keys for Object SEARCH which `neofs-cli object search` - accepts, e.g. `--oid` bearer (optional, str): path to Bearer Token file, appends to `--bearer` key filters (optional, dict): key=value pairs to filter Objects expected_objects_list (optional, list): a list of ObjectIDs to compare found Objects with @@ -202,19 +191,12 @@ def search_object(wallet: str, cid: str, keys: str = "", bearer: str = "", filte Returns: (list): list of found ObjectIDs """ - filters_result = "" - if filters: - filters_result += "--filters " - logger.info(filters) - filters_result += ','.join( - map(lambda i: f"'{i} EQ {filters[i]}'", filters)) - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} ' - f'object search {keys} --cid {cid} {filters_result} --config {wallet_config} ' - f'{"--bearer " + bearer if bearer else ""} {options}' - ) - output = _cmd_run(cmd) + cli = NeofsCli(config=wallet_config) + output = cli.object.search( + rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, bearer=bearer, + filters=[f'{filter_key} EQ {filter_val}' for filter_key, filter_val in filters.items()] if filters else None, + **options or {}) found_objects = re.findall(r'(\w{43,44})', output) @@ -231,7 +213,7 @@ def search_object(wallet: str, cid: str, keys: str = "", bearer: str = "", filte @keyword('Head object') def head_object(wallet: str, cid: str, oid: str, bearer_token: str = "", - options: str = "", endpoint: str = "", json_output: bool = True, + options: Optional[dict] = None, endpoint: str = None, json_output: bool = True, is_raw: bool = False, is_direct: bool = False, wallet_config: str = WALLET_CONFIG): """ HEAD an Object. @@ -256,20 +238,15 @@ def head_object(wallet: str, cid: str, oid: str, bearer_token: str = "", or (str): HEAD response as a plain text """ - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint if endpoint else NEOFS_ENDPOINT} ' - f'--wallet {wallet} --config {wallet_config} ' - f'object head --cid {cid} --oid {oid} {options} ' - f'{"--bearer " + bearer_token if bearer_token else ""} ' - f'{"--json" if json_output else ""} ' - f'{"--raw" if is_raw else ""} ' - f'{"--ttl 1" if is_direct else ""}' - ) - output = _cmd_run(cmd) + + cli = NeofsCli(config=wallet_config) + output = cli.object.head(rpc_endpoint=endpoint or NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid, + bearer=bearer_token, json_mode=json_output, raw=is_raw, + ttl=1 if is_direct else None, **options or {}) + if not json_output: return output - decoded = "" try: decoded = json.loads(output) except Exception as exc: