mmalygina/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Pull requests Activity

Implement neofs-cli lib for container and object

Browse source 
Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
This commit is contained in:
Vladimir Avdeev 2022-08-19 05:22:20 +03:00 committed by Vladimir Domnich
parent d935c2cafa
commit 3294299612
14 changed files with 680 additions and 171 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
pytest_tests/testsuites/container/test_container.py
View file

@ -1,13 +1,11 @@
import json
from time import sleep
import allure
import pytest
from epoch import tick_epoch
from grpc_responses import CONTAINER_NOT_FOUND, error_matches_status
from python_keywords.container import (create_container, delete_container, get_container,
list_containers)
from python_keywords.container import (create_container, delete_container, get_container, list_containers,
wait_for_container_creation, wait_for_container_deletion)
from utility import placement_policy_from_container
from wellknown_acl import PRIVATE_ACL_F
@ -24,13 +22,12 @@ def test_container_creation(prepare_wallet_and_deposit, name):
json_wallet = json.load(file)
placement_rule = 'REP 2 IN X CBF 1 SELECT 2 FROM * AS X'
options = f"--name {name}" if name else ""
cid = create_container(wallet, rule=placement_rule, options=options)
cid = create_container(wallet, rule=placement_rule, name=name)
containers = list_containers(wallet)
assert cid in containers, f'Expected container {cid} in containers: {containers}'
container_info: str = get_container(wallet, cid, flag='')
container_info: str = get_container(wallet, cid, json_mode=False)
container_info = container_info.casefold() # To ignore case when comparing with expected values
info_to_check = {
@ -58,17 +55,25 @@ def test_container_creation(prepare_wallet_and_deposit, name):
wait_for_container_deletion(wallet, cid)
@allure.step('Wait for container deletion')
def wait_for_container_deletion(wallet: str, cid: str) -> None:
attempts, sleep_interval = 10, 5
for _ in range(attempts):
try:
get_container(wallet, cid)
sleep(sleep_interval)
continue
except Exception as err:
if error_matches_status(err, CONTAINER_NOT_FOUND):
return
raise AssertionError(f'Expected "{CONTAINER_NOT_FOUND}" error, got\n{err}')
@allure.title('Parallel container creation and deletion')
@pytest.mark.sanity
@pytest.mark.container
def test_container_creation_deletion_parallel(prepare_wallet_and_deposit):
containers_count = 3
wallet = prepare_wallet_and_deposit
placement_rule = 'REP 2 IN X CBF 1 SELECT 2 FROM * AS X'
raise AssertionError(f'Container was not deleted within {attempts * sleep_interval} sec')
cids: list[str] = []
with allure.step(f'Create {containers_count} containers'):
for _ in range(containers_count):
cids.append(create_container(wallet, rule=placement_rule, await_mode=False, wait_for_creation=False))
with allure.step(f'Wait for containers occur in container list'):
for cid in cids:
wait_for_container_creation(wallet, cid, sleep_interval=containers_count)
with allure.step('Delete containers and check they were deleted'):
for cid in cids:
delete_container(wallet, cid)
tick_epoch()
wait_for_container_deletion(wallet, cid)

5
pytest_tests/testsuites/failovers/test_failover_network.py
View file

@ -4,9 +4,8 @@ from time import sleep
import allure
import pytest
from common import (STORAGE_NODE_SSH_PRIVATE_KEY_PATH, STORAGE_NODE_SSH_USER,
STORAGE_NODE_SSH_PASSWORD)
from common import (STORAGE_NODE_SSH_PASSWORD, STORAGE_NODE_SSH_PRIVATE_KEY_PATH,
STORAGE_NODE_SSH_USER)
from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes
from iptables_helper import IpTablesHelper
from python_keywords.container import create_container

11
pytest_tests/testsuites/failovers/test_failover_storage.py
View file

@ -2,8 +2,7 @@ import logging
import allure
import pytest
from common import (STORAGE_NODE_SSH_PRIVATE_KEY_PATH, STORAGE_NODE_SSH_USER,
STORAGE_NODE_SSH_PASSWORD)
from common import STORAGE_NODE_SSH_PASSWORD, STORAGE_NODE_SSH_PRIVATE_KEY_PATH, STORAGE_NODE_SSH_USER
from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes
from python_keywords.container import create_container
from python_keywords.neofs_verbs import get_object, put_object
@ -12,7 +11,6 @@ from sbercloud_helper import SberCloud, SberCloudConfig
from ssh_helper import HostClient
from wellknown_acl import PUBLIC_ACL
logger = logging.getLogger('NeoLogger')
stopped_hosts = []
@ -52,11 +50,11 @@ def return_all_storage_nodes(sbercloud_client: SberCloud) -> None:
wait_all_storage_node_returned()
@allure.title('Lost and return nodes')
@allure.title('Lost and returned nodes')
@pytest.mark.parametrize('hard_reboot', [True, False])
@pytest.mark.failover
def test_lost_storage_node(prepare_wallet_and_deposit, sbercloud_client: SberCloud,
cloud_infrastructure_check, hard_reboot: bool):
def test_lost_storage_node(prepare_wallet_and_deposit, sbercloud_client: SberCloud, cloud_infrastructure_check,
hard_reboot: bool):
wallet = prepare_wallet_and_deposit
placement_rule = 'REP 2 IN X CBF 2 SELECT 2 FROM * AS X'
source_file_path = generate_file()
@ -97,6 +95,7 @@ def test_panic_storage_node(prepare_wallet_and_deposit, cloud_infrastructure_che
oid = put_object(wallet, source_file_path, cid)
nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2)
new_nodes: list[str] = []
allure.attach('\n'.join(nodes), 'Current nodes with object', allure.attachment_type.TEXT)
for node in nodes:
with allure.step(f'Hard reboot host {node} via magic SysRq option'):

2
pytest_tests/testsuites/network/test_node_management.py
View file

@ -360,7 +360,7 @@ def test_shards(prepare_wallet_and_deposit, create_container_and_pick_node):
@allure.step('Validate object has {expected_copies} copies')
def validate_object_copies(wallet: str, placement_rule: str, file_path: str, expected_copies: int):
cid = create_container(wallet, rule=placement_rule, basic_acl=PUBLIC_ACL)
got_policy = placement_policy_from_container(get_container(wallet, cid, flag=''))
got_policy = placement_policy_from_container(get_container(wallet, cid, json_mode=False))
assert got_policy == placement_rule.replace('\'', ''), \
f'Expected \n{placement_rule} and got policy \n{got_policy} are the same'
oid = put_object(wallet, file_path, cid)

10
pytest_tests/testsuites/object/test_object_api.py
View file

@ -3,14 +3,12 @@ from time import sleep
import allure
import pytest
from common import SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
from container import create_container
from epoch import get_epoch, tick_epoch
from grpc_responses import OBJECT_ALREADY_REMOVED, OBJECT_NOT_FOUND, error_matches_status
from python_keywords.neofs_verbs import (delete_object, get_object, get_range,
get_range_hash, head_object,
put_object, search_object)
from python_keywords.neofs_verbs import (delete_object, get_object, get_range, get_range_hash, head_object, put_object,
search_object)
from python_keywords.storage_policy import get_simple_object_copies
from python_keywords.utility_keywords import generate_file, get_file_hash
from tombstone import verify_head_tombstone
@ -116,7 +114,7 @@ def test_object_api_lifetime(prepare_wallet_and_deposit, request, object_size):
file_hash = get_file_hash(file_path)
epoch = get_epoch()
oid = put_object(wallet, file_path, cid, options=f'--expire-at {epoch + 1}')
oid = put_object(wallet, file_path, cid, expire_at=epoch + 1)
got_file = get_object(wallet, cid, oid)
assert get_file_hash(got_file) == file_hash

1
robot/resources/lib/python_keywords/cli/__init__.py Normal file
View file

@ -0,0 +1 @@
from .cli import NeofsCli

25
robot/resources/lib/python_keywords/cli/accounting.py Normal file
View file

@ -0,0 +1,25 @@
from typing import Optional
from .cli_command import NeofsCliCommandBase
class NeofsCliAccounting(NeofsCliCommandBase):
def balance(self, wallet: str, rpc_endpoint: str, address: Optional[str] = None,
owner: Optional[str] = None) -> str:
"""Get internal balance of NeoFS account
Args:
address: address of wallet account
owner: owner of balance account (omit to use owner from private key)
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
wallet: WIF (NEP-2) string or path to the wallet or binary key
Returns:
str: Command string
"""
return self._execute(
'accounting balance',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)

32
robot/resources/lib/python_keywords/cli/cli.py Normal file
View file

@ -0,0 +1,32 @@
from typing import Optional
from common import NEOFS_CLI_EXEC
from .accounting import NeofsCliAccounting
from .cli_command import NeofsCliCommandBase
from .container import NeofsCliContainer
from .object import NeofsCliObject
class NeofsCli:
neofs_cli_exec_path: Optional[str] = None
config: Optional[str] = None
accounting: Optional[NeofsCliAccounting] = None
container: Optional[NeofsCliContainer] = None
object: Optional[NeofsCliObject] = None
def __init__(self, neofs_cli_exec_path: Optional[str] = None, config: Optional[str] = None, timeout: int = 30):
self.config = config # config(str): config file (default is $HOME/.config/neofs-cli/config.yaml)
self.neofs_cli_exec_path = neofs_cli_exec_path or NEOFS_CLI_EXEC
self.accounting = NeofsCliAccounting(self.neofs_cli_exec_path, timeout=timeout, config=config)
self.container = NeofsCliContainer(self.neofs_cli_exec_path, timeout=timeout, config=config)
self.object = NeofsCliObject(self.neofs_cli_exec_path, timeout=timeout, config=config)
def version(self) -> str:
"""Application version and NeoFS API compatibility
Returns:
str: Command string
"""
return NeofsCliCommandBase(self.neofs_cli_exec_path, config=self.config)._execute(command=None, version=True)

39
robot/resources/lib/python_keywords/cli/cli_command.py Normal file
View file

@ -0,0 +1,39 @@
from typing import Optional
from cli_helpers import _cmd_run
class NeofsCliCommandBase:
neofs_cli_exec: Optional[str] = None
timeout: Optional[int] = None
__base_params: Optional[str] = None
map_params = {'json_mode': 'json', 'await_mode': 'await', 'hash_type': 'hash'}
def __init__(self, neofs_cli_exec: str, timeout: int = 30, **base_params):
self.neofs_cli_exec = neofs_cli_exec
self.timeout = timeout
self.__base_params = ' '.join([f'--{param} {value}' for param, value in base_params.items() if value])
def _format_command(self, command: str, **params) -> str:
param_str = []
for param, value in params.items():
if param in self.map_params.keys():
param = self.map_params[param]
param = param.replace('_', '-')
if not value:
continue
if isinstance(value, bool):
param_str.append(f'--{param}')
elif isinstance(value, list):
param_str.append(f'--{param} \'{",".join(value)}\'')
elif isinstance(value, dict):
param_str.append(f'--{param} \'{",".join(f"{key}={val}" for key, val in value.items())}\'')
else:
value_str = str(value).replace("'", "\\'")
param_str.append(f"--{param} '{value_str}'")
param_str = ' '.join(param_str)
return f'{self.neofs_cli_exec} {self.__base_params} {command or ""} {param_str}'
def _execute(self, command: Optional[str], **params) -> str:
return _cmd_run(self._format_command(command, **params), timeout=self.timeout)

184
robot/resources/lib/python_keywords/cli/container.py Normal file
View file

@ -0,0 +1,184 @@
from typing import Optional
from .cli_command import NeofsCliCommandBase
class NeofsCliContainer(NeofsCliCommandBase):
def create(self, rpc_endpoint: str, wallet: str, address: Optional[str] = None, attributes: Optional[dict] = None,
basic_acl: Optional[str] = None, await_mode: bool = False, disable_timestamp: bool = False,
name: Optional[str] = None, nonce: Optional[str] = None, policy: Optional[str] = None,
session: Optional[str] = None, subnet: Optional[str] = None, ttl: Optional[int] = None,
xhdr: Optional[list] = None) -> str:
"""Create a new container and register it in the NeoFS.
It will be stored in the sidechain when the Inner Ring accepts it.
Args:
address: address of wallet account
attributes: comma separated pairs of container attributes in form of Key1=Value1,Key2=Value2
await_mode: block execution until container is persisted
basic_acl: hex encoded basic ACL value or keywords like 'public-read-write', 'private',
'eacl-public-read' (default "private")
disable_timestamp: disable timestamp container attribute
name: container name attribute
nonce: UUIDv4 nonce value for container
policy: QL-encoded or JSON-encoded placement policy or path to file with it
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
subnet: string representation of container subnetwork
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container create',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def delete(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None, await_mode: bool = False,
session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None) -> str:
"""Delete an existing container.
Only the owner of the container has permission to remove the container.
Args:
address: address of wallet account
await_mode: block execution until container is removed
cid: container ID
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container delete',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def get(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None, await_mode: bool = False,
to: Optional[str] = None, json_mode: bool = False, ttl: Optional[int] = None,
xhdr: Optional[dict] = None) -> str:
"""Get container field info
Args:
address: address of wallet account
await_mode: block execution until container is removed
cid: container ID
json_mode: print or dump container in JSON format
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
to: path to dump encoded container
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container get',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def get_eacl(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None,
await_mode: bool = False, to: Optional[str] = None, session: Optional[str] = None,
ttl: Optional[int] = None, xhdr: Optional[dict] = None) -> str:
"""Get extended ACL talbe of container
Args:
address: address of wallet account
await_mode: block execution until container is removed
cid: container ID
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
to: path to dump encoded container
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container get-eacl',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def list(self, rpc_endpoint: str, wallet: str, address: Optional[str] = None,
owner: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, **params) -> str:
"""List all created containers
Args:
address: address of wallet account
owner: owner of containers (omit to use owner from private key)
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container list',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def list_objects(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None,
ttl: Optional[int] = None, xhdr: Optional[dict] = None) -> str:
"""List existing objects in container
Args:
address: address of wallet account
cid: container ID
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container list-objects',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def set_eacl(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None,
await_mode: bool = False, table: Optional[str] = None, session: Optional[str] = None,
ttl: Optional[int] = None, xhdr: Optional[dict] = None) -> str:
"""Set a new extended ACL table for the container.
Container ID in the EACL table will be substituted with the ID from the CLI.
Args:
address: address of wallet account
await_mode: block execution until container is removed
cid: container ID
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
table: path to file with JSON or binary encoded EACL table
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container set-eacl',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)

240
robot/resources/lib/python_keywords/cli/object.py Normal file
View file

@ -0,0 +1,240 @@
from typing import Optional
from .cli_command import NeofsCliCommandBase
class NeofsCliObject(NeofsCliCommandBase):
def delete(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None,
bearer: Optional[str] = None, session: Optional[str] = None, ttl: Optional[int] = None,
xhdr: Optional[list] = None, **params) -> str:
"""Delete object from NeoFS
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
oid: Object ID
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object delete',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def get(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None,
bearer: Optional[str] = None, file: Optional[str] = None,
header: Optional[str] = None, no_progress: bool = False, raw: bool = False,
session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str:
"""Get object from NeoFS
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
file: File to write object payload to. Default: stdout.
header: File to write header to. Default: stdout.
no_progress: Do not show progress bar
oid: Object ID
raw: Set raw request option
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object get',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def hash(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None,
bearer: Optional[str] = None, range: Optional[str] = None, salt: Optional[str] = None,
ttl: Optional[int] = None, hash_type: Optional[str] = None, xhdr: Optional[list] = None,
**params) -> str:
"""Get object hash
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
oid: Object ID
range: Range to take hash from in the form offset1:length1,...
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
salt: Salt in hex format
ttl: TTL value in request meta header (default 2)
hash_type: Hash type. Either 'sha256' or 'tz' (default "sha256")
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object hash',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def head(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None,
bearer: Optional[str] = None, file: Optional[str] = None,
json_mode: bool = False, main_only: bool = False, proto: bool = False, raw: bool = False,
session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str:
"""Get object header
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
file: File to write object payload to. Default: stdout.
json_mode: Marshal output in JSON
main_only: Return only main fields
oid: Object ID
proto: Marshal output in Protobuf
raw: Set raw request option
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object head',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def lock(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, lifetime: int, address: Optional[str] = None,
bearer: Optional[str] = None, session: Optional[str] = None,
ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str:
"""Lock object in container
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
oid: Object ID
lifetime: Object lifetime
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object lock',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def put(self, rpc_endpoint: str, wallet: str, cid: str, file: str, address: Optional[str] = None,
attributes: Optional[dict] = None, bearer: Optional[str] = None, disable_filename: bool = False,
disable_timestamp: bool = False, expire_at: Optional[int] = None, no_progress: bool = False,
notify: Optional[str] = None, session: Optional[str] = None, ttl: Optional[int] = None,
xhdr: Optional[list] = None, **params) -> str:
"""Put object to NeoFS
Args:
address: address of wallet account
attributes: User attributes in form of Key1=Value1,Key2=Value2
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
disable_filename: Do not set well-known filename attribute
disable_timestamp: Do not set well-known timestamp attribute
expire_at: Last epoch in the life of the object
file: File with object payload
no_progress: Do not show progress bar
notify: Object notification in the form of *epoch*:*topic*; '-' topic means using default
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object put',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def range(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, range: str, address: Optional[str] = None,
bearer: Optional[str] = None, file: Optional[str] = None, json_mode: bool = False, raw: bool = False,
session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str:
"""Get payload range data of an object
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
file: File to write object payload to. Default: stdout.
json_mode: Marshal output in JSON
oid: Object ID
range: Range to take data from in the form offset:length
raw: Set raw request option
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object range',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def search(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None,
bearer: Optional[str] = None, filters: Optional[list] = None, oid: Optional[str] = None,
phy: bool = False, root: bool = False, session: Optional[str] = None, ttl: Optional[int] = None,
xhdr: Optional[list] = None, **params) -> str:
"""Search object
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
filters: Repeated filter expressions or files with protobuf JSON
oid: Object ID
phy: Search physically stored objects
root: Search for user objects
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object search',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)

16
robot/resources/lib/python_keywords/cli_helpers.py
View file

@ -42,22 +42,26 @@ def _cmd_run(cmd: str, timeout: int = 30) -> str:
return output
except subprocess.CalledProcessError as exc:
logger.info(f"Error:\nreturn code: {exc.returncode} "
logger.info(f"Command: {cmd}\n"
f"Error:\nreturn code: {exc.returncode} "
f"\nOutput: {exc.output}")
end_time = datetime.now()
return_code, cmd_output = subprocess.getstatusoutput(cmd)
_attach_allure_log(cmd, cmd_output, return_code, start_time, end_time)
raise RuntimeError(f"Error:\nreturn code: {exc.returncode} "
f"\nOutput: {exc.output}") from exc
raise RuntimeError(f"Command: {cmd}\n"
f"Error:\nreturn code: {exc.returncode}\n"
f"Output: {exc.output}") from exc
except OSError as exc:
raise RuntimeError(f"Output: {exc.strerror}") from exc
raise RuntimeError(f"Command: {cmd}\n"
f"Output: {exc.strerror}") from exc
except Exception as exc:
return_code, cmd_output = subprocess.getstatusoutput(cmd)
end_time = datetime.now()
_attach_allure_log(cmd, cmd_output, return_code, start_time, end_time)
logger.info(f"Error:\nreturn code: {return_code}\nOutput: "
f"{exc.output.decode('utf-8') if type(exc.output) is bytes else exc.output}")
logger.info(f"Command: {cmd}\n"
f"Error:\nreturn code: {return_code}\n"
f"Output: {exc.output.decode('utf-8') if type(exc.output) is bytes else exc.output}")
raise

102
robot/resources/lib/python_keywords/container.py
View file

@ -5,24 +5,24 @@
"""
import json
import time
from typing import Optional
from time import sleep
from typing import Optional, Union
import json_transformers
from data_formatters import dict_to_attrs
from cli_helpers import _cmd_run
from common import NEOFS_ENDPOINT, NEOFS_CLI_EXEC, WALLET_CONFIG
from cli import NeofsCli
from common import NEOFS_ENDPOINT, WALLET_CONFIG
from robot.api import logger
from robot.api.deco import keyword
ROBOT_AUTO_KEYWORDS = False
DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
@keyword('Create Container')
def create_container(wallet: str, rule: str = DEFAULT_PLACEMENT_RULE, basic_acl: str = '',
attributes: Optional[dict] = None, session_token: str = '',
session_wallet: str = '', options: str = '') -> str:
session_wallet: str = '', name: str = None, options: dict = None,
await_mode: bool = True, wait_for_creation: bool = True) -> str:
"""
A wrapper for `neofs-cli container create` call.
@ -37,39 +37,51 @@ def create_container(wallet: str, rule: str = DEFAULT_PLACEMENT_RULE, basic_acl:
session_wallet(optional, str): a path to the wallet which signed
the session token; this parameter makes sense
when paired with `session_token`
options (optional, str): any other options to pass to the call
options (optional, dict): any other options to pass to the call
name (optional, str): container name attribute
await_mode (bool): block execution until container is persisted
wait_for_creation (): Wait for container shows in container list
Returns:
(str): CID of the created container
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} container create '
f'--wallet {session_wallet if session_wallet else wallet} '
f'--config {WALLET_CONFIG} --policy "{rule}" '
f'{"--basic-acl " + basic_acl if basic_acl else ""} '
f'{"--attributes " + dict_to_attrs(attributes) if attributes else ""} '
f'{"--session " + session_token if session_token else ""} '
f'{options} --await'
)
output = _cmd_run(cmd, timeout=60)
cli = NeofsCli(config=WALLET_CONFIG, timeout=60)
output = cli.container.create(rpc_endpoint=NEOFS_ENDPOINT, wallet=session_wallet if session_wallet else wallet,
policy=rule, basic_acl=basic_acl, attributes=attributes, name=name,
session=session_token, await_mode=await_mode, **options or {})
cid = _parse_cid(output)
logger.info("Container created; waiting until it is persisted in sidechain")
logger.info("Container created; waiting until it is persisted in the sidechain")
deadline_to_persist = 15 # seconds
for i in range(0, deadline_to_persist):
time.sleep(1)
if wait_for_creation:
wait_for_container_creation(wallet, cid)
return cid
def wait_for_container_creation(wallet: str, cid: str, attempts: int = 15, sleep_interval: int = 1):
for _ in range(attempts):
containers = list_containers(wallet)
if cid in containers:
break
logger.info(f"There is no {cid} in {containers} yet; continue")
if i + 1 == deadline_to_persist:
raise RuntimeError(
f"After {deadline_to_persist} seconds the container "
f"{cid} hasn't been persisted; exiting"
)
return cid
return
logger.info(f"There is no {cid} in {containers} yet; sleep {sleep_interval} and continue")
sleep(sleep_interval)
raise RuntimeError(f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting")
def wait_for_container_deletion(wallet: str, cid: str, attempts: int = 30, sleep_interval: int = 1):
for _ in range(attempts):
try:
get_container(wallet, cid)
sleep(sleep_interval)
continue
except Exception as err:
if 'container not found' not in str(err):
raise AssertionError(f'Expected "container not found" in error, got\n{err}')
return
raise AssertionError(f'Expected container deleted during {attempts * sleep_interval} sec.')
@keyword('List Containers')
@ -82,33 +94,30 @@ def list_containers(wallet: str) -> list[str]:
Returns:
(list): list of containers
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_CONFIG} container list'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=WALLET_CONFIG)
output = cli.container.list(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet)
logger.info(f"Containers: \n{output}")
return output.split()
@keyword('Get Container')
def get_container(wallet: str, cid: str, flag: str = '--json') -> dict:
def get_container(wallet: str, cid: str, json_mode: bool = True) -> Union[dict, str]:
"""
A wrapper for `neofs-cli container get` call. It extracts container's
attributes and rearranges them into a more compact view.
Args:
wallet (str): path to a wallet on whose behalf we get the container
cid (str): ID of the container to get
flag (str): output as json or plain text
json_mode (bool): return container in JSON format
Returns:
(dict, str): dict of container attributes
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_CONFIG} --cid {cid} container get {flag}'
)
output = _cmd_run(cmd)
if flag != '--json':
cli = NeofsCli(config=WALLET_CONFIG)
output = cli.container.get(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, json_mode=json_mode)
if not json_mode:
return output
container_info = json.loads(output)
attributes = dict()
for attr in container_info['attributes']:
@ -130,11 +139,8 @@ def delete_container(wallet: str, cid: str) -> None:
This function doesn't return anything.
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_CONFIG} container delete --cid {cid}'
)
_cmd_run(cmd)
cli = NeofsCli(config=WALLET_CONFIG)
cli.container.delete(wallet=wallet, cid=cid, rpc_endpoint=NEOFS_ENDPOINT)
def _parse_cid(output: str) -> str:

139
robot/resources/lib/python_keywords/neofs_verbs.py
View file

@ -1,32 +1,28 @@
#!/usr/bin/python3
'''
"""
This module contains wrappers for NeoFS verbs executed via neofs-cli.
'''
"""
import json
import os
import random
import re
import uuid
from typing import Optional
import json_transformers
from cli_helpers import _cmd_run
from cli import NeofsCli
from common import ASSETS_DIR, NEOFS_ENDPOINT, NEOFS_NETMAP, WALLET_CONFIG
from data_formatters import dict_to_attrs
from robot.api import logger
from robot.api.deco import keyword
ROBOT_AUTO_KEYWORDS = False
# path to neofs-cli executable
NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli')
@keyword('Get object')
def get_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
write_object: str = "", endpoint: str = "", options: str = "",
wallet_config: str = WALLET_CONFIG):
def get_object(wallet: str, cid: str, oid: str, bearer_token: Optional[str] = None, write_object: str = "",
endpoint: str = "", options: Optional[dict] = None, wallet_config: str = WALLET_CONFIG,
no_progress: bool = True):
"""
GET from NeoFS.
@ -38,6 +34,7 @@ def get_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
write_object (optional, str): path to downloaded file, appends to `--file` key
endpoint (optional, str): NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar
options (optional, str): any options which `neofs-cli object get` accepts
Returns:
(str): path to downloaded file
@ -50,20 +47,17 @@ def get_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
if not endpoint:
endpoint = random.sample(NEOFS_NETMAP, 1)[0]
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint} --wallet {wallet} '
f'object get --cid {cid} --oid {oid} --file {file_path} --config {wallet_config} '
f'{"--bearer " + bearer_token if bearer_token else ""} '
f'{options}'
)
_cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
cli.object.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, oid=oid, file=file_path,
bearer=bearer_token, no_progress=no_progress, **options or {})
return file_path
# TODO: make `bearer_token` optional
@keyword('Get Range Hash')
def get_range_hash(wallet: str, cid: str, oid: str, bearer_token: str, range_cut: str,
wallet_config: str = WALLET_CONFIG, options: str = ""):
wallet_config: str = WALLET_CONFIG, options: Optional[dict] = None):
"""
GETRANGEHASH of given Object.
@ -79,20 +73,19 @@ def get_range_hash(wallet: str, cid: str, oid: str, bearer_token: str, range_cut
Returns:
None
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object hash --cid {cid} --oid {oid} --range {range_cut} --config {wallet_config} '
f'{"--bearer " + bearer_token if bearer_token else ""} '
f'{options}'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
output = cli.object.hash(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid, range=range_cut,
bearer=bearer_token, **options or {})
# cutting off output about range offset and length
return output.split(':')[1].strip()
@keyword('Put object')
def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers: dict = {},
endpoint: str = "", wallet_config: str = WALLET_CONFIG, options: str = ""):
def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers: Optional[dict] = None,
endpoint: str = "", wallet_config: str = WALLET_CONFIG, expire_at: Optional[int] = None,
no_progress: bool = True, options: Optional[dict] = None):
"""
PUT of given file.
@ -104,7 +97,9 @@ def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers:
user_headers (optional, dict): Object attributes, append to `--attributes` key
endpoint(optional, str): NeoFS endpoint to send request to
wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar
options (optional, str): any options which `neofs-cli object put` accepts
expire_at (optional, int): Last epoch in the life of the object
Returns:
(str): ID of uploaded Object
"""
@ -112,13 +107,12 @@ def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers:
endpoint = random.sample(NEOFS_NETMAP, 1)[0]
if not endpoint:
logger.info(f'---DEB:\n{NEOFS_NETMAP}')
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint} --wallet {wallet} '
f'object put --file {path} --cid {cid} {options} --config {wallet_config} '
f'{"--bearer " + bearer if bearer else ""} '
f'{"--attributes " + dict_to_attrs(user_headers) if user_headers else ""}'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
output = cli.object.put(rpc_endpoint=endpoint, wallet=wallet, file=path, cid=cid, bearer=bearer,
expire_at=expire_at, no_progress=no_progress,
attributes=user_headers or {}, **options or {})
# splitting CLI output to lines and taking the penultimate line
id_str = output.strip().split('\n')[-2]
oid = id_str.split(':')[1]
@ -127,7 +121,7 @@ def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers:
@keyword('Delete object')
def delete_object(wallet: str, cid: str, oid: str, bearer: str = "", wallet_config: str = WALLET_CONFIG,
options: str = ""):
options: Optional[dict] = None):
"""
DELETE an Object.
@ -137,16 +131,15 @@ def delete_object(wallet: str, cid: str, oid: str, bearer: str = "", wallet_conf
oid (str): ID of Object we are going to delete
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
wallet_config(optional, str): path to the wallet config
options (optional, str): any options which `neofs-cli object delete` accepts
options (optional, dict): any options which `neofs-cli object delete` accepts
Returns:
(str): Tombstone ID
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object delete --cid {cid} --oid {oid} {options} --config {wallet_config} '
f'{"--bearer " + bearer if bearer else ""}'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
output = cli.object.delete(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid, bearer=bearer,
**options or {})
id_str = output.split('\n')[1]
tombstone = id_str.split(':')[1]
return tombstone.strip()
@ -154,7 +147,7 @@ def delete_object(wallet: str, cid: str, oid: str, bearer: str = "", wallet_conf
@keyword('Get Range')
def get_range(wallet: str, cid: str, oid: str, range_cut: str, wallet_config: str = WALLET_CONFIG,
bearer: str = "", options: str = ""):
bearer: str = "", options: Optional[dict] = None):
"""
GETRANGE an Object.
@ -165,35 +158,31 @@ def get_range(wallet: str, cid: str, oid: str, range_cut: str, wallet_config: st
range_cut (str): range to take data from in the form offset:length
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
wallet_config(optional, str): path to the wallet config
options (optional, str): any options which `neofs-cli object range` accepts
options (optional, dict): any options which `neofs-cli object range` accepts
Returns:
(str, bytes) - path to the file with range content and content of this file as bytes
"""
range_file = f"{ASSETS_DIR}/{uuid.uuid4()}"
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object range --cid {cid} --oid {oid} --range {range_cut} --config {wallet_config} '
f'{options} --file {range_file} '
f'{"--bearer " + bearer if bearer else ""} '
)
_cmd_run(cmd)
content = ''
cli = NeofsCli(config=wallet_config)
cli.object.range(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid, range=range_cut, file=range_file,
bearer=bearer, **options or {})
with open(range_file, 'rb') as fout:
content = fout.read()
return range_file, content
@keyword('Search object')
def search_object(wallet: str, cid: str, keys: str = "", bearer: str = "", filters: dict = {},
expected_objects_list=[], wallet_config: str = WALLET_CONFIG, options: str = ""):
def search_object(wallet: str, cid: str, bearer: str = "", filters: Optional[dict] = None,
expected_objects_list: Optional[list] = None, wallet_config: str = WALLET_CONFIG,
options: Optional[dict] = None):
"""
SEARCH an Object.
Args:
wallet (str): wallet on whose behalf SEARCH is done
cid (str): ID of Container where we get the Object from
keys(optional, str): any keys for Object SEARCH which `neofs-cli object search`
accepts, e.g. `--oid`
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
filters (optional, dict): key=value pairs to filter Objects
expected_objects_list (optional, list): a list of ObjectIDs to compare found Objects with
@ -202,19 +191,12 @@ def search_object(wallet: str, cid: str, keys: str = "", bearer: str = "", filte
Returns:
(list): list of found ObjectIDs
"""
filters_result = ""
if filters:
filters_result += "--filters "
logger.info(filters)
filters_result += ','.join(
map(lambda i: f"'{i} EQ {filters[i]}'", filters))
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object search {keys} --cid {cid} {filters_result} --config {wallet_config} '
f'{"--bearer " + bearer if bearer else ""} {options}'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
output = cli.object.search(
rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, bearer=bearer,
filters=[f'{filter_key} EQ {filter_val}' for filter_key, filter_val in filters.items()] if filters else None,
**options or {})
found_objects = re.findall(r'(\w{43,44})', output)
@ -231,7 +213,7 @@ def search_object(wallet: str, cid: str, keys: str = "", bearer: str = "", filte
@keyword('Head object')
def head_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
options: str = "", endpoint: str = "", json_output: bool = True,
options: Optional[dict] = None, endpoint: str = None, json_output: bool = True,
is_raw: bool = False, is_direct: bool = False, wallet_config: str = WALLET_CONFIG):
"""
HEAD an Object.
@ -256,20 +238,15 @@ def head_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
or
(str): HEAD response as a plain text
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint if endpoint else NEOFS_ENDPOINT} '
f'--wallet {wallet} --config {wallet_config} '
f'object head --cid {cid} --oid {oid} {options} '
f'{"--bearer " + bearer_token if bearer_token else ""} '
f'{"--json" if json_output else ""} '
f'{"--raw" if is_raw else ""} '
f'{"--ttl 1" if is_direct else ""}'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
output = cli.object.head(rpc_endpoint=endpoint or NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid,
bearer=bearer_token, json_mode=json_output, raw=is_raw,
ttl=1 if is_direct else None, **options or {})
if not json_output:
return output
decoded = ""
try:
decoded = json.loads(output)
except Exception as exc: