Implement neofs-cli lib for container and object

Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
This commit is contained in:
Vladimir Avdeev 2022-08-19 05:22:20 +03:00 committed by Vladimir Domnich
parent d935c2cafa
commit 3294299612
14 changed files with 680 additions and 171 deletions

View file

@ -1,13 +1,11 @@
import json
from time import sleep
import allure
import pytest
from epoch import tick_epoch
from grpc_responses import CONTAINER_NOT_FOUND, error_matches_status
from python_keywords.container import (create_container, delete_container, get_container,
list_containers)
from python_keywords.container import (create_container, delete_container, get_container, list_containers,
wait_for_container_creation, wait_for_container_deletion)
from utility import placement_policy_from_container
from wellknown_acl import PRIVATE_ACL_F
@ -24,13 +22,12 @@ def test_container_creation(prepare_wallet_and_deposit, name):
json_wallet = json.load(file)
placement_rule = 'REP 2 IN X CBF 1 SELECT 2 FROM * AS X'
options = f"--name {name}" if name else ""
cid = create_container(wallet, rule=placement_rule, options=options)
cid = create_container(wallet, rule=placement_rule, name=name)
containers = list_containers(wallet)
assert cid in containers, f'Expected container {cid} in containers: {containers}'
container_info: str = get_container(wallet, cid, flag='')
container_info: str = get_container(wallet, cid, json_mode=False)
container_info = container_info.casefold() # To ignore case when comparing with expected values
info_to_check = {
@ -58,17 +55,25 @@ def test_container_creation(prepare_wallet_and_deposit, name):
wait_for_container_deletion(wallet, cid)
@allure.step('Wait for container deletion')
def wait_for_container_deletion(wallet: str, cid: str) -> None:
attempts, sleep_interval = 10, 5
for _ in range(attempts):
try:
get_container(wallet, cid)
sleep(sleep_interval)
continue
except Exception as err:
if error_matches_status(err, CONTAINER_NOT_FOUND):
return
raise AssertionError(f'Expected "{CONTAINER_NOT_FOUND}" error, got\n{err}')
@allure.title('Parallel container creation and deletion')
@pytest.mark.sanity
@pytest.mark.container
def test_container_creation_deletion_parallel(prepare_wallet_and_deposit):
containers_count = 3
wallet = prepare_wallet_and_deposit
placement_rule = 'REP 2 IN X CBF 1 SELECT 2 FROM * AS X'
raise AssertionError(f'Container was not deleted within {attempts * sleep_interval} sec')
cids: list[str] = []
with allure.step(f'Create {containers_count} containers'):
for _ in range(containers_count):
cids.append(create_container(wallet, rule=placement_rule, await_mode=False, wait_for_creation=False))
with allure.step(f'Wait for containers occur in container list'):
for cid in cids:
wait_for_container_creation(wallet, cid, sleep_interval=containers_count)
with allure.step('Delete containers and check they were deleted'):
for cid in cids:
delete_container(wallet, cid)
tick_epoch()
wait_for_container_deletion(wallet, cid)

View file

@ -4,9 +4,8 @@ from time import sleep
import allure
import pytest
from common import (STORAGE_NODE_SSH_PRIVATE_KEY_PATH, STORAGE_NODE_SSH_USER,
STORAGE_NODE_SSH_PASSWORD)
from common import (STORAGE_NODE_SSH_PASSWORD, STORAGE_NODE_SSH_PRIVATE_KEY_PATH,
STORAGE_NODE_SSH_USER)
from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes
from iptables_helper import IpTablesHelper
from python_keywords.container import create_container

View file

@ -2,8 +2,7 @@ import logging
import allure
import pytest
from common import (STORAGE_NODE_SSH_PRIVATE_KEY_PATH, STORAGE_NODE_SSH_USER,
STORAGE_NODE_SSH_PASSWORD)
from common import STORAGE_NODE_SSH_PASSWORD, STORAGE_NODE_SSH_PRIVATE_KEY_PATH, STORAGE_NODE_SSH_USER
from failover_utils import wait_all_storage_node_returned, wait_object_replication_on_nodes
from python_keywords.container import create_container
from python_keywords.neofs_verbs import get_object, put_object
@ -12,7 +11,6 @@ from sbercloud_helper import SberCloud, SberCloudConfig
from ssh_helper import HostClient
from wellknown_acl import PUBLIC_ACL
logger = logging.getLogger('NeoLogger')
stopped_hosts = []
@ -52,11 +50,11 @@ def return_all_storage_nodes(sbercloud_client: SberCloud) -> None:
wait_all_storage_node_returned()
@allure.title('Lost and return nodes')
@allure.title('Lost and returned nodes')
@pytest.mark.parametrize('hard_reboot', [True, False])
@pytest.mark.failover
def test_lost_storage_node(prepare_wallet_and_deposit, sbercloud_client: SberCloud,
cloud_infrastructure_check, hard_reboot: bool):
def test_lost_storage_node(prepare_wallet_and_deposit, sbercloud_client: SberCloud, cloud_infrastructure_check,
hard_reboot: bool):
wallet = prepare_wallet_and_deposit
placement_rule = 'REP 2 IN X CBF 2 SELECT 2 FROM * AS X'
source_file_path = generate_file()
@ -97,6 +95,7 @@ def test_panic_storage_node(prepare_wallet_and_deposit, cloud_infrastructure_che
oid = put_object(wallet, source_file_path, cid)
nodes = wait_object_replication_on_nodes(wallet, cid, oid, 2)
new_nodes: list[str] = []
allure.attach('\n'.join(nodes), 'Current nodes with object', allure.attachment_type.TEXT)
for node in nodes:
with allure.step(f'Hard reboot host {node} via magic SysRq option'):

View file

@ -360,7 +360,7 @@ def test_shards(prepare_wallet_and_deposit, create_container_and_pick_node):
@allure.step('Validate object has {expected_copies} copies')
def validate_object_copies(wallet: str, placement_rule: str, file_path: str, expected_copies: int):
cid = create_container(wallet, rule=placement_rule, basic_acl=PUBLIC_ACL)
got_policy = placement_policy_from_container(get_container(wallet, cid, flag=''))
got_policy = placement_policy_from_container(get_container(wallet, cid, json_mode=False))
assert got_policy == placement_rule.replace('\'', ''), \
f'Expected \n{placement_rule} and got policy \n{got_policy} are the same'
oid = put_object(wallet, file_path, cid)

View file

@ -3,14 +3,12 @@ from time import sleep
import allure
import pytest
from common import SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
from container import create_container
from epoch import get_epoch, tick_epoch
from grpc_responses import OBJECT_ALREADY_REMOVED, OBJECT_NOT_FOUND, error_matches_status
from python_keywords.neofs_verbs import (delete_object, get_object, get_range,
get_range_hash, head_object,
put_object, search_object)
from python_keywords.neofs_verbs import (delete_object, get_object, get_range, get_range_hash, head_object, put_object,
search_object)
from python_keywords.storage_policy import get_simple_object_copies
from python_keywords.utility_keywords import generate_file, get_file_hash
from tombstone import verify_head_tombstone
@ -116,7 +114,7 @@ def test_object_api_lifetime(prepare_wallet_and_deposit, request, object_size):
file_hash = get_file_hash(file_path)
epoch = get_epoch()
oid = put_object(wallet, file_path, cid, options=f'--expire-at {epoch + 1}')
oid = put_object(wallet, file_path, cid, expire_at=epoch + 1)
got_file = get_object(wallet, cid, oid)
assert get_file_hash(got_file) == file_hash

View file

@ -0,0 +1 @@
from .cli import NeofsCli

View file

@ -0,0 +1,25 @@
from typing import Optional
from .cli_command import NeofsCliCommandBase
class NeofsCliAccounting(NeofsCliCommandBase):
def balance(self, wallet: str, rpc_endpoint: str, address: Optional[str] = None,
owner: Optional[str] = None) -> str:
"""Get internal balance of NeoFS account
Args:
address: address of wallet account
owner: owner of balance account (omit to use owner from private key)
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
wallet: WIF (NEP-2) string or path to the wallet or binary key
Returns:
str: Command string
"""
return self._execute(
'accounting balance',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)

View file

@ -0,0 +1,32 @@
from typing import Optional
from common import NEOFS_CLI_EXEC
from .accounting import NeofsCliAccounting
from .cli_command import NeofsCliCommandBase
from .container import NeofsCliContainer
from .object import NeofsCliObject
class NeofsCli:
neofs_cli_exec_path: Optional[str] = None
config: Optional[str] = None
accounting: Optional[NeofsCliAccounting] = None
container: Optional[NeofsCliContainer] = None
object: Optional[NeofsCliObject] = None
def __init__(self, neofs_cli_exec_path: Optional[str] = None, config: Optional[str] = None, timeout: int = 30):
self.config = config # config(str): config file (default is $HOME/.config/neofs-cli/config.yaml)
self.neofs_cli_exec_path = neofs_cli_exec_path or NEOFS_CLI_EXEC
self.accounting = NeofsCliAccounting(self.neofs_cli_exec_path, timeout=timeout, config=config)
self.container = NeofsCliContainer(self.neofs_cli_exec_path, timeout=timeout, config=config)
self.object = NeofsCliObject(self.neofs_cli_exec_path, timeout=timeout, config=config)
def version(self) -> str:
"""Application version and NeoFS API compatibility
Returns:
str: Command string
"""
return NeofsCliCommandBase(self.neofs_cli_exec_path, config=self.config)._execute(command=None, version=True)

View file

@ -0,0 +1,39 @@
from typing import Optional
from cli_helpers import _cmd_run
class NeofsCliCommandBase:
neofs_cli_exec: Optional[str] = None
timeout: Optional[int] = None
__base_params: Optional[str] = None
map_params = {'json_mode': 'json', 'await_mode': 'await', 'hash_type': 'hash'}
def __init__(self, neofs_cli_exec: str, timeout: int = 30, **base_params):
self.neofs_cli_exec = neofs_cli_exec
self.timeout = timeout
self.__base_params = ' '.join([f'--{param} {value}' for param, value in base_params.items() if value])
def _format_command(self, command: str, **params) -> str:
param_str = []
for param, value in params.items():
if param in self.map_params.keys():
param = self.map_params[param]
param = param.replace('_', '-')
if not value:
continue
if isinstance(value, bool):
param_str.append(f'--{param}')
elif isinstance(value, list):
param_str.append(f'--{param} \'{",".join(value)}\'')
elif isinstance(value, dict):
param_str.append(f'--{param} \'{",".join(f"{key}={val}" for key, val in value.items())}\'')
else:
value_str = str(value).replace("'", "\\'")
param_str.append(f"--{param} '{value_str}'")
param_str = ' '.join(param_str)
return f'{self.neofs_cli_exec} {self.__base_params} {command or ""} {param_str}'
def _execute(self, command: Optional[str], **params) -> str:
return _cmd_run(self._format_command(command, **params), timeout=self.timeout)

View file

@ -0,0 +1,184 @@
from typing import Optional
from .cli_command import NeofsCliCommandBase
class NeofsCliContainer(NeofsCliCommandBase):
def create(self, rpc_endpoint: str, wallet: str, address: Optional[str] = None, attributes: Optional[dict] = None,
basic_acl: Optional[str] = None, await_mode: bool = False, disable_timestamp: bool = False,
name: Optional[str] = None, nonce: Optional[str] = None, policy: Optional[str] = None,
session: Optional[str] = None, subnet: Optional[str] = None, ttl: Optional[int] = None,
xhdr: Optional[list] = None) -> str:
"""Create a new container and register it in the NeoFS.
It will be stored in the sidechain when the Inner Ring accepts it.
Args:
address: address of wallet account
attributes: comma separated pairs of container attributes in form of Key1=Value1,Key2=Value2
await_mode: block execution until container is persisted
basic_acl: hex encoded basic ACL value or keywords like 'public-read-write', 'private',
'eacl-public-read' (default "private")
disable_timestamp: disable timestamp container attribute
name: container name attribute
nonce: UUIDv4 nonce value for container
policy: QL-encoded or JSON-encoded placement policy or path to file with it
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
subnet: string representation of container subnetwork
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container create',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def delete(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None, await_mode: bool = False,
session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None) -> str:
"""Delete an existing container.
Only the owner of the container has permission to remove the container.
Args:
address: address of wallet account
await_mode: block execution until container is removed
cid: container ID
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container delete',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def get(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None, await_mode: bool = False,
to: Optional[str] = None, json_mode: bool = False, ttl: Optional[int] = None,
xhdr: Optional[dict] = None) -> str:
"""Get container field info
Args:
address: address of wallet account
await_mode: block execution until container is removed
cid: container ID
json_mode: print or dump container in JSON format
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
to: path to dump encoded container
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container get',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def get_eacl(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None,
await_mode: bool = False, to: Optional[str] = None, session: Optional[str] = None,
ttl: Optional[int] = None, xhdr: Optional[dict] = None) -> str:
"""Get extended ACL talbe of container
Args:
address: address of wallet account
await_mode: block execution until container is removed
cid: container ID
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
to: path to dump encoded container
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container get-eacl',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def list(self, rpc_endpoint: str, wallet: str, address: Optional[str] = None,
owner: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[dict] = None, **params) -> str:
"""List all created containers
Args:
address: address of wallet account
owner: owner of containers (omit to use owner from private key)
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container list',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def list_objects(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None,
ttl: Optional[int] = None, xhdr: Optional[dict] = None) -> str:
"""List existing objects in container
Args:
address: address of wallet account
cid: container ID
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container list-objects',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)
def set_eacl(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None,
await_mode: bool = False, table: Optional[str] = None, session: Optional[str] = None,
ttl: Optional[int] = None, xhdr: Optional[dict] = None) -> str:
"""Set a new extended ACL table for the container.
Container ID in the EACL table will be substituted with the ID from the CLI.
Args:
address: address of wallet account
await_mode: block execution until container is removed
cid: container ID
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
table: path to file with JSON or binary encoded EACL table
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'container set-eacl',
**{param: param_value for param, param_value in locals().items() if param not in ['self']}
)

View file

@ -0,0 +1,240 @@
from typing import Optional
from .cli_command import NeofsCliCommandBase
class NeofsCliObject(NeofsCliCommandBase):
def delete(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None,
bearer: Optional[str] = None, session: Optional[str] = None, ttl: Optional[int] = None,
xhdr: Optional[list] = None, **params) -> str:
"""Delete object from NeoFS
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
oid: Object ID
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object delete',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def get(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None,
bearer: Optional[str] = None, file: Optional[str] = None,
header: Optional[str] = None, no_progress: bool = False, raw: bool = False,
session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str:
"""Get object from NeoFS
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
file: File to write object payload to. Default: stdout.
header: File to write header to. Default: stdout.
no_progress: Do not show progress bar
oid: Object ID
raw: Set raw request option
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object get',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def hash(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None,
bearer: Optional[str] = None, range: Optional[str] = None, salt: Optional[str] = None,
ttl: Optional[int] = None, hash_type: Optional[str] = None, xhdr: Optional[list] = None,
**params) -> str:
"""Get object hash
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
oid: Object ID
range: Range to take hash from in the form offset1:length1,...
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
salt: Salt in hex format
ttl: TTL value in request meta header (default 2)
hash_type: Hash type. Either 'sha256' or 'tz' (default "sha256")
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object hash',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def head(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, address: Optional[str] = None,
bearer: Optional[str] = None, file: Optional[str] = None,
json_mode: bool = False, main_only: bool = False, proto: bool = False, raw: bool = False,
session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str:
"""Get object header
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
file: File to write object payload to. Default: stdout.
json_mode: Marshal output in JSON
main_only: Return only main fields
oid: Object ID
proto: Marshal output in Protobuf
raw: Set raw request option
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object head',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def lock(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, lifetime: int, address: Optional[str] = None,
bearer: Optional[str] = None, session: Optional[str] = None,
ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str:
"""Lock object in container
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
oid: Object ID
lifetime: Object lifetime
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object lock',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def put(self, rpc_endpoint: str, wallet: str, cid: str, file: str, address: Optional[str] = None,
attributes: Optional[dict] = None, bearer: Optional[str] = None, disable_filename: bool = False,
disable_timestamp: bool = False, expire_at: Optional[int] = None, no_progress: bool = False,
notify: Optional[str] = None, session: Optional[str] = None, ttl: Optional[int] = None,
xhdr: Optional[list] = None, **params) -> str:
"""Put object to NeoFS
Args:
address: address of wallet account
attributes: User attributes in form of Key1=Value1,Key2=Value2
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
disable_filename: Do not set well-known filename attribute
disable_timestamp: Do not set well-known timestamp attribute
expire_at: Last epoch in the life of the object
file: File with object payload
no_progress: Do not show progress bar
notify: Object notification in the form of *epoch*:*topic*; '-' topic means using default
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object put',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def range(self, rpc_endpoint: str, wallet: str, cid: str, oid: str, range: str, address: Optional[str] = None,
bearer: Optional[str] = None, file: Optional[str] = None, json_mode: bool = False, raw: bool = False,
session: Optional[str] = None, ttl: Optional[int] = None, xhdr: Optional[list] = None, **params) -> str:
"""Get payload range data of an object
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
file: File to write object payload to. Default: stdout.
json_mode: Marshal output in JSON
oid: Object ID
range: Range to take data from in the form offset:length
raw: Set raw request option
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object range',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)
def search(self, rpc_endpoint: str, wallet: str, cid: str, address: Optional[str] = None,
bearer: Optional[str] = None, filters: Optional[list] = None, oid: Optional[str] = None,
phy: bool = False, root: bool = False, session: Optional[str] = None, ttl: Optional[int] = None,
xhdr: Optional[list] = None, **params) -> str:
"""Search object
Args:
address: address of wallet account
bearer: File with signed JSON or binary encoded bearer token
cid: Container ID
filters: Repeated filter expressions or files with protobuf JSON
oid: Object ID
phy: Search physically stored objects
root: Search for user objects
rpc_endpoint: remote node address (as 'multiaddr' or '<host>:<port>')
session: path to a JSON-encoded container session token
ttl: TTL value in request meta header (default 2)
wallet: WIF (NEP-2) string or path to the wallet or binary key
xhdr: Request X-Headers in form of Key=Value
Returns:
str: Command string
"""
return self._execute(
'object search',
**{param: param_value for param, param_value in locals().items() if param not in ['self', 'params']}
)

View file

@ -42,22 +42,26 @@ def _cmd_run(cmd: str, timeout: int = 30) -> str:
return output
except subprocess.CalledProcessError as exc:
logger.info(f"Error:\nreturn code: {exc.returncode} "
logger.info(f"Command: {cmd}\n"
f"Error:\nreturn code: {exc.returncode} "
f"\nOutput: {exc.output}")
end_time = datetime.now()
return_code, cmd_output = subprocess.getstatusoutput(cmd)
_attach_allure_log(cmd, cmd_output, return_code, start_time, end_time)
raise RuntimeError(f"Error:\nreturn code: {exc.returncode} "
f"\nOutput: {exc.output}") from exc
raise RuntimeError(f"Command: {cmd}\n"
f"Error:\nreturn code: {exc.returncode}\n"
f"Output: {exc.output}") from exc
except OSError as exc:
raise RuntimeError(f"Output: {exc.strerror}") from exc
raise RuntimeError(f"Command: {cmd}\n"
f"Output: {exc.strerror}") from exc
except Exception as exc:
return_code, cmd_output = subprocess.getstatusoutput(cmd)
end_time = datetime.now()
_attach_allure_log(cmd, cmd_output, return_code, start_time, end_time)
logger.info(f"Error:\nreturn code: {return_code}\nOutput: "
f"{exc.output.decode('utf-8') if type(exc.output) is bytes else exc.output}")
logger.info(f"Command: {cmd}\n"
f"Error:\nreturn code: {return_code}\n"
f"Output: {exc.output.decode('utf-8') if type(exc.output) is bytes else exc.output}")
raise

View file

@ -5,24 +5,24 @@
"""
import json
import time
from typing import Optional
from time import sleep
from typing import Optional, Union
import json_transformers
from data_formatters import dict_to_attrs
from cli_helpers import _cmd_run
from common import NEOFS_ENDPOINT, NEOFS_CLI_EXEC, WALLET_CONFIG
from cli import NeofsCli
from common import NEOFS_ENDPOINT, WALLET_CONFIG
from robot.api import logger
from robot.api.deco import keyword
ROBOT_AUTO_KEYWORDS = False
DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
@keyword('Create Container')
def create_container(wallet: str, rule: str = DEFAULT_PLACEMENT_RULE, basic_acl: str = '',
attributes: Optional[dict] = None, session_token: str = '',
session_wallet: str = '', options: str = '') -> str:
session_wallet: str = '', name: str = None, options: dict = None,
await_mode: bool = True, wait_for_creation: bool = True) -> str:
"""
A wrapper for `neofs-cli container create` call.
@ -37,39 +37,51 @@ def create_container(wallet: str, rule: str = DEFAULT_PLACEMENT_RULE, basic_acl:
session_wallet(optional, str): a path to the wallet which signed
the session token; this parameter makes sense
when paired with `session_token`
options (optional, str): any other options to pass to the call
options (optional, dict): any other options to pass to the call
name (optional, str): container name attribute
await_mode (bool): block execution until container is persisted
wait_for_creation (): Wait for container shows in container list
Returns:
(str): CID of the created container
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} container create '
f'--wallet {session_wallet if session_wallet else wallet} '
f'--config {WALLET_CONFIG} --policy "{rule}" '
f'{"--basic-acl " + basic_acl if basic_acl else ""} '
f'{"--attributes " + dict_to_attrs(attributes) if attributes else ""} '
f'{"--session " + session_token if session_token else ""} '
f'{options} --await'
)
output = _cmd_run(cmd, timeout=60)
cli = NeofsCli(config=WALLET_CONFIG, timeout=60)
output = cli.container.create(rpc_endpoint=NEOFS_ENDPOINT, wallet=session_wallet if session_wallet else wallet,
policy=rule, basic_acl=basic_acl, attributes=attributes, name=name,
session=session_token, await_mode=await_mode, **options or {})
cid = _parse_cid(output)
logger.info("Container created; waiting until it is persisted in sidechain")
logger.info("Container created; waiting until it is persisted in the sidechain")
deadline_to_persist = 15 # seconds
for i in range(0, deadline_to_persist):
time.sleep(1)
if wait_for_creation:
wait_for_container_creation(wallet, cid)
return cid
def wait_for_container_creation(wallet: str, cid: str, attempts: int = 15, sleep_interval: int = 1):
for _ in range(attempts):
containers = list_containers(wallet)
if cid in containers:
break
logger.info(f"There is no {cid} in {containers} yet; continue")
if i + 1 == deadline_to_persist:
raise RuntimeError(
f"After {deadline_to_persist} seconds the container "
f"{cid} hasn't been persisted; exiting"
)
return cid
return
logger.info(f"There is no {cid} in {containers} yet; sleep {sleep_interval} and continue")
sleep(sleep_interval)
raise RuntimeError(f"After {attempts * sleep_interval} seconds container {cid} hasn't been persisted; exiting")
def wait_for_container_deletion(wallet: str, cid: str, attempts: int = 30, sleep_interval: int = 1):
for _ in range(attempts):
try:
get_container(wallet, cid)
sleep(sleep_interval)
continue
except Exception as err:
if 'container not found' not in str(err):
raise AssertionError(f'Expected "container not found" in error, got\n{err}')
return
raise AssertionError(f'Expected container deleted during {attempts * sleep_interval} sec.')
@keyword('List Containers')
@ -82,33 +94,30 @@ def list_containers(wallet: str) -> list[str]:
Returns:
(list): list of containers
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_CONFIG} container list'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=WALLET_CONFIG)
output = cli.container.list(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet)
logger.info(f"Containers: \n{output}")
return output.split()
@keyword('Get Container')
def get_container(wallet: str, cid: str, flag: str = '--json') -> dict:
def get_container(wallet: str, cid: str, json_mode: bool = True) -> Union[dict, str]:
"""
A wrapper for `neofs-cli container get` call. It extracts container's
attributes and rearranges them into a more compact view.
Args:
wallet (str): path to a wallet on whose behalf we get the container
cid (str): ID of the container to get
flag (str): output as json or plain text
json_mode (bool): return container in JSON format
Returns:
(dict, str): dict of container attributes
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_CONFIG} --cid {cid} container get {flag}'
)
output = _cmd_run(cmd)
if flag != '--json':
cli = NeofsCli(config=WALLET_CONFIG)
output = cli.container.get(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, json_mode=json_mode)
if not json_mode:
return output
container_info = json.loads(output)
attributes = dict()
for attr in container_info['attributes']:
@ -130,11 +139,8 @@ def delete_container(wallet: str, cid: str) -> None:
This function doesn't return anything.
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_CONFIG} container delete --cid {cid}'
)
_cmd_run(cmd)
cli = NeofsCli(config=WALLET_CONFIG)
cli.container.delete(wallet=wallet, cid=cid, rpc_endpoint=NEOFS_ENDPOINT)
def _parse_cid(output: str) -> str:

View file

@ -1,32 +1,28 @@
#!/usr/bin/python3
'''
"""
This module contains wrappers for NeoFS verbs executed via neofs-cli.
'''
"""
import json
import os
import random
import re
import uuid
from typing import Optional
import json_transformers
from cli_helpers import _cmd_run
from cli import NeofsCli
from common import ASSETS_DIR, NEOFS_ENDPOINT, NEOFS_NETMAP, WALLET_CONFIG
from data_formatters import dict_to_attrs
from robot.api import logger
from robot.api.deco import keyword
ROBOT_AUTO_KEYWORDS = False
# path to neofs-cli executable
NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli')
@keyword('Get object')
def get_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
write_object: str = "", endpoint: str = "", options: str = "",
wallet_config: str = WALLET_CONFIG):
def get_object(wallet: str, cid: str, oid: str, bearer_token: Optional[str] = None, write_object: str = "",
endpoint: str = "", options: Optional[dict] = None, wallet_config: str = WALLET_CONFIG,
no_progress: bool = True):
"""
GET from NeoFS.
@ -38,6 +34,7 @@ def get_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
write_object (optional, str): path to downloaded file, appends to `--file` key
endpoint (optional, str): NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar
options (optional, str): any options which `neofs-cli object get` accepts
Returns:
(str): path to downloaded file
@ -50,20 +47,17 @@ def get_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
if not endpoint:
endpoint = random.sample(NEOFS_NETMAP, 1)[0]
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint} --wallet {wallet} '
f'object get --cid {cid} --oid {oid} --file {file_path} --config {wallet_config} '
f'{"--bearer " + bearer_token if bearer_token else ""} '
f'{options}'
)
_cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
cli.object.get(rpc_endpoint=endpoint, wallet=wallet, cid=cid, oid=oid, file=file_path,
bearer=bearer_token, no_progress=no_progress, **options or {})
return file_path
# TODO: make `bearer_token` optional
@keyword('Get Range Hash')
def get_range_hash(wallet: str, cid: str, oid: str, bearer_token: str, range_cut: str,
wallet_config: str = WALLET_CONFIG, options: str = ""):
wallet_config: str = WALLET_CONFIG, options: Optional[dict] = None):
"""
GETRANGEHASH of given Object.
@ -79,20 +73,19 @@ def get_range_hash(wallet: str, cid: str, oid: str, bearer_token: str, range_cut
Returns:
None
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object hash --cid {cid} --oid {oid} --range {range_cut} --config {wallet_config} '
f'{"--bearer " + bearer_token if bearer_token else ""} '
f'{options}'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
output = cli.object.hash(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid, range=range_cut,
bearer=bearer_token, **options or {})
# cutting off output about range offset and length
return output.split(':')[1].strip()
@keyword('Put object')
def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers: dict = {},
endpoint: str = "", wallet_config: str = WALLET_CONFIG, options: str = ""):
def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers: Optional[dict] = None,
endpoint: str = "", wallet_config: str = WALLET_CONFIG, expire_at: Optional[int] = None,
no_progress: bool = True, options: Optional[dict] = None):
"""
PUT of given file.
@ -104,7 +97,9 @@ def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers:
user_headers (optional, dict): Object attributes, append to `--attributes` key
endpoint(optional, str): NeoFS endpoint to send request to
wallet_config(optional, str): path to the wallet config
no_progress(optional, bool): do not show progress bar
options (optional, str): any options which `neofs-cli object put` accepts
expire_at (optional, int): Last epoch in the life of the object
Returns:
(str): ID of uploaded Object
"""
@ -112,13 +107,12 @@ def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers:
endpoint = random.sample(NEOFS_NETMAP, 1)[0]
if not endpoint:
logger.info(f'---DEB:\n{NEOFS_NETMAP}')
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint} --wallet {wallet} '
f'object put --file {path} --cid {cid} {options} --config {wallet_config} '
f'{"--bearer " + bearer if bearer else ""} '
f'{"--attributes " + dict_to_attrs(user_headers) if user_headers else ""}'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
output = cli.object.put(rpc_endpoint=endpoint, wallet=wallet, file=path, cid=cid, bearer=bearer,
expire_at=expire_at, no_progress=no_progress,
attributes=user_headers or {}, **options or {})
# splitting CLI output to lines and taking the penultimate line
id_str = output.strip().split('\n')[-2]
oid = id_str.split(':')[1]
@ -127,7 +121,7 @@ def put_object(wallet: str, path: str, cid: str, bearer: str = "", user_headers:
@keyword('Delete object')
def delete_object(wallet: str, cid: str, oid: str, bearer: str = "", wallet_config: str = WALLET_CONFIG,
options: str = ""):
options: Optional[dict] = None):
"""
DELETE an Object.
@ -137,16 +131,15 @@ def delete_object(wallet: str, cid: str, oid: str, bearer: str = "", wallet_conf
oid (str): ID of Object we are going to delete
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
wallet_config(optional, str): path to the wallet config
options (optional, str): any options which `neofs-cli object delete` accepts
options (optional, dict): any options which `neofs-cli object delete` accepts
Returns:
(str): Tombstone ID
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object delete --cid {cid} --oid {oid} {options} --config {wallet_config} '
f'{"--bearer " + bearer if bearer else ""}'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
output = cli.object.delete(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid, bearer=bearer,
**options or {})
id_str = output.split('\n')[1]
tombstone = id_str.split(':')[1]
return tombstone.strip()
@ -154,7 +147,7 @@ def delete_object(wallet: str, cid: str, oid: str, bearer: str = "", wallet_conf
@keyword('Get Range')
def get_range(wallet: str, cid: str, oid: str, range_cut: str, wallet_config: str = WALLET_CONFIG,
bearer: str = "", options: str = ""):
bearer: str = "", options: Optional[dict] = None):
"""
GETRANGE an Object.
@ -165,35 +158,31 @@ def get_range(wallet: str, cid: str, oid: str, range_cut: str, wallet_config: st
range_cut (str): range to take data from in the form offset:length
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
wallet_config(optional, str): path to the wallet config
options (optional, str): any options which `neofs-cli object range` accepts
options (optional, dict): any options which `neofs-cli object range` accepts
Returns:
(str, bytes) - path to the file with range content and content of this file as bytes
"""
range_file = f"{ASSETS_DIR}/{uuid.uuid4()}"
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object range --cid {cid} --oid {oid} --range {range_cut} --config {wallet_config} '
f'{options} --file {range_file} '
f'{"--bearer " + bearer if bearer else ""} '
)
_cmd_run(cmd)
content = ''
cli = NeofsCli(config=wallet_config)
cli.object.range(rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid, range=range_cut, file=range_file,
bearer=bearer, **options or {})
with open(range_file, 'rb') as fout:
content = fout.read()
return range_file, content
@keyword('Search object')
def search_object(wallet: str, cid: str, keys: str = "", bearer: str = "", filters: dict = {},
expected_objects_list=[], wallet_config: str = WALLET_CONFIG, options: str = ""):
def search_object(wallet: str, cid: str, bearer: str = "", filters: Optional[dict] = None,
expected_objects_list: Optional[list] = None, wallet_config: str = WALLET_CONFIG,
options: Optional[dict] = None):
"""
SEARCH an Object.
Args:
wallet (str): wallet on whose behalf SEARCH is done
cid (str): ID of Container where we get the Object from
keys(optional, str): any keys for Object SEARCH which `neofs-cli object search`
accepts, e.g. `--oid`
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
filters (optional, dict): key=value pairs to filter Objects
expected_objects_list (optional, list): a list of ObjectIDs to compare found Objects with
@ -202,19 +191,12 @@ def search_object(wallet: str, cid: str, keys: str = "", bearer: str = "", filte
Returns:
(list): list of found ObjectIDs
"""
filters_result = ""
if filters:
filters_result += "--filters "
logger.info(filters)
filters_result += ','.join(
map(lambda i: f"'{i} EQ {filters[i]}'", filters))
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object search {keys} --cid {cid} {filters_result} --config {wallet_config} '
f'{"--bearer " + bearer if bearer else ""} {options}'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
output = cli.object.search(
rpc_endpoint=NEOFS_ENDPOINT, wallet=wallet, cid=cid, bearer=bearer,
filters=[f'{filter_key} EQ {filter_val}' for filter_key, filter_val in filters.items()] if filters else None,
**options or {})
found_objects = re.findall(r'(\w{43,44})', output)
@ -231,7 +213,7 @@ def search_object(wallet: str, cid: str, keys: str = "", bearer: str = "", filte
@keyword('Head object')
def head_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
options: str = "", endpoint: str = "", json_output: bool = True,
options: Optional[dict] = None, endpoint: str = None, json_output: bool = True,
is_raw: bool = False, is_direct: bool = False, wallet_config: str = WALLET_CONFIG):
"""
HEAD an Object.
@ -256,20 +238,15 @@ def head_object(wallet: str, cid: str, oid: str, bearer_token: str = "",
or
(str): HEAD response as a plain text
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint if endpoint else NEOFS_ENDPOINT} '
f'--wallet {wallet} --config {wallet_config} '
f'object head --cid {cid} --oid {oid} {options} '
f'{"--bearer " + bearer_token if bearer_token else ""} '
f'{"--json" if json_output else ""} '
f'{"--raw" if is_raw else ""} '
f'{"--ttl 1" if is_direct else ""}'
)
output = _cmd_run(cmd)
cli = NeofsCli(config=wallet_config)
output = cli.object.head(rpc_endpoint=endpoint or NEOFS_ENDPOINT, wallet=wallet, cid=cid, oid=oid,
bearer=bearer_token, json_mode=json_output, raw=is_raw,
ttl=1 if is_direct else None, **options or {})
if not json_output:
return output
decoded = ""
try:
decoded = json.loads(output)
except Exception as exc: