realloc/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Issues Pull requests Projects Releases Packages Wiki Activity

added keywords library container.py extracted from neofs.py

Browse source 
Signed-off-by: anastasia prasolova <anastasia@nspcc.ru>
This commit is contained in:
anastasia prasolova 2022-04-25 12:53:20 +03:00 • committed by Anastasia Prasolova
parent 14183dabfc
commit 3e749abcd0
18 changed files with 317 additions and 303 deletions
Download patch file Download diff file Expand all files Collapse all files

158
robot/resources/lib/python/container.py Normal file
View file

@ -0,0 +1,158 @@
#!/usr/bin/python3
"""
This module contains keywords which utilize `neofs-cli container`
commands.
"""
import json
import time
from common import NEOFS_ENDPOINT, COMMON_PLACEMENT_RULE, NEOFS_CLI_EXEC, WALLET_PASS
from cli_helpers import _cmd_run
from data_formatters import dict_to_attrs
from robot.api.deco import keyword
from robot.api import logger
ROBOT_AUTO_KEYWORDS = False
@keyword('Create Container')
def create_container(wallet: str, rule: str=COMMON_PLACEMENT_RULE, basic_acl: str='',
attributes: dict={}, session_token: str='', session_wallet: str='',
options: str=''):
"""
A wrapper for `neofs-cli container create` call.
Args:
wallet (str): a wallet on whose behalf a container is created
rule (optional, str): placement rule for container
basic_acl (optional, str): an ACL for container, will be
appended to `--basic-acl` key
attributes (optional, dict): container attributes , will be
appended to `--attributes` key
session_token (optional, str): a path to session token file
session_wallet(optional, str): a path to the wallet which signed
the session token; this parameter makes sense
when paired with `session_token`
options (optional, str): any other options to pass to the call
Returns:
(str): CID of the created container
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} container create '
f'--wallet {session_wallet if session_wallet else wallet} '
f'--config {WALLET_PASS} --policy "{rule}" '
f'{"--basic-acl " + basic_acl if basic_acl else ""} '
f'{"--attributes " + dict_to_attrs(attributes) if attributes else ""} '
f'{"--session " + session_token if session_token else ""} '
f'{options} --await'
)
output = _cmd_run(cmd, timeout=60)
cid = _parse_cid(output)
logger.info("Container created; waiting until it is persisted in sidechain")
deadline_to_persist = 15 # seconds
for i in range(0, deadline_to_persist):
time.sleep(1)
containers = list_containers(wallet)
if cid in containers:
break
logger.info(f"There is no {cid} in {containers} yet; continue")
if i+1 == deadline_to_persist:
raise RuntimeError(
f"After {deadline_to_persist} seconds the container "
f"{cid} hasn't been persisted; exiting"
)
return cid
@keyword('List Containers')
def list_containers(wallet: str):
"""
A wrapper for `neofs-cli container list` call. It returns all the
available containers for the given wallet.
Args:
wallet (str): a wallet on whose behalf we list the containers
Returns:
(list): list of containers
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_PASS} container list'
)
output = _cmd_run(cmd)
return output.split()
@keyword('Get Container Attributes')
def get_container_attributes(wallet: str, cid: str):
"""
A wrapper for `neofs-cli container get` call. It extracts
container attributes and rearranges them to more compact view.
Args:
wallet (str): a wallet on whose behalf we get the container
cid (str): ID of the container to get
Returns:
(dict): dict of container attributes
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_PASS} --cid {cid} container get --json'
)
output = _cmd_run(cmd)
container_info = json.loads(output)
attributes = dict()
for attr in container_info['attributes']:
attributes[attr['key']] = attr['value']
return attributes
@keyword('Delete Container')
# TODO: make the error message about a non-found container more user-friendly
# https://github.com/nspcc-dev/neofs-contract/issues/121
def delete_container(wallet: str, cid: str):
"""
A wrapper for `neofs-cli container delete` call.
Args:
wallet (str): a wallet on whose behalf we delete the container
cid (str): ID of the container to delete
This function doesn't return anything.
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_PASS} container delete --cid {cid}'
)
_cmd_run(cmd)
def _parse_cid(ouptut: str):
"""
This function parses CID from given CLI output. The input string we
expect:
container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN
awaiting...
container has been persisted on sidechain
We want to take 'container ID' value from the string.
Args:
ouptut (str): a command run output
Returns:
(str): extracted CID
"""
try:
# taking first string from command output
fst_str = ouptut.split('\n')[0]
except Exception:
logger.error(f"Got empty output: {ouptut}")
splitted = fst_str.split(": ")
if len(splitted) != 2:
raise ValueError(f"no CID was parsed from command output: \t{fst_str}")
return splitted[1]

21
robot/resources/lib/python/data_formatters.py Normal file
View file

@ -0,0 +1,21 @@
"""
A bunch of functions which might rearrange some data or
change their representation.
"""
from functools import reduce
def dict_to_attrs(attrs: dict):
'''
This function takes dictionary of object attributes and converts them
into the string. The string is passed to `--attibutes` key of the
neofs-cli.
Args:
attrs (dict): object attirbutes in {"a": "b", "c": "d"} format.
Returns:
(str): string in "a=b,c=d" format.
'''
return reduce(lambda a,b: f"{a},{b}", map(lambda i: f"{i}={attrs[i]}", attrs))

127
robot/resources/lib/python/neofs.py
View file

@ -17,7 +17,6 @@ from robot.api.deco import keyword
from robot.api import logger
from cli_helpers import _run_with_passwd, _cmd_run
import neofs_verbs
import json_transformers
ROBOT_AUTO_KEYWORDS = False
@ -123,53 +122,6 @@ def validate_storage_policy_for_object(wallet: str, expected_copies: int, cid, o
raise Exception(f"Found node list '{found_nodes}' is not equal to expected list '{expected_node_list}'")
@keyword('Create container')
def create_container(wallet: str, basic_acl:str, rule:str, user_headers: str='', session: str=''):
if rule == "":
logger.error("Cannot create container with empty placement rule")
if basic_acl:
basic_acl = f"--basic-acl {basic_acl}"
if user_headers:
user_headers = f"--attributes {user_headers}"
if session:
session = f"--session {session}"
createContainerCmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'container create --policy "{rule}" {basic_acl} {user_headers} {session} --config {WALLET_PASS} --await'
)
output = _cmd_run(createContainerCmd)
cid = _parse_cid(output)
logger.info(f"Created container {cid} with rule {rule}")
return cid
@keyword('Container List')
def container_list(wallet: str):
Cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'container list --config {WALLET_PASS}'
)
output = _cmd_run(Cmd)
container_list = re.findall(r'(\w{43,44})', output)
logger.info(f"Containers list: {container_list}")
return container_list
@keyword('Container Existing')
def container_existing(wallet: str, cid: str):
Cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'container list --config {WALLET_PASS}'
)
output = _cmd_run(Cmd)
_find_cid(output, cid)
return
@keyword('Verify Head Tombstone')
def verify_head_tombstone(wallet: str, cid: str, oid_ts: str, oid: str, addr: str):
# TODO: replace with HEAD from neofs_verbs.py
@ -219,50 +171,6 @@ def verify_head_tombstone(wallet: str, cid: str, oid_ts: str, oid: str, addr: st
raise Exception("Header Session OID (deleted object) is not expected.")
@keyword('Get container attributes')
def get_container_attributes(wallet: str, cid: str, endpoint: str="", json_output: bool = False):
if endpoint == "":
endpoint = NEOFS_ENDPOINT
container_cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint} --wallet {wallet} --config {WALLET_PASS} '
f'--cid {cid} container get {"--json" if json_output else ""}'
)
output = _cmd_run(container_cmd)
return output
@keyword('Decode Container Attributes Json')
def decode_container_attributes_json(header):
result_header = dict()
json_header = json.loads(header)
attributes = []
attribute_list = json_header["attributes"]
if attribute_list is not None:
for e in attribute_list:
values_list = list(e.values())
attribute = values_list[0] + '=' + values_list[1]
attributes.append(attribute)
result_header["Attributes"] = attributes
else:
raise Exception(f"no Attributes were parsed from header: \t{header}")
return result_header
@keyword('Delete Container')
# TODO: make the error message about a non-found container more user-friendly https://github.com/nspcc-dev/neofs-contract/issues/121
def delete_container(cid: str, wallet: str):
deleteContainerCmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'container delete --cid {cid} --config {WALLET_PASS}'
)
_cmd_run(deleteContainerCmd)
@keyword('Get file hash')
def get_file_hash(filename : str):
file_hash = _get_file_hash(filename)
@ -286,6 +194,7 @@ def get_control_endpoint_with_wif(endpoint_number: str = ''):
return endpoint_num, endpoint_control, wif
@keyword('Get Locode')
def get_locode():
endpoint_values = random.choice(list(NEOFS_NETMAP_DICT.values()))
@ -438,6 +347,7 @@ def delete_storagegroup(wallet: str, cid: str, oid: str, bearer_token: str=""):
raise Exception(f"no Tombstone ID was parsed from command output: \t{output}")
return oid
@keyword('Generate Session Token')
def generate_session_token(owner: str, pub_key: str, cid: str = "", wildcard: bool = False) -> str:
@ -497,17 +407,6 @@ def _get_file_hash(filename):
logger.info(f"Hash: {hash.hexdigest()}")
return hash.hexdigest()
def _find_cid(output: str, cid: str):
"""
This function parses CID from given CLI output.
Parameters:
- output: a string with command run output
"""
if re.search(fr'({cid})', output):
logger.info(f"CID {cid} was parsed from command output: \t{output}")
else:
raise Exception(f"no CID {cid} was parsed from command output: \t{output}")
return cid
def _parse_oid(input_str: str):
"""
@ -531,28 +430,6 @@ def _parse_oid(input_str: str):
raise Exception(f"no OID was parsed from command output: \t{snd_str}")
return splitted[1]
def _parse_cid(input_str: str):
"""
This function parses CID from given CLI output. The input string we
expect:
container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN
awaiting...
container has been persisted on sidechain
We want to take 'container ID' value from the string.
Parameters:
- input_str: a string with command run output
"""
try:
# taking first string from command output
fst_str = input_str.split('\n')[0]
except:
logger.error(f"Got empty output: {input_str}")
splitted = fst_str.split(": ")
if len(splitted) != 2:
raise Exception(f"no CID was parsed from command output: \t{fst_str}")
return splitted[1]
def _search_object(node:str, wallet: str, cid:str, oid: str):
Cmd = (

19
robot/resources/lib/python/neofs_verbs.py
View file

@ -9,11 +9,11 @@ import os
import re
import random
import uuid
from functools import reduce
from common import NEOFS_ENDPOINT, ASSETS_DIR, NEOFS_NETMAP, WALLET_PASS
from cli_helpers import _cmd_run
import json_transformers
from data_formatters import dict_to_attrs
from robot.api.deco import keyword
from robot.api import logger
@ -108,7 +108,7 @@ def put_object(wallet: str, path: str, cid: str, bearer: str="", user_headers: d
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint} --wallet {wallet} '
f'object put --file {path} --cid {cid} {options} --config {WALLET_PASS} '
f'{"--bearer " + bearer if bearer else ""} '
f'{"--attributes " + _dict_to_attrs(user_headers) if user_headers else ""}'
f'{"--attributes " + dict_to_attrs(user_headers) if user_headers else ""}'
)
output = _cmd_run(cmd)
# splitting CLI output to lines and taking the penultimate line
@ -282,18 +282,3 @@ def head_object(wallet: str, cid: str, oid: str, bearer_token: str="",
logger.info("decoding simple header")
return json_transformers.decode_simple_header(decoded)
def _dict_to_attrs(attrs: dict):
'''
This function takes dictionary of object attributes and converts them
into the string. The string is passed to `--attibutes` key of the
neofs-cli.
Args:
attrs (dict): object attirbutes in {"a": "b", "c": "d"} format.
Returns:
(str): string in "a=b,c=d" format.
'''
return reduce(lambda a,b: f"{a},{b}", map(lambda i: f"{i}={attrs[i]}", attrs))