TrueCloudLab/frostfs-testcases
26
0
Fork 21
Code Issues 2 Pull requests 3 Releases Activity Actions

robot/resources/lib/python -> robot/resources/lib/python_keywords

Browse source 
Signed-off-by: anastasia prasolova <anastasia@nspcc.ru>
This commit is contained in:
anastasia prasolova 2022-05-30 18:51:59 +03:00 • committed by Anastasia Prasolova
parent a97e1ee1e9
commit 3e31c527d2
17 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

167
robot/resources/lib/python_keywords/acl.py Normal file
View file

@ -0,0 +1,167 @@
#!/usr/bin/python3.8
from enum import Enum, auto
import json
import os
import re
import uuid
import base64
import base58
from cli_helpers import _cmd_run
from common import ASSETS_DIR, NEOFS_ENDPOINT, WALLET_PASS
from robot.api.deco import keyword
from robot.api import logger
"""
Robot Keywords and helper functions for work with NeoFS ACL.
"""
ROBOT_AUTO_KEYWORDS = False
# path to neofs-cli executable
NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli')
EACL_LIFETIME = 100500
class AutoName(Enum):
def _generate_next_value_(name, start, count, last_values):
return name
class Role(AutoName):
USER = auto()
SYSTEM = auto()
OTHERS = auto()
@keyword('Get eACL')
def get_eacl(wallet: str, cid: str):
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'container get-eacl --cid {cid} --config {WALLET_PASS}'
)
try:
output = _cmd_run(cmd)
if re.search(r'extended ACL table is not set for this container', output):
return None
return output
except RuntimeError as exc:
logger.info("Extended ACL table is not set for this container")
logger.info(f"Got exception while getting eacl: {exc}")
return None
@keyword('Set eACL')
def set_eacl(wallet: str, cid: str, eacl_table_path: str):
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'container set-eacl --cid {cid} --table {eacl_table_path} --config {WALLET_PASS} --await'
)
_cmd_run(cmd)
def _encode_cid_for_eacl(cid: str) -> str:
cid_base58 = base58.b58decode(cid)
return base64.b64encode(cid_base58).decode("utf-8")
@keyword('Create eACL')
def create_eacl(cid: str, rules_list: list):
table = f"{os.getcwd()}/{ASSETS_DIR}/eacl_table_{str(uuid.uuid4())}.json"
rules = ""
for rule in rules_list:
# TODO: check if $Object: is still necessary for filtering in the newest releases
rules += f"--rule '{rule}' "
cmd = (
f"{NEOFS_CLI_EXEC} acl extended create --cid {cid} "
f"{rules}--out {table}"
)
_cmd_run(cmd)
return table
@keyword('Form BearerToken File')
def form_bearertoken_file(wif: str, cid: str, eacl_records: list) -> str:
"""
This function fetches eACL for given <cid> on behalf of <wif>,
then extends it with filters taken from <eacl_records>, signs
with bearer token and writes to file
"""
enc_cid = _encode_cid_for_eacl(cid)
file_path = f"{os.getcwd()}/{ASSETS_DIR}/{str(uuid.uuid4())}"
eacl = get_eacl(wif, cid)
json_eacl = dict()
if eacl:
eacl = eacl.replace('eACL: ', '')
eacl = eacl.split('Signature')[0]
json_eacl = json.loads(eacl)
logger.info(json_eacl)
eacl_result = {
"body":
{
"eaclTable":
{
"containerID":
{
"value": enc_cid
},
"records": []
},
"lifetime":
{
"exp": EACL_LIFETIME,
"nbf": "1",
"iat": "0"
}
}
}
if not eacl_records:
raise(f"Got empty eacl_records list: {eacl_records}")
for record in eacl_records:
op_data = {
"operation": record['Operation'],
"action": record['Access'],
"filters": [],
"targets": []
}
if Role(record['Role']):
op_data['targets'] = [
{
"role": record['Role']
}
]
else:
op_data['targets'] = [
{
"keys": [ record['Role'] ]
}
]
if 'Filters' in record.keys():
op_data["filters"].append(record['Filters'])
eacl_result["body"]["eaclTable"]["records"].append(op_data)
# Add records from current eACL
if "records" in json_eacl.keys():
for record in json_eacl["records"]:
eacl_result["body"]["eaclTable"]["records"].append(record)
with open(file_path, 'w', encoding='utf-8') as eacl_file:
json.dump(eacl_result, eacl_file, ensure_ascii=False, indent=4)
logger.info(f"Got these extended ACL records: {eacl_result}")
sign_bearer_token(wif, file_path)
return file_path
def sign_bearer_token(wallet: str, eacl_rules_file: str):
cmd = (
f'{NEOFS_CLI_EXEC} util sign bearer-token --from {eacl_rules_file} '
f'--to {eacl_rules_file} --wallet {wallet} --config {WALLET_PASS} --json'
)
_cmd_run(cmd)

44
robot/resources/lib/python_keywords/cli_helpers.py Normal file
View file

@ -0,0 +1,44 @@
#!/usr/bin/python3.8
"""
Helper functions to use with `neofs-cli`, `neo-go`
and other CLIs.
"""
import subprocess
import pexpect
from robot.api import logger
ROBOT_AUTO_KEYWORDS = False
def _cmd_run(cmd, timeout=30):
"""
Runs given shell command <cmd>, in case of success returns its stdout,
in case of failure returns error message.
"""
try:
logger.info(f"Executing command: {cmd}")
compl_proc = subprocess.run(cmd, check=True, universal_newlines=True,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout,
shell=True)
output = compl_proc.stdout
logger.info(f"Output: {output}")
return output
except subprocess.CalledProcessError as exc:
raise RuntimeError(f"Error:\nreturn code: {exc.returncode} "
f"\nOutput: {exc.output}") from exc
except Exception as exc:
return_code, _ = subprocess.getstatusoutput(cmd)
logger.info(f"Error:\nreturn code: {return_code}\nOutput: "
f"{exc.output.decode('utf-8') if type(exc.output) is bytes else exc.output}")
raise
def _run_with_passwd(cmd):
child = pexpect.spawn(cmd)
child.expect(".*")
child.sendline('\r')
child.wait()
cmd = child.read()
return cmd.decode()

23
robot/resources/lib/python_keywords/cli_keywords.py Normal file
View file

@ -0,0 +1,23 @@
#!/usr/bin/python3.8
import pexpect
from robot.api.deco import keyword
ROBOT_AUTO_KEYWORDS = False
@keyword('Run Process And Enter Empty Password')
def run_proccess_and_interact(cmd: str) -> str:
p = pexpect.spawn(cmd)
p.expect("[pP]assword")
# enter empty password
p.sendline('\r')
p.wait()
# throw a string with password prompt
first = p.readline()
# take all output
child_output = p.readline()
p.close()
if p.exitstatus != 0:
raise Exception(f"{first}\n{child_output}")
return child_output

76
robot/resources/lib/python_keywords/complex_object_actions.py Normal file
View file

@ -0,0 +1,76 @@
#!/usr/bin/python3
"""
This module contains functions which are used for Large Object assemling:
getting Last Object and split and getting Link Object. It is not enough to
simply perform a "raw" HEAD request, as noted in the issue:
https://github.com/nspcc-dev/neofs-node/issues/1304. Therefore, the reliable
retrival of the aforementioned objects must be done this way: send direct
"raw" HEAD request to the every Storage Node and return the desired OID on
first non-null response.
"""
from common import NEOFS_NETMAP
import neofs_verbs
from robot.api.deco import keyword
from robot.api import logger
from robot.libraries.BuiltIn import BuiltIn
ROBOT_AUTO_KEYWORDS = False
@keyword('Get Link Object')
def get_link_object(wallet: str, cid: str, oid: str, bearer_token: str=""):
"""
Args:
wallet (str): path to the wallet on whose behalf the Storage Nodes
are requested
cid (str): Container ID which stores the Large Object
oid (str): Large Object ID
bearer_token (optional, str): path to Bearer token file
Returns:
(str): Link Object ID
When no Link Object ID is found after all Storage Nodes polling,
the function throws a native robot error.
"""
for node in NEOFS_NETMAP:
try:
resp = neofs_verbs.head_object(wallet, cid, oid,
endpoint=node,
is_raw=True,
is_direct=True,
bearer_token=bearer_token)
if resp['link']:
return resp['link']
except Exception:
logger.info(f"No Link Object found on {node}; continue")
BuiltIn().fail(f"No Link Object for {cid}/{oid} found among all Storage Nodes")
return None
@keyword('Get Last Object')
def get_last_object(wallet: str, cid: str, oid: str):
"""
Args:
wallet (str): path to the wallet on whose behalf the Storage Nodes
are requested
cid (str): Container ID which stores the Large Object
oid (str): Large Object ID
Returns:
(str): Last Object ID
When no Last Object ID is found after all Storage Nodes polling,
the function throws a native robot error.
"""
for node in NEOFS_NETMAP:
try:
resp = neofs_verbs.head_object(wallet, cid, oid,
endpoint=node,
is_raw=True,
is_direct=True)
if resp['lastPart']:
return resp['lastPart']
except Exception:
logger.info(f"No Last Object found on {node}; continue")
BuiltIn().fail(f"No Last Object for {cid}/{oid} found among all Storage Nodes")
return None

158
robot/resources/lib/python_keywords/container.py Normal file
View file

@ -0,0 +1,158 @@
#!/usr/bin/python3
"""
This module contains keywords which utilize `neofs-cli container`
commands.
"""
import json
import time
from common import NEOFS_ENDPOINT, COMMON_PLACEMENT_RULE, NEOFS_CLI_EXEC, WALLET_PASS
from cli_helpers import _cmd_run
from data_formatters import dict_to_attrs
from robot.api.deco import keyword
from robot.api import logger
ROBOT_AUTO_KEYWORDS = False
@keyword('Create Container')
def create_container(wallet: str, rule: str=COMMON_PLACEMENT_RULE, basic_acl: str='',
attributes: dict={}, session_token: str='', session_wallet: str='',
options: str=''):
"""
A wrapper for `neofs-cli container create` call.
Args:
wallet (str): a wallet on whose behalf a container is created
rule (optional, str): placement rule for container
basic_acl (optional, str): an ACL for container, will be
appended to `--basic-acl` key
attributes (optional, dict): container attributes , will be
appended to `--attributes` key
session_token (optional, str): a path to session token file
session_wallet(optional, str): a path to the wallet which signed
the session token; this parameter makes sense
when paired with `session_token`
options (optional, str): any other options to pass to the call
Returns:
(str): CID of the created container
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} container create '
f'--wallet {session_wallet if session_wallet else wallet} '
f'--config {WALLET_PASS} --policy "{rule}" '
f'{"--basic-acl " + basic_acl if basic_acl else ""} '
f'{"--attributes " + dict_to_attrs(attributes) if attributes else ""} '
f'{"--session " + session_token if session_token else ""} '
f'{options} --await'
)
output = _cmd_run(cmd, timeout=60)
cid = _parse_cid(output)
logger.info("Container created; waiting until it is persisted in sidechain")
deadline_to_persist = 15 # seconds
for i in range(0, deadline_to_persist):
time.sleep(1)
containers = list_containers(wallet)
if cid in containers:
break
logger.info(f"There is no {cid} in {containers} yet; continue")
if i+1 == deadline_to_persist:
raise RuntimeError(
f"After {deadline_to_persist} seconds the container "
f"{cid} hasn't been persisted; exiting"
)
return cid
@keyword('List Containers')
def list_containers(wallet: str):
"""
A wrapper for `neofs-cli container list` call. It returns all the
available containers for the given wallet.
Args:
wallet (str): a wallet on whose behalf we list the containers
Returns:
(list): list of containers
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_PASS} container list'
)
output = _cmd_run(cmd)
return output.split()
@keyword('Get Container Attributes')
def get_container_attributes(wallet: str, cid: str):
"""
A wrapper for `neofs-cli container get` call. It extracts
container attributes and rearranges them to more compact view.
Args:
wallet (str): a wallet on whose behalf we get the container
cid (str): ID of the container to get
Returns:
(dict): dict of container attributes
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_PASS} --cid {cid} container get --json'
)
output = _cmd_run(cmd)
container_info = json.loads(output)
attributes = dict()
for attr in container_info['attributes']:
attributes[attr['key']] = attr['value']
return attributes
@keyword('Delete Container')
# TODO: make the error message about a non-found container more user-friendly
# https://github.com/nspcc-dev/neofs-contract/issues/121
def delete_container(wallet: str, cid: str):
"""
A wrapper for `neofs-cli container delete` call.
Args:
wallet (str): a wallet on whose behalf we delete the container
cid (str): ID of the container to delete
This function doesn't return anything.
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_PASS} container delete --cid {cid}'
)
_cmd_run(cmd)
def _parse_cid(ouptut: str):
"""
This function parses CID from given CLI output. The input string we
expect:
container ID: 2tz86kVTDpJxWHrhw3h6PbKMwkLtBEwoqhHQCKTre1FN
awaiting...
container has been persisted on sidechain
We want to take 'container ID' value from the string.
Args:
ouptut (str): a command run output
Returns:
(str): extracted CID
"""
try:
# taking first string from command output
fst_str = ouptut.split('\n')[0]
except Exception:
logger.error(f"Got empty output: {ouptut}")
splitted = fst_str.split(": ")
if len(splitted) != 2:
raise ValueError(f"no CID was parsed from command output: \t{fst_str}")
return splitted[1]

21
robot/resources/lib/python_keywords/data_formatters.py Normal file
View file

@ -0,0 +1,21 @@
"""
A bunch of functions which might rearrange some data or
change their representation.
"""
from functools import reduce
def dict_to_attrs(attrs: dict):
'''
This function takes dictionary of object attributes and converts them
into the string. The string is passed to `--attibutes` key of the
neofs-cli.
Args:
attrs (dict): object attirbutes in {"a": "b", "c": "d"} format.
Returns:
(str): string in "a=b,c=d" format.
'''
return reduce(lambda a,b: f"{a},{b}", map(lambda i: f"{i}={attrs[i]}", attrs))

35
robot/resources/lib/python_keywords/http_gate.py Normal file
View file

@ -0,0 +1,35 @@
#!/usr/bin/python3
import shutil
import requests
from common import HTTP_GATE
from robot.api.deco import keyword
from robot.api import logger
from robot.libraries.BuiltIn import BuiltIn
ROBOT_AUTO_KEYWORDS = False
ASSETS_DIR = BuiltIn().get_variable_value("${ASSETS_DIR}")
@keyword('Get via HTTP Gate')
def get_via_http_gate(cid: str, oid: str):
"""
This function gets given object from HTTP gate
:param cid: CID to get object from
:param oid: object OID
"""
request = f'{HTTP_GATE}/get/{cid}/{oid}'
resp = requests.get(request, stream=True)
if not resp.ok:
raise Exception(f"""Failed to get object via HTTP gate:
request: {resp.request.path_url},
response: {resp.text},
status code: {resp.status_code} {resp.reason}""")
logger.info(f'Request: {request}')
filename = f"{ASSETS_DIR}/{cid}_{oid}"
with open(filename, "wb") as get_file:
shutil.copyfileobj(resp.raw, get_file)
return filename

119
robot/resources/lib/python_keywords/json_transformers.py Normal file
View file

@ -0,0 +1,119 @@
#!/usr/bin/python3
'''
When doing requests to NeoFS, we get JSON output as an automatically decoded
structure from protobuf. Some fields are decoded with boilerplates and binary
values are Base64-encoded.
This module contains functions which rearrange the structure and reencode binary
data from Base64 to Base58.
'''
import base64
import base58
ROBOT_AUTO_KEYWORDS = False
def decode_simple_header(data: dict):
'''
This function reencodes Simple Object header and its attributes.
'''
try:
data = decode_common_fields(data)
# object attributes view normalization
ugly_attrs = data["header"]["attributes"]
data["header"]["attributes"] = {}
for attr in ugly_attrs:
data["header"]["attributes"][attr["key"]] = attr["value"]
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_split_header(data: dict):
'''
This function rearranges Complex Object header.
The header holds SplitID, a random unique
number, which is common among all splitted objects, and IDs of the Linking
Object and the last splitted Object.
'''
try:
data["splitId"] = json_reencode(data["splitId"])
data["lastPart"] = (
json_reencode(data["lastPart"]["value"])
if data["lastPart"] else None
)
data["link"] = (
json_reencode(data["link"]["value"])
if data["link"] else None
)
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_linking_object(data: dict):
'''
This function reencodes Linking Object header.
It contains IDs of child Objects and Split Chain data.
'''
try:
data = decode_simple_header(data)
# reencoding Child Object IDs
# { 'value': <Base58 encoded OID> } -> <Base64 encoded OID>
for ind, val in enumerate(data['header']['split']['children']):
data['header']['split']['children'][ind] = json_reencode(val['value'])
data['header']['split']['splitID'] = json_reencode(data['header']['split']['splitID'])
data['header']['split']['previous'] = (
json_reencode(data['header']['split']['previous']['value'])
if data['header']['split']['previous'] else None
)
data['header']['split']['parent'] = (
json_reencode(data['header']['split']['parent']['value'])
if data['header']['split']['parent'] else None
)
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def decode_storage_group(data: dict):
'''
This function reencodes Storage Group header.
'''
try:
data = decode_common_fields(data)
except Exception as exc:
raise ValueError(f"failed to decode JSON output: {exc}") from exc
return data
def json_reencode(data: str):
'''
According to JSON protocol, binary data (Object/Container/Storage Group IDs, etc)
is converted to string via Base58 encoder. But we usually operate with Base64-encoded
format.
This function reencodes given Base58 string into the Base64 one.
'''
return base58.b58encode(base64.b64decode(data)).decode("utf-8")
def decode_common_fields(data: dict):
'''
Despite of type (simple/complex Object, Storage Group, etc) every Object
header contains several common fields.
This function rearranges these fields.
'''
# reencoding binary IDs
data["objectID"] = json_reencode(data["objectID"]["value"])
data["header"]["containerID"] = json_reencode(data["header"]["containerID"]["value"])
data["header"]["ownerID"] = json_reencode(data["header"]["ownerID"]["value"])
data["header"]["homomorphicHash"] = json_reencode(data["header"]["homomorphicHash"]["sum"])
data["header"]["payloadHash"] = json_reencode(data["header"]["payloadHash"]["sum"])
data["header"]["version"] = (
f"{data['header']['version']['major']}{data['header']['version']['minor']}"
)
return data

155
robot/resources/lib/python_keywords/neofs.py Normal file
View file

@ -0,0 +1,155 @@
#!/usr/bin/python3
import base64
import json
import os
import re
import random
import uuid
import base58
from neo3 import wallet
from common import (NEOFS_NETMAP, WALLET_PASS, NEOFS_ENDPOINT,
NEOFS_NETMAP_DICT, ASSETS_DIR)
from cli_helpers import _cmd_run
import json_transformers
from robot.api.deco import keyword
from robot.api import logger
ROBOT_AUTO_KEYWORDS = False
# path to neofs-cli executable
NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli')
# TODO: move to neofs-keywords
@keyword('Get ScriptHash')
def get_scripthash(wif: str):
acc = wallet.Account.from_wif(wif, '')
return str(acc.script_hash)
@keyword('Verify Head Tombstone')
def verify_head_tombstone(wallet: str, cid: str, oid_ts: str, oid: str, addr: str):
# TODO: replace with HEAD from neofs_verbs.py
object_cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'--config {WALLET_PASS} object head --cid {cid} --oid {oid_ts} --json'
)
output = _cmd_run(object_cmd)
full_headers = json.loads(output)
logger.info(f"Output: {full_headers}")
# Header verification
header_cid = full_headers["header"]["containerID"]["value"]
if json_transformers.json_reencode(header_cid) == cid:
logger.info(f"Header CID is expected: {cid} ({header_cid} in the output)")
else:
raise Exception("Header CID is not expected.")
header_owner = full_headers["header"]["ownerID"]["value"]
if json_transformers.json_reencode(header_owner) == addr:
logger.info(f"Header ownerID is expected: {addr} ({header_owner} in the output)")
else:
raise Exception("Header ownerID is not expected.")
header_type = full_headers["header"]["objectType"]
if header_type == "TOMBSTONE":
logger.info(f"Header Type is expected: {header_type}")
else:
raise Exception("Header Type is not expected.")
header_session_type = full_headers["header"]["sessionToken"]["body"]["object"]["verb"]
if header_session_type == "DELETE":
logger.info(f"Header Session Type is expected: {header_session_type}")
else:
raise Exception("Header Session Type is not expected.")
header_session_cid = full_headers["header"]["sessionToken"]["body"]["object"]["address"]["containerID"]["value"]
if json_transformers.json_reencode(header_session_cid) == cid:
logger.info(f"Header ownerID is expected: {addr} ({header_session_cid} in the output)")
else:
raise Exception("Header Session CID is not expected.")
header_session_oid = full_headers["header"]["sessionToken"]["body"]["object"]["address"]["objectID"]["value"]
if json_transformers.json_reencode(header_session_oid) == oid:
logger.info(f"Header Session OID (deleted object) is expected: {oid} ({header_session_oid} in the output)")
else:
raise Exception("Header Session OID (deleted object) is not expected.")
@keyword('Get control endpoint with wif')
def get_control_endpoint_with_wif(endpoint_number: str = ''):
if endpoint_number == '':
netmap = []
for key in NEOFS_NETMAP_DICT.keys():
netmap.append(key)
endpoint_num = random.sample(netmap, 1)[0]
logger.info(f'Random node chosen: {endpoint_num}')
else:
endpoint_num = endpoint_number
endpoint_values = NEOFS_NETMAP_DICT[f'{endpoint_num}']
endpoint_control = endpoint_values['control']
wif = endpoint_values['wif']
return endpoint_num, endpoint_control, wif
@keyword('Get Locode')
def get_locode():
endpoint_values = random.choice(list(NEOFS_NETMAP_DICT.values()))
locode = endpoint_values['UN-LOCODE']
logger.info(f'Random locode chosen: {locode}')
return locode
@keyword('Generate Session Token')
def generate_session_token(owner: str, pub_key: str, cid: str = "", wildcard: bool = False) -> str:
file_path = f"{os.getcwd()}/{ASSETS_DIR}/{str(uuid.uuid4())}"
owner_64 = base64.b64encode(base58.b58decode(owner)).decode('utf-8')
cid_64 = base64.b64encode(cid.encode('utf-8')).decode('utf-8')
pub_key_64 = base64.b64encode(bytes.fromhex(pub_key)).decode('utf-8')
id_64 = base64.b64encode(uuid.uuid4().bytes).decode('utf-8')
session_token = {
"body":{
"id":f"{id_64}",
"ownerID":{
"value":f"{owner_64}"
},
"lifetime":{
"exp":"100000000",
"nbf":"0",
"iat":"0"
},
"sessionKey":f"{pub_key_64}",
"container":{
"verb":"PUT",
"wildcard": wildcard,
**({ "containerID":{"value":f"{cid_64}"} } if not wildcard else {})
}
}
}
logger.info(f"Got this Session Token: {session_token}")
with open(file_path, 'w', encoding='utf-8') as session_token_file:
json.dump(session_token, session_token_file, ensure_ascii=False, indent=4)
return file_path
@keyword ('Sign Session Token')
def sign_session_token(session_token: str, wallet: str, to_file: str=''):
if to_file:
to_file = f'--to {to_file}'
cmd = (
f'{NEOFS_CLI_EXEC} util sign session-token --from {session_token} '
f'-w {wallet} {to_file} --config {WALLET_PASS}'
)
logger.info(f"cmd: {cmd}")
_cmd_run(cmd)

289
robot/resources/lib/python_keywords/neofs_verbs.py Normal file
View file

@ -0,0 +1,289 @@
#!/usr/bin/python3
'''
This module contains wrappers for NeoFS verbs executed via neofs-cli.
'''
import json
import os
import re
import random
import uuid
from common import NEOFS_ENDPOINT, ASSETS_DIR, NEOFS_NETMAP, WALLET_PASS
from cli_helpers import _cmd_run
import json_transformers
from data_formatters import dict_to_attrs
from robot.api.deco import keyword
from robot.api import logger
ROBOT_AUTO_KEYWORDS = False
# path to neofs-cli executable
NEOFS_CLI_EXEC = os.getenv('NEOFS_CLI_EXEC', 'neofs-cli')
@keyword('Get object')
def get_object(wallet: str, cid: str, oid: str, bearer_token: str="",
write_object: str="", endpoint: str="", options: str="" ):
'''
GET from NeoFS.
Args:
wif (str): WIF of the wallet on whose behalf GET is done
cid (str): ID of Container where we get the Object from
oid (str): Object ID
bearer_token (optional, str): path to Bearer Token file, appends to `--bearer` key
write_object (optional, str): path to downloaded file, appends to `--file` key
endpoint (optional, str): NeoFS endpoint to send request to, appends to `--rpc-endpoint` key
options (optional, str): any options which `neofs-cli object get` accepts
Returns:
(str): path to downloaded file
'''
if not write_object:
write_object = str(uuid.uuid4())
file_path = f"{ASSETS_DIR}/{write_object}"
if not endpoint:
endpoint = random.sample(NEOFS_NETMAP, 1)[0]
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint} --wallet {wallet} '
f'object get --cid {cid} --oid {oid} --file {file_path} --config {WALLET_PASS} '
f'{"--bearer " + bearer_token if bearer_token else ""} '
f'{options}'
)
_cmd_run(cmd)
return file_path
@keyword('Get Range Hash')
def get_range_hash(wallet: str, cid: str, oid: str, bearer_token: str,
range_cut: str, options: str=""):
'''
GETRANGEHASH of given Object.
Args:
wif (str): WIF of the wallet on whose behalf GETRANGEHASH is done
cid (str): ID of Container where we get the Object from
oid (str): Object ID
bearer_token (str): path to Bearer Token file, appends to `--bearer` key
range_cut (str): Range to take hash from in the form offset1:length1,...,
value to pass to the `--range` parameter
options (optional, str): any options which `neofs-cli object hash` accepts
Returns:
None
'''
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object hash --cid {cid} --oid {oid} --range {range_cut} --config {WALLET_PASS} '
f'{"--bearer " + bearer_token if bearer_token else ""} '
f'{options}'
)
_cmd_run(cmd)
@keyword('Put object')
def put_object(wallet: str, path: str, cid: str, bearer: str="", user_headers: dict={},
endpoint: str="", options: str="" ):
'''
PUT of given file.
Args:
wif (str): WIF of the wallet on whose behalf PUT is done
path (str): path to file to be PUT
cid (str): ID of Container where we get the Object from
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
user_headers (optional, dict): Object attributes, append to `--attributes` key
endpoint(optional, str): NeoFS endpoint to send request to
options (optional, str): any options which `neofs-cli object put` accepts
Returns:
(str): ID of uploaded Object
'''
if not endpoint:
endpoint = random.sample(NEOFS_NETMAP, 1)[0]
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint} --wallet {wallet} '
f'object put --file {path} --cid {cid} {options} --config {WALLET_PASS} '
f'{"--bearer " + bearer if bearer else ""} '
f'{"--attributes " + dict_to_attrs(user_headers) if user_headers else ""}'
)
output = _cmd_run(cmd)
# splitting CLI output to lines and taking the penultimate line
id_str = output.strip().split('\n')[-2]
oid = id_str.split(':')[1]
return oid.strip()
@keyword('Delete object')
def delete_object(wallet: str, cid: str, oid: str, bearer: str="", options: str=""):
'''
DELETE an Object.
Args:
wif (str): WIF of the wallet on whose behalf DELETE is done
cid (str): ID of Container where we get the Object from
oid (str): ID of Object we are going to delete
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
options (optional, str): any options which `neofs-cli object delete` accepts
Returns:
(str): Tombstone ID
'''
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object delete --cid {cid} --oid {oid} {options} --config {WALLET_PASS} '
f'{"--bearer " + bearer if bearer else ""}'
)
output = _cmd_run(cmd)
id_str = output.split('\n')[1]
tombstone = id_str.split(':')[1]
return tombstone.strip()
@keyword('Get Range')
def get_range(wallet: str, cid: str, oid: str, range_file: str, bearer: str,
range_cut: str, options:str=""):
'''
GETRANGE an Object.
Args:
wif (str): WIF of the wallet on whose behalf GETRANGE is done
cid (str): ID of Container where we get the Object from
oid (str): ID of Object we are going to request
range_file (str): file where payload range data will be written
bearer (str): path to Bearer Token file, appends to `--bearer` key
range_cut (str): range to take data from in the form offset:length
options (optional, str): any options which `neofs-cli object range` accepts
Returns:
None
'''
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object range --cid {cid} --oid {oid} --range {range_cut} --config {WALLET_PASS} '
f'--file {ASSETS_DIR}/{range_file} {options} '
f'{"--bearer " + bearer if bearer else ""} '
)
_cmd_run(cmd)
@keyword('Search object')
def search_object(wallet: str, cid: str, keys: str="", bearer: str="", filters: dict={},
expected_objects_list=[], options:str=""):
'''
GETRANGE an Object.
Args:
wif (str): WIF of the wallet on whose behalf SEARCH is done
cid (str): ID of Container where we get the Object from
keys(optional, str): any keys for Object SEARCH which `neofs-cli object search`
accepts, e.g. `--oid`
bearer (optional, str): path to Bearer Token file, appends to `--bearer` key
filters (optional, dict): key=value pairs to filter Objects
expected_objects_list (optional, list): a list of ObjectIDs to compare found Objects with
options (optional, str): any options which `neofs-cli object search` accepts
Returns:
(list): list of found ObjectIDs
'''
filters_result = ""
if filters:
filters_result += "--filters "
logger.info(filters)
filters_result += ','.join(map(lambda i: f"'{i} EQ {filters[i]}'", filters))
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wallet {wallet} '
f'object search {keys} --cid {cid} {filters_result} {options} --config {WALLET_PASS} '
f'{"--bearer " + bearer if bearer else ""}'
)
output = _cmd_run(cmd)
found_objects = re.findall(r'(\w{43,44})', output)
if expected_objects_list:
if sorted(found_objects) == sorted(expected_objects_list):
logger.info(f"Found objects list '{found_objects}' ",
f"is equal for expected list '{expected_objects_list}'")
else:
raise Exception(f"Found object list {found_objects} ",
f"is not equal to expected list '{expected_objects_list}'")
return found_objects
@keyword('Head object')
def head_object(wallet: str, cid: str, oid: str, bearer_token: str="",
options:str="", endpoint: str="", json_output: bool = True,
is_raw: bool = False, is_direct: bool = False):
'''
HEAD an Object.
Args:
wif (str): WIF of the wallet on whose behalf HEAD is done
cid (str): ID of Container where we get the Object from
oid (str): ObjectID to HEAD
bearer_token (optional, str): path to Bearer Token file, appends to `--bearer` key
options (optional, str): any options which `neofs-cli object head` accepts
endpoint(optional, str): NeoFS endpoint to send request to
json_output(optional, bool): return reponse in JSON format or not; this flag
turns into `--json` key
is_raw(optional, bool): send "raw" request or not; this flag
turns into `--raw` key
is_direct(optional, bool): send request directly to the node or not; this flag
turns into `--ttl 1` key
Returns:
depending on the `json_output` parameter value, the function returns
(dict): HEAD response in JSON format
or
(str): HEAD response as a plain text
'''
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {endpoint if endpoint else NEOFS_ENDPOINT} '
f'--wallet {wallet} --config {WALLET_PASS} '
f'object head --cid {cid} --oid {oid} {options} '
f'{"--bearer " + bearer_token if bearer_token else ""} '
f'{"--json" if json_output else ""} '
f'{"--raw" if is_raw else ""} '
f'{"--ttl 1" if is_direct else ""}'
)
output = None
try:
output = _cmd_run(cmd)
except Exception as exc:
logger.info(f"Head request failed with output: {output}")
return None
if not json_output:
return output
decoded = ""
try:
decoded = json.loads(output)
except Exception as exc:
# If we failed to parse output as JSON, the cause might be
# the plain text string in the beginning of the output.
# Here we cut off first string and try to parse again.
logger.info(f"failed to parse output: {exc}")
logger.info("parsing output in another way")
fst_line_idx = output.find('\n')
decoded = json.loads(output[fst_line_idx:])
# If response is Complex Object header, it has `splitId` key
if 'splitId' in decoded.keys():
logger.info("decoding split header")
return json_transformers.decode_split_header(decoded)
# If response is Last or Linking Object header,
# it has `header` dictionary and non-null `split` dictionary
if 'split' in decoded['header'].keys():
if decoded['header']['split']:
logger.info("decoding linking object")
return json_transformers.decode_linking_object(decoded)
if decoded['header']['objectType'] == 'STORAGE_GROUP':
logger.info("decoding storage group")
return json_transformers.decode_storage_group(decoded)
logger.info("decoding simple header")
return json_transformers.decode_simple_header(decoded)

47
robot/resources/lib/python_keywords/nodes_management.py Normal file
View file

@ -0,0 +1,47 @@
#!/usr/bin/python3
"""
This module contains keywords for management test stand
nodes. It assumes that nodes are docker containers.
"""
import random
import docker
from robot.api.deco import keyword
ROBOT_AUTO_KEYWORDS = False
@keyword('Stop Nodes')
def stop_nodes(number: int, nodes: list):
"""
The function shuts down the given number of randomly
selected nodes in docker.
Args:
number (int): the number of nodes to shut down
nodes (list): the list of nodes for possible shut down
Returns:
(list): the list of nodes which have been shut down
"""
nodes = random.sample(nodes, number)
client = docker.APIClient()
for node in nodes:
node = node.split('.')[0]
client.stop(node)
return nodes
@keyword('Start Nodes')
def start_nodes(nodes: list):
"""
The function raises the given nodes.
Args:
nodes (list): the list of nodes to raise
Returns:
(void)
"""
client = docker.APIClient()
for node in nodes:
node = node.split('.')[0]
client.start(node)

95
robot/resources/lib/python_keywords/payment_neogo.py Normal file
View file

@ -0,0 +1,95 @@
#!/usr/bin/python3
import os
import pexpect
import re
from robot.api.deco import keyword
from robot.api import logger
from neo3 import wallet
from common import *
import rpc_client
import contract
from wrappers import run_sh_with_passwd_contract
ROBOT_AUTO_KEYWORDS = False
MORPH_TOKEN_POWER = 12
morph_rpc_cli = rpc_client.RPCClient(NEOFS_NEO_API_ENDPOINT)
mainnet_rpc_cli = rpc_client.RPCClient(NEO_MAINNET_ENDPOINT)
@keyword('Withdraw Mainnet Gas')
def withdraw_mainnet_gas(wallet: str, address: str, scripthash: str, amount: int):
cmd = (
f"{NEOGO_CLI_EXEC} contract invokefunction -w {wallet} -a {address} "
f"-r {NEO_MAINNET_ENDPOINT} {NEOFS_CONTRACT} withdraw {scripthash} "
f"int:{amount} -- {scripthash}:Global"
)
logger.info(f"Executing command: {cmd}")
out = (run_sh_with_passwd_contract('', cmd, expect_confirmation=True)).decode('utf-8')
logger.info(f"Command completed with output: {out}")
m = re.match(r'^Sent invocation transaction (\w{64})$', out)
if m is None:
raise Exception("Can not get Tx.")
tx = m.group(1)
return tx
@keyword('Transaction accepted in block')
def transaction_accepted_in_block(tx_id: str):
"""
This function return True in case of accepted TX.
Parameters:
:param tx_id: transaction ID
"""
try:
resp = mainnet_rpc_cli.get_transaction_height(tx_id)
if resp is not None:
logger.info(f"got block height: {resp}")
return True
except Exception as e:
logger.info(f"request failed with error: {e}")
raise e
@keyword('Get NeoFS Balance')
def get_balance(wif: str):
"""
This function returns NeoFS balance for given WIF.
"""
acc = wallet.Account.from_wif(wif, '')
payload = [
{
'type': 'Hash160',
'value': str(acc.script_hash)
}
]
try:
resp = morph_rpc_cli.invoke_function(
contract.get_balance_contract_hash(NEOFS_NEO_API_ENDPOINT),
'balanceOf',
payload
)
logger.info(resp)
value = int(resp['stack'][0]['value'])
return value/(10**MORPH_TOKEN_POWER)
except Exception as e:
logger.error(f"failed to get {wif} balance: {e}")
raise e
def _run_sh_with_passwd(passwd, cmd):
p = pexpect.spawn(cmd)
p.expect(".*")
p.sendline(passwd + '\r')
p.wait()
# throw a string with password prompt
# take a string with tx hash
tx_hash = p.read().splitlines()[-1]
return tx_hash.decode()

236
robot/resources/lib/python_keywords/s3_gate.py Normal file
View file

@ -0,0 +1,236 @@
#!/usr/bin/python3
import json
import os
import uuid
import boto3
import botocore
from cli_helpers import _run_with_passwd
from common import GATE_PUB_KEY, NEOFS_ENDPOINT, S3_GATE
import urllib3
from robot.api.deco import keyword
from robot.api import logger
##########################################################
# Disabling warnings on self-signed certificate which the
# boto library produces on requests to S3-gate in dev-env.
urllib3.disable_warnings()
##########################################################
ROBOT_AUTO_KEYWORDS = False
CREDENTIALS_CREATE_TIMEOUT = '30s'
NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate')
@keyword('Init S3 Credentials')
def init_s3_credentials(wallet):
bucket = str(uuid.uuid4())
s3_bearer_rules = "robot/resources/files/s3_bearer_rules.json"
cmd = (
f'{NEOFS_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} '
f'issue-secret --wallet {wallet} --gate-public-key={GATE_PUB_KEY} '
f'--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} '
f'--bearer-rules {s3_bearer_rules}'
)
logger.info(f"Executing command: {cmd}")
try:
output = _run_with_passwd(cmd)
logger.info(f"Command completed with output: {output}")
# first five string are log output, cutting them off and parse
# the rest of the output as JSON
output = '\n'.join(output.split('\n')[5:])
output_dict = json.loads(output)
return (output_dict['container_id'],
bucket,
output_dict['access_key_id'],
output_dict['secret_access_key'],
output_dict['owner_private_key'])
except Exception as exc:
raise RuntimeError("failed to init s3 credentials") from exc
@keyword('Config S3 client')
def config_s3_client(access_key_id, secret_access_key):
try:
session = boto3.session.Session()
s3_client = session.client(
service_name='s3',
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
endpoint_url=S3_GATE, verify=False
)
return s3_client
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('List objects S3 v2')
def list_objects_s3_v2(s3_client, bucket):
try:
response = s3_client.list_objects_v2(Bucket=bucket)
logger.info(f"S3 v2 List objects result: {response['Contents']}")
obj_list = []
for obj in response['Contents']:
obj_list.append(obj['Key'])
logger.info(f"Found s3 objects: {obj_list}")
return obj_list
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('List objects S3')
def list_objects_s3(s3_client, bucket):
try:
response = s3_client.list_objects(Bucket=bucket)
logger.info(f"S3 List objects result: {response['Contents']}")
obj_list = []
for obj in response['Contents']:
obj_list.append(obj['Key'])
logger.info(f"Found s3 objects: {obj_list}")
return obj_list
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Create bucket S3')
def create_bucket_s3(s3_client):
bucket_name = str(uuid.uuid4())
try:
s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
logger.info(f"Created S3 bucket: {s3_bucket}")
return bucket_name
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('List buckets S3')
def list_buckets_s3(s3_client):
found_buckets = []
try:
response = s3_client.list_buckets()
logger.info(f"S3 List buckets result: {response}")
for bucket in response['Buckets']:
found_buckets.append(bucket['Name'])
return found_buckets
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Delete bucket S3')
def delete_bucket_s3(s3_client, bucket):
try:
response = s3_client.delete_bucket(Bucket=bucket)
logger.info(f"S3 Delete bucket result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Head bucket S3')
def head_bucket(s3_client, bucket):
try:
response = s3_client.head_bucket(Bucket=bucket)
logger.info(f"S3 Head bucket result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Put object S3')
def put_object_s3(s3_client, bucket, filepath):
filename = os.path.basename(filepath)
with open(filepath, "rb") as put_file:
file_content = put_file.read()
try:
response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename)
logger.info(f"S3 Put object result: {response}")
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Head object S3')
def head_object_s3(s3_client, bucket, object_key):
try:
response = s3_client.head_object(Bucket=bucket, Key=object_key)
logger.info(f"S3 Head object result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Delete object S3')
def delete_object_s3(s3_client, bucket, object_key):
try:
response = s3_client.delete_object(Bucket=bucket, Key=object_key)
logger.info(f"S3 Put object result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Copy object S3')
def copy_object_s3(s3_client, bucket, object_key):
filename = f"{os.getcwd()}/{uuid.uuid4()}"
try:
response = s3_client.copy_object(Bucket=bucket,
CopySource=f"{bucket}/{object_key}",
Key=filename)
logger.info(f"S3 Copy object result: {response}")
return filename
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Get object S3')
def get_object_s3(s3_client, bucket, object_key):
filename = f"{os.getcwd()}/{uuid.uuid4()}"
try:
response = s3_client.get_object(Bucket=bucket, Key=object_key)
with open(f"{filename}", 'wb') as get_file:
chunk = response['Body'].read(1024)
while chunk:
get_file.write(chunk)
chunk = response['Body'].read(1024)
return filename
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err

126
robot/resources/lib/python_keywords/storage_group.py Normal file
View file

@ -0,0 +1,126 @@
#!/usr/bin/python3
"""
This module contains keywords for work with Storage Groups.
It contains wrappers for `neofs-cli storagegroup` verbs.
"""
from cli_helpers import _cmd_run
from common import NEOFS_CLI_EXEC, NEOFS_ENDPOINT, WALLET_PASS
from robot.api.deco import keyword
ROBOT_AUTO_KEYWORDS = False
@keyword('Put Storagegroup')
def put_storagegroup(wallet: str, cid: str, objects: list, bearer_token: str=""):
"""
Wrapper for `neofs-cli storagegroup put`. Before the SG is created,
neofs-cli performs HEAD on `objects`, so this verb must be allowed
for `wallet` in `cid`.
Args:
wallet (str): path to wallet on whose behalf the SG is created
cid (str): ID of Container to put SG to
objects (list): list of Object IDs to include into the SG
bearer_token (optional, str): path to Bearer token file
Returns:
(str): Object ID of created Storage Group
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} '
f'--wallet {wallet} --config {WALLET_PASS} '
f'storagegroup put --cid {cid} '
f'--members {",".join(objects)} '
f'{"--bearer " + bearer_token if bearer_token else ""}'
)
output = _cmd_run(cmd)
oid = output.split('\n')[1].split(': ')[1]
return oid
@keyword('List Storagegroup')
def list_storagegroup(wallet: str, cid: str, bearer_token: str=""):
"""
Wrapper for `neofs-cli storagegroup list`. This operation
requires SEARCH allowed for `wallet` in `cid`.
Args:
wallet (str): path to wallet on whose behalf the SGs are
listed in the container
cid (str): ID of Container to list
bearer_token (optional, str): path to Bearer token file
Returns:
(list): Object IDs of found Storage Groups
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} '
f'--wallet {wallet} --config {WALLET_PASS} storagegroup list '
f'--cid {cid} {"--bearer " + bearer_token if bearer_token else ""}'
)
output = _cmd_run(cmd)
# throwing off the first string of output
found_objects = output.split('\n')[1:]
return found_objects
@keyword('Get Storagegroup')
def get_storagegroup(wallet: str, cid: str, oid: str, bearer_token: str=''):
"""
Wrapper for `neofs-cli storagegroup get`.
Args:
wallet (str): path to wallet on whose behalf the SG is got
cid (str): ID of Container where SG is stored
oid (str): ID of the Storage Group
bearer_token (optional, str): path to Bearer token file
Returns:
(dict): detailed information on the Storage Group
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} '
f'--wallet {wallet} --config {WALLET_PASS} '
f'storagegroup get --cid {cid} --id {oid} '
f'{"--bearer " + bearer_token if bearer_token else ""}'
)
output = _cmd_run(cmd)
# TODO: temporary solution for parsing output. Needs to be replaced with
# JSON parsing when https://github.com/nspcc-dev/neofs-node/issues/1355
# is done.
strings = output.strip().split('\n')
# first three strings go to `data`;
# skip the 'Members:' string;
# the rest of strings go to `members`
data, members = strings[:3], strings[3:]
sg_dict = {}
for i in data:
key, val = i.split(': ')
sg_dict[key] = val
sg_dict['Members'] = []
for member in members[1:]:
sg_dict['Members'].append(member.strip())
return sg_dict
@keyword('Delete Storagegroup')
def delete_storagegroup(wallet: str, cid: str, oid: str, bearer_token: str=""):
"""
Wrapper for `neofs-cli storagegroup delete`.
Args:
wallet (str): path to wallet on whose behalf the SG is deleted
cid (str): ID of Container where SG is stored
oid (str): ID of the Storage Group
bearer_token (optional, str): path to Bearer token file
Returns:
(str): Tombstone ID of the deleted Storage Group
"""
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} '
f'--wallet {wallet} --config {WALLET_PASS} '
f'storagegroup delete --cid {cid} --id {oid} '
f'{"--bearer " + bearer_token if bearer_token else ""}'
)
output = _cmd_run(cmd)
tombstone_id = output.strip().split('\n')[1].split(': ')[1]
return tombstone_id

125
robot/resources/lib/python_keywords/storage_policy.py Normal file
View file

@ -0,0 +1,125 @@
#!/usr/bin/python3
'''
This module contains keywords which are used for asserting
that storage policies are kept.
'''
from common import NEOFS_NETMAP
import complex_object_actions
import neofs_verbs
from robot.api.deco import keyword
ROBOT_AUTO_KEYWORDS = False
@keyword('Get Object Copies')
def get_object_copies(complexity: str, wallet: str, cid: str, oid: str):
"""
The function performs requests to all nodes of the container and
finds out if they store a copy of the object. The procedure is
different for simple and complex object, so the function requires
a sign of object complexity.
Args:
complexity (str): the tag of object size and complexity,
[Simple|Complex]
wallet (str): the path to the wallet on whose behalf the
copies are got
cid (str): ID of the container
oid (str): ID of the Object
Returns:
(int): the number of object copies in the container
"""
return (get_simple_object_copies(wallet, cid, oid) if complexity == "Simple"
else get_complex_object_copies(wallet, cid, oid))
@keyword('Get Simple Object Copies')
def get_simple_object_copies(wallet: str, cid: str, oid: str):
"""
To figure out the number of a simple object copies, only direct
HEAD requests should be made to the every node of the container.
We consider non-empty HEAD response as a stored object copy.
Args:
wallet (str): the path to the wallet on whose behalf the
copies are got
cid (str): ID of the container
oid (str): ID of the Object
Returns:
(int): the number of object copies in the container
"""
copies = 0
for node in NEOFS_NETMAP:
response = neofs_verbs.head_object(wallet, cid, oid,
endpoint=node,
is_direct=True)
if response:
copies += 1
return copies
@keyword('Get Complex Object Copies')
def get_complex_object_copies(wallet: str, cid: str, oid: str):
"""
To figure out the number of a complex object copies, we firstly
need to retrieve its Last object. We consider that the number of
complex object copies is equal to the number of its last object
copies. When we have the Last object ID, the task is reduced
to getting simple object copies.
Args:
wallet (str): the path to the wallet on whose behalf the
copies are got
cid (str): ID of the container
oid (str): ID of the Object
Returns:
(int): the number of object copies in the container
"""
last_oid = complex_object_actions.get_last_object(wallet, cid, oid)
return get_simple_object_copies(wallet, cid, last_oid)
@keyword('Get Nodes With Object')
def get_nodes_with_object(wallet: str, cid: str, oid: str):
"""
The function returns list of nodes which store
the given object.
Args:
wallet (str): the path to the wallet on whose behalf
we request the nodes
cid (str): ID of the container which store the object
oid (str): object ID
Returns:
(list): nodes which store the object
"""
nodes_list = []
for node in NEOFS_NETMAP:
res = neofs_verbs.head_object(wallet, cid, oid,
endpoint=node,
is_direct=True)
if res is not None:
nodes_list.append(node)
return nodes_list
@keyword('Get Nodes Without Object')
def get_nodes_without_object(wallet: str, cid: str, oid: str):
"""
The function returns list of nodes which do not store
the given object.
Args:
wallet (str): the path to the wallet on whose behalf
we request the nodes
cid (str): ID of the container which store the object
oid (str): object ID
Returns:
(list): nodes which do not store the object
"""
nodes_list = []
for node in NEOFS_NETMAP:
res = neofs_verbs.head_object(wallet, cid, oid,
endpoint=node,
is_direct=True)
if res is None:
nodes_list.append(node)
return nodes_list

108
robot/resources/lib/python_keywords/utility_keywords.py Normal file
View file

@ -0,0 +1,108 @@
#!/usr/bin/python3.8
import os
import tarfile
import uuid
import hashlib
import docker
from common import SIMPLE_OBJ_SIZE, ASSETS_DIR
from cli_helpers import _cmd_run
from robot.api.deco import keyword
from robot.api import logger
from robot.libraries.BuiltIn import BuiltIn
ROBOT_AUTO_KEYWORDS = False
@keyword('Generate file')
def generate_file_and_file_hash(size: int) -> str:
"""
Function generates a big binary file with the specified size in bytes and its hash.
Args:
size (int): the size in bytes, can be declared as 6e+6 for example
Returns:
(str): the path to the generated file
(str): the hash of the generated file
"""
filename = f"{os.getcwd()}/{ASSETS_DIR}/{str(uuid.uuid4())}"
with open(filename, 'wb') as fout:
fout.write(os.urandom(size))
logger.info(f"file with size {size} bytes has been generated: {filename}")
file_hash = get_file_hash(filename)
return filename, file_hash
@keyword('Get File Hash')
def get_file_hash(filename: str):
"""
This function generates hash for the specified file.
Args:
filename (str): the path to the file to generate hash for
Returns:
(str): the hash of the file
"""
blocksize = 65536
file_hash = hashlib.md5()
with open(filename, "rb") as out:
for block in iter(lambda: out.read(blocksize), b""):
file_hash.update(block)
return file_hash.hexdigest()
@keyword('Get Docker Logs')
def get_container_logs(testcase_name: str) -> None:
client = docker.APIClient(base_url='unix://var/run/docker.sock')
logs_dir = BuiltIn().get_variable_value("${OUTPUT_DIR}")
tar_name = f"{logs_dir}/dockerlogs({testcase_name}).tar.gz"
tar = tarfile.open(tar_name, "w:gz")
for container in client.containers():
container_name = container['Names'][0][1:]
if client.inspect_container(container_name)['Config']['Domainname'] == "neofs.devenv":
file_name = f"{logs_dir}/docker_log_{container_name}"
with open(file_name,'wb') as out:
out.write(client.logs(container_name))
logger.info(f"Collected logs from container {container_name}")
tar.add(file_name)
os.remove(file_name)
tar.close()
@keyword('Make Up')
def make_up(services: list=[], config_dict: dict={}):
test_path = os.getcwd()
dev_path = os.getenv('DEVENV_PATH', '../neofs-dev-env')
os.chdir(dev_path)
if len(services) > 0:
for service in services:
if config_dict != {}:
with open(f"{dev_path}/.int_test.env", "a") as out:
for key, value in config_dict.items():
out.write(f'{key}={value}')
cmd = f'make up/{service}'
_cmd_run(cmd)
else:
cmd = f'make up/basic; make update.max_object_size val={SIMPLE_OBJ_SIZE}'
_cmd_run(cmd, timeout=120)
os.chdir(test_path)
@keyword('Make Down')
def make_down(services: list=[]):
test_path = os.getcwd()
dev_path = os.getenv('DEVENV_PATH', '../neofs-dev-env')
os.chdir(dev_path)
if len(services) > 0:
for service in services:
cmd = f'make down/{service}'
_cmd_run(cmd)
with open(f"{dev_path}/.int_test.env", "w"):
pass
else:
cmd = 'make down; make clean'
_cmd_run(cmd, timeout=60)
os.chdir(test_path)