forked from TrueCloudLab/frostfs-testcases
[#127]: Large Object assrtions moved from python to robot
Signed-off-by: anastasia prasolova <anastasia@nspcc.ru>
This commit is contained in:
anastasia prasolova
Anastasia Prasolova
parent
040f648c61
commit
15dd59ddad
12 changed files with 240 additions and 218 deletions
74
robot/resources/lib/python/complex_object_actions.py
Normal file
74
robot/resources/lib/python/complex_object_actions.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
"""
|
||||
This module contains functions which are used for Large Object assemling:
|
||||
getting Last Object and split and getting Link Object. It is not enough to
|
||||
simply perform a "raw" HEAD request, as noted in the issue:
|
||||
https://github.com/nspcc-dev/neofs-node/issues/1304. Therefore, the reliable
|
||||
retrival of the aforementioned objects must be done this way: send direct
|
||||
"raw" HEAD request to the every Storage Node and return the desired OID on
|
||||
first non-null response.
|
||||
"""
|
||||
|
||||
from common import NEOFS_NETMAP
|
||||
import neofs_verbs
|
||||
|
||||
from robot.api.deco import keyword
|
||||
from robot.api import logger
|
||||
from robot.libraries.BuiltIn import BuiltIn
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
|
||||
|
||||
@keyword('Get Link Object')
|
||||
def get_link_object(wallet: str, cid: str, oid: str):
|
||||
"""
|
||||
Args:
|
||||
wallet (str): path to the wallet on whose behalf the Storage Nodes
|
||||
are requested
|
||||
cid (str): Container ID which stores the Large Object
|
||||
oid (str): Large Object ID
|
||||
Returns:
|
||||
(str): Link Object ID
|
||||
When no Link Object ID is found after all Storage Nodes polling,
|
||||
the function throws a native robot error.
|
||||
"""
|
||||
for node in NEOFS_NETMAP:
|
||||
try:
|
||||
resp = neofs_verbs.head_object(wallet, cid, oid,
|
||||
endpoint=node,
|
||||
is_raw=True,
|
||||
is_direct=True)
|
||||
if resp['link']:
|
||||
return resp['link']
|
||||
except Exception:
|
||||
logger.info(f"No Link Object found on {node}; continue")
|
||||
BuiltIn().fail(f"No Link Object for {cid}/{oid} found among all Storage Nodes")
|
||||
return None
|
||||
|
||||
|
||||
@keyword('Get Last Object')
|
||||
def get_last_object(wallet: str, cid: str, oid: str):
|
||||
"""
|
||||
Args:
|
||||
wallet (str): path to the wallet on whose behalf the Storage Nodes
|
||||
are requested
|
||||
cid (str): Container ID which stores the Large Object
|
||||
oid (str): Large Object ID
|
||||
Returns:
|
||||
(str): Last Object ID
|
||||
When no Last Object ID is found after all Storage Nodes polling,
|
||||
the function throws a native robot error.
|
||||
"""
|
||||
for node in NEOFS_NETMAP:
|
||||
try:
|
||||
resp = neofs_verbs.head_object(wallet, cid, oid,
|
||||
endpoint=node,
|
||||
is_raw=True,
|
||||
is_direct=True)
|
||||
if resp['lastPart']:
|
||||
return resp['lastPart']
|
||||
except Exception:
|
||||
logger.info(f"No Last Object found on {node}; continue")
|
||||
BuiltIn().fail(f"No Last Object for {cid}/{oid} found among all Storage Nodes")
|
||||
return None
|
||||
|
|
@ -174,183 +174,6 @@ def container_existing(private_key: str, cid: str):
|
|||
return
|
||||
|
||||
|
||||
|
||||
@keyword('Get Split objects')
|
||||
def get_component_objects(private_key: str, cid: str, oid: str):
|
||||
logger.info("Collect Split objects list from Linked object.")
|
||||
split_id = ""
|
||||
for node in NEOFS_NETMAP:
|
||||
try:
|
||||
parsed_header_virtual = neofs_verbs.head_object(private_key, cid, oid, options=' --ttl 1',
|
||||
endpoint=node, is_raw=True)
|
||||
|
||||
if 'link' in parsed_header_virtual.keys():
|
||||
return _collect_split_objects_from_header(private_key, cid, parsed_header_virtual)
|
||||
|
||||
elif 'split' in parsed_header_virtual['header'].keys():
|
||||
logger.info(f"parsed_header_virtual: !@ {parsed_header_virtual}" )
|
||||
split_id = parsed_header_virtual['header']['splitID']
|
||||
|
||||
except:
|
||||
logger.warn("Linking object has not been found.")
|
||||
|
||||
# Get all existing objects
|
||||
full_obj_list = neofs_verbs.search_object(private_key, cid, None, None, None, None, '--phy')
|
||||
|
||||
# Search expected Linking object
|
||||
for targer_oid in full_obj_list:
|
||||
header_parsed = neofs_verbs.head_object(private_key, cid, targer_oid, is_raw=True)
|
||||
if header_parsed['header']['split']:
|
||||
if header_parsed['header']['split']['splitID'] == split_id and 'children' in header_parsed['header']['split'].keys():
|
||||
logger.info("Linking object has been found in additional check (head of all objects).")
|
||||
return _collect_split_objects_from_header(private_key, cid, parsed_header_virtual)
|
||||
|
||||
|
||||
raise Exception("Linking object is not found at all - all existed objects have been headed.")
|
||||
|
||||
def _collect_split_objects_from_header(private_key, cid, parsed_header):
|
||||
header_link_parsed = neofs_verbs.head_object(private_key, cid, parsed_header['link'])
|
||||
return header_link_parsed['header']['split']['children']
|
||||
|
||||
|
||||
@keyword('Verify Split Chain')
|
||||
def verify_split_chain(wif: str, cid: str, oid: str):
|
||||
|
||||
header_virtual_parsed = dict()
|
||||
header_last_parsed = dict()
|
||||
|
||||
marker_last_obj = 0
|
||||
marker_link_obj = 0
|
||||
|
||||
final_verif_data = dict()
|
||||
|
||||
# Get Latest object
|
||||
logger.info("Collect Split objects information and verify chain of the objects.")
|
||||
for node in NEOFS_NETMAP:
|
||||
try:
|
||||
parsed_header_virtual = neofs_verbs.head_object(wif, cid, oid, options=' --ttl 1',
|
||||
endpoint=node, is_raw=True)
|
||||
logger.info(f"header {parsed_header_virtual}")
|
||||
|
||||
if 'lastPart' in parsed_header_virtual.keys():
|
||||
header_last_parsed = neofs_verbs.head_object(wif, cid,
|
||||
parsed_header_virtual['lastPart'],
|
||||
is_raw=True)
|
||||
marker_last_obj = 1
|
||||
|
||||
# Recursive chain validation up to the first object
|
||||
final_verif_data = _verify_child_link(wif, cid, oid, header_last_parsed['header'], final_verif_data)
|
||||
break
|
||||
logger.info(f"Found Split Object with header:\n\t{parsed_header_virtual}")
|
||||
logger.info("Continue to search Last Split Object")
|
||||
|
||||
except Exception as exc:
|
||||
logger.info(f"Failed while collectiong Split Objects: {exc}")
|
||||
continue
|
||||
|
||||
if marker_last_obj == 0:
|
||||
raise Exception("Last object has not been found")
|
||||
|
||||
# Get Linking object
|
||||
logger.info("Compare Split objects result information with Linking object.")
|
||||
for node in NEOFS_NETMAP:
|
||||
try:
|
||||
parsed_header_virtual = neofs_verbs.head_object(wif, cid, oid, options=' --ttl 1',
|
||||
endpoint=node, is_raw=True)
|
||||
if 'link' in parsed_header_virtual.keys():
|
||||
header_link_parsed = neofs_verbs.head_object(wif, cid,
|
||||
parsed_header_virtual['link'],
|
||||
is_raw=True)
|
||||
marker_link_obj = 1
|
||||
|
||||
reversed_list = final_verif_data['header']['split']['children'][::-1]
|
||||
|
||||
if header_link_parsed['header']['split']['children'] == reversed_list:
|
||||
logger.info(f"Split objects list from Linked Object is equal to expected "
|
||||
f"{', '.join(header_link_parsed['header']['split']['children'])}")
|
||||
else:
|
||||
raise Exception(f"Split objects list from Linking Object "
|
||||
f"({', '.join(header_link_parsed['header']['split']['children'])}) "
|
||||
f"is not equal to expected ({', '.join(reversed_list)})")
|
||||
|
||||
if int(header_link_parsed['header']['payloadLength']) == 0:
|
||||
logger.info("Linking object Payload is equal to expected - zero size.")
|
||||
else:
|
||||
raise Exception("Linking object Payload is not equal to expected. Should be zero.")
|
||||
|
||||
if header_link_parsed['header']['objectType'] == 'REGULAR':
|
||||
logger.info("Linking Object Type is 'regular' as expected.")
|
||||
else:
|
||||
raise Exception("Object Type is not 'regular'.")
|
||||
|
||||
if header_link_parsed['header']['split']['children'] == final_verif_data['split']['children']:
|
||||
logger.info(f"Linking Object Split ID is equal to expected {final_verif_data['split']['children']}.")
|
||||
else:
|
||||
raise Exception(f"Split ID from Linking Object ({header_link_parsed['header']['split']['children']}) "
|
||||
f"is not equal to expected ({final_verif_data['split']['children']})")
|
||||
|
||||
break
|
||||
logger.info(f"Found Linking Object with header:\n\t{parsed_header_virtual}")
|
||||
logger.info("Continue to search Linking Object")
|
||||
except RuntimeError as e:
|
||||
logger.info(f"Failed while collecting Split Object: {e}")
|
||||
continue
|
||||
|
||||
if marker_link_obj == 0:
|
||||
raise Exception("Linked object has not been found")
|
||||
|
||||
|
||||
logger.info("Compare Split objects result information with Virtual object.")
|
||||
|
||||
header_virtual_parsed = neofs_verbs.head_object(wif, cid, oid)
|
||||
|
||||
if int(header_virtual_parsed['header']['payloadLength']) == int(final_verif_data['payloadLength']):
|
||||
logger.info(f"Split objects PayloadLength are equal to Virtual Object Payload "
|
||||
f"{header_virtual_parsed['header']['payloadLength']}")
|
||||
else:
|
||||
raise Exception(f"Split objects PayloadLength from Virtual Object "
|
||||
f"({header_virtual_parsed['header']['payloadLength']}) is not equal "
|
||||
f"to expected ({final_verif_data['payloadLength']})")
|
||||
|
||||
if header_link_parsed['header']['objectType'] == 'REGULAR':
|
||||
logger.info("Virtual Object Type is 'regular' as expected.")
|
||||
else:
|
||||
raise Exception("Object Type is not 'regular'.")
|
||||
|
||||
|
||||
def _verify_child_link(wif: str, cid: str, oid: str, header_last_parsed: dict, final_verif_data: dict):
|
||||
|
||||
if 'payloadLength' in final_verif_data.keys():
|
||||
final_verif_data['payloadLength'] = int(final_verif_data['payloadLength']) + int(header_last_parsed['payloadLength'])
|
||||
else:
|
||||
final_verif_data['payloadLength'] = int(header_last_parsed['payloadLength'])
|
||||
|
||||
if header_last_parsed['objectType'] != 'REGULAR':
|
||||
raise Exception("Object Type is not 'regular'.")
|
||||
|
||||
if 'split' in final_verif_data.keys():
|
||||
if final_verif_data['split']['splitID'] != header_last_parsed['split']['splitID']:
|
||||
raise Exception(f"Object Split ID ({header_last_parsed['split']['splitID']}) "
|
||||
f"is not expected ({final_verif_data['split']['splitID']}).")
|
||||
else:
|
||||
final_verif_data['split']['splitID'] = header_last_parsed['split']['splitID']
|
||||
|
||||
if 'children' in final_verif_data['split'].keys():
|
||||
final_verif_data['split']['children'].append(header_last_parsed['split']['children'])
|
||||
else:
|
||||
final_verif_data['split']['children'] = []
|
||||
final_verif_data['split']['children'].append(header_last_parsed['split']['children'])
|
||||
|
||||
if 'previous' in header_last_parsed['split'].keys():
|
||||
parsed_header_virtual = neofs_verbs.head_object(wif, cid, header_last_parsed['split']['previous'], is_raw=True)
|
||||
|
||||
final_verif_data = _verify_child_link(wif, cid, oid, parsed_header_virtual, final_verif_data)
|
||||
else:
|
||||
logger.info("Chain of the objects has been parsed from the last object ot the first.")
|
||||
|
||||
return final_verif_data
|
||||
|
||||
|
||||
@keyword('Verify Head Tombstone')
|
||||
def verify_head_tombstone(private_key: str, cid: str, oid_ts: str, oid: str, addr: str):
|
||||
# TODO: replace with HEAD from neofs_verbs.py
|
||||
|
|
@ -540,9 +363,6 @@ def find_in_nodes_Log(line: str, nodes_logs_time: dict):
|
|||
return 1
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@keyword('Put Storagegroup')
|
||||
def put_storagegroup(private_key: str, cid: str, bearer_token: str="", *oid_list):
|
||||
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ def search_object(wif: str, cid: str, keys: str="", bearer: str="", filters: dic
|
|||
@keyword('Head object')
|
||||
def head_object(wif: str, cid: str, oid: str, bearer_token: str="",
|
||||
options:str="", endpoint: str="", json_output: bool = True,
|
||||
is_raw: bool = False):
|
||||
is_raw: bool = False, is_direct: bool = False):
|
||||
'''
|
||||
HEAD an Object.
|
||||
|
||||
|
|
@ -230,6 +230,8 @@ def head_object(wif: str, cid: str, oid: str, bearer_token: str="",
|
|||
turns into `--json` key
|
||||
is_raw(optional, bool): send "raw" request or not; this flag
|
||||
turns into `--raw` key
|
||||
is_direct(optional, bool): send request directly to the node or not; this flag
|
||||
turns into `--ttl 1` key
|
||||
Returns:
|
||||
depending on the `json_output` parameter value, the function returns
|
||||
(dict): HEAD response in JSON format
|
||||
|
|
@ -242,7 +244,8 @@ def head_object(wif: str, cid: str, oid: str, bearer_token: str="",
|
|||
f'object head --cid {cid} --oid {oid} {options} '
|
||||
f'{"--bearer " + bearer_token if bearer_token else ""} '
|
||||
f'{"--json" if json_output else ""} '
|
||||
f'{"--raw" if is_raw else ""}'
|
||||
f'{"--raw" if is_raw else ""} '
|
||||
f'{"--ttl 1" if is_direct else ""}'
|
||||
)
|
||||
output = _cmd_run(cmd)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue