From 15dd59ddad47f4a72892441430bed3283670871a Mon Sep 17 00:00:00 2001 From: anastasia prasolova Date: Thu, 7 Apr 2022 21:48:03 +0300 Subject: [PATCH] [#127]: Large Object assrtions moved from python to robot Signed-off-by: anastasia prasolova --- .../lib/python/complex_object_actions.py | 74 +++++++ robot/resources/lib/python/neofs.py | 180 ------------------ robot/resources/lib/python/neofs_verbs.py | 7 +- .../lib/robot/complex_object_operations.robot | 20 ++ ...basic_private_container_storagegroup.robot | 16 +- ..._basic_public_container_storagegroup.robot | 11 +- .../acl/acl_basic_readonly_container.robot | 16 +- ...asic_readonly_container_storagegroup.robot | 16 +- .../acl/acl_bearer_allow_storagegroup.robot | 6 +- .../network/netmap_control_drop.robot | 3 +- .../integration/object/object_complex.robot | 104 +++++++++- .../object/object_storagegroup_complex.robot | 5 +- 12 files changed, 240 insertions(+), 218 deletions(-) create mode 100644 robot/resources/lib/python/complex_object_actions.py create mode 100644 robot/resources/lib/robot/complex_object_operations.robot diff --git a/robot/resources/lib/python/complex_object_actions.py b/robot/resources/lib/python/complex_object_actions.py new file mode 100644 index 0000000..f112442 --- /dev/null +++ b/robot/resources/lib/python/complex_object_actions.py @@ -0,0 +1,74 @@ +#!/usr/bin/python3 + +""" + This module contains functions which are used for Large Object assemling: + getting Last Object and split and getting Link Object. It is not enough to + simply perform a "raw" HEAD request, as noted in the issue: + https://github.com/nspcc-dev/neofs-node/issues/1304. Therefore, the reliable + retrival of the aforementioned objects must be done this way: send direct + "raw" HEAD request to the every Storage Node and return the desired OID on + first non-null response. +""" + +from common import NEOFS_NETMAP +import neofs_verbs + +from robot.api.deco import keyword +from robot.api import logger +from robot.libraries.BuiltIn import BuiltIn + +ROBOT_AUTO_KEYWORDS = False + + +@keyword('Get Link Object') +def get_link_object(wallet: str, cid: str, oid: str): + """ + Args: + wallet (str): path to the wallet on whose behalf the Storage Nodes + are requested + cid (str): Container ID which stores the Large Object + oid (str): Large Object ID + Returns: + (str): Link Object ID + When no Link Object ID is found after all Storage Nodes polling, + the function throws a native robot error. + """ + for node in NEOFS_NETMAP: + try: + resp = neofs_verbs.head_object(wallet, cid, oid, + endpoint=node, + is_raw=True, + is_direct=True) + if resp['link']: + return resp['link'] + except Exception: + logger.info(f"No Link Object found on {node}; continue") + BuiltIn().fail(f"No Link Object for {cid}/{oid} found among all Storage Nodes") + return None + + +@keyword('Get Last Object') +def get_last_object(wallet: str, cid: str, oid: str): + """ + Args: + wallet (str): path to the wallet on whose behalf the Storage Nodes + are requested + cid (str): Container ID which stores the Large Object + oid (str): Large Object ID + Returns: + (str): Last Object ID + When no Last Object ID is found after all Storage Nodes polling, + the function throws a native robot error. + """ + for node in NEOFS_NETMAP: + try: + resp = neofs_verbs.head_object(wallet, cid, oid, + endpoint=node, + is_raw=True, + is_direct=True) + if resp['lastPart']: + return resp['lastPart'] + except Exception: + logger.info(f"No Last Object found on {node}; continue") + BuiltIn().fail(f"No Last Object for {cid}/{oid} found among all Storage Nodes") + return None diff --git a/robot/resources/lib/python/neofs.py b/robot/resources/lib/python/neofs.py index 3108f11..c13e7cb 100644 --- a/robot/resources/lib/python/neofs.py +++ b/robot/resources/lib/python/neofs.py @@ -174,183 +174,6 @@ def container_existing(private_key: str, cid: str): return - -@keyword('Get Split objects') -def get_component_objects(private_key: str, cid: str, oid: str): - logger.info("Collect Split objects list from Linked object.") - split_id = "" - for node in NEOFS_NETMAP: - try: - parsed_header_virtual = neofs_verbs.head_object(private_key, cid, oid, options=' --ttl 1', - endpoint=node, is_raw=True) - - if 'link' in parsed_header_virtual.keys(): - return _collect_split_objects_from_header(private_key, cid, parsed_header_virtual) - - elif 'split' in parsed_header_virtual['header'].keys(): - logger.info(f"parsed_header_virtual: !@ {parsed_header_virtual}" ) - split_id = parsed_header_virtual['header']['splitID'] - - except: - logger.warn("Linking object has not been found.") - - # Get all existing objects - full_obj_list = neofs_verbs.search_object(private_key, cid, None, None, None, None, '--phy') - - # Search expected Linking object - for targer_oid in full_obj_list: - header_parsed = neofs_verbs.head_object(private_key, cid, targer_oid, is_raw=True) - if header_parsed['header']['split']: - if header_parsed['header']['split']['splitID'] == split_id and 'children' in header_parsed['header']['split'].keys(): - logger.info("Linking object has been found in additional check (head of all objects).") - return _collect_split_objects_from_header(private_key, cid, parsed_header_virtual) - - - raise Exception("Linking object is not found at all - all existed objects have been headed.") - -def _collect_split_objects_from_header(private_key, cid, parsed_header): - header_link_parsed = neofs_verbs.head_object(private_key, cid, parsed_header['link']) - return header_link_parsed['header']['split']['children'] - - -@keyword('Verify Split Chain') -def verify_split_chain(wif: str, cid: str, oid: str): - - header_virtual_parsed = dict() - header_last_parsed = dict() - - marker_last_obj = 0 - marker_link_obj = 0 - - final_verif_data = dict() - - # Get Latest object - logger.info("Collect Split objects information and verify chain of the objects.") - for node in NEOFS_NETMAP: - try: - parsed_header_virtual = neofs_verbs.head_object(wif, cid, oid, options=' --ttl 1', - endpoint=node, is_raw=True) - logger.info(f"header {parsed_header_virtual}") - - if 'lastPart' in parsed_header_virtual.keys(): - header_last_parsed = neofs_verbs.head_object(wif, cid, - parsed_header_virtual['lastPart'], - is_raw=True) - marker_last_obj = 1 - - # Recursive chain validation up to the first object - final_verif_data = _verify_child_link(wif, cid, oid, header_last_parsed['header'], final_verif_data) - break - logger.info(f"Found Split Object with header:\n\t{parsed_header_virtual}") - logger.info("Continue to search Last Split Object") - - except Exception as exc: - logger.info(f"Failed while collectiong Split Objects: {exc}") - continue - - if marker_last_obj == 0: - raise Exception("Last object has not been found") - - # Get Linking object - logger.info("Compare Split objects result information with Linking object.") - for node in NEOFS_NETMAP: - try: - parsed_header_virtual = neofs_verbs.head_object(wif, cid, oid, options=' --ttl 1', - endpoint=node, is_raw=True) - if 'link' in parsed_header_virtual.keys(): - header_link_parsed = neofs_verbs.head_object(wif, cid, - parsed_header_virtual['link'], - is_raw=True) - marker_link_obj = 1 - - reversed_list = final_verif_data['header']['split']['children'][::-1] - - if header_link_parsed['header']['split']['children'] == reversed_list: - logger.info(f"Split objects list from Linked Object is equal to expected " - f"{', '.join(header_link_parsed['header']['split']['children'])}") - else: - raise Exception(f"Split objects list from Linking Object " - f"({', '.join(header_link_parsed['header']['split']['children'])}) " - f"is not equal to expected ({', '.join(reversed_list)})") - - if int(header_link_parsed['header']['payloadLength']) == 0: - logger.info("Linking object Payload is equal to expected - zero size.") - else: - raise Exception("Linking object Payload is not equal to expected. Should be zero.") - - if header_link_parsed['header']['objectType'] == 'REGULAR': - logger.info("Linking Object Type is 'regular' as expected.") - else: - raise Exception("Object Type is not 'regular'.") - - if header_link_parsed['header']['split']['children'] == final_verif_data['split']['children']: - logger.info(f"Linking Object Split ID is equal to expected {final_verif_data['split']['children']}.") - else: - raise Exception(f"Split ID from Linking Object ({header_link_parsed['header']['split']['children']}) " - f"is not equal to expected ({final_verif_data['split']['children']})") - - break - logger.info(f"Found Linking Object with header:\n\t{parsed_header_virtual}") - logger.info("Continue to search Linking Object") - except RuntimeError as e: - logger.info(f"Failed while collecting Split Object: {e}") - continue - - if marker_link_obj == 0: - raise Exception("Linked object has not been found") - - - logger.info("Compare Split objects result information with Virtual object.") - - header_virtual_parsed = neofs_verbs.head_object(wif, cid, oid) - - if int(header_virtual_parsed['header']['payloadLength']) == int(final_verif_data['payloadLength']): - logger.info(f"Split objects PayloadLength are equal to Virtual Object Payload " - f"{header_virtual_parsed['header']['payloadLength']}") - else: - raise Exception(f"Split objects PayloadLength from Virtual Object " - f"({header_virtual_parsed['header']['payloadLength']}) is not equal " - f"to expected ({final_verif_data['payloadLength']})") - - if header_link_parsed['header']['objectType'] == 'REGULAR': - logger.info("Virtual Object Type is 'regular' as expected.") - else: - raise Exception("Object Type is not 'regular'.") - - -def _verify_child_link(wif: str, cid: str, oid: str, header_last_parsed: dict, final_verif_data: dict): - - if 'payloadLength' in final_verif_data.keys(): - final_verif_data['payloadLength'] = int(final_verif_data['payloadLength']) + int(header_last_parsed['payloadLength']) - else: - final_verif_data['payloadLength'] = int(header_last_parsed['payloadLength']) - - if header_last_parsed['objectType'] != 'REGULAR': - raise Exception("Object Type is not 'regular'.") - - if 'split' in final_verif_data.keys(): - if final_verif_data['split']['splitID'] != header_last_parsed['split']['splitID']: - raise Exception(f"Object Split ID ({header_last_parsed['split']['splitID']}) " - f"is not expected ({final_verif_data['split']['splitID']}).") - else: - final_verif_data['split']['splitID'] = header_last_parsed['split']['splitID'] - - if 'children' in final_verif_data['split'].keys(): - final_verif_data['split']['children'].append(header_last_parsed['split']['children']) - else: - final_verif_data['split']['children'] = [] - final_verif_data['split']['children'].append(header_last_parsed['split']['children']) - - if 'previous' in header_last_parsed['split'].keys(): - parsed_header_virtual = neofs_verbs.head_object(wif, cid, header_last_parsed['split']['previous'], is_raw=True) - - final_verif_data = _verify_child_link(wif, cid, oid, parsed_header_virtual, final_verif_data) - else: - logger.info("Chain of the objects has been parsed from the last object ot the first.") - - return final_verif_data - - @keyword('Verify Head Tombstone') def verify_head_tombstone(private_key: str, cid: str, oid_ts: str, oid: str, addr: str): # TODO: replace with HEAD from neofs_verbs.py @@ -540,9 +363,6 @@ def find_in_nodes_Log(line: str, nodes_logs_time: dict): return 1 - - - @keyword('Put Storagegroup') def put_storagegroup(private_key: str, cid: str, bearer_token: str="", *oid_list): diff --git a/robot/resources/lib/python/neofs_verbs.py b/robot/resources/lib/python/neofs_verbs.py index a0bbfa7..68ea470 100644 --- a/robot/resources/lib/python/neofs_verbs.py +++ b/robot/resources/lib/python/neofs_verbs.py @@ -215,7 +215,7 @@ def search_object(wif: str, cid: str, keys: str="", bearer: str="", filters: dic @keyword('Head object') def head_object(wif: str, cid: str, oid: str, bearer_token: str="", options:str="", endpoint: str="", json_output: bool = True, - is_raw: bool = False): + is_raw: bool = False, is_direct: bool = False): ''' HEAD an Object. @@ -230,6 +230,8 @@ def head_object(wif: str, cid: str, oid: str, bearer_token: str="", turns into `--json` key is_raw(optional, bool): send "raw" request or not; this flag turns into `--raw` key + is_direct(optional, bool): send request directly to the node or not; this flag + turns into `--ttl 1` key Returns: depending on the `json_output` parameter value, the function returns (dict): HEAD response in JSON format @@ -242,7 +244,8 @@ def head_object(wif: str, cid: str, oid: str, bearer_token: str="", f'object head --cid {cid} --oid {oid} {options} ' f'{"--bearer " + bearer_token if bearer_token else ""} ' f'{"--json" if json_output else ""} ' - f'{"--raw" if is_raw else ""}' + f'{"--raw" if is_raw else ""} ' + f'{"--ttl 1" if is_direct else ""}' ) output = _cmd_run(cmd) diff --git a/robot/resources/lib/robot/complex_object_operations.robot b/robot/resources/lib/robot/complex_object_operations.robot new file mode 100644 index 0000000..51bd753 --- /dev/null +++ b/robot/resources/lib/robot/complex_object_operations.robot @@ -0,0 +1,20 @@ +*** Settings *** +Variables common.py + +Library neofs_verbs.py +Library complex_object_operations.py + + +*** Keywords *** + +Get Object Parts By Link Object + [Documentation] The keyword accepts the ID of a Large Object, retrieves its split + ... header and returns all Part Object IDs from Link Object. + + [Arguments] ${WIF} ${CID} ${LARGE_OID} + + + &{RESPONSE} = Get Link Object ${WIF} ${CID} ${LARGE_OID} + &{LINK_HEADER} = Head Object ${WIF} ${CID} ${RESPONSE.link} is_raw=True + + [Return] ${LINK_HEADER.header.split.children} diff --git a/robot/testsuites/integration/acl/acl_basic_private_container_storagegroup.robot b/robot/testsuites/integration/acl/acl_basic_private_container_storagegroup.robot index 1026608..da323bf 100644 --- a/robot/testsuites/integration/acl/acl_basic_private_container_storagegroup.robot +++ b/robot/testsuites/integration/acl/acl_basic_private_container_storagegroup.robot @@ -9,6 +9,7 @@ Library Collections Resource common_steps_acl_basic.robot Resource payment_operations.robot Resource setup_teardown.robot +Resource complex_object_operations.robot *** Test cases *** @@ -47,8 +48,9 @@ Check Private Container ${SG_OID_INV} = Put Storagegroup ${USER_KEY} ${PRIV_CID} ${EMPTY} ${S_OID_USER} ${SG_OID} = Put Storagegroup ${USER_KEY} ${PRIV_CID} ${EMPTY} ${S_OID_USER} List Storagegroup ${USER_KEY} ${PRIV_CID} ${EMPTY} ${SG_OID} ${SG_OID_INV} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${USER_KEY} ${PRIV_CID} ${S_OID_USER} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${USER_KEY} ${PRIV_CID} ${S_OID_USER} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} Get Storagegroup ${USER_KEY} ${PRIV_CID} ${SG_OID} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Delete Storagegroup ${USER_KEY} ${PRIV_CID} ${SG_OID} ${EMPTY} @@ -67,8 +69,9 @@ Check Private Container # System group key (Storage Node) ${SG_OID_SN} = Put Storagegroup ${NEOFS_SN_WIF} ${PRIV_CID} ${EMPTY} ${S_OID_USER} List Storagegroup ${NEOFS_SN_WIF} ${PRIV_CID} ${EMPTY} ${SG_OID_SN} ${SG_OID_INV} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${NEOFS_SN_WIF} ${PRIV_CID} ${S_OID_USER} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${NEOFS_SN_WIF} ${PRIV_CID} ${S_OID_USER} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} Get Storagegroup ${NEOFS_SN_WIF} ${PRIV_CID} ${SG_OID_SN} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Run Keyword And Expect Error * ... Delete Storagegroup ${NEOFS_SN_WIF} ${PRIV_CID} ${SG_OID_SN} ${EMPTY} @@ -77,8 +80,9 @@ Check Private Container # System group key (Inner Ring Node) ${SG_OID_IR} = Put Storagegroup ${NEOFS_IR_WIF} ${PRIV_CID} ${EMPTY} ${S_OID_USER} List Storagegroup ${NEOFS_IR_WIF} ${PRIV_CID} ${EMPTY} ${SG_OID_SN} ${SG_OID_IR} ${SG_OID_INV} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${USER_KEY} ${PRIV_CID} ${S_OID_USER} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${USER_KEY} ${PRIV_CID} ${S_OID_USER} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} Get Storagegroup ${NEOFS_IR_WIF} ${PRIV_CID} ${SG_OID_IR} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Run Keyword And Expect Error * ... Delete Storagegroup ${NEOFS_IR_WIF} ${PRIV_CID} ${SG_OID_IR} ${EMPTY} diff --git a/robot/testsuites/integration/acl/acl_basic_public_container_storagegroup.robot b/robot/testsuites/integration/acl/acl_basic_public_container_storagegroup.robot index 1a0903d..860c6f7 100644 --- a/robot/testsuites/integration/acl/acl_basic_public_container_storagegroup.robot +++ b/robot/testsuites/integration/acl/acl_basic_public_container_storagegroup.robot @@ -9,6 +9,7 @@ Library contract_keywords.py Resource common_steps_acl_basic.robot Resource payment_operations.robot Resource setup_teardown.robot +Resource complex_object_operations.robot *** Test cases *** @@ -50,8 +51,9 @@ Check Public Container FOR ${ROLE_KEY} IN @{ROLES_KEYS_PASS} ${SG_OID_USERS} = Put Storagegroup ${ROLE_KEY} ${PUBLIC_CID} ${EMPTY} ${S_OID} List Storagegroup ${ROLE_KEY} ${PUBLIC_CID} ${EMPTY} ${SG_OID_USERS} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${ROLE_KEY} ${PUBLIC_CID} ${S_OID} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${ROLE_KEY} ${PUBLIC_CID} ${S_OID} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID} Get Storagegroup ${ROLE_KEY} ${PUBLIC_CID} ${SG_OID_USERS} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Delete Storagegroup ${ROLE_KEY} ${PUBLIC_CID} ${SG_OID_USERS} ${EMPTY} Tick Epoch @@ -59,8 +61,9 @@ Check Public Container FOR ${ROLE_KEY} IN @{ROLES_KEYS_SYS} ${SG_OID_SYS} = Put Storagegroup ${ROLE_KEY} ${PUBLIC_CID} ${EMPTY} ${S_OID} List Storagegroup ${ROLE_KEY} ${PUBLIC_CID} ${EMPTY} ${SG_OID_SYS} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${ROLE_KEY} ${PUBLIC_CID} ${S_OID} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${ROLE_KEY} ${PUBLIC_CID} ${S_OID} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID} Get Storagegroup ${ROLE_KEY} ${PUBLIC_CID} ${SG_OID_SYS} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Run Keyword And Expect Error * ... Delete Storagegroup ${ROLE_KEY} ${PUBLIC_CID} ${SG_OID_SYS} ${EMPTY} diff --git a/robot/testsuites/integration/acl/acl_basic_readonly_container.robot b/robot/testsuites/integration/acl/acl_basic_readonly_container.robot index 5ca6a1a..70c25c1 100644 --- a/robot/testsuites/integration/acl/acl_basic_readonly_container.robot +++ b/robot/testsuites/integration/acl/acl_basic_readonly_container.robot @@ -8,6 +8,7 @@ Library payment_neogo.py Resource common_steps_acl_basic.robot Resource payment_operations.robot Resource setup_teardown.robot +Resource complex_object_operations.robot *** Test cases *** @@ -50,24 +51,27 @@ Check Read-Only Container ${SG_OID_INV} = Put Storagegroup ${USER_KEY} ${READONLY_CID} ${EMPTY} ${S_OID_USER} ${SG_OID_1} = Put Storagegroup ${USER_KEY} ${READONLY_CID} ${EMPTY} ${S_OID_USER} List Storagegroup ${USER_KEY} ${READONLY_CID} ${EMPTY} ${SG_OID_1} ${SG_OID_INV} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${USER_KEY} ${READONLY_CID} ${S_OID_USER} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${USER_KEY} ${READONLY_CID} ${S_OID_USER} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} Get Storagegroup ${USER_KEY} ${READONLY_CID} ${SG_OID_1} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Delete Storagegroup ${USER_KEY} ${READONLY_CID} ${SG_OID_1} ${EMPTY} Run Keyword And Expect Error * ... Put Storagegroup ${OTHER_KEY} ${READONLY_CID} ${EMPTY} ${S_OID_USER} List Storagegroup ${OTHER_KEY} ${READONLY_CID} ${EMPTY} ${SG_OID_INV} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${USER_KEY} ${READONLY_CID} ${S_OID_USER} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${USER_KEY} ${READONLY_CID} ${S_OID_USER} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} Get Storagegroup ${OTHER_KEY} ${READONLY_CID} ${SG_OID_INV} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Run Keyword And Expect Error * ... Delete Storagegroup ${OTHER_KEY} ${READONLY_CID} ${SG_OID_INV} ${EMPTY} ${SG_OID_IR} = Put Storagegroup ${NEOFS_IR_WIF} ${READONLY_CID} ${EMPTY} ${S_OID_USER} List Storagegroup ${NEOFS_IR_WIF} ${READONLY_CID} ${EMPTY} ${SG_OID_INV} ${SG_OID_IR} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${USER_KEY} ${READONLY_CID} ${S_OID_USER} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${USER_KEY} ${READONLY_CID} ${S_OID_USER} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} Get Storagegroup ${NEOFS_IR_WIF} ${READONLY_CID} ${SG_OID_IR} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Run Keyword And Expect Error * ... Delete Storagegroup ${NEOFS_IR_WIF} ${READONLY_CID} ${SG_OID_IR} ${EMPTY} diff --git a/robot/testsuites/integration/acl/acl_basic_readonly_container_storagegroup.robot b/robot/testsuites/integration/acl/acl_basic_readonly_container_storagegroup.robot index 6badf00..3157d7b 100644 --- a/robot/testsuites/integration/acl/acl_basic_readonly_container_storagegroup.robot +++ b/robot/testsuites/integration/acl/acl_basic_readonly_container_storagegroup.robot @@ -8,6 +8,7 @@ Library payment_neogo.py Resource common_steps_acl_basic.robot Resource payment_operations.robot Resource setup_teardown.robot +Resource complex_object_operations.robot *** Test cases *** @@ -47,8 +48,9 @@ Check Read-Only Container ${SG_OID_INV} = Put Storagegroup ${USER_KEY} ${READONLY_CID} ${EMPTY} ${S_OID_USER} ${SG_OID_1} = Put Storagegroup ${USER_KEY} ${READONLY_CID} ${EMPTY} ${S_OID_USER} List Storagegroup ${USER_KEY} ${READONLY_CID} ${EMPTY} ${SG_OID_1} ${SG_OID_INV} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${USER_KEY} ${READONLY_CID} ${S_OID_USER} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${USER_KEY} ${READONLY_CID} ${S_OID_USER} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} Get Storagegroup ${USER_KEY} ${READONLY_CID} ${SG_OID_1} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Delete Storagegroup ${USER_KEY} ${READONLY_CID} ${SG_OID_1} ${EMPTY} @@ -56,8 +58,9 @@ Check Read-Only Container Run Keyword And Expect Error * ... Put Storagegroup ${OTHER_KEY} ${READONLY_CID} ${EMPTY} ${S_OID_USER} List Storagegroup ${OTHER_KEY} ${READONLY_CID} ${EMPTY} ${SG_OID_INV} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${USER_KEY} ${READONLY_CID} ${S_OID_USER} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${USER_KEY} ${READONLY_CID} ${S_OID_USER} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} Get Storagegroup ${OTHER_KEY} ${READONLY_CID} ${SG_OID_INV} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Run Keyword And Expect Error * ... Delete Storagegroup ${OTHER_KEY} ${READONLY_CID} ${SG_OID_INV} ${EMPTY} @@ -65,8 +68,9 @@ Check Read-Only Container ${SG_OID_IR} = Put Storagegroup ${NEOFS_IR_WIF} ${READONLY_CID} ${EMPTY} ${S_OID_USER} List Storagegroup ${NEOFS_IR_WIF} ${READONLY_CID} ${EMPTY} ${SG_OID_INV} ${SG_OID_IR} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${USER_KEY} ${READONLY_CID} ${S_OID_USER} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${USER_KEY} ${READONLY_CID} ${S_OID_USER} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} Get Storagegroup ${NEOFS_IR_WIF} ${READONLY_CID} ${SG_OID_IR} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Run Keyword And Expect Error * ... Delete Storagegroup ${NEOFS_IR_WIF} ${READONLY_CID} ${SG_OID_INV} ${EMPTY} diff --git a/robot/testsuites/integration/acl/acl_bearer_allow_storagegroup.robot b/robot/testsuites/integration/acl/acl_bearer_allow_storagegroup.robot index c1e8dd0..95cb519 100644 --- a/robot/testsuites/integration/acl/acl_bearer_allow_storagegroup.robot +++ b/robot/testsuites/integration/acl/acl_bearer_allow_storagegroup.robot @@ -11,6 +11,7 @@ Resource eacl_tables.robot Resource common_steps_acl_bearer.robot Resource payment_operations.robot Resource setup_teardown.robot +Resource complex_object_operations.robot *** Test cases *** @@ -50,8 +51,9 @@ Check eACL Deny and Allow All Bearer ${SG_OID_INV} = Put Storagegroup ${USER_KEY} ${CID} ${EMPTY} ${S_OID_USER} ${SG_OID_1} = Put Storagegroup ${USER_KEY} ${CID} ${EMPTY} ${S_OID_USER} List Storagegroup ${USER_KEY} ${CID} ${EMPTY} ${SG_OID_1} ${SG_OID_INV} - @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" Get Split objects ${USER_KEY} ${CID} ${S_OID_USER} - ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} + @{EXPECTED_OIDS} = Run Keyword If "${RUN_TYPE}" == "Complex" + ... Get Object Parts By Link Object ${USER_KEY} ${CID} ${S_OID_USER} + ... ELSE IF "${RUN_TYPE}" == "Simple" Create List ${S_OID_USER} Get Storagegroup ${USER_KEY} ${CID} ${SG_OID_1} ${EMPTY} ${EMPTY} @{EXPECTED_OIDS} Delete Storagegroup ${USER_KEY} ${CID} ${SG_OID_1} ${EMPTY} diff --git a/robot/testsuites/integration/network/netmap_control_drop.robot b/robot/testsuites/integration/network/netmap_control_drop.robot index 4603ea7..441c2f7 100644 --- a/robot/testsuites/integration/network/netmap_control_drop.robot +++ b/robot/testsuites/integration/network/netmap_control_drop.robot @@ -13,6 +13,7 @@ Library Process Resource setup_teardown.robot Resource payment_operations.robot Resource storage.robot +Resource complex_object_operations.robot *** Variables *** ${CONTAINER_WAIT_INTERVAL} = 1 min @@ -69,7 +70,7 @@ Drop command in control group Get object ${USER_KEY} ${PRIV_CID} ${C_OID} ${EMPTY} s_file_read Head object ${USER_KEY} ${PRIV_CID} ${C_OID} - @{SPLIT_OIDS} = Get Split objects ${USER_KEY} ${PRIV_CID} ${C_OID} + @{SPLIT_OIDS} = Get Object Parts By Link Object ${USER_KEY} ${PRIV_CID} ${C_OID} FOR ${CHILD_OID} IN @{SPLIT_OIDS} Drop object ${NODE} ${WIF} ${PRIV_CID} ${CHILD_OID} diff --git a/robot/testsuites/integration/object/object_complex.robot b/robot/testsuites/integration/object/object_complex.robot index 3b1c8a0..3682c56 100644 --- a/robot/testsuites/integration/object/object_complex.robot +++ b/robot/testsuites/integration/object/object_complex.robot @@ -2,6 +2,7 @@ Variables common.py Library neofs_verbs.py +Library complex_object_actions.py Library neofs.py Library payment_neogo.py Library contract_keywords.py @@ -15,6 +16,7 @@ Resource payment_operations.robot ${CLEANUP_TIMEOUT} = 10s &{FILE_USR_HEADER} = key1=1 key2=abc &{FILE_USR_HEADER_OTH} = key1=2 +${ALREADY_REMOVED_ERROR} = code = 1024 message = object already removed *** Test cases *** @@ -66,15 +68,23 @@ NeoFS Complex Object Operations Search object ${WIF} ${CID} --root filters=${FILE_USR_HEADER} expected_objects_list=${S_OBJ_H} Search object ${WIF} ${CID} --root filters=${FILE_USR_HEADER_OTH} expected_objects_list=${S_OBJ_H_OTH} - Head object ${WIF} ${CID} ${S_OID} - &{RESPONSE} = Head object ${WIF} ${CID} ${H_OID} + &{S_RESPONSE} = Head object ${WIF} ${CID} ${S_OID} + &{H_RESPONSE} = Head object ${WIF} ${CID} ${H_OID} Dictionary Should Contain Sub Dictionary - ... ${RESPONSE}[header][attributes] + ... ${H_RESPONSE}[header][attributes] ... ${FILE_USR_HEADER} ... msg="There are no User Headers in HEAD response" - Verify Split Chain ${WIF} ${CID} ${S_OID} - Verify Split Chain ${WIF} ${CID} ${H_OID} + ${PAYLOAD_LENGTH} ${SPLIT_ID} ${SPLIT_OBJECTS} = Restore Large Object By Last + ... ${WIF} ${CID} ${S_OID} + ${H_PAYLOAD_LENGTH} ${H_SPLIT_ID} ${H_SPLIT_OBJECTS} = Restore Large Object By Last + ... ${WIF} ${CID} ${H_OID} + + Compare With Link Object ${WIF} ${CID} ${S_OID} ${SPLIT_ID} ${SPLIT_OBJECTS} + Compare With Link Object ${WIF} ${CID} ${H_OID} ${H_SPLIT_ID} ${H_SPLIT_OBJECTS} + + Should Be Equal As Numbers ${S_RESPONSE.header.payloadLength} ${PAYLOAD_LENGTH} + Should Be Equal As Numbers ${H_RESPONSE.header.payloadLength} ${H_PAYLOAD_LENGTH} ${TOMBSTONE_S} = Delete object ${WIF} ${CID} ${S_OID} ${TOMBSTONE_H} = Delete object ${WIF} ${CID} ${H_OID} @@ -86,9 +96,85 @@ NeoFS Complex Object Operations # we assume that during this time objects must be deleted Sleep ${CLEANUP_TIMEOUT} - Run Keyword And Expect Error "rpc error: status: code = 1024 message = object already removed" - ... Get object ${WIF} ${CID} ${S_OID} ${EMPTY} ${GET_OBJ_S} - Run Keyword And Expect Error "rpc error: status: code = 1024 message = object already removed" - ... Get object ${WIF} ${CID} ${H_OID} ${EMPTY} ${GET_OBJ_H} + ${ERR_MSG} = Run Keyword And Expect Error * + ... Get object ${WIF} ${CID} ${S_OID} + Should Contain ${ERR_MSG} ${ALREADY_REMOVED_ERROR} + ${ERR_MSG} = Run Keyword And Expect Error * + ... Get object ${WIF} ${CID} ${H_OID} + Should Contain ${ERR_MSG} ${ALREADY_REMOVED_ERROR} [Teardown] Teardown object_complex + + +*** Keywords *** + +Restore Large Object By Last + [Documentation] In this keyword we assemble Large Object from its parts. First, we search for the + ... Last Object; then, we try to restore the Large Object using Split Chain. We check + ... that all Object Parts have identical SplitID, accumulate total payload length and + ... compile a list of Object Parts. For the first part of split we also check if is + ... has the only `splitID` field in the split header. + ... The keyword returns total payload length, SplitID and list of Part Objects for + ... these data might be verified by other keywords. + + [Arguments] ${WIF} ${CID} ${LARGE_OID} + + ${LAST_OID} = Get Last Object ${WIF} ${CID} ${LARGE_OID} + &{LAST_OBJ_HEADER} = Head Object ${WIF} ${CID} ${LAST_OID} is_raw=True + Should Be Equal ${LARGE_OID} ${LAST_OBJ_HEADER.header.split.parent} + + ${SPLIT_ID} = Set Variable ${LAST_OBJ_HEADER.header.split.splitID} + ${PART_OID} = Set Variable ${LAST_OBJ_HEADER.objectID} + ${PAYLOAD_LENGTH} = Set Variable 0 + @{PART_OBJECTS} = Create List + + FOR ${i} IN RANGE 1000 + &{SPLIT_HEADER} = Head object ${WIF} ${CID} ${PART_OID} is_raw=True + + ${PAYLOAD_LENGTH} = Evaluate ${PAYLOAD_LENGTH} + ${SPLIT_HEADER.header.payloadLength} + + # Every Object of the given split contains the same SplitID + Should Be Equal ${SPLIT_HEADER.header.split.splitID} ${SPLIT_ID} + Should Be Equal ${SPLIT_HEADER.header.objectType} REGULAR + + Append To List ${PART_OBJECTS} ${PART_OID} + + # If we have reached the First Object, it has no `previous` field. + # Asserting this condition and exiting the loop. + ${PASSED} = Run Keyword And Return Status + ... Should Be Equal + ... ${SPLIT_HEADER.header.split.previous} ${None} + + Exit For Loop If ${PASSED} + ${PART_OID} = Set Variable ${SPLIT_HEADER.header.split.previous} + END + + [Return] ${PAYLOAD_LENGTH} ${SPLIT_ID} ${PART_OBJECTS} + + +Compare With Link Object + [Documentation] The keyword accepts Large Object SplitID and its Part Objects as + ... a parameters. Then it requests the Link Object and verifies that + ... a Split Chain which it stores is equal to the Part Objects list. + ... In this way we check that Part Objects list restored from Last + ... Object and the Split Chain from Link Object are equal and the + ... system is able to restore the Large Object using any of these ways. + + [Arguments] ${WIF} ${CID} ${LARGE_OID} ${SPLIT_ID} ${SPLIT_OBJECTS} + + ${LINK_OID} = Get Link Object ${WIF} ${CID} ${LARGE_OID} + &{LINK_HEADER} = Head Object ${WIF} ${CID} ${LINK_OID} is_raw=True + + Reverse List ${SPLIT_OBJECTS} + Lists Should Be Equal + ... ${LINK_HEADER.header.split.children} + ... ${SPLIT_OBJECTS} + + Should Be Equal As Numbers + ... ${LINK_HEADER.header.payloadLength} 0 + + Should Be Equal + ... ${LINK_HEADER.header.objectType} REGULAR + + Should Be Equal + ... ${LINK_HEADER.header.split.splitID} ${SPLIT_ID} diff --git a/robot/testsuites/integration/object/object_storagegroup_complex.robot b/robot/testsuites/integration/object/object_storagegroup_complex.robot index 5fc13ef..50fda2a 100644 --- a/robot/testsuites/integration/object/object_storagegroup_complex.robot +++ b/robot/testsuites/integration/object/object_storagegroup_complex.robot @@ -9,6 +9,7 @@ Library Collections Resource common_steps_object.robot Resource setup_teardown.robot Resource payment_operations.robot +Resource complex_object_operations.robot *** Variables *** ${UNEXIST_OID} = B2DKvkHnLnPvapbDgfpU1oVUPuXQo5LTfKVxmNDZXQff @@ -38,7 +39,7 @@ NeoFS Complex Storagegroup Log Storage group with 1 object ${SG_OID_1} = Put Storagegroup ${WIF} ${CID} ${EMPTY} ${S_OID_1} List Storagegroup ${WIF} ${CID} ${EMPTY} ${SG_OID_1} - @{SPLIT_OBJ_1} = Get Split objects ${WIF} ${CID} ${S_OID_1} + @{SPLIT_OBJ_1} = Get Object Parts By Link Object ${WIF} ${CID} ${S_OID_1} Get Storagegroup ${WIF} ${CID} ${SG_OID_1} ${EMPTY} ${COMPLEX_OBJ_SIZE} @{SPLIT_OBJ_1} ${Tombstone} = Delete Storagegroup ${WIF} ${CID} ${SG_OID_1} ${EMPTY} Verify Head tombstone ${WIF} ${CID} ${Tombstone} ${SG_OID_1} ${ADDR} @@ -49,7 +50,7 @@ NeoFS Complex Storagegroup Log Storage group with 2 objects ${SG_OID_2} = Put Storagegroup ${WIF} ${CID} ${EMPTY} @{S_OBJ_ALL} List Storagegroup ${WIF} ${CID} ${EMPTY} ${SG_OID_2} - @{SPLIT_OBJ_2} = Get Split objects ${WIF} ${CID} ${S_OID_2} + @{SPLIT_OBJ_2} = Get Object Parts By Link Object ${WIF} ${CID} ${S_OID_2} @{SPLIT_OBJ_ALL} = Combine Lists ${SPLIT_OBJ_1} ${SPLIT_OBJ_2} ${EXPECTED_SIZE} = Evaluate 2*${COMPLEX_OBJ_SIZE} Get Storagegroup ${WIF} ${CID} ${SG_OID_2} ${EMPTY} ${EXPECTED_SIZE} @{SPLIT_OBJ_ALL}