diff --git a/robot/resources/lib/neofs.py b/robot/resources/lib/neofs.py index 20484b4e..e82c7719 100644 --- a/robot/resources/lib/neofs.py +++ b/robot/resources/lib/neofs.py @@ -1,10 +1,8 @@ -#!/usr/bin/python3.7 +#!/usr/bin/python3.8 import subprocess import os import re -import binascii -import uuid import hashlib from robot.api.deco import keyword from robot.api import logger @@ -14,9 +12,7 @@ import base58 import docker import json import tarfile -import shutil -import time from datetime import datetime from common import * @@ -107,8 +103,9 @@ def get_nodes_without_object(private_key: str, cid: str, oid: str): @keyword('Validate storage policy for object') -def validate_storage_policy_for_object(private_key: str, expected_copies: int, cid, oid, *expected_node_list): - storage_nodes = _get_storage_nodes() +def validate_storage_policy_for_object(private_key: str, expected_copies: int, cid, oid, + expected_node_list=[], storage_nodes=[]): + storage_nodes = storage_nodes if len(storage_nodes) != 0 else _get_storage_nodes() copies = 0 found_nodes = [] @@ -120,7 +117,8 @@ def validate_storage_policy_for_object(private_key: str, expected_copies: int, c found_nodes.append(node) if copies != expected_copies: - raise Exception("Object copies is not match storage policy. Found: %s, expexted: %s." % (copies, expected_copies)) + raise Exception(f"Object copies is not match storage policy.", + f"Found: {copies}, expected: {expected_copies}.") else: logger.info("Found copies: %s, expected: %s" % (copies, expected_copies)) @@ -156,22 +154,6 @@ def get_eacl(private_key: str, cid: str): raise Exception("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) -@keyword('Get Epoch') -def get_epoch(private_key: str): - cmd = ( - f'{NEOFS_CLI_EXEC} --rpc-endpoint {NEOFS_ENDPOINT} --wif {private_key} ' - f'netmap epoch' - ) - logger.info(f"Cmd: {cmd}") - try: - complProc = subprocess.run(cmd, check=True, universal_newlines=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=150, shell=True) - output = complProc.stdout - logger.info(f"Output: {output}") - return int(output) - except subprocess.CalledProcessError as e: - raise Exception(f"command '{e.cmd}' return with error (code {e.returncode}): {e.output}") - @keyword('Set eACL') def set_eacl(private_key: str, cid: str, eacl_table_path: str): cmd = ( @@ -1128,7 +1110,7 @@ def _parse_cid(input_str: str): def _get_storage_nodes(): # TODO: fix to get netmap from neofs-cli - logger.info("Storage nodes: %s" % NEOFS_NETMAP) + logger.info(f"Storage nodes: {NEOFS_NETMAP}") return NEOFS_NETMAP def _search_object(node:str, private_key: str, cid:str, oid: str): diff --git a/robot/testsuites/integration/network/replication.robot b/robot/testsuites/integration/network/replication.robot index f335d1f8..1137b561 100644 --- a/robot/testsuites/integration/network/replication.robot +++ b/robot/testsuites/integration/network/replication.robot @@ -1,16 +1,20 @@ *** Settings *** Variables ../../../variables/common.py -Library ../${RESOURCES}/neofs.py +Library Collections Library ../${RESOURCES}/payment_neogo.py -Library ${KEYWORDS}/wallet_keywords.py -Library ${KEYWORDS}/rpc_call_keywords.py +Library ../${RESOURCES}/neofs.py +Library wallet_keywords.py +Library rpc_call_keywords.py +Library contract_keywords.py Resource ../${RESOURCES}/payment_operations.robot Resource ../${RESOURCES}/setup_teardown.robot *** Variables *** ${PLACEMENT_RULE} = REP 2 IN X CBF 1 SELECT 4 FROM * AS X +${EXPECTED_COPIES} = ${2} +${CHECK_INTERVAL} = 1 min *** Test cases *** NeoFS Object Replication @@ -29,25 +33,40 @@ NeoFS Object Replication ${FILE} = Generate file of bytes ${SIMPLE_OBJ_SIZE} ${FILE_HASH} = Get file hash ${FILE} - ${S_OID} = Put object ${WIF} ${FILE} ${CID} ${EMPTY} ${EMPTY} - Validate storage policy for object ${WIF} 2 ${CID} ${S_OID} + ${S_OID} = Put object ${WIF} ${FILE} ${CID} ${EMPTY} ${EMPTY} + Validate storage policy for object ${WIF} ${EXPECTED_COPIES} ${CID} ${S_OID} @{NODES_OBJ} = Get nodes with object ${WIF} ${CID} ${S_OID} - ${NODES_LOG_TIME} = Get Nodes Log Latest Timestamp - @{NODES_OBJ_STOPPED} = Stop nodes 1 @{NODES_OBJ} + @{NODES_OBJ_STOPPED} = Stop nodes 1 @{NODES_OBJ} + @{NETMAP} = Convert To List ${NEOFS_NETMAP} + Remove Values From List ${NETMAP} @{NODES_OBJ_STOPPED} - ${state} ${output}= Run Keyword And Ignore Error - ... Wait Until Keyword Succeeds 10 min 2 min - ... Validate storage policy for object ${WIF} 2 ${CID} ${S_OID} + # We expect that during two epochs the missed copy will be replicated. + FOR ${i} IN RANGE 2 + ${PASSED} = Run Keyword And Return Status + ... Validate storage policy for object ${WIF} ${EXPECTED_COPIES} + ... ${CID} ${S_OID} ${EMPTY} ${NETMAP} + Exit For Loop If ${PASSED} + Tick Epoch + Sleep ${CHECK_INTERVAL} + END + Run Keyword Unless ${PASSED} Fail Keyword failed: Validate storage policy for object ${S_OID} - Run Keyword If '${state}'!='PASS' Log Warning: Keyword failed: Validate storage policy for object ${S_OID} {\n}${output} WARN - Find in Nodes Log object successfully replicated ${NODES_LOG_TIME} - Start nodes @{NODES_OBJ_STOPPED} + Find in Nodes Log object successfully replicated ${NODES_LOG_TIME} + Start nodes @{NODES_OBJ_STOPPED} + Tick Epoch - # We have 2 or 3 copies. Expected behaviour: after one epoch potential 3rd copy should be removed. - Sleep ${NEOFS_EPOCH_TIMEOUT} - Validate storage policy for object ${WIF} 2 ${CID} ${S_OID} + # We have 2 or 3 copies. Expected behaviour: during two epochs potential 3rd copy should be removed. + FOR ${i} IN RANGE 2 + ${PASSED} = Run Keyword And Return Status + ... Validate storage policy for object ${WIF} ${EXPECTED_COPIES} ${CID} ${S_OID} + Exit For Loop If ${PASSED} + Tick Epoch + Sleep ${CHECK_INTERVAL} + END + Run Keyword Unless ${PASSED} Fail Keyword failed: Validate storage policy for object ${S_OID} - [Teardown] Teardown replaication + + [Teardown] Teardown replication diff --git a/robot/testsuites/integration/object/object_complex.robot b/robot/testsuites/integration/object/object_complex.robot index d5ee0059..3d6f48b2 100644 --- a/robot/testsuites/integration/object/object_complex.robot +++ b/robot/testsuites/integration/object/object_complex.robot @@ -8,6 +8,8 @@ Resource common_steps_object.robot Resource ../${RESOURCES}/setup_teardown.robot Resource ../${RESOURCES}/payment_operations.robot +*** Variables *** +${CLEANUP_TIMEOUT} = 10s *** Test cases *** NeoFS Complex Object Operations @@ -68,7 +70,9 @@ NeoFS Complex Object Operations Verify Head tombstone ${WIF} ${CID} ${TOMBSTONE_S} ${S_OID} ${ADDR} Verify Head tombstone ${WIF} ${CID} ${TOMBSTONE_H} ${H_OID} ${ADDR} - Sleep 2min + Tick Epoch + # we assume that during this time objects must be deleted + Sleep ${CLEANUP_TIMEOUT} Run Keyword And Expect Error * ... Get object ${WIF} ${CID} ${S_OID} ${EMPTY} ${GET_OBJ_S} diff --git a/robot/testsuites/integration/object/object_expiration.robot b/robot/testsuites/integration/object/object_expiration.robot index 53860d51..4ee5dc84 100644 --- a/robot/testsuites/integration/object/object_expiration.robot +++ b/robot/testsuites/integration/object/object_expiration.robot @@ -8,6 +8,8 @@ Resource common_steps_object.robot Resource ../${RESOURCES}/setup_teardown.robot Resource ../${RESOURCES}/payment_operations.robot +*** Variables *** +${CLEANUP_TIMEOUT} = 10s *** Test cases *** NeoFS Simple Object Operations @@ -24,7 +26,7 @@ NeoFS Simple Object Operations ${FILE} = Generate file of bytes ${SIMPLE_OBJ_SIZE} ${FILE_HASH} = Get file hash ${FILE} - ${EPOCH} = Get Epoch ${WIF} + ${EPOCH} = Get Epoch ${EPOCH_PRE} = Evaluate ${EPOCH}-1 ${EPOCH_NEXT} = Evaluate ${EPOCH}+1 @@ -44,9 +46,10 @@ NeoFS Simple Object Operations Get object ${WIF} ${CID} ${OID_NXT} ${EMPTY} file_read_nxt Get object ${WIF} ${CID} ${OID_PST} ${EMPTY} file_read_pst - # Wait one epoch to check that expired objects (OID_CUR) will be removed - Sleep ${NEOFS_EPOCH_TIMEOUT} - + # Increment epoch to check that expired objects (OID_CUR) will be removed + Tick Epoch + # we assume that during this time objects must be deleted + Sleep ${CLEANUP_TIMEOUT} Run Keyword And Expect Error * ... Get object ${WIF} ${CID} ${OID_CUR} ${EMPTY} file_read @@ -54,9 +57,10 @@ NeoFS Simple Object Operations Get object ${WIF} ${CID} ${OID_NXT} ${EMPTY} file_read Get object ${WIF} ${CID} ${OID_PST} ${EMPTY} file_read_pst - # Wait one more epoch to check that expired object (OID_NXT) will be removed - Sleep ${NEOFS_EPOCH_TIMEOUT} - + # Increment one more epoch to check that expired object (OID_NXT) will be removed + Tick Epoch + # we assume that during this time objects must be deleted + Sleep ${CLEANUP_TIMEOUT} Run Keyword And Expect Error * ... Get object ${WIF} ${CID} ${OID_NXT} ${EMPTY} file_read diff --git a/robot/testsuites/integration/object/object_simple.robot b/robot/testsuites/integration/object/object_simple.robot index 78474600..bfdb9b0c 100644 --- a/robot/testsuites/integration/object/object_simple.robot +++ b/robot/testsuites/integration/object/object_simple.robot @@ -8,6 +8,8 @@ Resource common_steps_object.robot Resource ../${RESOURCES}/payment_operations.robot Resource ../${RESOURCES}/setup_teardown.robot +*** Variables *** +${CLEANUP_TIMEOUT} = 10s *** Test cases *** NeoFS Simple Object Operations @@ -62,7 +64,9 @@ NeoFS Simple Object Operations Verify Head tombstone ${WIF} ${CID} ${TOMBSTONE_S} ${S_OID} ${ADDR} Verify Head tombstone ${WIF} ${CID} ${TOMBSTONE_H} ${H_OID} ${ADDR} - Sleep 2min + Tick Epoch + # we assume that during this time objects must be deleted + Sleep ${CLEANUP_TIMEOUT} Run Keyword And Expect Error * ... Get object ${WIF} ${CID} ${S_OID} ${EMPTY} ${GET_OBJ_S}