dstepanov-yadro/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#199] Add non-global Generate File for the rest

Browse source 
Signed-off-by: Elizaveta Chichindaeva <elizaveta@nspcc.ru>
This commit is contained in:
Elizaveta Chichindaeva 2022-05-20 14:18:14 +03:00
parent 057caa54b0
commit 0799e15526
18 changed files with 82 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
robot/resources/lib/python/neofs.py
View file

@ -2,7 +2,6 @@
import base64
from datetime import datetime
import hashlib
import json
import os
import re
@ -12,13 +11,13 @@ import docker
import base58
from neo3 import wallet
from common import *
from common import (NEOFS_NETMAP, WALLET_PASS, NEOFS_ENDPOINT,
NEOFS_NETMAP_DICT, ASSETS_DIR)
from cli_helpers import _cmd_run
import json_transformers
from robot.api.deco import keyword
from robot.api import logger
from cli_helpers import _run_with_passwd, _cmd_run
import json_transformers
ROBOT_AUTO_KEYWORDS = False
# path to neofs-cli executable
@ -36,9 +35,9 @@ def get_scripthash(wif: str):
def stop_nodes(down_num: int, *nodes_list):
# select nodes to stop from list
stop_nodes = random.sample(nodes_list, down_num)
nodes = random.sample(nodes_list, down_num)
for node in stop_nodes:
for node in nodes:
m = re.search(r'(s\d+).', node)
node = m.group(1)
@ -60,14 +59,13 @@ def start_nodes(*nodes_list):
@keyword('Get nodes with object')
def get_nodes_with_object(wallet: str, cid: str, oid: str):
copies = 0
nodes_list = []
for node in NEOFS_NETMAP:
search_res = _search_object(node, wallet, cid, oid)
if search_res:
if oid in search_res:
res = _search_object(node, wallet, cid, oid)
if res:
if oid in res:
nodes_list.append(node)
logger.info(f"Nodes with object: {nodes_list}")
@ -76,7 +74,6 @@ def get_nodes_with_object(wallet: str, cid: str, oid: str):
@keyword('Get nodes without object')
def get_nodes_without_object(wallet: str, cid: str, oid: str):
copies = 0
nodes_list = []
@ -101,14 +98,14 @@ def validate_storage_policy_for_object(wallet: str, expected_copies: int, cid, o
oid = oid.strip()
for node in storage_nodes:
search_res = _search_object(node, wallet, cid, oid)
if search_res:
if oid in search_res:
res = _search_object(node, wallet, cid, oid)
if res:
if oid in res:
copies += 1
found_nodes.append(node)
if copies != expected_copies:
raise Exception(f"Object copies is not match storage policy.",
raise Exception("Object copies is not match storage policy."
f"Found: {copies}, expected: {expected_copies}.")
else:
logger.info(f"Found copies: {copies}, expected: {expected_copies}")
@ -117,9 +114,11 @@ def validate_storage_policy_for_object(wallet: str, expected_copies: int, cid, o
if expected_node_list:
if sorted(found_nodes) == sorted(expected_node_list):
logger.info(f"Found node list '{found_nodes}' is equal for expected list '{expected_node_list}'")
logger.info(f"Found node list '{found_nodes}' "
f"is equal for expected list '{expected_node_list}'")
else:
raise Exception(f"Found node list '{found_nodes}' is not equal to expected list '{expected_node_list}'")
raise Exception(f"Found node list '{found_nodes}' "
f"is not equal to expected list '{expected_node_list}'")
@keyword('Verify Head Tombstone')
@ -135,48 +134,42 @@ def verify_head_tombstone(wallet: str, cid: str, oid_ts: str, oid: str, addr: st
# Header verification
header_cid = full_headers["header"]["containerID"]["value"]
if (json_transformers.json_reencode(header_cid) == cid):
if json_transformers.json_reencode(header_cid) == cid:
logger.info(f"Header CID is expected: {cid} ({header_cid} in the output)")
else:
raise Exception("Header CID is not expected.")
header_owner = full_headers["header"]["ownerID"]["value"]
if (json_transformers.json_reencode(header_owner) == addr):
if json_transformers.json_reencode(header_owner) == addr:
logger.info(f"Header ownerID is expected: {addr} ({header_owner} in the output)")
else:
raise Exception("Header ownerID is not expected.")
header_type = full_headers["header"]["objectType"]
if (header_type == "TOMBSTONE"):
if header_type == "TOMBSTONE":
logger.info(f"Header Type is expected: {header_type}")
else:
raise Exception("Header Type is not expected.")
header_session_type = full_headers["header"]["sessionToken"]["body"]["object"]["verb"]
if (header_session_type == "DELETE"):
if header_session_type == "DELETE":
logger.info(f"Header Session Type is expected: {header_session_type}")
else:
raise Exception("Header Session Type is not expected.")
header_session_cid = full_headers["header"]["sessionToken"]["body"]["object"]["address"]["containerID"]["value"]
if (json_transformers.json_reencode(header_session_cid) == cid):
if json_transformers.json_reencode(header_session_cid) == cid:
logger.info(f"Header ownerID is expected: {addr} ({header_session_cid} in the output)")
else:
raise Exception("Header Session CID is not expected.")
header_session_oid = full_headers["header"]["sessionToken"]["body"]["object"]["address"]["objectID"]["value"]
if (json_transformers.json_reencode(header_session_oid) == oid):
if json_transformers.json_reencode(header_session_oid) == oid:
logger.info(f"Header Session OID (deleted object) is expected: {oid} ({header_session_oid} in the output)")
else:
raise Exception("Header Session OID (deleted object) is not expected.")
@keyword('Get file hash')
def get_file_hash(filename : str):
file_hash = _get_file_hash(filename)
return file_hash
@keyword('Get control endpoint with wif')
def get_control_endpoint_with_wif(endpoint_number: str = ''):
if endpoint_number == '':
@ -232,7 +225,7 @@ def get_logs_latest_timestamp():
@keyword('Find in Nodes Log')
def find_in_nodes_Log(line: str, nodes_logs_time: dict):
def find_in_nodes_log(line: str, nodes_logs_time: dict):
client_api = docker.APIClient()
container_names = list()
@ -314,16 +307,6 @@ def sign_session_token(session_token: str, wallet: str, to_file: str=''):
_cmd_run(cmd)
def _get_file_hash(filename):
blocksize = 65536
hash = hashlib.md5()
with open(filename, "rb") as f:
for block in iter(lambda: f.read(blocksize), b""):
hash.update(block)
logger.info(f"Hash: {hash.hexdigest()}")
return hash.hexdigest()
def _parse_oid(input_str: str):
"""
This function parses OID from given CLI output. The input string we
@ -348,9 +331,9 @@ def _parse_oid(input_str: str):
def _search_object(node:str, wallet: str, cid:str, oid: str):
Cmd = (
cmd = (
f'{NEOFS_CLI_EXEC} --rpc-endpoint {node} --wallet {wallet} --ttl 1 '
f'object search --root --cid {cid} --oid {oid} --config {WALLET_PASS}'
)
output = _cmd_run(Cmd)
output = _cmd_run(cmd)
return output

43
robot/resources/lib/python/utility_keywords.py
View file

@ -15,19 +15,6 @@ from robot.libraries.BuiltIn import BuiltIn
ROBOT_AUTO_KEYWORDS = False
@keyword('Generate file of bytes')
def generate_file_of_bytes(size: str) -> str:
"""
Function generates big binary file with the specified size in bytes.
:param size: the size in bytes, can be declared as 6e+6 for example
"""
size = int(float(size))
filename = f"{os.getcwd()}/{ASSETS_DIR}/{uuid.uuid4()}"
with open(filename, 'wb') as fout:
fout.write(os.urandom(size))
logger.info(f"file with size {size} bytes has been generated: {filename}")
return filename
@keyword('Generate file')
def generate_file_and_file_hash(size: str) -> str:
"""
@ -44,10 +31,29 @@ def generate_file_and_file_hash(size: str) -> str:
fout.write(os.urandom(size))
logger.info(f"file with size {size} bytes has been generated: {filename}")
file_hash = _get_file_hash(filename)
file_hash = get_file_hash(filename)
return filename, file_hash
@keyword('Get File Hash')
def get_file_hash(filename: str):
"""
This function generates hash for the specified file.
Args:
filename (str): the path to the file to generate hash for
Returns:
(str): the hash of the file
"""
blocksize = 65536
file_hash = hashlib.md5()
with open(filename, "rb") as out:
for block in iter(lambda: out.read(blocksize), b""):
file_hash.update(block)
logger.info(f"Hash: {file_hash.hexdigest()}")
return file_hash.hexdigest()
@keyword('Get Docker Logs')
def get_container_logs(testcase_name: str) -> None:
client = docker.APIClient(base_url='unix://var/run/docker.sock')
@ -102,12 +108,3 @@ def make_down(services: list=[]):
_cmd_run(cmd, timeout=60)
os.chdir(test_path)
def _get_file_hash(filename: str):
blocksize = 65536
file_hash = hashlib.md5()
with open(filename, "rb") as out:
for block in iter(lambda: out.read(blocksize), b""):
file_hash.update(block)
logger.info(f"Hash: {file_hash.hexdigest()}")
return file_hash.hexdigest()

5
robot/testsuites/integration/acl/object_attributes/creation_epoch_filter.robot
View file

@ -4,12 +4,9 @@ Variables eacl_object_filters.py
Library acl.py
Library container.py
Library neofs.py
Library Collections
Library contract_keywords.py
Library utility_keywords.py
Resource common_steps_acl_extended.robot
Resource common_steps_acl_basic.robot
Resource payment_operations.robot
Resource setup_teardown.robot

4
robot/testsuites/integration/acl/object_attributes/object_id_filter.robot
View file

@ -4,11 +4,9 @@ Variables eacl_object_filters.py
Library acl.py
Library container.py
Library neofs.py
Library Collections
Library utility_keywords.py
Resource common_steps_acl_extended.robot
Resource common_steps_acl_basic.robot
Resource payment_operations.robot
Resource setup_teardown.robot

4
robot/testsuites/integration/acl/object_attributes/payload_length_filter.robot
View file

@ -4,11 +4,9 @@ Variables eacl_object_filters.py
Library acl.py
Library container.py
Library neofs.py
Library Collections
Library utility_keywords.py
Resource common_steps_acl_extended.robot
Resource common_steps_acl_basic.robot
Resource payment_operations.robot
Resource setup_teardown.robot

1
robot/testsuites/integration/acl/storage_group/bearer_allow_sg.robot
View file

@ -6,6 +6,7 @@ Library acl.py
Library container.py
Library neofs.py
Library neofs_verbs.py
Library utility_keywords.py
Resource common_steps_acl_bearer.robot
Resource eacl_tables.robot

12
robot/testsuites/integration/network/netmap_control_drop.robot
View file

@ -3,13 +3,9 @@ Variables common.py
Variables wellknown_acl.py
Library container.py
Library contract_keywords.py
Library neofs.py
Library neofs_verbs.py
Library payment_neogo.py
Library String
Library Process
Library utility_keywords.py
Resource setup_teardown.robot
Resource payment_operations.robot
@ -27,12 +23,12 @@ Drop command in control group
${WALLET_STORAGE} ${_} = Prepare Wallet with WIF And Deposit ${WIF_STORAGE}
${LOCODE} = Get Locode
${FILE_SIMPLE} = Generate file of bytes ${SIMPLE_OBJ_SIZE}
${FILE_COMPLEX} = Generate file of bytes ${COMPLEX_OBJ_SIZE}
${FILE_SIMPLE} ${_} = Generate file ${SIMPLE_OBJ_SIZE}
${FILE_COMPLEX} ${_} = Generate file ${COMPLEX_OBJ_SIZE}
${WALLET} ${_} ${_} = Prepare Wallet And Deposit
${PRIV_CID} = Create container ${WALLET} basic_acl=${PRIVATE_ACL_F}
${PRIV_CID} = Create Container ${WALLET}
... rule=REP 1 CBF 1 SELECT 1 FROM * FILTER 'UN-LOCODE' EQ '${LOCODE}' AS LOC
#########################

6
robot/testsuites/integration/network/netmap_simple.robot
View file

@ -4,9 +4,7 @@ Variables common.py
Library container.py
Library neofs.py
Library neofs_verbs.py
Library payment_neogo.py
Library wallet_keywords.py
Library rpc_call_keywords.py
Library utility_keywords.py
Resource payment_operations.robot
Resource setup_teardown.robot
@ -23,7 +21,7 @@ NeoFS Simple Netmap
[Setup] Setup
${WALLET} ${_} ${_} = Prepare Wallet And Deposit
${FILE} = Generate file of bytes ${SIMPLE_OBJ_SIZE}
${FILE} ${_} = Generate file ${SIMPLE_OBJ_SIZE}
Validate Policy ${WALLET} ${FILE} REP 2 IN X CBF 2 SELECT 2 FROM * AS X 2 @{EMPTY}

7
robot/testsuites/integration/network/replication.robot
View file

@ -3,12 +3,10 @@ Variables common.py
Variables wellknown_acl.py
Library container.py
Library payment_neogo.py
Library neofs.py
Library neofs_verbs.py
Library wallet_keywords.py
Library rpc_call_keywords.py
Library contract_keywords.py
Library utility_keywords.py
Library Collections
@ -41,8 +39,7 @@ Check Replication
${WALLET} ${_} ${_} = Prepare Wallet And Deposit
${CID} = Create Container ${WALLET} basic_acl=${ACL}
${FILE} = Generate file of bytes ${SIMPLE_OBJ_SIZE}
${FILE_HASH} = Get file hash ${FILE}
${FILE} ${_} = Generate file ${SIMPLE_OBJ_SIZE}
${S_OID} = Put Object ${WALLET} ${FILE} ${CID}
Validate storage policy for object ${WALLET} ${EXPECTED_COPIES} ${CID} ${S_OID}

6
robot/testsuites/integration/object/object_attributes.robot
View file

@ -3,10 +3,8 @@ Variables common.py
Variables wellknown_acl.py
Library container.py
Library neofs.py
Library neofs_verbs.py
Library payment_neogo.py
Library String
Library utility_keywords.py
Library Collections
Resource setup_teardown.robot
@ -30,7 +28,7 @@ Duplicated Object Attributes
${WALLET} ${_} ${_} = Prepare Wallet And Deposit
${PUBLIC_CID} = Create Container ${WALLET} basic_acl=${PUBLIC_ACL_F}
${FILE_S} = Generate file of bytes ${SIMPLE_OBJ_SIZE}
${FILE_S} ${_} = Generate File ${SIMPLE_OBJ_SIZE}
###################################################

5
robot/testsuites/integration/object/object_complex.robot
View file

@ -4,9 +4,9 @@ Variables common.py
Library neofs_verbs.py
Library complex_object_actions.py
Library neofs.py
Library payment_neogo.py
Library contract_keywords.py
Library Collections
Library utility_keywords.py
Resource common_steps_object.robot
Resource setup_teardown.robot
@ -30,8 +30,7 @@ NeoFS Complex Object Operations
${WALLET} ${ADDR} ${WIF} = Prepare Wallet And Deposit
${CID} = Prepare container ${WIF} ${WALLET}
${FILE} = Generate file of bytes ${COMPLEX_OBJ_SIZE}
${FILE_HASH} = Get file hash ${FILE}
${FILE} ${FILE_HASH} = Generate file ${COMPLEX_OBJ_SIZE}
${S_OID} = Put object ${WALLET} ${FILE} ${CID}
${H_OID} = Put object ${WALLET} ${FILE} ${CID} user_headers=${FILE_USR_HEADER}

7
robot/testsuites/integration/object/object_expiration.robot
View file

@ -1,10 +1,9 @@
*** Settings ***
Variables common.py
Library neofs.py
Library neofs_verbs.py
Library payment_neogo.py
Library contract_keywords.py
Library utility_keywords.py
Resource common_steps_object.robot
Resource setup_teardown.robot
@ -24,9 +23,7 @@ NeoFS Simple Object Operations
${WALLET} ${_} ${WIF} = Prepare Wallet And Deposit
${CID} = Prepare container ${WIF} ${WALLET}
${FILE} = Generate file of bytes ${SIMPLE_OBJ_SIZE}
${FILE_HASH} = Get file hash ${FILE}
${FILE} ${FILE_HASH} = Generate File ${SIMPLE_OBJ_SIZE}
${EPOCH} = Get Epoch
${EPOCH_PRE} = Evaluate ${EPOCH}-1

6
robot/testsuites/integration/object/object_simple.robot
View file

@ -3,9 +3,9 @@ Variables common.py
Library neofs.py
Library neofs_verbs.py
Library payment_neogo.py
Library contract_keywords.py
Library Collections
Library utility_keywords.py
Resource common_steps_object.robot
Resource payment_operations.robot
@ -28,9 +28,7 @@ NeoFS Simple Object Operations
${WALLET} ${ADDR} ${WIF} = Prepare Wallet And Deposit
${CID} = Prepare container ${WIF} ${WALLET}
${FILE} = Generate file of bytes ${SIMPLE_OBJ_SIZE}
${FILE_HASH} = Get file hash ${FILE}
${FILE} ${FILE_HASH} = Generate file ${SIMPLE_OBJ_SIZE}
${S_OID} = Put object ${WALLET} ${FILE} ${CID}
${H_OID} = Put object ${WALLET} ${FILE} ${CID} user_headers=${FILE_USR_HEADER}

3
robot/testsuites/integration/object/storage_group/sg_of_complex_objects.robot
View file

@ -5,6 +5,7 @@ Library neofs.py
Library neofs_verbs.py
Library storage_group.py
Library Collections
Library utility_keywords.py
Resource common_steps_object.robot
Resource setup_teardown.robot
@ -25,7 +26,7 @@ NeoFS Complex Storagegroup
${WALLET} ${_} ${WIF} = Prepare Wallet And Deposit
${CID} = Prepare container ${WIF} ${WALLET}
${FILE} = Generate file of bytes ${COMPLEX_OBJ_SIZE}
${FILE} ${_} = Generate file ${COMPLEX_OBJ_SIZE}
${OID_1} = Put object ${WALLET} ${FILE} ${CID}
${OID_2} = Put object ${WALLET} ${FILE} ${CID}

5
robot/testsuites/integration/object/storage_group/sg_of_simple_objects.robot
View file

@ -1,10 +1,9 @@
*** Settings ***
Variables common.py
Library neofs.py
Library neofs_verbs.py
Library payment_neogo.py
Library storage_group.py
Library utility_keywords.py
Resource common_steps_object.robot
Resource setup_teardown.robot
@ -25,7 +24,7 @@ NeoFS Simple Storagegroup
${WALLET} ${_} ${WIF} = Prepare Wallet And Deposit
${CID} = Prepare container ${WIF} ${WALLET}
${FILE_S} = Generate file of bytes ${SIMPLE_OBJ_SIZE}
${FILE_S} ${_} = Generate file ${SIMPLE_OBJ_SIZE}
${OID_1} = Put object ${WALLET} ${FILE_S} ${CID}
${OID_2} = Put object ${WALLET} ${FILE_S} ${CID}

17
robot/testsuites/integration/services/http_gate.robot
View file

@ -6,6 +6,7 @@ Library container.py
Library neofs.py
Library neofs_verbs.py
Library http_gate.py
Library utility_keywords.py
Resource payment_operations.robot
Resource setup_teardown.robot
@ -24,11 +25,9 @@ NeoFS HTTP Gateway
Make Up ${INCLUDE_SVC}
${WALLET} ${_} ${_} = Prepare Wallet And Deposit
${CID} = Create container ${WALLET} rule=${PLACEMENT_RULE} basic_acl=${PUBLIC_ACL}
${FILE} = Generate file of bytes ${SIMPLE_OBJ_SIZE}
${FILE_L} = Generate file of bytes ${COMPLEX_OBJ_SIZE}
${FILE_HASH} = Get file hash ${FILE}
${FILE_L_HASH} = Get file hash ${FILE_L}
${CID} = Create container ${WALLET} rule=${PLACEMENT_RULE} basic_acl=${PUBLIC_ACL}
${FILE} ${HASH} = Generate file ${SIMPLE_OBJ_SIZE}
${FILE_L} ${L_HASH} = Generate file ${COMPLEX_OBJ_SIZE}
${S_OID} = Put object ${WALLET} ${FILE} ${CID}
${L_OID} = Put object ${WALLET} ${FILE_L} ${CID}
@ -43,8 +42,8 @@ NeoFS HTTP Gateway
${PLAIN_FILE_HASH} = Get file hash ${GET_OBJ_S}
${GATE_FILE_HASH} = Get file hash ${FILEPATH}
Should Be Equal ${FILE_HASH} ${PLAIN_FILE_HASH}
Should Be Equal ${FILE_HASH} ${GATE_FILE_HASH}
Should Be Equal ${HASH} ${PLAIN_FILE_HASH}
Should Be Equal ${HASH} ${GATE_FILE_HASH}
@{GET_NODE_LIST} = Get nodes without object ${WALLET} ${CID} ${L_OID}
${NODE} = Evaluate random.choice($GET_NODE_LIST) random
@ -54,7 +53,7 @@ NeoFS HTTP Gateway
${PLAIN_FILE_HASH} = Get file hash ${GET_OBJ_L}
${GATE_FILE_HASH} = Get file hash ${FILEPATH}
Should Be Equal ${FILE_L_HASH} ${PLAIN_FILE_HASH}
Should Be Equal ${FILE_L_HASH} ${GATE_FILE_HASH}
Should Be Equal ${L_HASH} ${PLAIN_FILE_HASH}
Should Be Equal ${L_HASH} ${GATE_FILE_HASH}
[Teardown] Teardown http_gate

3
robot/testsuites/integration/services/s3_gate_bucket.robot
View file

@ -7,6 +7,7 @@ Library OperatingSystem
Library neofs.py
Library s3_gate.py
Library contract_keywords.py
Library utility_keywords.py
Resource setup_teardown.robot
Resource payment_operations.robot
@ -23,7 +24,7 @@ Buckets in NeoFS S3 Gateway
Make Up ${INCLUDE_SVC}
${WALLET} ${_} ${WIF} = Prepare Wallet And Deposit
${FILE_S3} = Generate file of bytes ${COMPLEX_OBJ_SIZE}
${FILE_S3} ${_} = Generate file ${COMPLEX_OBJ_SIZE}
${_} ${S3_OBJECT_KEY} = Split Path ${FILE_S3}
${CID}

4
robot/testsuites/integration/services/s3_gate_object.robot
View file

@ -8,6 +8,7 @@ Library container.py
Library neofs.py
Library s3_gate.py
Library contract_keywords.py
Library utility_keywords.py
Resource payment_operations.robot
Resource setup_teardown.robot
@ -26,8 +27,7 @@ Objects in NeoFS S3 Gateway
${WALLET} ${_} ${_} = Prepare Wallet And Deposit
${FILE_S3} = Generate file of bytes ${COMPLEX_OBJ_SIZE}
${FILE_S3_HASH} = Get file hash ${FILE_S3}
${FILE_S3} ${FILE_S3_HASH} = Generate file ${COMPLEX_OBJ_SIZE}
${_} ${S3_OBJECT_KEY} = Split Path ${FILE_S3}
${CID}