From bdffe9d2dc188c1b884aaddacc1e1f78de6da056 Mon Sep 17 00:00:00 2001 From: "anatoly@nspcc.ru" Date: Fri, 4 Dec 2020 15:28:59 +0300 Subject: [PATCH 1/3] add http-gate testcase, fix issue with binary files in the get from gate. --- robot/resources/lib/gates.py | 20 +++++-- robot/resources/lib/neofs.py | 24 +++++--- robot/resources/lib/neofs_int_vars.py | 2 +- robot/testsuites/integration/http_gate.robot | 55 +++++++++++++++++++ .../integration/object_complex.robot | 4 +- .../integration/object_simple.robot | 6 +- 6 files changed, 94 insertions(+), 17 deletions(-) create mode 100644 robot/testsuites/integration/http_gate.robot diff --git a/robot/resources/lib/gates.py b/robot/resources/lib/gates.py index d33b9dd..88a835c 100644 --- a/robot/resources/lib/gates.py +++ b/robot/resources/lib/gates.py @@ -3,6 +3,7 @@ import logging import os import requests +import shutil from robot.api.deco import keyword from robot.api import logger @@ -27,15 +28,26 @@ def get_via_http_gate(cid: str, oid: str): :param cid: CID to get object from :param oid: object OID """ - resp = requests.get(f'{HTTP_GATE}/get/{cid}/{oid}') + request = f'{HTTP_GATE}/get/{cid}/{oid}' + resp = requests.get(request, stream=True) + if not resp.ok: - logger.info(f"""Failed to get object via HTTP gate: + raise Exception(f"""Failed to get object via HTTP gate: request: {resp.request.path_url}, response: {resp.text}, status code: {resp.status_code} {resp.reason}""") return + logger.info(f'Request: {request}') filename = os.path.curdir + f"/{cid}_{oid}" - with open(filename, "w+") as f: - f.write(resp.text) + with open(filename, "wb") as f: + shutil.copyfileobj(resp.raw, f) + del resp return filename + + +#url = 'http://example.com/img.png' +#response = requests.get(url, stream=True) +#with open('img.png', 'wb') as out_file: +# shutil.copyfileobj(response.raw, out_file) +#del response \ No newline at end of file diff --git a/robot/resources/lib/neofs.py b/robot/resources/lib/neofs.py index 6b82188..85a99ed 100644 --- a/robot/resources/lib/neofs.py +++ b/robot/resources/lib/neofs.py @@ -167,9 +167,6 @@ def validate_storage_policy_for_object(private_key: str, expected_copies: int, c raise Exception("Found node list '{}' is not equal to expected list '{}'".format(found_nodes, expected_node_list)) - - - @keyword('Get eACL') def get_eacl(private_key: str, cid: str): @@ -843,16 +840,19 @@ def cleanup_file(*filename_list): @keyword('Put object to NeoFS') -def put_object(private_key: str, path: str, cid: str, bearer: str, user_headers: str): +def put_object(private_key: str, path: str, cid: str, bearer: str, user_headers: str, endpoint: str="" ): logger.info("Going to put the object") + if not endpoint: + endpoint = random.sample(_get_storage_nodes(private_key), 1)[0] + if user_headers: user_headers = f"--attributes {user_headers}" if bearer: bearer = f"--bearer {bearer}" - putObjectCmd = f'neofs-cli --rpc-endpoint {NEOFS_ENDPOINT} --key {private_key} object put --file {path} --cid {cid} {bearer} {user_headers}' + putObjectCmd = f'neofs-cli --rpc-endpoint {endpoint} --key {private_key} object put --file {path} --cid {cid} {bearer} {user_headers}' logger.info("Cmd: %s" % putObjectCmd) try: @@ -882,13 +882,21 @@ def get_range_hash(private_key: str, cid: str, oid: str, bearer_token: str, rang except subprocess.CalledProcessError as e: raise Exception("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) -@keyword('Get object from NeoFS') -def get_object(private_key: str, cid: str, oid: str, bearer_token: str, read_object: str): +@keyword('Get object from NeoFS') +def get_object(private_key: str, cid: str, oid: str, bearer_token: str, read_object: str, endpoint: str="" ): + # TODO: add object return instead of read_object (uuid) + + logger.info("Going to put the object") + + if not endpoint: + endpoint = random.sample(_get_storage_nodes(private_key), 1)[0] + + if bearer_token: bearer_token = f"--bearer {bearer_token}" - ObjectCmd = f'neofs-cli --rpc-endpoint {NEOFS_ENDPOINT} --key {private_key} object get --cid {cid} --oid {oid} --file {read_object} {bearer_token}' + ObjectCmd = f'neofs-cli --rpc-endpoint {endpoint} --key {private_key} object get --cid {cid} --oid {oid} --file {read_object} {bearer_token}' logger.info("Cmd: %s" % ObjectCmd) try: diff --git a/robot/resources/lib/neofs_int_vars.py b/robot/resources/lib/neofs_int_vars.py index 7e2ea29..9dfb2a8 100644 --- a/robot/resources/lib/neofs_int_vars.py +++ b/robot/resources/lib/neofs_int_vars.py @@ -6,4 +6,4 @@ NEOGO_CLI_PREFIX = "docker exec -it main_chain neo-go" NEO_MAINNET_ENDPOINT = "main_chain.neofs.devenv:30333" NEOFS_NEO_API_ENDPOINT = 'http://main_chain.neofs.devenv:30333' -HTTP_GATE = '' +HTTP_GATE = 'http://http.neofs.devenv' diff --git a/robot/testsuites/integration/http_gate.robot b/robot/testsuites/integration/http_gate.robot new file mode 100644 index 0000000..829e94e --- /dev/null +++ b/robot/testsuites/integration/http_gate.robot @@ -0,0 +1,55 @@ +*** Settings *** +Variables ../../variables/common.py + + +Library ${RESOURCES}/neofs.py +Library ${RESOURCES}/payment_neogo.py +Library ${RESOURCES}/gates.py + + +*** Test cases *** + +NeoFS HTTP Gateway + [Documentation] Creates container and does PUT, GET via HTTP Gate + [Timeout] 5 min + + ${WALLET} = Init wallet + Generate wallet ${WALLET} + ${ADDR} = Dump Address ${WALLET} + ${PRIV_KEY} = Dump PrivKey ${WALLET} ${ADDR} + ${TX} = Transfer Mainnet Gas wallets/wallet.json NTrezR3C4X8aMLVg7vozt5wguyNfFhwuFx ${ADDR} 55 + + Wait Until Keyword Succeeds 1 min 15 sec + ... Transaction accepted in block ${TX} + Get Transaction ${TX} + Expexted Mainnet Balance ${ADDR} 55 + + ${SCRIPT_HASH} = Get ScripHash ${PRIV_KEY} + + ${TX_DEPOSIT} = NeoFS Deposit ${WALLET} ${ADDR} ${SCRIPT_HASH} 50 + Wait Until Keyword Succeeds 1 min 15 sec + ... Transaction accepted in block ${TX_DEPOSIT} + Get Transaction ${TX_DEPOSIT} + + ${CID} = Create container ${PRIV_KEY} public REP 1 IN X CBF 1 SELECT 1 FROM * AS X + Wait Until Keyword Succeeds 2 min 30 sec + ... Container Existing ${PRIV_KEY} ${CID} + + ${FILE} = Generate file of bytes 1024 + ${FILE_HASH} = Get file hash ${FILE} + + ${S_OID} = Put object to NeoFS ${PRIV_KEY} ${FILE} ${CID} ${EMPTY} ${EMPTY} + + # By request from Service team - try to GET object from the node without object + + @{GET_NODE_LIST} = Get nodes without object ${PRIV_KEY} ${CID} ${S_OID} + ${NODE} = Evaluate random.choice($GET_NODE_LIST) random + + Get object from NeoFS ${PRIV_KEY} ${CID} ${S_OID} ${EMPTY} s_file_read ${NODE} + ${FILEPATH} = Get via HTTP Gate ${CID} ${S_OID} + + Verify file hash s_file_read ${FILE_HASH} + Verify file hash ${FILEPATH} ${FILE_HASH} + + [Teardown] Cleanup Files ${FILEPATH} ${FILE} s_file_read + \ No newline at end of file diff --git a/robot/testsuites/integration/object_complex.robot b/robot/testsuites/integration/object_complex.robot index 1a0d799..e290b64 100644 --- a/robot/testsuites/integration/object_complex.robot +++ b/robot/testsuites/integration/object_complex.robot @@ -102,10 +102,12 @@ NeoFS Complex Object Operations [Teardown] Cleanup ${FILE} + *** Keywords *** + Cleanup [Arguments] ${FILE} - + @{CLEANUP_FILES} = Create List ${FILE} s_file_read h_file_read s_get_range h_get_range Cleanup Files @{CLEANUP_FILES} diff --git a/robot/testsuites/integration/object_simple.robot b/robot/testsuites/integration/object_simple.robot index 74c9286..fba23fc 100644 --- a/robot/testsuites/integration/object_simple.robot +++ b/robot/testsuites/integration/object_simple.robot @@ -48,9 +48,9 @@ NeoFS Simple Object Operations ${H_OID} = Put object to NeoFS ${PRIV_KEY} ${FILE} ${CID} ${EMPTY} ${FILE_USR_HEADER} ${H_OID_OTH} = Put object to NeoFS ${PRIV_KEY} ${FILE} ${CID} ${EMPTY} ${FILE_USR_HEADER_OTH} - Validate storage policy for object ${PRIV_KEY} 2 ${CID} ${S_OID} - Validate storage policy for object ${PRIV_KEY} 2 ${CID} ${H_OID} - Validate storage policy for object ${PRIV_KEY} 2 ${CID} ${H_OID_OTH} + Validate storage policy for object ${PRIV_KEY} 2 ${CID} ${S_OID} + Validate storage policy for object ${PRIV_KEY} 2 ${CID} ${H_OID} + Validate storage policy for object ${PRIV_KEY} 2 ${CID} ${H_OID_OTH} @{S_OBJ_ALL} = Create List ${S_OID} ${H_OID} ${H_OID_OTH} @{S_OBJ_H} = Create List ${H_OID} From 208602d96351fe6acf0329f9066f1eb4885d42c5 Mon Sep 17 00:00:00 2001 From: "anatoly@nspcc.ru" Date: Fri, 4 Dec 2020 15:32:29 +0300 Subject: [PATCH 2/3] fix comments --- robot/resources/lib/gates.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/robot/resources/lib/gates.py b/robot/resources/lib/gates.py index 88a835c..fd09842 100644 --- a/robot/resources/lib/gates.py +++ b/robot/resources/lib/gates.py @@ -43,11 +43,4 @@ def get_via_http_gate(cid: str, oid: str): with open(filename, "wb") as f: shutil.copyfileobj(resp.raw, f) del resp - return filename - - -#url = 'http://example.com/img.png' -#response = requests.get(url, stream=True) -#with open('img.png', 'wb') as out_file: -# shutil.copyfileobj(response.raw, out_file) -#del response \ No newline at end of file + return filename \ No newline at end of file From b5166662a4e90412dc5377e45d6333deb3b28e9a Mon Sep 17 00:00:00 2001 From: "anatoly@nspcc.ru" Date: Fri, 4 Dec 2020 15:34:06 +0300 Subject: [PATCH 3/3] fix --- robot/resources/lib/gates.py | 2 +- robot/testsuites/integration/http_gate.robot | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/robot/resources/lib/gates.py b/robot/resources/lib/gates.py index fd09842..d6b4f26 100644 --- a/robot/resources/lib/gates.py +++ b/robot/resources/lib/gates.py @@ -43,4 +43,4 @@ def get_via_http_gate(cid: str, oid: str): with open(filename, "wb") as f: shutil.copyfileobj(resp.raw, f) del resp - return filename \ No newline at end of file + return filename diff --git a/robot/testsuites/integration/http_gate.robot b/robot/testsuites/integration/http_gate.robot index 829e94e..750d929 100644 --- a/robot/testsuites/integration/http_gate.robot +++ b/robot/testsuites/integration/http_gate.robot @@ -52,4 +52,3 @@ NeoFS HTTP Gateway Verify file hash ${FILEPATH} ${FILE_HASH} [Teardown] Cleanup Files ${FILEPATH} ${FILE} s_file_read - \ No newline at end of file