Merge pull request #3 from nspcc-dev/feature/http-gate-case

Add http-gate testcase and fix the issue with getting binary files from the gate

robot/resources/lib/gates.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import requests
+import shutil
 
 from robot.api.deco import keyword
 from robot.api import logger
@@ -27,15 +28,19 @@ def get_via_http_gate(cid: str, oid: str):
     :param cid:      CID to get object from
     :param oid:      object OID
     """
-    resp = requests.get(f'{HTTP_GATE}/get/{cid}/{oid}')
+    request = f'{HTTP_GATE}/get/{cid}/{oid}'
+    resp = requests.get(request, stream=True)
+
     if not resp.ok:
-        logger.info(f"""Failed to get object via HTTP gate:
+        raise Exception(f"""Failed to get object via HTTP gate:
                 request: {resp.request.path_url},
                 response: {resp.text},
                 status code: {resp.status_code} {resp.reason}""")
         return
 
+    logger.info(f'Request: {request}')
     filename = os.path.curdir + f"/{cid}_{oid}"
-    with open(filename, "w+") as f:
+    with open(filename, "wb") as f:
-        f.write(resp.text)
+        shutil.copyfileobj(resp.raw, f)
+    del resp
     return filename

robot/resources/lib/neofs.py

@@ -167,9 +167,6 @@ def validate_storage_policy_for_object(private_key: str, expected_copies: int, c
             raise Exception("Found node list '{}' is not equal to expected list '{}'".format(found_nodes, expected_node_list))
 
 
-
-
-
 @keyword('Get eACL')
 def get_eacl(private_key: str, cid: str):
 
@@ -843,16 +840,19 @@ def cleanup_file(*filename_list):
 
 
 @keyword('Put object to NeoFS')
-def put_object(private_key: str, path: str, cid: str, bearer: str, user_headers: str):
+def put_object(private_key: str, path: str, cid: str, bearer: str, user_headers: str, endpoint: str="" ):
     logger.info("Going to put the object")
 
+    if not endpoint:
+      endpoint = random.sample(_get_storage_nodes(private_key), 1)[0]
+
     if user_headers:
         user_headers = f"--attributes {user_headers}"
 
     if bearer:
         bearer = f"--bearer {bearer}"
 
-    putObjectCmd = f'neofs-cli --rpc-endpoint {NEOFS_ENDPOINT} --key {private_key} object put --file {path} --cid {cid} {bearer} {user_headers}'
+    putObjectCmd = f'neofs-cli --rpc-endpoint {endpoint} --key {private_key} object put --file {path} --cid {cid} {bearer} {user_headers}'
     logger.info("Cmd: %s" % putObjectCmd)
 
     try:
@@ -882,13 +882,21 @@ def get_range_hash(private_key: str, cid: str, oid: str, bearer_token: str, rang
     except subprocess.CalledProcessError as e:
         raise Exception("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output))
 
+
 @keyword('Get object from NeoFS') 
-def get_object(private_key: str, cid: str, oid: str, bearer_token: str, read_object: str):
+def get_object(private_key: str, cid: str, oid: str, bearer_token: str, read_object: str, endpoint: str="" ):
+    # TODO: add object return instead of read_object (uuid)
+
+    logger.info("Going to put the object")
+
+    if not endpoint:
+      endpoint = random.sample(_get_storage_nodes(private_key), 1)[0]
+
     
     if bearer_token:
         bearer_token = f"--bearer {bearer_token}"
 
-    ObjectCmd = f'neofs-cli --rpc-endpoint {NEOFS_ENDPOINT} --key {private_key} object get --cid {cid} --oid {oid} --file {read_object} {bearer_token}'
+    ObjectCmd = f'neofs-cli --rpc-endpoint {endpoint} --key {private_key} object get --cid {cid} --oid {oid} --file {read_object} {bearer_token}'
 
     logger.info("Cmd: %s" % ObjectCmd)
     try:

robot/resources/lib/neofs_int_vars.py

@@ -6,4 +6,4 @@ NEOGO_CLI_PREFIX = "docker exec -it main_chain neo-go"
 NEO_MAINNET_ENDPOINT = "main_chain.neofs.devenv:30333"
 
 NEOFS_NEO_API_ENDPOINT = 'http://main_chain.neofs.devenv:30333'
-HTTP_GATE = ''
+HTTP_GATE = 'http://http.neofs.devenv'

robot/testsuites/integration/http_gate.robot

@@ -0,0 +1,54 @@
+*** Settings ***
+Variables   ../../variables/common.py
+
+
+Library     ${RESOURCES}/neofs.py
+Library     ${RESOURCES}/payment_neogo.py
+Library     ${RESOURCES}/gates.py
+
+
+*** Test cases ***
+
+NeoFS HTTP Gateway 
+    [Documentation]     Creates container and does PUT, GET via HTTP Gate
+    [Timeout]           5 min
+
+    ${WALLET} =         Init wallet
+                        Generate wallet         ${WALLET}
+    ${ADDR} =           Dump Address            ${WALLET}
+    ${PRIV_KEY} =       Dump PrivKey            ${WALLET}              ${ADDR}
+    ${TX} =             Transfer Mainnet Gas    wallets/wallet.json    NTrezR3C4X8aMLVg7vozt5wguyNfFhwuFx    ${ADDR}    55
+
+                        Wait Until Keyword Succeeds         1 min      15 sec        
+                        ...  Transaction accepted in block  ${TX}
+                        Get Transaction                     ${TX}
+                        Expexted Mainnet Balance            ${ADDR}    55
+
+    ${SCRIPT_HASH} =    Get ScripHash                       ${PRIV_KEY}  
+
+    ${TX_DEPOSIT} =     NeoFS Deposit                       ${WALLET}    ${ADDR}    ${SCRIPT_HASH}    50      
+                        Wait Until Keyword Succeeds         1 min        15 sec
+                        ...  Transaction accepted in block  ${TX_DEPOSIT}
+                        Get Transaction                     ${TX_DEPOSIT}
+
+    ${CID} =            Create container                    ${PRIV_KEY}    public    REP 1 IN X CBF 1 SELECT 1 FROM * AS X
+                        Wait Until Keyword Succeeds         2 min          30 sec
+                        ...  Container Existing             ${PRIV_KEY}    ${CID}
+
+    ${FILE} =           Generate file of bytes              1024
+    ${FILE_HASH} =      Get file hash                       ${FILE}
+
+    ${S_OID} =          Put object to NeoFS                 ${PRIV_KEY}    ${FILE}    ${CID}    ${EMPTY}    ${EMPTY}       
+  
+    # By request from Service team - try to GET object from the node without object
+
+    @{GET_NODE_LIST} =  Get nodes without object            ${PRIV_KEY}    ${CID}    ${S_OID}
+    ${NODE} =           Evaluate                            random.choice($GET_NODE_LIST)    random
+
+                        Get object from NeoFS               ${PRIV_KEY}    ${CID}    ${S_OID}    ${EMPTY}    s_file_read    ${NODE}
+    ${FILEPATH} =       Get via HTTP Gate                   ${CID}         ${S_OID}
+
+                        Verify file hash                    s_file_read    ${FILE_HASH} 
+                        Verify file hash                    ${FILEPATH}    ${FILE_HASH} 
+                        
+    [Teardown]          Cleanup Files    ${FILEPATH}    ${FILE}    s_file_read

robot/testsuites/integration/object_complex.robot

@@ -102,7 +102,9 @@ NeoFS Complex Object Operations
     [Teardown]          Cleanup                             ${FILE}
 
 
+
 *** Keywords ***
+
 Cleanup
     [Arguments]         ${FILE}
 

robot/testsuites/integration/object_simple.robot

@@ -48,9 +48,9 @@ NeoFS Simple Object Operations
     ${H_OID} =          Put object to NeoFS                 ${PRIV_KEY}    ${FILE}       ${CID}            ${EMPTY}         ${FILE_USR_HEADER} 
     ${H_OID_OTH} =      Put object to NeoFS                 ${PRIV_KEY}    ${FILE}       ${CID}            ${EMPTY}         ${FILE_USR_HEADER_OTH}
 
-                        Validate storage policy for object  ${PRIV_KEY}    2             ${CID}         ${S_OID}    
+                        Validate storage policy for object  ${PRIV_KEY}    2             ${CID}            ${S_OID}    
-                        Validate storage policy for object  ${PRIV_KEY}    2             ${CID}         ${H_OID}    
+                        Validate storage policy for object  ${PRIV_KEY}    2             ${CID}            ${H_OID}    
-                        Validate storage policy for object  ${PRIV_KEY}    2             ${CID}         ${H_OID_OTH}    
+                        Validate storage policy for object  ${PRIV_KEY}    2             ${CID}            ${H_OID_OTH}    
 
     @{S_OBJ_ALL} =	    Create List	                        ${S_OID}       ${H_OID}      ${H_OID_OTH}
     @{S_OBJ_H} =	    Create List	                        ${H_OID}