From 2b51542ca2253d57f6754b162cba81e64381737b Mon Sep 17 00:00:00 2001 From: anastasia prasolova Date: Fri, 11 Mar 2022 19:08:14 +0300 Subject: [PATCH] fixes and refactoring in HTTP and S3 tests Signed-off-by: anastasia prasolova --- .gitmodules | 1 + ca/id_rsa | 27 ---- ca/id_rsa.pub | 1 - ca/known_hosts | 2 - ca/nspcc-ca.pem | 21 --- keys/s3_docker_hcs.pub.key | 1 - keys/s3_selectel_hcs.pub.key | 1 - robot/resources/files/s3_bearer_rules.json | 41 +++++ robot/resources/lib/python/http_gate.py | 35 ++++ robot/resources/lib/python/neofs.py | 12 +- .../lib/python/{gates.py => s3_gate.py} | 151 ++++++++---------- .../resources/lib/python/utility_keywords.py | 4 +- .../integration/services/http_gate.robot | 12 +- .../integration/services/s3_gate_bucket.robot | 36 ++--- run.sh | 38 ----- 15 files changed, 164 insertions(+), 219 deletions(-) delete mode 100644 ca/id_rsa delete mode 100644 ca/id_rsa.pub delete mode 100644 ca/known_hosts delete mode 100644 ca/nspcc-ca.pem delete mode 100644 keys/s3_docker_hcs.pub.key delete mode 100644 keys/s3_selectel_hcs.pub.key create mode 100644 robot/resources/files/s3_bearer_rules.json create mode 100644 robot/resources/lib/python/http_gate.py rename robot/resources/lib/python/{gates.py => s3_gate.py} (61%) delete mode 100755 run.sh diff --git a/.gitmodules b/.gitmodules index 31873174..ef40c8c7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "neofs-keywords"] path = neofs-keywords url = ssh://git@github.com/nspcc-dev/neofs-keywords.git + ignore = all diff --git a/ca/id_rsa b/ca/id_rsa deleted file mode 100644 index 8542365d..00000000 --- a/ca/id_rsa +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAt28xenSOhtLv5sj2gMgp/SWNcIkTHP/OiHvZe2mgdXXDpyIG -iUVGR35c1bep5sHvDLnL2IeFqrWSuajCn2Sf2FTgb/a7dWrTd10fNDlCJK5ibXm9 -CvXXYmsGc+CVDH4E/44qkcm5kqur70D8N9MfnSwrhFlLlwuCXbweyt+cqzFFQeby -47lChQ5Aeim+r0zPVcjqCG66TF1llCSxMbDNYIVONYPQeTWDLut8azgVIUE4bhlR -B44w1mYS6BjMRcYKFZ672os6DLLc/qsRPquXR54AKuD2gg5sn9VgHplFrr2llPzL -Iwoiky6plIfQVgHn3BMGmePx/cNe51mq09AQwQIDAQABAoIBAQCCfJrZ3Wg2CH+X -0IVp/vm/lqMS1q++BUrKVC/VVsJKTEet8Mptg9YGraEkds5p1LNUfibAFUfEs/14 -DNDFyjLbFSXC/+VCFYfwdVHpOIIQzew+rEcKMO/Slwe0DqJ4jHzJvjwSEUntSCm6 -vKOuooTurakXMN5QyGMogtX0wzUToXajl8VPjGy1B2xlIM8KKmvUzsMMrIv4W157 -xV/9OnJ0vPn+3gh90F5Daxk069xNMDefpSbfx5DcZemhnZLLWddX+QmnsSOl5tfQ -8zRx2fkjmjqDCJJ2e4KqV1+7ePhJqARLPnHYxmxbgsfaRGU/RizZFI90Ou3Fma8E -9cc/mfABAoGBAPMtAachdkkuxMB4btTAsW5PgKod0JQhMcXajZJqsl35WczVHHPP -zPXm03Ggx+Yk4BLFBoQT7xFfQjQOo4uUQ30hgS5+lkpGRtJa5p2kLVoDlyuGi0ig -0V+1u/h7wUNfuxncFcFmT8CCjhLrv9WMH2aNGdeRMIHVuxbVIqCftqThAoGBAMEb -pcppZ4Wj7poN033Tq4i9vXoJ2p1d5AwEKWKiYxoHvGRthD/NAjekVrRQrmzVQd1f -/gIOtnfO39WVN7ocH6wRWTi/KBMIrR0oPhtor4MPsB+19JvDmNtq48b94lVz+Z4L -hOSqNdGzlsuy1vZagRzHkxD7VU7PlT+UBdDHpwfhAoGBAPHjZO+Ao46sTN4/bc+H -VXcq8gtF2QJf+oiam5R3ObGspRzRJ5ozq+c2kkFG81EEgTdqcM7UnUuke9AYd6oR -8wf3We6L0KdVPIFmFlvcwZf2VlrfXJEEFwCjX7UONPH1ucFBYQqd4NrXgsdjZdDf -ryRtWrVJIP0lQxK1M9qexClBAoGAI3SxFx4NTONRjuWU/Fhd+WhlHsAqbJRtp6sn -8h1AtunOtF3LV2+Lxa2d4dOigwcQ5dWXLMeIxyyrumqAZeJ+CjjROfMXJ4+DQYQ/ -CwdImnbJ7riY5fSe30Kb+dBpuyjlHxicWOPLp+oieNooT+lEJYWbQhXzjtncXGUQ -QEo4J+ECgYA1UiQr+C3NbMr27HOynXInv09f86HvHsHGooXuGoJ7GB518ZjtoiA7 -xefoSUR10NLmx8QhdGydaFH+wIdgAAwB7RjpsrA5XGnTEKj5yV00o272jXj2jygu -U5700lxwQFHbd5mNZXoywBP0kU/5I98A5OHGhpYthp8lGC+Vk1NZFA== ------END RSA PRIVATE KEY----- diff --git a/ca/id_rsa.pub b/ca/id_rsa.pub deleted file mode 100644 index c83e20f4..00000000 --- a/ca/id_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3bzF6dI6G0u/myPaAyCn9JY1wiRMc/86Ie9l7aaB1dcOnIgaJRUZHflzVt6nmwe8MucvYh4WqtZK5qMKfZJ/YVOBv9rt1atN3XR80OUIkrmJteb0K9ddiawZz4JUMfgT/jiqRybmSq6vvQPw30x+dLCuEWUuXC4JdvB7K35yrMUVB5vLjuUKFDkB6Kb6vTM9VyOoIbrpMXWWUJLExsM1ghU41g9B5NYMu63xrOBUhQThuGVEHjjDWZhLoGMxFxgoVnrvaizoMstz+qxE+q5dHngAq4PaCDmyf1WAemUWuvaWU/MsjCiKTLqmUh9BWAefcEwaZ4/H9w17nWarT0BDB neospcc@neospcc diff --git a/ca/known_hosts b/ca/known_hosts deleted file mode 100644 index aa95d5b2..00000000 --- a/ca/known_hosts +++ /dev/null @@ -1,2 +0,0 @@ -|1|UPdjsdJ58lY6EqUaP7n833Fj7Ic=|hQkKlD6Ui6Ne03QrjHpsL7NU0qA= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw== -|1|C5OFm8ugKVhcy7PRLErwNGAV/jw=|qMuNNLMrE8rDdleXMmX8N7YBEkM= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw== \ No newline at end of file diff --git a/ca/nspcc-ca.pem b/ca/nspcc-ca.pem deleted file mode 100644 index eb940cb9..00000000 --- a/ca/nspcc-ca.pem +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDhDCCAmygAwIBAgIBATANBgkqhkiG9w0BAQsFADB2MQswCQYDVQQGEwJSVTEM -MAoGA1UECBMDU1BCMQwwCgYDVQQHEwNTUEIxDjAMBgNVBAoTBU5TUENDMQ4wDAYD -VQQLEwVOU1BDQzEOMAwGA1UEAxMFTlNQQ0MxGzAZBgkqhkiG9w0BCQEWDG9wc0Bu -c3BjYy5ydTAeFw0xOTEyMzAxNjQ0MDBaFw0yMDEyMzAxNjQ0MDBaMHYxCzAJBgNV -BAYTAlJVMQwwCgYDVQQIEwNTUEIxDDAKBgNVBAcTA1NQQjEOMAwGA1UEChMFTlNQ -Q0MxDjAMBgNVBAsTBU5TUENDMQ4wDAYDVQQDEwVOU1BDQzEbMBkGCSqGSIb3DQEJ -ARYMb3BzQG5zcGNjLnJ1MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA -q14BlfD4ZcNJz3HZGLj4ksqbi2tzogpnIlC2ccVcd8sk7mrkAjUFI1VqdJg57F7Y -kv8qYsyYwTaUQB5NAmSOQxNFKKsRc4LI3J8b7dE2q5+dDCWRN5tz8sIRfXryXyF2 -VpW1RnuxMGJ6SwgTn9Wb+R6o1WkwNtrDhhNkdWsBnrJzL37GIk6nhQJxm3OlrHgQ -IPz0bt2QlSYQwuTTd1V6vMgOpHfqkQiAJZLBvQ39SImAO+pNS1+CrJuh+zWqhKzq -0vagW0EA4mq7Z98WqGF/g9d2yj8WRpxoAwpJzUoExPstHyDYSvKhr6R8+QV0zhpF -KdjqggmAM9ixSonYC3txawIDAQABox0wGzAMBgNVHRMEBTADAQH/MAsGA1UdDwQE -AwICjDANBgkqhkiG9w0BAQsFAAOCAQEAQ3gBirQTNU9rWIT9Bwh4Hj1ft5jIepQ7 -yXmqFiOFKUli850t1yCre5/oGyOhQy8huUDL+qQDFeX63g5m3r1B3MsBfxKsukn6 -I2HqwNVEUrsnRtWDr/NXpwpRGfjg1zG9tUxg6Tbo5VD4HG7dUGBuyOXA+/tBYvUQ -3vr9bGFUpuYGJ6/KmnoEbR6B1cB4MwW+xorJFODCaEcd2aAuAABL7o84nx3UFTsb -CguS26u5dpA+32rwV9US17sJfJ301DirapG3CxlhxhUPMXqxuQH1Am8x4zaMz3dN -fRvPLar6g33xKcd6T/VAAp7kMpxyzWoenOYXxGfpgs7Rj/X6xBX3sw== ------END CERTIFICATE----- diff --git a/keys/s3_docker_hcs.pub.key b/keys/s3_docker_hcs.pub.key deleted file mode 100644 index d286f7b5..00000000 --- a/keys/s3_docker_hcs.pub.key +++ /dev/null @@ -1 +0,0 @@ -­d#°ªBA˜\EºËïïë‡Ï¼-^rª =n± * \ No newline at end of file diff --git a/keys/s3_selectel_hcs.pub.key b/keys/s3_selectel_hcs.pub.key deleted file mode 100644 index 3c6b4982..00000000 --- a/keys/s3_selectel_hcs.pub.key +++ /dev/null @@ -1 +0,0 @@ -=b­–•|vn¾b®%ü“ÐþaåÙÍ-ie×ðŒ= \ No newline at end of file diff --git a/robot/resources/files/s3_bearer_rules.json b/robot/resources/files/s3_bearer_rules.json new file mode 100644 index 00000000..5967b161 --- /dev/null +++ b/robot/resources/files/s3_bearer_rules.json @@ -0,0 +1,41 @@ +{ + "records": + [ + { + "operation":"PUT", + "action":"ALLOW", + "filters":[], + "targets": + [ + { + "role":"OTHERS", + "keys":[] + } + ] + }, + { + "operation":"SEARCH", + "action":"ALLOW", + "filters":[], + "targets": + [ + { + "role":"OTHERS", + "keys":[] + } + ] + }, + { + "operation":"GET", + "action":"ALLOW", + "filters":[], + "targets": + [ + { + "role":"OTHERS", + "keys":[] + } + ] + } + ] +} diff --git a/robot/resources/lib/python/http_gate.py b/robot/resources/lib/python/http_gate.py new file mode 100644 index 00000000..eb756edf --- /dev/null +++ b/robot/resources/lib/python/http_gate.py @@ -0,0 +1,35 @@ +#!/usr/bin/python3 + +import shutil + +import requests + +from common import HTTP_GATE +from robot.api.deco import keyword +from robot.api import logger +from robot.libraries.BuiltIn import BuiltIn + +ROBOT_AUTO_KEYWORDS = False +ASSETS_DIR = BuiltIn().get_variable_value("${ASSETS_DIR}") + +@keyword('Get via HTTP Gate') +def get_via_http_gate(cid: str, oid: str): + """ + This function gets given object from HTTP gate + :param cid: CID to get object from + :param oid: object OID + """ + request = f'{HTTP_GATE}/get/{cid}/{oid}' + resp = requests.get(request, stream=True) + + if not resp.ok: + raise Exception(f"""Failed to get object via HTTP gate: + request: {resp.request.path_url}, + response: {resp.text}, + status code: {resp.status_code} {resp.reason}""") + + logger.info(f'Request: {request}') + filename = f"{ASSETS_DIR}/{cid}_{oid}" + with open(filename, "wb") as get_file: + shutil.copyfileobj(resp.raw, get_file) + return filename diff --git a/robot/resources/lib/python/neofs.py b/robot/resources/lib/python/neofs.py index 58818271..7a4e2211 100644 --- a/robot/resources/lib/python/neofs.py +++ b/robot/resources/lib/python/neofs.py @@ -54,7 +54,7 @@ def start_nodes(*nodes_list): node = m.group(1) client = docker.APIClient() client.start(node) - + @keyword('Get nodes with object') def get_nodes_with_object(private_key: str, cid: str, oid: str): @@ -712,12 +712,6 @@ def delete_container(cid: str, private_key: str): _cmd_run(deleteContainerCmd) -@keyword('Get file name') -def get_file_name(filepath): - filename = os.path.basename(filepath) - return filename - - @keyword('Get file hash') def get_file_hash(filename : str): file_hash = _get_file_hash(filename) @@ -771,7 +765,7 @@ def get_control_endpoint_with_wif(endpoint_number: str = ''): endpoint_values = NEOFS_NETMAP_DICT[f'{endpoint_num}'] endpoint_control = endpoint_values['control'] wif = endpoint_values['wif'] - + return endpoint_num, endpoint_control, wif @keyword('Get Locode') @@ -1012,7 +1006,7 @@ def generate_session_token(owner: str, pub_key: str, cid: str = "", wildcard: bo @keyword ('Sign Session Token') def sign_session_token(session_token: str, wallet: str, to_file: str=''): if to_file: - to_file = f'--to {to_file}' + to_file = f'--to {to_file}' cmd = ( f'{NEOFS_CLI_EXEC} util sign session-token --from {session_token} ' f'-w {wallet} {to_file}' diff --git a/robot/resources/lib/python/gates.py b/robot/resources/lib/python/s3_gate.py similarity index 61% rename from robot/resources/lib/python/gates.py rename to robot/resources/lib/python/s3_gate.py index a607212c..22fec5d0 100644 --- a/robot/resources/lib/python/gates.py +++ b/robot/resources/lib/python/s3_gate.py @@ -1,19 +1,23 @@ -#!/usr/bin/python3.8 +#!/usr/bin/python3 +import json import os -import re -import shutil -import subprocess import uuid -import requests -import botocore -import boto3 -from common import * -from robot.api.deco import keyword -from robot.api import logger +import boto3 +import botocore from cli_helpers import _run_with_passwd +from common import GATE_PUB_KEY, NEOFS_ENDPOINT, S3_GATE +import urllib3 +from robot.api.deco import keyword +from robot.api import logger + +########################################################## +# Disabling warnings on self-signed certificate which the +# boto library produces on requests to S3-gate in dev-env. +urllib3.disable_warnings() +########################################################## ROBOT_AUTO_KEYWORDS = False @@ -22,44 +26,37 @@ NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate') @keyword('Init S3 Credentials') def init_s3_credentials(wallet): bucket = str(uuid.uuid4()) - records = ' \' {"records":[{"operation":"PUT","action":"ALLOW","filters":[],"targets":[{"role":"OTHERS","keys":[]}]}, {"operation":"SEARCH","action":"ALLOW","filters":[],"targets":[{"role":"OTHERS","keys":[]}]}, {"operation":"GET","action":"ALLOW","filters":[],"targets":[{"role":"OTHERS","keys":[]}]}]} \' ' - Cmd = ( + s3_bearer_rules = "robot/resources/files/s3_bearer_rules.json" + cmd = ( f'{NEOFS_EXEC} --debug --with-log issue-secret --wallet {wallet} ' f'--gate-public-key={GATE_PUB_KEY} --peer {NEOFS_ENDPOINT} ' - f'--container-friendly-name {bucket} --create-session-token ' - f'--bearer-rules {records}' + f'--container-friendly-name {bucket} ' + f'--bearer-rules {s3_bearer_rules}' ) - logger.info(f"Executing command: {Cmd}") + logger.info(f"Executing command: {cmd}") try: - output = _run_with_passwd(Cmd) + output = _run_with_passwd(cmd) logger.info(f"Command completed with output: {output}") + # first five string are log output, cutting them off and parse + # the rest of the output as JSON + output = '\n'.join(output.split('\n')[5:]) + output_dict = json.loads(output) - m = re.search(r'"container_id":\s+"(\w+)"', output) - cid = m.group(1) - logger.info("cid: %s" % cid) + return (output_dict['container_id'], + bucket, + output_dict['access_key_id'], + output_dict['secret_access_key'], + output_dict['owner_private_key']) - m = re.search(r'"access_key_id":\s+"([\w\/]+)"', output) - access_key_id = m.group(1) - logger.info("access_key_id: %s" % access_key_id) - - m = re.search(r'"secret_access_key":\s+"(\w+)"', output) - secret_access_key = m.group(1) - logger.info("secret_access_key: %s" % secret_access_key) - - m = re.search(r'"owner_private_key":\s+"(\w+)"', output) - owner_private_key = m.group(1) - logger.info("owner_private_key: %s" % owner_private_key) - - return cid, bucket, access_key_id, secret_access_key, owner_private_key - - except subprocess.CalledProcessError as e: - raise Exception(f"Error: \nreturn code: {e.returncode}. \nOutput: {e.stderr}") + except Exception as exc: + raise RuntimeError("failed to init s3 credentials") from exc @keyword('Config S3 client') def config_s3_client(access_key_id, secret_access_key): try: + session = boto3.session.Session() s3_client = session.client( @@ -80,11 +77,11 @@ def config_s3_client(access_key_id, secret_access_key): def list_objects_s3_v2(s3_client, bucket): try: response = s3_client.list_objects_v2(Bucket=bucket) - logger.info("S3 v2 List objects result: %s" % response['Contents']) + logger.info(f"S3 v2 List objects result: {response['Contents']}") obj_list = [] for obj in response['Contents']: obj_list.append(obj['Key']) - logger.info("Found s3 objects: %s" % obj_list) + logger.info(f"Found s3 objects: {obj_list}") return obj_list except botocore.exceptions.ClientError as err: @@ -96,11 +93,11 @@ def list_objects_s3_v2(s3_client, bucket): def list_objects_s3(s3_client, bucket): try: response = s3_client.list_objects(Bucket=bucket) - logger.info("S3 List objects result: %s" % response['Contents']) + logger.info(f"S3 List objects result: {response['Contents']}") obj_list = [] for obj in response['Contents']: obj_list.append(obj['Key']) - logger.info("Found s3 objects: %s" % obj_list) + logger.info(f"Found s3 objects: {obj_list}") return obj_list except botocore.exceptions.ClientError as err: @@ -114,12 +111,12 @@ def create_bucket_s3(s3_client): try: s3_bucket = s3_client.create_bucket(Bucket=bucket_name) - logger.info("Created S3 bucket: %s" % s3_bucket) + logger.info(f"Created S3 bucket: {s3_bucket}") return bucket_name except botocore.exceptions.ClientError as err: raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err + f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err @keyword('List buckets S3') @@ -127,7 +124,7 @@ def list_buckets_s3(s3_client): found_buckets = [] try: response = s3_client.list_buckets() - logger.info("S3 List buckets result: %s" % response) + logger.info(f"S3 List buckets result: {response}") for bucket in response['Buckets']: found_buckets.append(bucket['Name']) @@ -136,7 +133,7 @@ def list_buckets_s3(s3_client): except botocore.exceptions.ClientError as err: raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err + f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err @keyword('Delete bucket S3') @@ -144,7 +141,7 @@ def delete_bucket_s3(s3_client, bucket): try: response = s3_client.delete_bucket(Bucket=bucket) logger.info(f"S3 Delete bucket result: {response}") - + return response except botocore.exceptions.ClientError as err: @@ -152,11 +149,11 @@ def delete_bucket_s3(s3_client, bucket): f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err -@keyword('HeadBucket S3') -def headbucket(bucket, s3_client): +@keyword('Head bucket S3') +def head_bucket(s3_client, bucket): try: response = s3_client.head_bucket(Bucket=bucket) - logger.info(f"S3 HeadBucket result: {response}") + logger.info(f"S3 Head bucket result: {response}") return response except botocore.exceptions.ClientError as err: @@ -168,12 +165,12 @@ def headbucket(bucket, s3_client): def put_object_s3(s3_client, bucket, filepath): filename = os.path.basename(filepath) - with open(filepath, "rb") as f: - fileContent = f.read() + with open(filepath, "rb") as put_file: + file_content = put_file.read() try: - response = s3_client.put_object(Body=fileContent, Bucket=bucket, Key=filename) - logger.info("S3 Put object result: %s" % response) + response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename) + logger.info(f"S3 Put object result: {response}") except botocore.exceptions.ClientError as err: raise Exception(f"Error Message: {err.response['Error']['Message']}\n" f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err @@ -184,19 +181,19 @@ def head_object_s3(s3_client, bucket, object_key): try: response = s3_client.head_object(Bucket=bucket, Key=object_key) - logger.info("S3 Head object result: %s" % response) + logger.info(f"S3 Head object result: {response}") return response except botocore.exceptions.ClientError as err: raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err + f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err @keyword('Delete object S3') def delete_object_s3(s3_client, bucket, object_key): try: response = s3_client.delete_object(Bucket=bucket, Key=object_key) - logger.info("S3 Put object result: %s" % response) + logger.info(f"S3 Put object result: {response}") return response except botocore.exceptions.ClientError as err: @@ -205,11 +202,14 @@ def delete_object_s3(s3_client, bucket, object_key): @keyword('Copy object S3') -def copy_object_s3(s3_client, bucket, object_key, new_object): +def copy_object_s3(s3_client, bucket, object_key): + filename = f"{os.getcwd()}/{uuid.uuid4()}" try: - response = s3_client.copy_object(Bucket=bucket, CopySource=bucket+"/"+object_key, Key=new_object) - logger.info("S3 Copy object result: %s" % response) - return response + response = s3_client.copy_object(Bucket=bucket, + CopySource=f"{bucket}/{object_key}", + Key=filename) + logger.info(f"S3 Copy object result: {response}") + return filename except botocore.exceptions.ClientError as err: raise Exception(f"Error Message: {err.response['Error']['Message']}\n" @@ -217,44 +217,19 @@ def copy_object_s3(s3_client, bucket, object_key, new_object): @keyword('Get object S3') -def get_object_s3(s3_client, bucket, object_key, target_file): +def get_object_s3(s3_client, bucket, object_key): + filename = f"{os.getcwd()}/{uuid.uuid4()}" try: response = s3_client.get_object(Bucket=bucket, Key=object_key) - with open(f"{target_file}", 'wb') as f: + with open(f"{filename}", 'wb') as get_file: chunk = response['Body'].read(1024) while chunk: - f.write(chunk) + get_file.write(chunk) chunk = response['Body'].read(1024) - return target_file + return filename except botocore.exceptions.ClientError as err: raise Exception(f"Error Message: {err.response['Error']['Message']}\n" f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('Get via HTTP Gate') -def get_via_http_gate(cid: str, oid: str): - """ - This function gets given object from HTTP gate - :param cid: CID to get object from - :param oid: object OID - """ - request = f'{HTTP_GATE}/get/{cid}/{oid}' - resp = requests.get(request, stream=True) - - if not resp.ok: - raise Exception(f"""Failed to get object via HTTP gate: - request: {resp.request.path_url}, - response: {resp.text}, - status code: {resp.status_code} {resp.reason}""") - return - - logger.info(f'Request: {request}') - filename = os.path.curdir + f"/{cid}_{oid}" - with open(filename, "wb") as f: - shutil.copyfileobj(resp.raw, f) - del resp - return filename - diff --git a/robot/resources/lib/python/utility_keywords.py b/robot/resources/lib/python/utility_keywords.py index 7f977f77..46388e87 100644 --- a/robot/resources/lib/python/utility_keywords.py +++ b/robot/resources/lib/python/utility_keywords.py @@ -54,12 +54,12 @@ def wif_to_binary(wif: str) -> str: return path @keyword('Make Up') -def make_up(services=['']): +def make_up(services: list=[]): test_path = os.getcwd() dev_path = os.getenv('DEVENV_PATH', '../neofs-dev-env') os.chdir(dev_path) - if services != ['']: + if len(services) > 0: for service in services: cmd = f'make up/{service}' logger.info(f"Cmd: {cmd}") diff --git a/robot/testsuites/integration/services/http_gate.robot b/robot/testsuites/integration/services/http_gate.robot index 2a6fdb81..c8ed46e3 100644 --- a/robot/testsuites/integration/services/http_gate.robot +++ b/robot/testsuites/integration/services/http_gate.robot @@ -3,19 +3,13 @@ Variables common.py Variables wellknown_acl.py Library neofs.py -Library payment_neogo.py -Library gates.py -Library wallet_keywords.py -Library rpc_call_keywords.py -Library utility_keywords.py +Library http_gate.py Resource payment_operations.robot Resource setup_teardown.robot *** Variables *** ${PLACEMENT_RULE} = REP 1 IN X CBF 1 SELECT 1 FROM * AS X -${TRANSFER_AMOUNT} = ${6} -${DEPOSIT_AMOUNT} = ${5} ${CONTAINER_WAIT_INTERVAL} = 1 min @{INCLUDE_SVC} = http_gate @@ -25,9 +19,9 @@ NeoFS HTTP Gateway [Documentation] Creates container and does PUT, GET via HTTP Gate [Timeout] 5 min - [Setup] Setup + [Setup] Setup Make Up ${INCLUDE_SVC} - + ${WALLET} ${ADDR} ${WIF} = Prepare Wallet And Deposit ${CID} = Create container ${WIF} ${PUBLIC_ACL} ${PLACEMENT_RULE} diff --git a/robot/testsuites/integration/services/s3_gate_bucket.robot b/robot/testsuites/integration/services/s3_gate_bucket.robot index 17a5a48e..7434a86c 100644 --- a/robot/testsuites/integration/services/s3_gate_bucket.robot +++ b/robot/testsuites/integration/services/s3_gate_bucket.robot @@ -2,33 +2,29 @@ Variables common.py Library Collections +Library OperatingSystem + Library neofs.py -Library payment_neogo.py -Library gates.py -Library wallet_keywords.py +Library s3_gate.py Library contract_keywords.py -Library Process Resource setup_teardown.robot Resource payment_operations.robot *** Variables *** -${DEPOSIT} = ${30} -${WIF} = ${MAINNET_WALLET_WIF} -${DEPOSIT_TIMEOUT}= 30s -@{INCLUDE_SVC} = s3_gate +@{INCLUDE_SVC} = s3_gate coredns *** Test cases *** Buckets in NeoFS S3 Gateway [Documentation] Execute operations with bucket via S3 Gate [Timeout] 10 min - [Setup] Setup + [Setup] Setup Make Up ${INCLUDE_SVC} - ${WALLET} ${ADDR} ${WIF} = Prepare Wallet And Deposit - ${FILE_S3} = Generate file of bytes ${COMPLEX_OBJ_SIZE} - ${FILE_S3_NAME} = Get file name ${FILE_S3} + ${WALLET} ${_} ${WIF} = Prepare Wallet And Deposit + ${FILE_S3} = Generate file of bytes ${COMPLEX_OBJ_SIZE} + ${_} ${S3_OBJECT_KEY} = Split Path ${FILE_S3} ${CID} ... ${BUCKET} @@ -44,24 +40,24 @@ Buckets in NeoFS S3 Gateway ${NEW_BUCKET} = Create Bucket S3 ${S3_CLIENT} ${NEW_BUCKET_EMPTY} = Create Bucket S3 ${S3_CLIENT} - HeadBucket S3 ${BUCKET} ${S3_CLIENT} - HeadBucket S3 ${NEW_BUCKET} ${S3_CLIENT} + Head bucket S3 ${S3_CLIENT} ${BUCKET} + Head bucket S3 ${S3_CLIENT} ${NEW_BUCKET} Put object S3 ${S3_CLIENT} ${NEW_BUCKET} ${FILE_S3} - Head object S3 ${S3_CLIENT} ${NEW_BUCKET} ${FILE_S3_NAME} + Head object S3 ${S3_CLIENT} ${NEW_BUCKET} ${S3_OBJECT_KEY} ${LIST_S3_OBJECTS} = List objects S3 ${S3_CLIENT} ${NEW_BUCKET} - List Should Contain Value ${LIST_S3_OBJECTS} ${FILE_S3_NAME} - + List Should Contain Value ${LIST_S3_OBJECTS} ${S3_OBJECT_KEY} + Run Keyword and Expect Error * ... Delete Bucket S3 ${S3_CLIENT} ${NEW_BUCKET} - HeadBucket S3 ${NEW_BUCKET} ${S3_CLIENT} + Head bucket S3 ${S3_CLIENT} ${NEW_BUCKET} Delete Bucket S3 ${S3_CLIENT} ${NEW_BUCKET_EMPTY} Tick Epoch Run Keyword And Expect Error * - ... HeadBucket S3 ${NEW_BUCKET_EMPTY} ${S3_CLIENT} - + ... Head bucket S3 ${S3_CLIENT} ${NEW_BUCKET_EMPTY} + ${BUCKET_LIST} = List Buckets S3 ${S3_CLIENT} Tick Epoch List Should Contain Value ${BUCKET_LIST} ${NEW_BUCKET} diff --git a/run.sh b/run.sh deleted file mode 100755 index 7d824fff..00000000 --- a/run.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Before each test execution dev-env should be build down and up. - -xmls='' - -if [ $# -eq 0 ]; then - test_list=$(find ./robot/testsuites -regex '.*robot') -else - test_list=$(find $@ -regex '.*robot') -fi - -echo Tests to execute: $test_list - -for test in $test_list; do - pushd $DEVENV_PATH - if [[ $test =~ 's3_gate' || $test =~ 'http_gate' ]]; then - sed -i -e '/coredns/d' .services - else - sed -i -e '/coredns/d' -e '/s3_gate/d' -e '/http_gate/d' .services - fi - make down - make clean - make up - make update.max_object_size val=1000 - popd - test_addr=`echo $test | sed "s/\//_/g" | sed "s/.robot//"` - robot --outputdir artifacts/ --output ${test_addr}_output.xml --log ${test_addr}_log.html --report ${test_addr}_report.html $test - xmls+=" ./artifacts/${test_addr}_output.xml" - pushd $DEVENV_PATH - echo 'coredns' >> .services - if [ -z $(cat .services | grep 's3_gate') ]; then echo 's3_gate' >> .services; fi - if [ -z $(cat .services | grep 'http_gate') ]; then echo 'http_gate' >> .services; fi - popd -done - -rebot ${xmls} -