fixes and refactoring in HTTP and S3 tests
Signed-off-by: anastasia prasolova <anastasia@nspcc.ru>
This commit is contained in:
parent
40db81216a
commit
2b51542ca2
15 changed files with 164 additions and 219 deletions
1
.gitmodules
vendored
1
.gitmodules
vendored
|
@ -1,3 +1,4 @@
|
||||||
[submodule "neofs-keywords"]
|
[submodule "neofs-keywords"]
|
||||||
path = neofs-keywords
|
path = neofs-keywords
|
||||||
url = ssh://git@github.com/nspcc-dev/neofs-keywords.git
|
url = ssh://git@github.com/nspcc-dev/neofs-keywords.git
|
||||||
|
ignore = all
|
||||||
|
|
27
ca/id_rsa
27
ca/id_rsa
|
@ -1,27 +0,0 @@
|
||||||
-----BEGIN RSA PRIVATE KEY-----
|
|
||||||
MIIEpAIBAAKCAQEAt28xenSOhtLv5sj2gMgp/SWNcIkTHP/OiHvZe2mgdXXDpyIG
|
|
||||||
iUVGR35c1bep5sHvDLnL2IeFqrWSuajCn2Sf2FTgb/a7dWrTd10fNDlCJK5ibXm9
|
|
||||||
CvXXYmsGc+CVDH4E/44qkcm5kqur70D8N9MfnSwrhFlLlwuCXbweyt+cqzFFQeby
|
|
||||||
47lChQ5Aeim+r0zPVcjqCG66TF1llCSxMbDNYIVONYPQeTWDLut8azgVIUE4bhlR
|
|
||||||
B44w1mYS6BjMRcYKFZ672os6DLLc/qsRPquXR54AKuD2gg5sn9VgHplFrr2llPzL
|
|
||||||
Iwoiky6plIfQVgHn3BMGmePx/cNe51mq09AQwQIDAQABAoIBAQCCfJrZ3Wg2CH+X
|
|
||||||
0IVp/vm/lqMS1q++BUrKVC/VVsJKTEet8Mptg9YGraEkds5p1LNUfibAFUfEs/14
|
|
||||||
DNDFyjLbFSXC/+VCFYfwdVHpOIIQzew+rEcKMO/Slwe0DqJ4jHzJvjwSEUntSCm6
|
|
||||||
vKOuooTurakXMN5QyGMogtX0wzUToXajl8VPjGy1B2xlIM8KKmvUzsMMrIv4W157
|
|
||||||
xV/9OnJ0vPn+3gh90F5Daxk069xNMDefpSbfx5DcZemhnZLLWddX+QmnsSOl5tfQ
|
|
||||||
8zRx2fkjmjqDCJJ2e4KqV1+7ePhJqARLPnHYxmxbgsfaRGU/RizZFI90Ou3Fma8E
|
|
||||||
9cc/mfABAoGBAPMtAachdkkuxMB4btTAsW5PgKod0JQhMcXajZJqsl35WczVHHPP
|
|
||||||
zPXm03Ggx+Yk4BLFBoQT7xFfQjQOo4uUQ30hgS5+lkpGRtJa5p2kLVoDlyuGi0ig
|
|
||||||
0V+1u/h7wUNfuxncFcFmT8CCjhLrv9WMH2aNGdeRMIHVuxbVIqCftqThAoGBAMEb
|
|
||||||
pcppZ4Wj7poN033Tq4i9vXoJ2p1d5AwEKWKiYxoHvGRthD/NAjekVrRQrmzVQd1f
|
|
||||||
/gIOtnfO39WVN7ocH6wRWTi/KBMIrR0oPhtor4MPsB+19JvDmNtq48b94lVz+Z4L
|
|
||||||
hOSqNdGzlsuy1vZagRzHkxD7VU7PlT+UBdDHpwfhAoGBAPHjZO+Ao46sTN4/bc+H
|
|
||||||
VXcq8gtF2QJf+oiam5R3ObGspRzRJ5ozq+c2kkFG81EEgTdqcM7UnUuke9AYd6oR
|
|
||||||
8wf3We6L0KdVPIFmFlvcwZf2VlrfXJEEFwCjX7UONPH1ucFBYQqd4NrXgsdjZdDf
|
|
||||||
ryRtWrVJIP0lQxK1M9qexClBAoGAI3SxFx4NTONRjuWU/Fhd+WhlHsAqbJRtp6sn
|
|
||||||
8h1AtunOtF3LV2+Lxa2d4dOigwcQ5dWXLMeIxyyrumqAZeJ+CjjROfMXJ4+DQYQ/
|
|
||||||
CwdImnbJ7riY5fSe30Kb+dBpuyjlHxicWOPLp+oieNooT+lEJYWbQhXzjtncXGUQ
|
|
||||||
QEo4J+ECgYA1UiQr+C3NbMr27HOynXInv09f86HvHsHGooXuGoJ7GB518ZjtoiA7
|
|
||||||
xefoSUR10NLmx8QhdGydaFH+wIdgAAwB7RjpsrA5XGnTEKj5yV00o272jXj2jygu
|
|
||||||
U5700lxwQFHbd5mNZXoywBP0kU/5I98A5OHGhpYthp8lGC+Vk1NZFA==
|
|
||||||
-----END RSA PRIVATE KEY-----
|
|
|
@ -1 +0,0 @@
|
||||||
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3bzF6dI6G0u/myPaAyCn9JY1wiRMc/86Ie9l7aaB1dcOnIgaJRUZHflzVt6nmwe8MucvYh4WqtZK5qMKfZJ/YVOBv9rt1atN3XR80OUIkrmJteb0K9ddiawZz4JUMfgT/jiqRybmSq6vvQPw30x+dLCuEWUuXC4JdvB7K35yrMUVB5vLjuUKFDkB6Kb6vTM9VyOoIbrpMXWWUJLExsM1ghU41g9B5NYMu63xrOBUhQThuGVEHjjDWZhLoGMxFxgoVnrvaizoMstz+qxE+q5dHngAq4PaCDmyf1WAemUWuvaWU/MsjCiKTLqmUh9BWAefcEwaZ4/H9w17nWarT0BDB neospcc@neospcc
|
|
|
@ -1,2 +0,0 @@
|
||||||
|1|UPdjsdJ58lY6EqUaP7n833Fj7Ic=|hQkKlD6Ui6Ne03QrjHpsL7NU0qA= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw==
|
|
||||||
|1|C5OFm8ugKVhcy7PRLErwNGAV/jw=|qMuNNLMrE8rDdleXMmX8N7YBEkM= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw==
|
|
|
@ -1,21 +0,0 @@
|
||||||
-----BEGIN CERTIFICATE-----
|
|
||||||
MIIDhDCCAmygAwIBAgIBATANBgkqhkiG9w0BAQsFADB2MQswCQYDVQQGEwJSVTEM
|
|
||||||
MAoGA1UECBMDU1BCMQwwCgYDVQQHEwNTUEIxDjAMBgNVBAoTBU5TUENDMQ4wDAYD
|
|
||||||
VQQLEwVOU1BDQzEOMAwGA1UEAxMFTlNQQ0MxGzAZBgkqhkiG9w0BCQEWDG9wc0Bu
|
|
||||||
c3BjYy5ydTAeFw0xOTEyMzAxNjQ0MDBaFw0yMDEyMzAxNjQ0MDBaMHYxCzAJBgNV
|
|
||||||
BAYTAlJVMQwwCgYDVQQIEwNTUEIxDDAKBgNVBAcTA1NQQjEOMAwGA1UEChMFTlNQ
|
|
||||||
Q0MxDjAMBgNVBAsTBU5TUENDMQ4wDAYDVQQDEwVOU1BDQzEbMBkGCSqGSIb3DQEJ
|
|
||||||
ARYMb3BzQG5zcGNjLnJ1MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
|
|
||||||
q14BlfD4ZcNJz3HZGLj4ksqbi2tzogpnIlC2ccVcd8sk7mrkAjUFI1VqdJg57F7Y
|
|
||||||
kv8qYsyYwTaUQB5NAmSOQxNFKKsRc4LI3J8b7dE2q5+dDCWRN5tz8sIRfXryXyF2
|
|
||||||
VpW1RnuxMGJ6SwgTn9Wb+R6o1WkwNtrDhhNkdWsBnrJzL37GIk6nhQJxm3OlrHgQ
|
|
||||||
IPz0bt2QlSYQwuTTd1V6vMgOpHfqkQiAJZLBvQ39SImAO+pNS1+CrJuh+zWqhKzq
|
|
||||||
0vagW0EA4mq7Z98WqGF/g9d2yj8WRpxoAwpJzUoExPstHyDYSvKhr6R8+QV0zhpF
|
|
||||||
KdjqggmAM9ixSonYC3txawIDAQABox0wGzAMBgNVHRMEBTADAQH/MAsGA1UdDwQE
|
|
||||||
AwICjDANBgkqhkiG9w0BAQsFAAOCAQEAQ3gBirQTNU9rWIT9Bwh4Hj1ft5jIepQ7
|
|
||||||
yXmqFiOFKUli850t1yCre5/oGyOhQy8huUDL+qQDFeX63g5m3r1B3MsBfxKsukn6
|
|
||||||
I2HqwNVEUrsnRtWDr/NXpwpRGfjg1zG9tUxg6Tbo5VD4HG7dUGBuyOXA+/tBYvUQ
|
|
||||||
3vr9bGFUpuYGJ6/KmnoEbR6B1cB4MwW+xorJFODCaEcd2aAuAABL7o84nx3UFTsb
|
|
||||||
CguS26u5dpA+32rwV9US17sJfJ301DirapG3CxlhxhUPMXqxuQH1Am8x4zaMz3dN
|
|
||||||
fRvPLar6g33xKcd6T/VAAp7kMpxyzWoenOYXxGfpgs7Rj/X6xBX3sw==
|
|
||||||
-----END CERTIFICATE-----
|
|
|
@ -1 +0,0 @@
|
||||||
╜d#╟╙BA≤\E╨кООК┤о╪-^r╙═=n╠
*
|
|
|
@ -1 +0,0 @@
|
||||||
=b–•|vn¾b®%ü“ÐþaåÙÍ-ie×ðŒ=
|
|
41
robot/resources/files/s3_bearer_rules.json
Normal file
41
robot/resources/files/s3_bearer_rules.json
Normal file
|
@ -0,0 +1,41 @@
|
||||||
|
{
|
||||||
|
"records":
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"operation":"PUT",
|
||||||
|
"action":"ALLOW",
|
||||||
|
"filters":[],
|
||||||
|
"targets":
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"role":"OTHERS",
|
||||||
|
"keys":[]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"operation":"SEARCH",
|
||||||
|
"action":"ALLOW",
|
||||||
|
"filters":[],
|
||||||
|
"targets":
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"role":"OTHERS",
|
||||||
|
"keys":[]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"operation":"GET",
|
||||||
|
"action":"ALLOW",
|
||||||
|
"filters":[],
|
||||||
|
"targets":
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"role":"OTHERS",
|
||||||
|
"keys":[]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
35
robot/resources/lib/python/http_gate.py
Normal file
35
robot/resources/lib/python/http_gate.py
Normal file
|
@ -0,0 +1,35 @@
|
||||||
|
#!/usr/bin/python3
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from common import HTTP_GATE
|
||||||
|
from robot.api.deco import keyword
|
||||||
|
from robot.api import logger
|
||||||
|
from robot.libraries.BuiltIn import BuiltIn
|
||||||
|
|
||||||
|
ROBOT_AUTO_KEYWORDS = False
|
||||||
|
ASSETS_DIR = BuiltIn().get_variable_value("${ASSETS_DIR}")
|
||||||
|
|
||||||
|
@keyword('Get via HTTP Gate')
|
||||||
|
def get_via_http_gate(cid: str, oid: str):
|
||||||
|
"""
|
||||||
|
This function gets given object from HTTP gate
|
||||||
|
:param cid: CID to get object from
|
||||||
|
:param oid: object OID
|
||||||
|
"""
|
||||||
|
request = f'{HTTP_GATE}/get/{cid}/{oid}'
|
||||||
|
resp = requests.get(request, stream=True)
|
||||||
|
|
||||||
|
if not resp.ok:
|
||||||
|
raise Exception(f"""Failed to get object via HTTP gate:
|
||||||
|
request: {resp.request.path_url},
|
||||||
|
response: {resp.text},
|
||||||
|
status code: {resp.status_code} {resp.reason}""")
|
||||||
|
|
||||||
|
logger.info(f'Request: {request}')
|
||||||
|
filename = f"{ASSETS_DIR}/{cid}_{oid}"
|
||||||
|
with open(filename, "wb") as get_file:
|
||||||
|
shutil.copyfileobj(resp.raw, get_file)
|
||||||
|
return filename
|
|
@ -54,7 +54,7 @@ def start_nodes(*nodes_list):
|
||||||
node = m.group(1)
|
node = m.group(1)
|
||||||
client = docker.APIClient()
|
client = docker.APIClient()
|
||||||
client.start(node)
|
client.start(node)
|
||||||
|
|
||||||
|
|
||||||
@keyword('Get nodes with object')
|
@keyword('Get nodes with object')
|
||||||
def get_nodes_with_object(private_key: str, cid: str, oid: str):
|
def get_nodes_with_object(private_key: str, cid: str, oid: str):
|
||||||
|
@ -712,12 +712,6 @@ def delete_container(cid: str, private_key: str):
|
||||||
_cmd_run(deleteContainerCmd)
|
_cmd_run(deleteContainerCmd)
|
||||||
|
|
||||||
|
|
||||||
@keyword('Get file name')
|
|
||||||
def get_file_name(filepath):
|
|
||||||
filename = os.path.basename(filepath)
|
|
||||||
return filename
|
|
||||||
|
|
||||||
|
|
||||||
@keyword('Get file hash')
|
@keyword('Get file hash')
|
||||||
def get_file_hash(filename : str):
|
def get_file_hash(filename : str):
|
||||||
file_hash = _get_file_hash(filename)
|
file_hash = _get_file_hash(filename)
|
||||||
|
@ -771,7 +765,7 @@ def get_control_endpoint_with_wif(endpoint_number: str = ''):
|
||||||
endpoint_values = NEOFS_NETMAP_DICT[f'{endpoint_num}']
|
endpoint_values = NEOFS_NETMAP_DICT[f'{endpoint_num}']
|
||||||
endpoint_control = endpoint_values['control']
|
endpoint_control = endpoint_values['control']
|
||||||
wif = endpoint_values['wif']
|
wif = endpoint_values['wif']
|
||||||
|
|
||||||
return endpoint_num, endpoint_control, wif
|
return endpoint_num, endpoint_control, wif
|
||||||
|
|
||||||
@keyword('Get Locode')
|
@keyword('Get Locode')
|
||||||
|
@ -1012,7 +1006,7 @@ def generate_session_token(owner: str, pub_key: str, cid: str = "", wildcard: bo
|
||||||
@keyword ('Sign Session Token')
|
@keyword ('Sign Session Token')
|
||||||
def sign_session_token(session_token: str, wallet: str, to_file: str=''):
|
def sign_session_token(session_token: str, wallet: str, to_file: str=''):
|
||||||
if to_file:
|
if to_file:
|
||||||
to_file = f'--to {to_file}'
|
to_file = f'--to {to_file}'
|
||||||
cmd = (
|
cmd = (
|
||||||
f'{NEOFS_CLI_EXEC} util sign session-token --from {session_token} '
|
f'{NEOFS_CLI_EXEC} util sign session-token --from {session_token} '
|
||||||
f'-w {wallet} {to_file}'
|
f'-w {wallet} {to_file}'
|
||||||
|
|
|
@ -1,19 +1,23 @@
|
||||||
#!/usr/bin/python3.8
|
#!/usr/bin/python3
|
||||||
|
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import uuid
|
import uuid
|
||||||
import requests
|
|
||||||
import botocore
|
|
||||||
import boto3
|
|
||||||
|
|
||||||
from common import *
|
import boto3
|
||||||
from robot.api.deco import keyword
|
import botocore
|
||||||
from robot.api import logger
|
|
||||||
from cli_helpers import _run_with_passwd
|
from cli_helpers import _run_with_passwd
|
||||||
|
|
||||||
|
from common import GATE_PUB_KEY, NEOFS_ENDPOINT, S3_GATE
|
||||||
|
import urllib3
|
||||||
|
from robot.api.deco import keyword
|
||||||
|
from robot.api import logger
|
||||||
|
|
||||||
|
##########################################################
|
||||||
|
# Disabling warnings on self-signed certificate which the
|
||||||
|
# boto library produces on requests to S3-gate in dev-env.
|
||||||
|
urllib3.disable_warnings()
|
||||||
|
##########################################################
|
||||||
|
|
||||||
ROBOT_AUTO_KEYWORDS = False
|
ROBOT_AUTO_KEYWORDS = False
|
||||||
|
|
||||||
|
@ -22,44 +26,37 @@ NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate')
|
||||||
@keyword('Init S3 Credentials')
|
@keyword('Init S3 Credentials')
|
||||||
def init_s3_credentials(wallet):
|
def init_s3_credentials(wallet):
|
||||||
bucket = str(uuid.uuid4())
|
bucket = str(uuid.uuid4())
|
||||||
records = ' \' {"records":[{"operation":"PUT","action":"ALLOW","filters":[],"targets":[{"role":"OTHERS","keys":[]}]}, {"operation":"SEARCH","action":"ALLOW","filters":[],"targets":[{"role":"OTHERS","keys":[]}]}, {"operation":"GET","action":"ALLOW","filters":[],"targets":[{"role":"OTHERS","keys":[]}]}]} \' '
|
s3_bearer_rules = "robot/resources/files/s3_bearer_rules.json"
|
||||||
Cmd = (
|
cmd = (
|
||||||
f'{NEOFS_EXEC} --debug --with-log issue-secret --wallet {wallet} '
|
f'{NEOFS_EXEC} --debug --with-log issue-secret --wallet {wallet} '
|
||||||
f'--gate-public-key={GATE_PUB_KEY} --peer {NEOFS_ENDPOINT} '
|
f'--gate-public-key={GATE_PUB_KEY} --peer {NEOFS_ENDPOINT} '
|
||||||
f'--container-friendly-name {bucket} --create-session-token '
|
f'--container-friendly-name {bucket} '
|
||||||
f'--bearer-rules {records}'
|
f'--bearer-rules {s3_bearer_rules}'
|
||||||
)
|
)
|
||||||
logger.info(f"Executing command: {Cmd}")
|
logger.info(f"Executing command: {cmd}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
output = _run_with_passwd(Cmd)
|
output = _run_with_passwd(cmd)
|
||||||
logger.info(f"Command completed with output: {output}")
|
logger.info(f"Command completed with output: {output}")
|
||||||
|
# first five string are log output, cutting them off and parse
|
||||||
|
# the rest of the output as JSON
|
||||||
|
output = '\n'.join(output.split('\n')[5:])
|
||||||
|
output_dict = json.loads(output)
|
||||||
|
|
||||||
m = re.search(r'"container_id":\s+"(\w+)"', output)
|
return (output_dict['container_id'],
|
||||||
cid = m.group(1)
|
bucket,
|
||||||
logger.info("cid: %s" % cid)
|
output_dict['access_key_id'],
|
||||||
|
output_dict['secret_access_key'],
|
||||||
|
output_dict['owner_private_key'])
|
||||||
|
|
||||||
m = re.search(r'"access_key_id":\s+"([\w\/]+)"', output)
|
except Exception as exc:
|
||||||
access_key_id = m.group(1)
|
raise RuntimeError("failed to init s3 credentials") from exc
|
||||||
logger.info("access_key_id: %s" % access_key_id)
|
|
||||||
|
|
||||||
m = re.search(r'"secret_access_key":\s+"(\w+)"', output)
|
|
||||||
secret_access_key = m.group(1)
|
|
||||||
logger.info("secret_access_key: %s" % secret_access_key)
|
|
||||||
|
|
||||||
m = re.search(r'"owner_private_key":\s+"(\w+)"', output)
|
|
||||||
owner_private_key = m.group(1)
|
|
||||||
logger.info("owner_private_key: %s" % owner_private_key)
|
|
||||||
|
|
||||||
return cid, bucket, access_key_id, secret_access_key, owner_private_key
|
|
||||||
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
raise Exception(f"Error: \nreturn code: {e.returncode}. \nOutput: {e.stderr}")
|
|
||||||
|
|
||||||
|
|
||||||
@keyword('Config S3 client')
|
@keyword('Config S3 client')
|
||||||
def config_s3_client(access_key_id, secret_access_key):
|
def config_s3_client(access_key_id, secret_access_key):
|
||||||
try:
|
try:
|
||||||
|
|
||||||
session = boto3.session.Session()
|
session = boto3.session.Session()
|
||||||
|
|
||||||
s3_client = session.client(
|
s3_client = session.client(
|
||||||
|
@ -80,11 +77,11 @@ def config_s3_client(access_key_id, secret_access_key):
|
||||||
def list_objects_s3_v2(s3_client, bucket):
|
def list_objects_s3_v2(s3_client, bucket):
|
||||||
try:
|
try:
|
||||||
response = s3_client.list_objects_v2(Bucket=bucket)
|
response = s3_client.list_objects_v2(Bucket=bucket)
|
||||||
logger.info("S3 v2 List objects result: %s" % response['Contents'])
|
logger.info(f"S3 v2 List objects result: {response['Contents']}")
|
||||||
obj_list = []
|
obj_list = []
|
||||||
for obj in response['Contents']:
|
for obj in response['Contents']:
|
||||||
obj_list.append(obj['Key'])
|
obj_list.append(obj['Key'])
|
||||||
logger.info("Found s3 objects: %s" % obj_list)
|
logger.info(f"Found s3 objects: {obj_list}")
|
||||||
return obj_list
|
return obj_list
|
||||||
|
|
||||||
except botocore.exceptions.ClientError as err:
|
except botocore.exceptions.ClientError as err:
|
||||||
|
@ -96,11 +93,11 @@ def list_objects_s3_v2(s3_client, bucket):
|
||||||
def list_objects_s3(s3_client, bucket):
|
def list_objects_s3(s3_client, bucket):
|
||||||
try:
|
try:
|
||||||
response = s3_client.list_objects(Bucket=bucket)
|
response = s3_client.list_objects(Bucket=bucket)
|
||||||
logger.info("S3 List objects result: %s" % response['Contents'])
|
logger.info(f"S3 List objects result: {response['Contents']}")
|
||||||
obj_list = []
|
obj_list = []
|
||||||
for obj in response['Contents']:
|
for obj in response['Contents']:
|
||||||
obj_list.append(obj['Key'])
|
obj_list.append(obj['Key'])
|
||||||
logger.info("Found s3 objects: %s" % obj_list)
|
logger.info(f"Found s3 objects: {obj_list}")
|
||||||
return obj_list
|
return obj_list
|
||||||
|
|
||||||
except botocore.exceptions.ClientError as err:
|
except botocore.exceptions.ClientError as err:
|
||||||
|
@ -114,12 +111,12 @@ def create_bucket_s3(s3_client):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
|
s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
|
||||||
logger.info("Created S3 bucket: %s" % s3_bucket)
|
logger.info(f"Created S3 bucket: {s3_bucket}")
|
||||||
return bucket_name
|
return bucket_name
|
||||||
|
|
||||||
except botocore.exceptions.ClientError as err:
|
except botocore.exceptions.ClientError as err:
|
||||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||||
|
|
||||||
|
|
||||||
@keyword('List buckets S3')
|
@keyword('List buckets S3')
|
||||||
|
@ -127,7 +124,7 @@ def list_buckets_s3(s3_client):
|
||||||
found_buckets = []
|
found_buckets = []
|
||||||
try:
|
try:
|
||||||
response = s3_client.list_buckets()
|
response = s3_client.list_buckets()
|
||||||
logger.info("S3 List buckets result: %s" % response)
|
logger.info(f"S3 List buckets result: {response}")
|
||||||
|
|
||||||
for bucket in response['Buckets']:
|
for bucket in response['Buckets']:
|
||||||
found_buckets.append(bucket['Name'])
|
found_buckets.append(bucket['Name'])
|
||||||
|
@ -136,7 +133,7 @@ def list_buckets_s3(s3_client):
|
||||||
|
|
||||||
except botocore.exceptions.ClientError as err:
|
except botocore.exceptions.ClientError as err:
|
||||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||||
|
|
||||||
|
|
||||||
@keyword('Delete bucket S3')
|
@keyword('Delete bucket S3')
|
||||||
|
@ -144,7 +141,7 @@ def delete_bucket_s3(s3_client, bucket):
|
||||||
try:
|
try:
|
||||||
response = s3_client.delete_bucket(Bucket=bucket)
|
response = s3_client.delete_bucket(Bucket=bucket)
|
||||||
logger.info(f"S3 Delete bucket result: {response}")
|
logger.info(f"S3 Delete bucket result: {response}")
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
except botocore.exceptions.ClientError as err:
|
except botocore.exceptions.ClientError as err:
|
||||||
|
@ -152,11 +149,11 @@ def delete_bucket_s3(s3_client, bucket):
|
||||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||||
|
|
||||||
|
|
||||||
@keyword('HeadBucket S3')
|
@keyword('Head bucket S3')
|
||||||
def headbucket(bucket, s3_client):
|
def head_bucket(s3_client, bucket):
|
||||||
try:
|
try:
|
||||||
response = s3_client.head_bucket(Bucket=bucket)
|
response = s3_client.head_bucket(Bucket=bucket)
|
||||||
logger.info(f"S3 HeadBucket result: {response}")
|
logger.info(f"S3 Head bucket result: {response}")
|
||||||
return response
|
return response
|
||||||
|
|
||||||
except botocore.exceptions.ClientError as err:
|
except botocore.exceptions.ClientError as err:
|
||||||
|
@ -168,12 +165,12 @@ def headbucket(bucket, s3_client):
|
||||||
def put_object_s3(s3_client, bucket, filepath):
|
def put_object_s3(s3_client, bucket, filepath):
|
||||||
filename = os.path.basename(filepath)
|
filename = os.path.basename(filepath)
|
||||||
|
|
||||||
with open(filepath, "rb") as f:
|
with open(filepath, "rb") as put_file:
|
||||||
fileContent = f.read()
|
file_content = put_file.read()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = s3_client.put_object(Body=fileContent, Bucket=bucket, Key=filename)
|
response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename)
|
||||||
logger.info("S3 Put object result: %s" % response)
|
logger.info(f"S3 Put object result: {response}")
|
||||||
except botocore.exceptions.ClientError as err:
|
except botocore.exceptions.ClientError as err:
|
||||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||||
|
@ -184,19 +181,19 @@ def head_object_s3(s3_client, bucket, object_key):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = s3_client.head_object(Bucket=bucket, Key=object_key)
|
response = s3_client.head_object(Bucket=bucket, Key=object_key)
|
||||||
logger.info("S3 Head object result: %s" % response)
|
logger.info(f"S3 Head object result: {response}")
|
||||||
return response
|
return response
|
||||||
|
|
||||||
except botocore.exceptions.ClientError as err:
|
except botocore.exceptions.ClientError as err:
|
||||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||||
|
|
||||||
|
|
||||||
@keyword('Delete object S3')
|
@keyword('Delete object S3')
|
||||||
def delete_object_s3(s3_client, bucket, object_key):
|
def delete_object_s3(s3_client, bucket, object_key):
|
||||||
try:
|
try:
|
||||||
response = s3_client.delete_object(Bucket=bucket, Key=object_key)
|
response = s3_client.delete_object(Bucket=bucket, Key=object_key)
|
||||||
logger.info("S3 Put object result: %s" % response)
|
logger.info(f"S3 Put object result: {response}")
|
||||||
return response
|
return response
|
||||||
|
|
||||||
except botocore.exceptions.ClientError as err:
|
except botocore.exceptions.ClientError as err:
|
||||||
|
@ -205,11 +202,14 @@ def delete_object_s3(s3_client, bucket, object_key):
|
||||||
|
|
||||||
|
|
||||||
@keyword('Copy object S3')
|
@keyword('Copy object S3')
|
||||||
def copy_object_s3(s3_client, bucket, object_key, new_object):
|
def copy_object_s3(s3_client, bucket, object_key):
|
||||||
|
filename = f"{os.getcwd()}/{uuid.uuid4()}"
|
||||||
try:
|
try:
|
||||||
response = s3_client.copy_object(Bucket=bucket, CopySource=bucket+"/"+object_key, Key=new_object)
|
response = s3_client.copy_object(Bucket=bucket,
|
||||||
logger.info("S3 Copy object result: %s" % response)
|
CopySource=f"{bucket}/{object_key}",
|
||||||
return response
|
Key=filename)
|
||||||
|
logger.info(f"S3 Copy object result: {response}")
|
||||||
|
return filename
|
||||||
|
|
||||||
except botocore.exceptions.ClientError as err:
|
except botocore.exceptions.ClientError as err:
|
||||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||||
|
@ -217,44 +217,19 @@ def copy_object_s3(s3_client, bucket, object_key, new_object):
|
||||||
|
|
||||||
|
|
||||||
@keyword('Get object S3')
|
@keyword('Get object S3')
|
||||||
def get_object_s3(s3_client, bucket, object_key, target_file):
|
def get_object_s3(s3_client, bucket, object_key):
|
||||||
|
filename = f"{os.getcwd()}/{uuid.uuid4()}"
|
||||||
try:
|
try:
|
||||||
response = s3_client.get_object(Bucket=bucket, Key=object_key)
|
response = s3_client.get_object(Bucket=bucket, Key=object_key)
|
||||||
|
|
||||||
with open(f"{target_file}", 'wb') as f:
|
with open(f"{filename}", 'wb') as get_file:
|
||||||
chunk = response['Body'].read(1024)
|
chunk = response['Body'].read(1024)
|
||||||
while chunk:
|
while chunk:
|
||||||
f.write(chunk)
|
get_file.write(chunk)
|
||||||
chunk = response['Body'].read(1024)
|
chunk = response['Body'].read(1024)
|
||||||
|
|
||||||
return target_file
|
return filename
|
||||||
|
|
||||||
except botocore.exceptions.ClientError as err:
|
except botocore.exceptions.ClientError as err:
|
||||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||||
|
|
||||||
|
|
||||||
@keyword('Get via HTTP Gate')
|
|
||||||
def get_via_http_gate(cid: str, oid: str):
|
|
||||||
"""
|
|
||||||
This function gets given object from HTTP gate
|
|
||||||
:param cid: CID to get object from
|
|
||||||
:param oid: object OID
|
|
||||||
"""
|
|
||||||
request = f'{HTTP_GATE}/get/{cid}/{oid}'
|
|
||||||
resp = requests.get(request, stream=True)
|
|
||||||
|
|
||||||
if not resp.ok:
|
|
||||||
raise Exception(f"""Failed to get object via HTTP gate:
|
|
||||||
request: {resp.request.path_url},
|
|
||||||
response: {resp.text},
|
|
||||||
status code: {resp.status_code} {resp.reason}""")
|
|
||||||
return
|
|
||||||
|
|
||||||
logger.info(f'Request: {request}')
|
|
||||||
filename = os.path.curdir + f"/{cid}_{oid}"
|
|
||||||
with open(filename, "wb") as f:
|
|
||||||
shutil.copyfileobj(resp.raw, f)
|
|
||||||
del resp
|
|
||||||
return filename
|
|
||||||
|
|
|
@ -54,12 +54,12 @@ def wif_to_binary(wif: str) -> str:
|
||||||
return path
|
return path
|
||||||
|
|
||||||
@keyword('Make Up')
|
@keyword('Make Up')
|
||||||
def make_up(services=['']):
|
def make_up(services: list=[]):
|
||||||
test_path = os.getcwd()
|
test_path = os.getcwd()
|
||||||
dev_path = os.getenv('DEVENV_PATH', '../neofs-dev-env')
|
dev_path = os.getenv('DEVENV_PATH', '../neofs-dev-env')
|
||||||
os.chdir(dev_path)
|
os.chdir(dev_path)
|
||||||
|
|
||||||
if services != ['']:
|
if len(services) > 0:
|
||||||
for service in services:
|
for service in services:
|
||||||
cmd = f'make up/{service}'
|
cmd = f'make up/{service}'
|
||||||
logger.info(f"Cmd: {cmd}")
|
logger.info(f"Cmd: {cmd}")
|
||||||
|
|
|
@ -3,19 +3,13 @@ Variables common.py
|
||||||
Variables wellknown_acl.py
|
Variables wellknown_acl.py
|
||||||
|
|
||||||
Library neofs.py
|
Library neofs.py
|
||||||
Library payment_neogo.py
|
Library http_gate.py
|
||||||
Library gates.py
|
|
||||||
Library wallet_keywords.py
|
|
||||||
Library rpc_call_keywords.py
|
|
||||||
Library utility_keywords.py
|
|
||||||
|
|
||||||
Resource payment_operations.robot
|
Resource payment_operations.robot
|
||||||
Resource setup_teardown.robot
|
Resource setup_teardown.robot
|
||||||
|
|
||||||
*** Variables ***
|
*** Variables ***
|
||||||
${PLACEMENT_RULE} = REP 1 IN X CBF 1 SELECT 1 FROM * AS X
|
${PLACEMENT_RULE} = REP 1 IN X CBF 1 SELECT 1 FROM * AS X
|
||||||
${TRANSFER_AMOUNT} = ${6}
|
|
||||||
${DEPOSIT_AMOUNT} = ${5}
|
|
||||||
${CONTAINER_WAIT_INTERVAL} = 1 min
|
${CONTAINER_WAIT_INTERVAL} = 1 min
|
||||||
@{INCLUDE_SVC} = http_gate
|
@{INCLUDE_SVC} = http_gate
|
||||||
|
|
||||||
|
@ -25,9 +19,9 @@ NeoFS HTTP Gateway
|
||||||
[Documentation] Creates container and does PUT, GET via HTTP Gate
|
[Documentation] Creates container and does PUT, GET via HTTP Gate
|
||||||
[Timeout] 5 min
|
[Timeout] 5 min
|
||||||
|
|
||||||
[Setup] Setup
|
[Setup] Setup
|
||||||
Make Up ${INCLUDE_SVC}
|
Make Up ${INCLUDE_SVC}
|
||||||
|
|
||||||
${WALLET} ${ADDR} ${WIF} = Prepare Wallet And Deposit
|
${WALLET} ${ADDR} ${WIF} = Prepare Wallet And Deposit
|
||||||
|
|
||||||
${CID} = Create container ${WIF} ${PUBLIC_ACL} ${PLACEMENT_RULE}
|
${CID} = Create container ${WIF} ${PUBLIC_ACL} ${PLACEMENT_RULE}
|
||||||
|
|
|
@ -2,33 +2,29 @@
|
||||||
Variables common.py
|
Variables common.py
|
||||||
|
|
||||||
Library Collections
|
Library Collections
|
||||||
|
Library OperatingSystem
|
||||||
|
|
||||||
Library neofs.py
|
Library neofs.py
|
||||||
Library payment_neogo.py
|
Library s3_gate.py
|
||||||
Library gates.py
|
|
||||||
Library wallet_keywords.py
|
|
||||||
Library contract_keywords.py
|
Library contract_keywords.py
|
||||||
Library Process
|
|
||||||
|
|
||||||
Resource setup_teardown.robot
|
Resource setup_teardown.robot
|
||||||
Resource payment_operations.robot
|
Resource payment_operations.robot
|
||||||
|
|
||||||
*** Variables ***
|
*** Variables ***
|
||||||
${DEPOSIT} = ${30}
|
@{INCLUDE_SVC} = s3_gate coredns
|
||||||
${WIF} = ${MAINNET_WALLET_WIF}
|
|
||||||
${DEPOSIT_TIMEOUT}= 30s
|
|
||||||
@{INCLUDE_SVC} = s3_gate
|
|
||||||
|
|
||||||
*** Test cases ***
|
*** Test cases ***
|
||||||
Buckets in NeoFS S3 Gateway
|
Buckets in NeoFS S3 Gateway
|
||||||
[Documentation] Execute operations with bucket via S3 Gate
|
[Documentation] Execute operations with bucket via S3 Gate
|
||||||
[Timeout] 10 min
|
[Timeout] 10 min
|
||||||
|
|
||||||
[Setup] Setup
|
[Setup] Setup
|
||||||
Make Up ${INCLUDE_SVC}
|
Make Up ${INCLUDE_SVC}
|
||||||
|
|
||||||
${WALLET} ${ADDR} ${WIF} = Prepare Wallet And Deposit
|
${WALLET} ${_} ${WIF} = Prepare Wallet And Deposit
|
||||||
${FILE_S3} = Generate file of bytes ${COMPLEX_OBJ_SIZE}
|
${FILE_S3} = Generate file of bytes ${COMPLEX_OBJ_SIZE}
|
||||||
${FILE_S3_NAME} = Get file name ${FILE_S3}
|
${_} ${S3_OBJECT_KEY} = Split Path ${FILE_S3}
|
||||||
|
|
||||||
${CID}
|
${CID}
|
||||||
... ${BUCKET}
|
... ${BUCKET}
|
||||||
|
@ -44,24 +40,24 @@ Buckets in NeoFS S3 Gateway
|
||||||
${NEW_BUCKET} = Create Bucket S3 ${S3_CLIENT}
|
${NEW_BUCKET} = Create Bucket S3 ${S3_CLIENT}
|
||||||
${NEW_BUCKET_EMPTY} = Create Bucket S3 ${S3_CLIENT}
|
${NEW_BUCKET_EMPTY} = Create Bucket S3 ${S3_CLIENT}
|
||||||
|
|
||||||
HeadBucket S3 ${BUCKET} ${S3_CLIENT}
|
Head bucket S3 ${S3_CLIENT} ${BUCKET}
|
||||||
HeadBucket S3 ${NEW_BUCKET} ${S3_CLIENT}
|
Head bucket S3 ${S3_CLIENT} ${NEW_BUCKET}
|
||||||
|
|
||||||
Put object S3 ${S3_CLIENT} ${NEW_BUCKET} ${FILE_S3}
|
Put object S3 ${S3_CLIENT} ${NEW_BUCKET} ${FILE_S3}
|
||||||
Head object S3 ${S3_CLIENT} ${NEW_BUCKET} ${FILE_S3_NAME}
|
Head object S3 ${S3_CLIENT} ${NEW_BUCKET} ${S3_OBJECT_KEY}
|
||||||
|
|
||||||
${LIST_S3_OBJECTS} = List objects S3 ${S3_CLIENT} ${NEW_BUCKET}
|
${LIST_S3_OBJECTS} = List objects S3 ${S3_CLIENT} ${NEW_BUCKET}
|
||||||
List Should Contain Value ${LIST_S3_OBJECTS} ${FILE_S3_NAME}
|
List Should Contain Value ${LIST_S3_OBJECTS} ${S3_OBJECT_KEY}
|
||||||
|
|
||||||
Run Keyword and Expect Error *
|
Run Keyword and Expect Error *
|
||||||
... Delete Bucket S3 ${S3_CLIENT} ${NEW_BUCKET}
|
... Delete Bucket S3 ${S3_CLIENT} ${NEW_BUCKET}
|
||||||
HeadBucket S3 ${NEW_BUCKET} ${S3_CLIENT}
|
Head bucket S3 ${S3_CLIENT} ${NEW_BUCKET}
|
||||||
|
|
||||||
Delete Bucket S3 ${S3_CLIENT} ${NEW_BUCKET_EMPTY}
|
Delete Bucket S3 ${S3_CLIENT} ${NEW_BUCKET_EMPTY}
|
||||||
Tick Epoch
|
Tick Epoch
|
||||||
Run Keyword And Expect Error *
|
Run Keyword And Expect Error *
|
||||||
... HeadBucket S3 ${NEW_BUCKET_EMPTY} ${S3_CLIENT}
|
... Head bucket S3 ${S3_CLIENT} ${NEW_BUCKET_EMPTY}
|
||||||
|
|
||||||
${BUCKET_LIST} = List Buckets S3 ${S3_CLIENT}
|
${BUCKET_LIST} = List Buckets S3 ${S3_CLIENT}
|
||||||
Tick Epoch
|
Tick Epoch
|
||||||
List Should Contain Value ${BUCKET_LIST} ${NEW_BUCKET}
|
List Should Contain Value ${BUCKET_LIST} ${NEW_BUCKET}
|
||||||
|
|
38
run.sh
38
run.sh
|
@ -1,38 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Before each test execution dev-env should be build down and up.
|
|
||||||
|
|
||||||
xmls=''
|
|
||||||
|
|
||||||
if [ $# -eq 0 ]; then
|
|
||||||
test_list=$(find ./robot/testsuites -regex '.*robot')
|
|
||||||
else
|
|
||||||
test_list=$(find $@ -regex '.*robot')
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo Tests to execute: $test_list
|
|
||||||
|
|
||||||
for test in $test_list; do
|
|
||||||
pushd $DEVENV_PATH
|
|
||||||
if [[ $test =~ 's3_gate' || $test =~ 'http_gate' ]]; then
|
|
||||||
sed -i -e '/coredns/d' .services
|
|
||||||
else
|
|
||||||
sed -i -e '/coredns/d' -e '/s3_gate/d' -e '/http_gate/d' .services
|
|
||||||
fi
|
|
||||||
make down
|
|
||||||
make clean
|
|
||||||
make up
|
|
||||||
make update.max_object_size val=1000
|
|
||||||
popd
|
|
||||||
test_addr=`echo $test | sed "s/\//_/g" | sed "s/.robot//"`
|
|
||||||
robot --outputdir artifacts/ --output ${test_addr}_output.xml --log ${test_addr}_log.html --report ${test_addr}_report.html $test
|
|
||||||
xmls+=" ./artifacts/${test_addr}_output.xml"
|
|
||||||
pushd $DEVENV_PATH
|
|
||||||
echo 'coredns' >> .services
|
|
||||||
if [ -z $(cat .services | grep 's3_gate') ]; then echo 's3_gate' >> .services; fi
|
|
||||||
if [ -z $(cat .services | grep 'http_gate') ]; then echo 'http_gate' >> .services; fi
|
|
||||||
popd
|
|
||||||
done
|
|
||||||
|
|
||||||
rebot ${xmls}
|
|
||||||
|
|
Loading…
Reference in a new issue