forked from TrueCloudLab/frostfs-testcases
Add pytest tests.
Change AWS cli v1 to cli v2. Improve allure attachments. Add tests for S3 API. Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
This commit is contained in:
Anastasia Prasolova
parent
7abb2761c9
commit
6b1e1ab28d
24 changed files with 2210 additions and 247 deletions
12
.flake8
Normal file
12
.flake8
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
[flake8]
|
||||
exclude =
|
||||
.git,
|
||||
__pycache__,
|
||||
.idea,
|
||||
.pytest_cache,
|
||||
venv
|
||||
per-file-ignores =
|
||||
# imported but unused
|
||||
__init__.py: F401
|
||||
max-line-length = 120
|
||||
disable-noqa
|
||||
36
Makefile
36
Makefile
|
|
@ -7,6 +7,13 @@ SHELL = bash
|
|||
OUTPUT_DIR = artifacts/
|
||||
KEYWORDS_REPO = git@github.com:nspcc-dev/neofs-keywords.git
|
||||
VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a)
|
||||
ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
|
||||
DEV_IMAGE_PY ?= registry.spb.yadro.com/tools/pytest-neofs-x86_64:4
|
||||
|
||||
ifeq ($(shell uname -s),Darwin)
|
||||
DOCKER_NETWORK = --network bridge -p 389:389 -p 636:636
|
||||
endif
|
||||
|
||||
|
||||
.PHONY: all
|
||||
all: venvs
|
||||
|
|
@ -30,5 +37,34 @@ submodules:
|
|||
clean:
|
||||
rm -rf venv.*
|
||||
|
||||
pytest-local:
|
||||
@echo "⇒ Run Pytest"
|
||||
export PYTHONPATH=$(ROOT_DIR)/neofs-keywords/lib:$(ROOT_DIR)/neofs-keywords/robot:$(ROOT_DIR)/robot/resources/lib:$(ROOT_DIR)/robot/resources/lib/python_keywords:$(ROOT_DIR)/robot/variables && \
|
||||
python -m pytest pytest_tests/testsuites/
|
||||
|
||||
help:
|
||||
@echo "⇒ run Run testcases ${R}"
|
||||
|
||||
.PHONY: pytest-docker
|
||||
pytest-docker:
|
||||
-docker ps
|
||||
-docker rm neofs_tests_py
|
||||
-docker pull $(DEV_IMAGE_PY)
|
||||
docker run -t --rm \
|
||||
-w /tests \
|
||||
--name neofs_tests_py \
|
||||
-e PYTHONPATH="/tests/neofs-keywords/lib:/tests/neofs-keywords/robot:/tests/robot/resources/lib:/tests/robot/resources/lib/python_keywords:/tests/robot/variables:/tests/pytest_tests/helpers" \
|
||||
-v $(CURDIR):/tests \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v $(NEO_BIN_DIR):/neofs \
|
||||
--privileged \
|
||||
$(DOCKER_NETWORK) \
|
||||
--env-file $(CURDIR)/.env \
|
||||
$(DEV_IMAGE_PY) \
|
||||
-v \
|
||||
-m "$(CI_MARKERS)" \
|
||||
--color=no \
|
||||
--junitxml=/tests/xunit_results.xml \
|
||||
--alluredir=/tests/allure_results \
|
||||
--setup-show \
|
||||
/tests/pytest_tests/testsuites
|
||||
|
|
|
|||
40
pytest_tests/Dockerfile
Normal file
40
pytest_tests/Dockerfile
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
ARG ARCH
|
||||
|
||||
FROM python:3.9-slim
|
||||
|
||||
ARG ARCH
|
||||
|
||||
RUN apt-get -y update && apt-get -y install \
|
||||
gcc \
|
||||
make \
|
||||
git \
|
||||
curl \
|
||||
wget \
|
||||
openssh-client \
|
||||
iputils-ping \
|
||||
unzip \
|
||||
vim \
|
||||
dbus \
|
||||
lsof \
|
||||
tini \
|
||||
libssl-dev \
|
||||
expect \
|
||||
runc \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& apt-get clean
|
||||
|
||||
COPY requirements.txt ./
|
||||
|
||||
# install aws cli utility v2 from pip
|
||||
RUN python3 -m pip install awscliv2
|
||||
RUN ln /usr/local/bin/awscliv2 /usr/bin/aws
|
||||
RUN awsv2 --install
|
||||
RUN aws --version
|
||||
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
ENV PATH=$PATH:/root:/neofs
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
ENTRYPOINT ["tini", "--", "pytest"]
|
||||
0
pytest_tests/helpers/__init__.py
Normal file
0
pytest_tests/helpers/__init__.py
Normal file
49
pytest_tests/helpers/utility.py
Normal file
49
pytest_tests/helpers/utility.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import os
|
||||
import uuid
|
||||
from typing import List
|
||||
|
||||
from common import ASSETS_DIR, SIMPLE_OBJ_SIZE
|
||||
|
||||
|
||||
def create_file_with_content(file_path: str = None, content: str = None) -> str:
|
||||
mode = 'w+'
|
||||
if not content:
|
||||
content = os.urandom(SIMPLE_OBJ_SIZE)
|
||||
mode = 'wb'
|
||||
|
||||
if not file_path:
|
||||
file_path = f"{os.getcwd()}/{ASSETS_DIR}/{str(uuid.uuid4())}"
|
||||
else:
|
||||
if not os.path.exists(os.path.dirname(file_path)):
|
||||
os.makedirs(os.path.dirname(file_path))
|
||||
|
||||
with open(file_path, mode) as out_file:
|
||||
out_file.write(content)
|
||||
|
||||
return file_path
|
||||
|
||||
|
||||
def get_file_content(file_path: str) -> str:
|
||||
with open(file_path, 'r') as out_file:
|
||||
content = out_file.read()
|
||||
|
||||
return content
|
||||
|
||||
|
||||
def split_file(file_path: str, parts: int) -> List[str]:
|
||||
files = []
|
||||
with open(file_path, 'rb') as in_file:
|
||||
data = in_file.read()
|
||||
|
||||
content_size = len(data)
|
||||
|
||||
chunk_size = int((content_size + parts) / parts)
|
||||
part_id = 1
|
||||
for start_position in range(0, content_size + 1, chunk_size):
|
||||
part_file_name = f'{file_path}_part_{part_id}'
|
||||
files.append(part_file_name)
|
||||
with open(part_file_name, 'wb') as out_file:
|
||||
out_file.write(data[start_position:start_position + chunk_size])
|
||||
part_id += 1
|
||||
|
||||
return files
|
||||
17
pytest_tests/pytest.ini
Normal file
17
pytest_tests/pytest.ini
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
[pytest]
|
||||
log_cli = 1
|
||||
log_cli_level = DEBUG
|
||||
log_cli_format = %(asctime)s [%(levelname)4s] %(message)s
|
||||
log_format = %(asctime)s [%(levelname)4s] %(message)s
|
||||
log_cli_date_format = %Y-%m-%d %H:%M:%S
|
||||
log_date_format = %H:%M:%S
|
||||
markers =
|
||||
# special markers
|
||||
sanity: small tests subset
|
||||
staging: test to be excluded from run in verifier/pr-validation/sanity jobs and run test in staging job
|
||||
# functional markers
|
||||
grpc_api: standard gRPC API tests
|
||||
http_gate: HTTP gate contract
|
||||
s3_gate: S3 gate tests
|
||||
curl: tests for HTTP gate with curl utility
|
||||
long: long tests (with long execution time)
|
||||
63
pytest_tests/requirements.txt
Normal file
63
pytest_tests/requirements.txt
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
aiodns==3.0.0
|
||||
aiohttp==3.7.4.post0
|
||||
aioresponses==0.7.2
|
||||
allure-pytest==2.9.45
|
||||
allure-python-commons==2.9.45
|
||||
async-timeout==3.0.1
|
||||
asynctest==0.13.0
|
||||
attrs==21.4.0
|
||||
base58==2.1.0
|
||||
bitarray==2.3.4
|
||||
boto3==1.16.33
|
||||
botocore==1.19.33
|
||||
certifi==2022.5.18
|
||||
cffi==1.15.0
|
||||
chardet==4.0.0
|
||||
charset-normalizer==2.0.12
|
||||
coverage==6.3.3
|
||||
docker==4.4.0
|
||||
docutils==0.17.1
|
||||
Events==0.4
|
||||
flake8==4.0.1
|
||||
idna==3.3
|
||||
iniconfig==1.1.1
|
||||
isort==5.10.1
|
||||
jmespath==0.10.0
|
||||
jsonschema==4.5.1
|
||||
lz4==3.1.3
|
||||
mccabe==0.6.1
|
||||
mmh3==3.0.0
|
||||
multidict==6.0.2
|
||||
mypy==0.950
|
||||
mypy-extensions==0.4.3
|
||||
neo-mamba==0.10.0
|
||||
neo3crypto==0.2.1
|
||||
neo3vm==0.9.0
|
||||
neo3vm-stubs==0.9.0
|
||||
netaddr==0.8.0
|
||||
orjson==3.6.8
|
||||
packaging==21.3
|
||||
pexpect==4.8.0
|
||||
pluggy==1.0.0
|
||||
ptyprocess==0.7.0
|
||||
py==1.11.0
|
||||
pybiginteger==1.2.6
|
||||
pybiginteger-stubs==1.2.6
|
||||
pycares==4.1.2
|
||||
pycodestyle==2.8.0
|
||||
pycparser==2.21
|
||||
pycryptodome==3.11.0
|
||||
pyflakes==2.4.0
|
||||
pyparsing==3.0.9
|
||||
pyrsistent==0.18.1
|
||||
pytest==7.1.2
|
||||
python-dateutil==2.8.2
|
||||
requests==2.27.1
|
||||
robotframework==4.1.2
|
||||
s3transfer==0.3.7
|
||||
six==1.16.0
|
||||
tomli==2.0.1
|
||||
typing-extensions==4.2.0
|
||||
urllib3==1.26.9
|
||||
websocket-client==1.3.2
|
||||
yarl==1.7.2
|
||||
0
pytest_tests/testsuites/__init__.py
Normal file
0
pytest_tests/testsuites/__init__.py
Normal file
148
pytest_tests/testsuites/conftest.py
Normal file
148
pytest_tests/testsuites/conftest.py
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from re import search
|
||||
from time import sleep
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
import rpc_client
|
||||
import wallet
|
||||
from cli_helpers import _cmd_run
|
||||
from common import (ASSETS_DIR, COMMON_PLACEMENT_RULE, COMPLEX_OBJ_SIZE,
|
||||
MAINNET_WALLET_WIF, NEO_MAINNET_ENDPOINT, SIMPLE_OBJ_SIZE)
|
||||
from python_keywords.container import create_container
|
||||
from python_keywords.payment_neogo import get_balance
|
||||
from python_keywords.utility_keywords import generate_file_and_file_hash
|
||||
from robot.api import deco
|
||||
from wallet_keywords import neofs_deposit, transfer_mainnet_gas
|
||||
from wellknown_acl import PUBLIC_ACL
|
||||
|
||||
deco.keyword = allure.step
|
||||
|
||||
|
||||
logger = logging.getLogger('NeoLogger')
|
||||
|
||||
|
||||
@pytest.fixture(scope='session', autouse=True)
|
||||
@allure.title('Check binary versions')
|
||||
def check_binary_versions(request):
|
||||
environment_dir = request.config.getoption('--alluredir')
|
||||
binaries = ['neo-go', 'neofs-cli', 'neofs-authmate', 'aws']
|
||||
env_out = {}
|
||||
for binary in binaries:
|
||||
out = _cmd_run(f'{binary} --version')
|
||||
version = search(r'(v?\d.*)\s+', out)
|
||||
version = version.group(1) if version else 'Unknown'
|
||||
env_out[binary.upper()] = version
|
||||
|
||||
if environment_dir:
|
||||
with open(f'{environment_dir}/environment.properties', 'w') as out_file:
|
||||
for env, env_value in env_out.items():
|
||||
out_file.write(f'{env}={env_value}\n')
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
@allure.title('Init wallet with address')
|
||||
def init_wallet_with_address():
|
||||
full_path = f'{os.getcwd()}/{ASSETS_DIR}'
|
||||
os.mkdir(full_path)
|
||||
|
||||
yield wallet.init_wallet_w_addr(ASSETS_DIR)
|
||||
|
||||
shutil.rmtree(full_path)
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
@allure.title('Prepare wallet and deposit')
|
||||
def prepare_wallet_and_deposit(init_wallet_with_address):
|
||||
deposit = 30
|
||||
wallet, addr, wif = init_wallet_with_address
|
||||
logger.info(f'Init wallet: {wallet},\naddr: {addr},\nwif: {wif}')
|
||||
|
||||
txid = transfer_mainnet_gas(MAINNET_WALLET_WIF, addr, deposit + 1)
|
||||
wait_unitl_transaction_accepted_in_block(txid)
|
||||
deposit_tx = neofs_deposit(wif, deposit)
|
||||
wait_unitl_transaction_accepted_in_block(deposit_tx)
|
||||
|
||||
return wallet, wif
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
@allure.title('Create Container')
|
||||
def prepare_container(prepare_wallet_and_deposit):
|
||||
wallet, wif = prepare_wallet_and_deposit
|
||||
return prepare_container_impl(wallet, wif)
|
||||
|
||||
|
||||
@pytest.fixture(scope='module')
|
||||
@allure.title('Create Public Container')
|
||||
def prepare_public_container(prepare_wallet_and_deposit):
|
||||
placement_rule = 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X'
|
||||
wallet, wif = prepare_wallet_and_deposit
|
||||
return prepare_container_impl(wallet, wif, rule=placement_rule, basic_acl=PUBLIC_ACL)
|
||||
|
||||
|
||||
def prepare_container_impl(wallet: str, wif: str, rule=COMMON_PLACEMENT_RULE, basic_acl: str = ''):
|
||||
balance = get_balance(wif)
|
||||
assert balance > 0, f'Expected balance is greater than 0. Got {balance}'
|
||||
|
||||
cid = create_container(wallet, rule=rule, basic_acl=basic_acl)
|
||||
|
||||
new_balance = get_balance(wif)
|
||||
assert new_balance < balance, 'Expected some fee has charged'
|
||||
|
||||
return cid, wallet
|
||||
|
||||
|
||||
@allure.step('Wait until transaction accepted in block')
|
||||
def wait_unitl_transaction_accepted_in_block(tx_id: str):
|
||||
"""
|
||||
This function return True in case of accepted TX.
|
||||
Parameters:
|
||||
:param tx_id: transaction ID
|
||||
"""
|
||||
mainnet_rpc_cli = rpc_client.RPCClient(NEO_MAINNET_ENDPOINT)
|
||||
|
||||
if isinstance(tx_id, bytes):
|
||||
tx_id = tx_id.decode()
|
||||
|
||||
sleep_interval, attempts = 5, 10
|
||||
|
||||
for __attempt in range(attempts):
|
||||
try:
|
||||
resp = mainnet_rpc_cli.get_transaction_height(tx_id)
|
||||
if resp is not None:
|
||||
logger.info(f"got block height: {resp}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.info(f"request failed with error: {e}")
|
||||
raise e
|
||||
sleep(sleep_interval)
|
||||
raise TimeoutError(f'Timeout {sleep_interval * attempts} sec. reached on waiting for transaction accepted')
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
@allure.title('Generate files')
|
||||
def generate_files():
|
||||
file_name_simple, _ = generate_file_and_file_hash(SIMPLE_OBJ_SIZE)
|
||||
large_file_name, _ = generate_file_and_file_hash(COMPLEX_OBJ_SIZE)
|
||||
|
||||
return file_name_simple, large_file_name
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
@allure.title('Generate file')
|
||||
def generate_file():
|
||||
file_name_simple, _ = generate_file_and_file_hash(SIMPLE_OBJ_SIZE)
|
||||
|
||||
return file_name_simple
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
@allure.title('Generate large file')
|
||||
def generate_large_file():
|
||||
obj_size = int(os.getenv('BIG_OBJ_SIZE', COMPLEX_OBJ_SIZE))
|
||||
file_path, file_hash = generate_file_and_file_hash(obj_size)
|
||||
|
||||
return file_path, file_hash
|
||||
93
pytest_tests/testsuites/object/test_object_api.py
Normal file
93
pytest_tests/testsuites/object/test_object_api.py
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import logging
|
||||
from time import sleep
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
from contract_keywords import tick_epoch
|
||||
from python_keywords.neofs import verify_head_tombstone
|
||||
from python_keywords.neofs_verbs import (delete_object, get_object, get_range,
|
||||
get_range_hash, head_object,
|
||||
put_object, search_object)
|
||||
from python_keywords.storage_policy import get_simple_object_copies
|
||||
from python_keywords.utility_keywords import get_file_hash
|
||||
|
||||
logger = logging.getLogger('NeoLogger')
|
||||
|
||||
CLEANUP_TIMEOUT = 10
|
||||
|
||||
|
||||
@allure.title('Test native object API')
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.grpc_api
|
||||
def test_object_api(prepare_container, generate_file):
|
||||
cid, wallet = prepare_container
|
||||
wallet_cid = {'wallet': wallet, 'cid': cid}
|
||||
file_usr_header = {'key1': 1, 'key2': 'abc'}
|
||||
file_usr_header_oth = {'key1': 2}
|
||||
range_cut = '0:10'
|
||||
oids = []
|
||||
|
||||
file_name = generate_file
|
||||
file_hash = get_file_hash(file_name)
|
||||
|
||||
search_object(**wallet_cid, expected_objects_list=oids)
|
||||
|
||||
with allure.step('Put objects'):
|
||||
oids.append(put_object(wallet=wallet, path=file_name, cid=cid))
|
||||
oids.append(put_object(wallet=wallet, path=file_name, cid=cid, user_headers=file_usr_header))
|
||||
oids.append(put_object(wallet=wallet, path=file_name, cid=cid, user_headers=file_usr_header_oth))
|
||||
|
||||
with allure.step('Validate storage policy for objects'):
|
||||
for oid_to_check in oids:
|
||||
assert get_simple_object_copies(wallet=wallet, cid=cid, oid=oid_to_check) == 2, 'Expected 2 copies'
|
||||
|
||||
with allure.step('Get objects and compare hashes'):
|
||||
for oid_to_check in oids:
|
||||
got_file_path = get_object(wallet=wallet, cid=cid, oid=oid_to_check)
|
||||
got_file_hash = get_file_hash(got_file_path)
|
||||
assert file_hash == got_file_hash
|
||||
|
||||
with allure.step('Get range/range hash'):
|
||||
get_range_hash(**wallet_cid, oid=oids[0], bearer_token='', range_cut=range_cut)
|
||||
get_range_hash(**wallet_cid, oid=oids[1], bearer_token='', range_cut=range_cut)
|
||||
get_range(**wallet_cid, oid=oids[1], file_path='s_get_range', bearer='', range_cut=range_cut)
|
||||
|
||||
with allure.step('Search objects'):
|
||||
search_object(**wallet_cid, expected_objects_list=oids)
|
||||
search_object(**wallet_cid, filters=file_usr_header, expected_objects_list=oids[1:2])
|
||||
search_object(**wallet_cid, filters=file_usr_header_oth, expected_objects_list=oids[2:3])
|
||||
|
||||
with allure.step('Head object and validate'):
|
||||
head_object(**wallet_cid, oid=oids[0])
|
||||
head_info = head_object(**wallet_cid, oid=oids[1])
|
||||
check_header_is_presented(head_info, file_usr_header)
|
||||
|
||||
with allure.step('Delete objects'):
|
||||
tombstone_s = delete_object(**wallet_cid, oid=oids[0])
|
||||
tombstone_h = delete_object(**wallet_cid, oid=oids[1])
|
||||
|
||||
verify_head_tombstone(wallet_path=wallet, cid=cid, oid_ts=tombstone_s, oid=oids[0])
|
||||
verify_head_tombstone(wallet_path=wallet, cid=cid, oid_ts=tombstone_h, oid=oids[1])
|
||||
|
||||
tick_epoch()
|
||||
sleep(CLEANUP_TIMEOUT)
|
||||
|
||||
with allure.step('Get objects and check errors'):
|
||||
get_object_and_check_error(**wallet_cid, oid=oids[0], err_msg='object already removed')
|
||||
get_object_and_check_error(**wallet_cid, oid=oids[1], err_msg='object already removed')
|
||||
|
||||
|
||||
def get_object_and_check_error(wallet: str, cid: str, oid: str, err_msg: str):
|
||||
try:
|
||||
get_object(wallet=wallet, cid=cid, oid=oid)
|
||||
raise AssertionError(f'Expected object {oid} removed, but it is not')
|
||||
except Exception as err:
|
||||
logger.info(f'Error is {err}')
|
||||
assert err_msg in str(err), f'Expected message {err_msg} in error: {err}'
|
||||
|
||||
|
||||
def check_header_is_presented(head_info: dict, object_header: dict):
|
||||
for key_to_check, val_to_check in object_header.items():
|
||||
assert key_to_check in head_info['header']['attributes'], f'Key {key_to_check} is found in {head_object}'
|
||||
assert head_info['header']['attributes'].get(key_to_check) == str(
|
||||
val_to_check), f'Value {val_to_check} is equal'
|
||||
235
pytest_tests/testsuites/services/test_http_gate.py
Normal file
235
pytest_tests/testsuites/services/test_http_gate.py
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
import logging
|
||||
from random import choice
|
||||
from time import sleep
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
from contract_keywords import get_epoch, tick_epoch
|
||||
from python_keywords.http_gate import (get_via_http_curl, get_via_http_gate,
|
||||
get_via_http_gate_by_attribute,
|
||||
get_via_zip_http_gate,
|
||||
upload_via_http_gate,
|
||||
upload_via_http_gate_curl)
|
||||
from python_keywords.neofs_verbs import get_object, put_object
|
||||
from python_keywords.storage_policy import get_nodes_without_object
|
||||
from python_keywords.utility_keywords import get_file_hash
|
||||
|
||||
logger = logging.getLogger('NeoLogger')
|
||||
|
||||
CLEANUP_TIMEOUT = 10
|
||||
|
||||
|
||||
@allure.link('https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway', name='neofs-http-gateway')
|
||||
@allure.link('https://github.com/nspcc-dev/neofs-http-gw#uploading', name='uploading')
|
||||
@allure.link('https://github.com/nspcc-dev/neofs-http-gw#downloading', name='downloading')
|
||||
@pytest.mark.http_gate
|
||||
class TestHttpGate:
|
||||
|
||||
@allure.title('Test Put over gRPC, Get over HTTP')
|
||||
def test_put_grpc_get_http(self, prepare_public_container, generate_files):
|
||||
"""
|
||||
Test that object can be put using gRPC interface and get using HTTP.
|
||||
|
||||
Steps:
|
||||
1. Create simple and large objects.
|
||||
2. Put objects using gRPC (neofs-cli).
|
||||
3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading).
|
||||
4. Get objects using gRPC (neofs-cli).
|
||||
5. Compare hashes for got objects.
|
||||
6. Compare hashes for got and original objects.
|
||||
|
||||
Expected result:
|
||||
Hashes must be the same.
|
||||
"""
|
||||
cid, wallet = prepare_public_container
|
||||
file_name_simple, large_file_name = generate_files
|
||||
|
||||
with allure.step('Put objects using gRPC'):
|
||||
oid_simple = put_object(wallet=wallet, path=file_name_simple, cid=cid)
|
||||
oid_large = put_object(wallet=wallet, path=large_file_name, cid=cid)
|
||||
|
||||
for oid, file_name in ((oid_simple, file_name_simple), (oid_large, large_file_name)):
|
||||
self.get_object_and_verify_hashes(oid, file_name, wallet, cid)
|
||||
|
||||
@allure.link('https://github.com/nspcc-dev/neofs-http-gw#uploading', name='uploading')
|
||||
@allure.link('https://github.com/nspcc-dev/neofs-http-gw#downloading', name='downloading')
|
||||
@pytest.mark.sanity
|
||||
@allure.title('Test Put over HTTP, Get over HTTP')
|
||||
def test_put_http_get_http(self, prepare_public_container, generate_files):
|
||||
"""
|
||||
Test that object can be put and get using HTTP interface.
|
||||
|
||||
Steps:
|
||||
1. Create simple and large objects.
|
||||
2. Upload objects using HTTP (https://github.com/nspcc-dev/neofs-http-gw#uploading).
|
||||
3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading).
|
||||
4. Compare hashes for got and original objects.
|
||||
|
||||
Expected result:
|
||||
Hashes must be the same.
|
||||
"""
|
||||
cid, wallet = prepare_public_container
|
||||
file_name_simple, large_file_name = generate_files
|
||||
|
||||
with allure.step('Put objects using HTTP'):
|
||||
oid_simple = upload_via_http_gate(cid=cid, path=file_name_simple)
|
||||
oid_large = upload_via_http_gate(cid=cid, path=large_file_name)
|
||||
|
||||
for oid, file_name in ((oid_simple, file_name_simple), (oid_large, large_file_name)):
|
||||
self.get_object_and_verify_hashes(oid, file_name, wallet, cid)
|
||||
|
||||
@allure.link('https://github.com/nspcc-dev/neofs-http-gw#by-attributes', name='download by attributes')
|
||||
@allure.title('Test Put over HTTP, Get over HTTP with headers')
|
||||
@pytest.mark.parametrize('attributes',
|
||||
[
|
||||
{'fileName': 'simple_obj_filename'},
|
||||
{'file-Name': 'simple obj filename'},
|
||||
{'cat%jpeg': 'cat%jpeg'}
|
||||
], ids=['simple', 'hyphen', 'percent']
|
||||
)
|
||||
def test_put_http_get_http_with_headers(self, prepare_public_container, generate_files, attributes):
|
||||
"""
|
||||
Test that object can be downloaded using different attributes in HTTP header.
|
||||
|
||||
Steps:
|
||||
1. Create simple and large objects.
|
||||
2. Upload objects using HTTP with particular attributes in the header.
|
||||
3. Download objects by attributes using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#by-attributes).
|
||||
4. Compare hashes for got and original objects.
|
||||
|
||||
Expected result:
|
||||
Hashes must be the same.
|
||||
"""
|
||||
cid, wallet = prepare_public_container
|
||||
file_name_simple, _ = generate_files
|
||||
|
||||
with allure.step('Put objects using HTTP with attribute'):
|
||||
oid_simple = upload_via_http_gate(cid=cid, path=file_name_simple,
|
||||
headers=self._attr_into_header(attributes))
|
||||
|
||||
self.get_object_by_attr_and_verify_hashes(oid_simple, file_name_simple, cid, attributes)
|
||||
|
||||
@allure.title('Test Expiration-Epoch in HTTP header')
|
||||
def test_expiration_epoch_in_http(self, prepare_public_container, generate_file):
|
||||
cid, wallet = prepare_public_container
|
||||
file_name_simple = generate_file
|
||||
object_not_found_err = 'object not found'
|
||||
oids = []
|
||||
|
||||
curr_epoch = get_epoch()
|
||||
epochs = (curr_epoch, curr_epoch + 1, curr_epoch + 2, curr_epoch + 100)
|
||||
|
||||
for epoch in epochs:
|
||||
headers = {'X-Attribute-Neofs-Expiration-Epoch': str(epoch)}
|
||||
|
||||
with allure.step('Put objects using HTTP with attribute Expiration-Epoch'):
|
||||
oids.append(upload_via_http_gate(cid=cid, path=file_name_simple, headers=headers))
|
||||
|
||||
assert len(oids) == len(epochs), 'Expected all objects has been put successfully'
|
||||
|
||||
with allure.step('All objects can be get'):
|
||||
for oid in oids:
|
||||
get_via_http_gate(cid=cid, oid=oid)
|
||||
|
||||
for expired_objects, not_expired_objects in [(oids[:1], oids[1:]), (oids[:2], oids[2:])]:
|
||||
tick_epoch()
|
||||
sleep(CLEANUP_TIMEOUT)
|
||||
|
||||
for oid in expired_objects:
|
||||
self.try_to_get_object_and_expect_error(cid=cid, oid=oid, expected_err=object_not_found_err)
|
||||
|
||||
with allure.step('Other objects can be get'):
|
||||
for oid in not_expired_objects:
|
||||
get_via_http_gate(cid=cid, oid=oid)
|
||||
|
||||
@allure.title('Test Zip in HTTP header')
|
||||
def test_zip_in_http(self, prepare_public_container, generate_files):
|
||||
cid, wallet = prepare_public_container
|
||||
file_name_simple, file_name_complex = generate_files
|
||||
common_prefix = 'my_files'
|
||||
|
||||
headers1 = {'X-Attribute-FilePath': f'{common_prefix}/file1'}
|
||||
headers2 = {'X-Attribute-FilePath': f'{common_prefix}/file2'}
|
||||
|
||||
upload_via_http_gate(cid=cid, path=file_name_simple, headers=headers1)
|
||||
upload_via_http_gate(cid=cid, path=file_name_complex, headers=headers2)
|
||||
|
||||
dir_path = get_via_zip_http_gate(cid=cid, prefix=common_prefix)
|
||||
|
||||
with allure.step('Verify hashes'):
|
||||
assert get_file_hash(f'{dir_path}/file1') == get_file_hash(file_name_simple)
|
||||
assert get_file_hash(f'{dir_path}/file2') == get_file_hash(file_name_complex)
|
||||
|
||||
@pytest.mark.curl
|
||||
@pytest.mark.long
|
||||
@allure.title('Test Put over HTTP/Curl, Get over HTTP/Curl for large object')
|
||||
def test_put_http_get_http_large_file(self, prepare_public_container, generate_large_file):
|
||||
"""
|
||||
This test checks upload and download using curl with 'large' object. Large is object with size up to 20Mb.
|
||||
"""
|
||||
cid, wallet = prepare_public_container
|
||||
file_path, file_hash = generate_large_file
|
||||
|
||||
with allure.step('Put objects using HTTP'):
|
||||
oid_simple = upload_via_http_gate(cid=cid, path=file_path)
|
||||
oid_curl = upload_via_http_gate_curl(cid=cid, filepath=file_path, large_object=True)
|
||||
|
||||
self.get_object_and_verify_hashes(oid_simple, file_path, wallet, cid)
|
||||
self.get_object_and_verify_hashes(oid_curl, file_path, wallet, cid, object_getter=get_via_http_curl)
|
||||
|
||||
@pytest.mark.curl
|
||||
@allure.title('Test Put/Get over HTTP using Curl utility')
|
||||
def test_put_http_get_http_curl(self, prepare_public_container, generate_files):
|
||||
"""
|
||||
Test checks upload and download over HTTP using curl utility.
|
||||
"""
|
||||
cid, wallet = prepare_public_container
|
||||
file_name_simple, large_file_name = generate_files
|
||||
|
||||
with allure.step('Put objects using curl utility'):
|
||||
oid_simple = upload_via_http_gate_curl(cid=cid, filepath=file_name_simple)
|
||||
oid_large = upload_via_http_gate_curl(cid=cid, filepath=large_file_name)
|
||||
|
||||
for oid, file_name in ((oid_simple, file_name_simple), (oid_large, large_file_name)):
|
||||
self.get_object_and_verify_hashes(oid, file_name, wallet, cid, object_getter=get_via_http_curl)
|
||||
|
||||
@staticmethod
|
||||
@allure.step('Try to get object and expect error')
|
||||
def try_to_get_object_and_expect_error(cid: str, oid: str, expected_err: str):
|
||||
try:
|
||||
get_via_http_gate(cid=cid, oid=oid)
|
||||
raise AssertionError(f'Expected error on getting object with cid: {cid}')
|
||||
except Exception as err:
|
||||
assert expected_err in str(err), f'Expected error {expected_err} in {err}'
|
||||
|
||||
@staticmethod
|
||||
@allure.step('Verify object can be get using HTTP header attribute')
|
||||
def get_object_by_attr_and_verify_hashes(oid: str, file_name: str, cid: str, attrs: dict):
|
||||
|
||||
got_file_path_http = get_via_http_gate(cid=cid, oid=oid)
|
||||
got_file_path_http_attr = get_via_http_gate_by_attribute(cid=cid, attribute=attrs)
|
||||
|
||||
TestHttpGate._assert_hashes_the_same(file_name, got_file_path_http, got_file_path_http_attr)
|
||||
|
||||
@staticmethod
|
||||
@allure.step('Verify object can be get using HTTP')
|
||||
def get_object_and_verify_hashes(oid: str, file_name: str, wallet: str, cid: str, object_getter=None):
|
||||
nodes = get_nodes_without_object(wallet=wallet, cid=cid, oid=oid)
|
||||
random_node = choice(nodes)
|
||||
object_getter = object_getter or get_via_http_gate
|
||||
|
||||
got_file_path = get_object(wallet=wallet, cid=cid, oid=oid, endpoint=random_node)
|
||||
got_file_path_http = object_getter(cid=cid, oid=oid)
|
||||
|
||||
TestHttpGate._assert_hashes_the_same(file_name, got_file_path, got_file_path_http)
|
||||
|
||||
@staticmethod
|
||||
def _assert_hashes_the_same(orig_file_name: str, got_file_1: str, got_file_2: str):
|
||||
msg = 'Expected hashes are equal for files {f1} and {f2}'
|
||||
got_file_hash_http = get_file_hash(got_file_1)
|
||||
assert get_file_hash(got_file_2) == got_file_hash_http, msg.format(f1=got_file_2, f2=got_file_1)
|
||||
assert get_file_hash(orig_file_name) == got_file_hash_http, msg.format(f1=orig_file_name, f2=got_file_1)
|
||||
|
||||
@staticmethod
|
||||
def _attr_into_header(attrs: dict) -> dict:
|
||||
return {f'X-Attribute-{_key}': _value for _key, _value in attrs.items()}
|
||||
525
pytest_tests/testsuites/services/test_s3_gate.py
Normal file
525
pytest_tests/testsuites/services/test_s3_gate.py
Normal file
|
|
@ -0,0 +1,525 @@
|
|||
import logging
|
||||
import os
|
||||
from random import choice, choices
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
|
||||
from contract_keywords import tick_epoch
|
||||
from python_keywords import s3_gate_bucket, s3_gate_object
|
||||
from python_keywords.aws_cli_client import AwsCliClient
|
||||
from python_keywords.container import list_containers
|
||||
from python_keywords.utility_keywords import (generate_file_and_file_hash,
|
||||
get_file_hash)
|
||||
from utility import create_file_with_content, get_file_content, split_file
|
||||
|
||||
logger = logging.getLogger('NeoLogger')
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize("s3_client", ['aws cli', 'boto3'], indirect=True)
|
||||
|
||||
|
||||
@allure.link('https://github.com/nspcc-dev/neofs-s3-gw#neofs-s3-gateway', name='neofs-s3-gateway')
|
||||
@pytest.mark.s3_gate
|
||||
class TestS3Gate:
|
||||
s3_client = None
|
||||
|
||||
@pytest.fixture(scope='class', autouse=True)
|
||||
@allure.title('[Class/Autouse]: Create S3 client')
|
||||
def s3_client(self, prepare_wallet_and_deposit, request):
|
||||
wallet, wif = prepare_wallet_and_deposit
|
||||
s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json"
|
||||
|
||||
cid, bucket, access_key_id, secret_access_key, owner_private_key = \
|
||||
s3_gate_bucket.init_s3_credentials(wallet, s3_bearer_rules_file=s3_bearer_rules_file)
|
||||
containers_list = list_containers(wallet)
|
||||
assert cid in containers_list, f'Expected cid {cid} in {containers_list}'
|
||||
|
||||
if request.param == 'aws cli':
|
||||
try:
|
||||
client = AwsCliClient(access_key_id, secret_access_key)
|
||||
except Exception as err:
|
||||
if 'command was not found or was not executable' in str(err):
|
||||
pytest.skip('AWS CLI was not found')
|
||||
else:
|
||||
raise RuntimeError('Error on creating instance for AwsCliClient') from err
|
||||
else:
|
||||
client = s3_gate_bucket.config_s3_client(access_key_id, secret_access_key)
|
||||
TestS3Gate.s3_client = client
|
||||
|
||||
@pytest.fixture
|
||||
@allure.title('Create two buckets')
|
||||
def create_buckets(self):
|
||||
bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
return bucket_1, bucket_2
|
||||
|
||||
@pytest.fixture
|
||||
@allure.title('Create/delete bucket')
|
||||
def bucket(self):
|
||||
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
yield bucket
|
||||
objects = s3_gate_object.list_objects_s3(self.s3_client, bucket)
|
||||
if objects:
|
||||
s3_gate_object.delete_objects_s3(self.s3_client, bucket, objects)
|
||||
s3_gate_bucket.delete_bucket_s3(self.s3_client, bucket)
|
||||
|
||||
@allure.title('Test S3 Bucket API')
|
||||
def test_s3_buckets(self, generate_files):
|
||||
"""
|
||||
Test base S3 Bucket API (Create/List/Head/Delete).
|
||||
"""
|
||||
|
||||
file_name_simple, file_name_large = generate_files
|
||||
file_name = self.object_key_from_file_path(file_name_simple)
|
||||
|
||||
with allure.step('Create buckets'):
|
||||
bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
|
||||
with allure.step('Check buckets are presented in the system'):
|
||||
buckets = s3_gate_bucket.list_buckets_s3(self.s3_client)
|
||||
assert bucket_1 in buckets, f'Expected bucket {bucket_1} is in the list'
|
||||
assert bucket_2 in buckets, f'Expected bucket {bucket_2} is in the list'
|
||||
|
||||
with allure.step('Bucket must be empty'):
|
||||
for bucket in (bucket_1, bucket_2):
|
||||
objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket)
|
||||
assert not objects_list, f'Expected empty bucket, got {objects_list}'
|
||||
|
||||
with allure.step('Check buckets are visible with S3 head command'):
|
||||
s3_gate_bucket.head_bucket(self.s3_client, bucket_1)
|
||||
s3_gate_bucket.head_bucket(self.s3_client, bucket_2)
|
||||
|
||||
with allure.step('Check we can put/list object with S3 commands'):
|
||||
s3_gate_object.put_object_s3(self.s3_client, bucket_1, file_name_simple)
|
||||
s3_gate_object.head_object_s3(self.s3_client, bucket_1, file_name)
|
||||
|
||||
bucket_objects = s3_gate_object.list_objects_s3(self.s3_client, bucket_1)
|
||||
assert file_name in bucket_objects, \
|
||||
f'Expected file {file_name} in objects list {bucket_objects}'
|
||||
|
||||
with allure.step('Try to delete not empty bucket and get error'):
|
||||
with pytest.raises(Exception, match=r'.*The bucket you tried to delete is not empty.*'):
|
||||
s3_gate_bucket.delete_bucket_s3(self.s3_client, bucket_1)
|
||||
|
||||
s3_gate_bucket.head_bucket(self.s3_client, bucket_1)
|
||||
|
||||
with allure.step(f'Delete empty bucket {bucket_2}'):
|
||||
s3_gate_bucket.delete_bucket_s3(self.s3_client, bucket_2)
|
||||
tick_epoch()
|
||||
|
||||
with allure.step(f'Check bucket {bucket_2} deleted'):
|
||||
with pytest.raises(Exception, match=r'.*Not Found.*'):
|
||||
s3_gate_bucket.head_bucket(self.s3_client, bucket_2)
|
||||
|
||||
buckets = s3_gate_bucket.list_buckets_s3(self.s3_client)
|
||||
assert bucket_1 in buckets, f'Expected bucket {bucket_1} is in the list'
|
||||
assert bucket_2 not in buckets, f'Expected bucket {bucket_2} is not in the list'
|
||||
|
||||
@allure.title('Test S3 Object API')
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.parametrize('file_type', ['simple', 'large'], ids=['Simple object', 'Large object'])
|
||||
def test_s3_api_object(self, generate_files, file_type):
|
||||
"""
|
||||
Test base S3 Object API (Put/Head/List) for simple and large objects.
|
||||
"""
|
||||
file_name_simple, file_name_large = generate_files
|
||||
file_name_path = file_name_simple if file_type == 'simple' else file_name_large
|
||||
file_name = self.object_key_from_file_path(file_name_path)
|
||||
|
||||
bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
|
||||
for bucket in (bucket_1, bucket_2):
|
||||
with allure.step('Bucket must be empty'):
|
||||
objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket)
|
||||
assert not objects_list, f'Expected empty bucket, got {objects_list}'
|
||||
|
||||
s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_path)
|
||||
s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_large)
|
||||
s3_gate_object.head_object_s3(self.s3_client, bucket, file_name)
|
||||
|
||||
bucket_objects = s3_gate_object.list_objects_s3(self.s3_client, bucket)
|
||||
assert file_name in bucket_objects, \
|
||||
f'Expected file {file_name} in objects list {bucket_objects}'
|
||||
|
||||
with allure.step("Check object's attributes"):
|
||||
for attrs in (['ETag'], ['ObjectSize', 'StorageClass']):
|
||||
s3_gate_object.get_object_attributes(self.s3_client, bucket, file_name, *attrs)
|
||||
|
||||
@allure.title('Test S3 Sync directory')
|
||||
def test_s3_sync_dir(self, bucket):
|
||||
"""
|
||||
Test checks sync directory with AWS CLI utility.
|
||||
"""
|
||||
file_path_1 = f"{os.getcwd()}/{ASSETS_DIR}/test_sync/test_file_1"
|
||||
file_path_2 = f"{os.getcwd()}/{ASSETS_DIR}/test_sync/test_file_2"
|
||||
key_to_path = {'test_file_1': file_path_1, 'test_file_2': file_path_2}
|
||||
|
||||
if not isinstance(self.s3_client, AwsCliClient):
|
||||
pytest.skip('This test is not supported with boto3 client')
|
||||
|
||||
create_file_with_content(file_path=file_path_1)
|
||||
create_file_with_content(file_path=file_path_2)
|
||||
|
||||
self.s3_client.sync(bucket_name=bucket, dir_path=os.path.dirname(file_path_1))
|
||||
|
||||
with allure.step('Check objects are synced'):
|
||||
objects = s3_gate_object.list_objects_s3(self.s3_client, bucket)
|
||||
|
||||
with allure.step('Check these are the same objects'):
|
||||
assert set(key_to_path.keys()) == set(objects), f'Expected all abjects saved. Got {objects}'
|
||||
for obj_key in objects:
|
||||
got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, obj_key)
|
||||
assert get_file_hash(got_object) == get_file_hash(key_to_path.get(obj_key)), \
|
||||
'Expected hashes are the same'
|
||||
|
||||
@allure.title('Test S3 Object versioning')
|
||||
def test_s3_api_versioning(self, bucket):
|
||||
"""
|
||||
Test checks basic versioning functionality for S3 bucket.
|
||||
"""
|
||||
version_1_content = 'Version 1'
|
||||
version_2_content = 'Version 2'
|
||||
file_name_simple = create_file_with_content(content=version_1_content)
|
||||
obj_key = os.path.basename(file_name_simple)
|
||||
|
||||
with allure.step('Set versioning enable for bucket'):
|
||||
s3_gate_bucket.get_bucket_versioning_status(self.s3_client, bucket)
|
||||
|
||||
s3_gate_bucket.set_bucket_versioning(self.s3_client, bucket, status=s3_gate_bucket.VersioningStatus.ENABLED)
|
||||
status = s3_gate_bucket.get_bucket_versioning_status(self.s3_client, bucket)
|
||||
assert status == s3_gate_bucket.VersioningStatus.ENABLED.value, f'Expected enabled status. Got {status}'
|
||||
|
||||
with allure.step('Put several versions of object into bucket'):
|
||||
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
|
||||
create_file_with_content(file_path=file_name_simple, content=version_2_content)
|
||||
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
|
||||
|
||||
with allure.step('Check bucket shows all versions'):
|
||||
versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket)
|
||||
obj_versions = {version.get('VersionId') for version in versions if version.get('Key') == obj_key}
|
||||
assert obj_versions == {version_id_1, version_id_2}, \
|
||||
f'Expected object has versions: {version_id_1, version_id_2}'
|
||||
|
||||
with allure.step('Show information about particular version'):
|
||||
for version_id in (version_id_1, version_id_2):
|
||||
response = s3_gate_object.head_object_s3(self.s3_client, bucket, obj_key, version_id=version_id)
|
||||
assert 'LastModified' in response, 'Expected LastModified field'
|
||||
assert 'ETag' in response, 'Expected ETag field'
|
||||
assert response.get('VersionId') == version_id, f'Expected VersionId is {version_id}'
|
||||
assert response.get('ContentLength') != 0, 'Expected ContentLength is not zero'
|
||||
|
||||
with allure.step("Check object's attributes"):
|
||||
for version_id in (version_id_1, version_id_2):
|
||||
got_attrs = s3_gate_object.get_object_attributes(self.s3_client, bucket, obj_key, 'ETag',
|
||||
version_id=version_id)
|
||||
if got_attrs:
|
||||
assert got_attrs.get('VersionId') == version_id, f'Expected VersionId is {version_id}'
|
||||
|
||||
with allure.step('Delete object and check it was deleted'):
|
||||
response = s3_gate_object.delete_object_s3(self.s3_client, bucket, obj_key)
|
||||
version_id_delete = response.get('VersionId')
|
||||
|
||||
with pytest.raises(Exception, match=r'.*Not Found.*'):
|
||||
s3_gate_object.head_object_s3(self.s3_client, bucket, obj_key)
|
||||
|
||||
with allure.step('Get content for all versions and check it is correct'):
|
||||
for version, content in ((version_id_2, version_2_content), (version_id_1, version_1_content)):
|
||||
file_name = s3_gate_object.get_object_s3(self.s3_client, bucket, obj_key, version_id=version)
|
||||
got_content = get_file_content(file_name)
|
||||
assert got_content == content, f'Expected object content is\n{content}\nGot\n{got_content}'
|
||||
|
||||
with allure.step('Restore previous object version'):
|
||||
s3_gate_object.delete_object_s3(self.s3_client, bucket, obj_key, version_id=version_id_delete)
|
||||
|
||||
file_name = s3_gate_object.get_object_s3(self.s3_client, bucket, obj_key)
|
||||
got_content = get_file_content(file_name)
|
||||
assert got_content == version_2_content, \
|
||||
f'Expected object content is\n{version_2_content}\nGot\n{got_content}'
|
||||
|
||||
@allure.title('Test S3 Object Multipart API')
|
||||
def test_s3_api_multipart(self, bucket):
|
||||
"""
|
||||
Test checks S3 Multipart API (Create multipart upload/Abort multipart upload/List multipart upload/
|
||||
Upload part/List parts/Complete multipart upload).
|
||||
"""
|
||||
parts_count = 3
|
||||
file_name_large, _ = generate_file_and_file_hash(SIMPLE_OBJ_SIZE * 1024 * 6 * parts_count) # 5Mb - min part
|
||||
# file_name_large, _ = generate_file_and_file_hash(SIMPLE_OBJ_SIZE * 1024 * 30 * parts_count) # 5Mb - min part
|
||||
object_key = self.object_key_from_file_path(file_name_large)
|
||||
part_files = split_file(file_name_large, parts_count)
|
||||
parts = []
|
||||
|
||||
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
|
||||
assert not uploads, f'Expected there is no uploads in bucket {bucket}'
|
||||
|
||||
with allure.step('Create and abort multipart upload'):
|
||||
upload_id = s3_gate_object.create_multipart_upload_s3(self.s3_client, bucket, object_key)
|
||||
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
|
||||
assert uploads, f'Expected there one upload in bucket {bucket}'
|
||||
assert uploads[0].get('Key') == object_key, f'Expected correct key {object_key} in upload {uploads}'
|
||||
assert uploads[0].get('UploadId') == upload_id, f'Expected correct UploadId {upload_id} in upload {uploads}'
|
||||
|
||||
s3_gate_object.abort_multipart_uploads_s3(self.s3_client, bucket, object_key, upload_id)
|
||||
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
|
||||
assert not uploads, f'Expected there is no uploads in bucket {bucket}'
|
||||
|
||||
with allure.step('Create new multipart upload and upload several parts'):
|
||||
upload_id = s3_gate_object.create_multipart_upload_s3(self.s3_client, bucket, object_key)
|
||||
for part_id, file_path in enumerate(part_files, start=1):
|
||||
etag = s3_gate_object.upload_part_s3(self.s3_client, bucket, object_key, upload_id, part_id, file_path)
|
||||
parts.append((part_id, etag))
|
||||
|
||||
with allure.step('Check all parts are visible in bucket'):
|
||||
got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
|
||||
assert len(got_parts) == len(part_files), f'Expected {parts_count} parts, got\n{got_parts}'
|
||||
|
||||
s3_gate_object.complete_multipart_upload_s3(self.s3_client, bucket, object_key, upload_id, parts)
|
||||
|
||||
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
|
||||
assert not uploads, f'Expected there is no uploads in bucket {bucket}'
|
||||
|
||||
with allure.step('Check we can get whole object from bucket'):
|
||||
got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, object_key)
|
||||
assert get_file_hash(got_object) == get_file_hash(file_name_large)
|
||||
|
||||
self.check_object_attributes(bucket, object_key, parts_count)
|
||||
|
||||
@allure.title('Test S3 Bucket tagging API')
|
||||
def test_s3_api_bucket_tagging(self, bucket):
|
||||
"""
|
||||
Test checks S3 Bucket tagging API (Put tag/Get tag).
|
||||
"""
|
||||
key_value_pair = [('some-key', 'some-value'), ('some-key-2', 'some-value-2')]
|
||||
|
||||
s3_gate_bucket.put_bucket_tagging(self.s3_client, bucket, key_value_pair)
|
||||
got_tags = s3_gate_bucket.get_bucket_tagging(self.s3_client, bucket)
|
||||
|
||||
with allure.step('Check all tags are presented'):
|
||||
assert got_tags, f'Expected tags, got {got_tags}'
|
||||
expected_tags = [{'Key': key, 'Value': value} for key, value in key_value_pair]
|
||||
for tag in expected_tags:
|
||||
assert tag in got_tags
|
||||
|
||||
s3_gate_bucket.delete_bucket_tagging(self.s3_client, bucket)
|
||||
tags = s3_gate_bucket.get_bucket_tagging(self.s3_client, bucket)
|
||||
assert not tags, f'Expected there is no tags for bucket {bucket}, got {tags}'
|
||||
|
||||
@allure.title('Test S3 Object tagging API')
|
||||
def test_s3_api_object_tagging(self, bucket):
|
||||
"""
|
||||
Test checks S3 Object tagging API (Put tag/Get tag/Update tag).
|
||||
"""
|
||||
key_value_pair_bucket = [('some-key', 'some-value'), ('some-key-2', 'some-value-2')]
|
||||
key_value_pair_obj = [('some-key-obj', 'some-value-obj'), ('some-key--obj2', 'some-value--obj2')]
|
||||
key_value_pair_obj_new = [('some-key-obj-new', 'some-value-obj-new')]
|
||||
file_name_simple, _ = generate_file_and_file_hash(SIMPLE_OBJ_SIZE)
|
||||
obj_key = self.object_key_from_file_path(file_name_simple)
|
||||
|
||||
s3_gate_bucket.put_bucket_tagging(self.s3_client, bucket, key_value_pair_bucket)
|
||||
|
||||
s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
|
||||
|
||||
for tags in (key_value_pair_obj, key_value_pair_obj_new):
|
||||
s3_gate_object.put_object_tagging(self.s3_client, bucket, obj_key, tags)
|
||||
|
||||
got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, obj_key)
|
||||
assert got_tags, f'Expected tags, got {got_tags}'
|
||||
expected_tags = [{'Key': key, 'Value': value} for key, value in tags]
|
||||
for tag in expected_tags:
|
||||
assert tag in got_tags
|
||||
|
||||
s3_gate_object.delete_object_tagging(self.s3_client, bucket, obj_key)
|
||||
got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, obj_key)
|
||||
assert not got_tags, f'Expected there is no tags for bucket {bucket}, got {got_tags}'
|
||||
|
||||
@allure.title('Test S3: Delete object & delete objects S3 API')
|
||||
def test_s3_api_delete(self, create_buckets):
|
||||
"""
|
||||
Check delete_object and delete_objects S3 API operation. From first bucket some objects deleted one by one.
|
||||
From second bucket some objects deleted all at once.
|
||||
"""
|
||||
max_obj_count = 20
|
||||
max_delete_objects = 17
|
||||
put_objects = []
|
||||
file_paths = []
|
||||
obj_sizes = [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE]
|
||||
|
||||
bucket_1, bucket_2 = create_buckets
|
||||
|
||||
with allure.step(f'Generate {max_obj_count} files'):
|
||||
for _ in range(max_obj_count):
|
||||
file_paths.append(generate_file_and_file_hash(choice(obj_sizes))[0])
|
||||
|
||||
for bucket in (bucket_1, bucket_2):
|
||||
with allure.step(f'Bucket {bucket} must be empty as it just created'):
|
||||
objects_list = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket)
|
||||
assert not objects_list, f'Expected empty bucket, got {objects_list}'
|
||||
|
||||
for file_path in file_paths:
|
||||
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
|
||||
put_objects.append(self.object_key_from_file_path(file_path))
|
||||
|
||||
with allure.step(f'Check all objects put in bucket {bucket} successfully'):
|
||||
bucket_objects = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket)
|
||||
assert set(put_objects) == set(bucket_objects), \
|
||||
f'Expected all objects {put_objects} in objects list {bucket_objects}'
|
||||
|
||||
with allure.step('Delete some objects from bucket_1 one by one'):
|
||||
objects_to_delete_b1 = choices(put_objects, k=max_delete_objects)
|
||||
for obj in objects_to_delete_b1:
|
||||
s3_gate_object.delete_object_s3(self.s3_client, bucket_1, obj)
|
||||
|
||||
with allure.step('Check deleted objects are not visible in bucket bucket_1'):
|
||||
bucket_objects = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket_1)
|
||||
assert set(put_objects).difference(set(objects_to_delete_b1)) == set(bucket_objects), \
|
||||
f'Expected all objects {put_objects} in objects list {bucket_objects}'
|
||||
self.try_to_get_object_and_got_error(bucket_1, objects_to_delete_b1)
|
||||
|
||||
with allure.step('Delete some objects from bucket_2 at once'):
|
||||
objects_to_delete_b2 = choices(put_objects, k=max_delete_objects)
|
||||
s3_gate_object.delete_objects_s3(self.s3_client, bucket_2, objects_to_delete_b2)
|
||||
|
||||
with allure.step('Check deleted objects are not visible in bucket bucket_2'):
|
||||
objects_list = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket_2)
|
||||
assert set(put_objects).difference(set(objects_to_delete_b2)) == set(objects_list), \
|
||||
f'Expected all objects {put_objects} in objects list {bucket_objects}'
|
||||
self.try_to_get_object_and_got_error(bucket_2, objects_to_delete_b2)
|
||||
|
||||
@allure.title('Test S3: Copy object to the same bucket')
|
||||
def test_s3_copy_same_bucket(self, generate_files):
|
||||
"""
|
||||
Test object can be copied to the same bucket.
|
||||
"""
|
||||
file_simple, file_large = generate_files
|
||||
file_name_simple = self.object_key_from_file_path(file_simple)
|
||||
file_name_large = self.object_key_from_file_path(file_large)
|
||||
bucket_objects = [file_name_simple, file_name_large]
|
||||
|
||||
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
|
||||
with allure.step('Bucket must be empty'):
|
||||
objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket)
|
||||
assert not objects_list, f'Expected empty bucket, got {objects_list}'
|
||||
|
||||
with allure.step('Put objects into bucket'):
|
||||
for obj in (file_simple, file_large):
|
||||
s3_gate_object.put_object_s3(self.s3_client, bucket, obj)
|
||||
|
||||
with allure.step('Copy one object into the same bucket'):
|
||||
copy_obj_path = s3_gate_object.copy_object_s3(self.s3_client, bucket, file_name_simple)
|
||||
bucket_objects.append(copy_obj_path)
|
||||
|
||||
self.check_objects_in_bucket(bucket, bucket_objects)
|
||||
|
||||
with allure.step('Check copied object has the same content'):
|
||||
got_copied_file = s3_gate_object.get_object_s3(self.s3_client, bucket, copy_obj_path)
|
||||
assert get_file_hash(file_simple) == get_file_hash(got_copied_file), 'Hashes must be the same'
|
||||
|
||||
with allure.step('Delete one object from bucket'):
|
||||
s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name_simple)
|
||||
bucket_objects.remove(file_name_simple)
|
||||
|
||||
self.check_objects_in_bucket(bucket, expected_objects=bucket_objects, unexpected_objects=[file_name_simple])
|
||||
|
||||
@allure.title('Test S3: Copy object to another bucket')
|
||||
def test_s3_copy_to_another_bucket(self, generate_files):
|
||||
"""
|
||||
Test object can be copied to another bucket.
|
||||
"""
|
||||
file_simple, file_large = generate_files
|
||||
file_name_simple = self.object_key_from_file_path(file_simple)
|
||||
file_name_large = self.object_key_from_file_path(file_large)
|
||||
bucket_1_objects = [file_name_simple, file_name_large]
|
||||
|
||||
bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
|
||||
with allure.step('Buckets must be empty'):
|
||||
for bucket in (bucket_1, bucket_2):
|
||||
objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket)
|
||||
assert not objects_list, f'Expected empty bucket, got {objects_list}'
|
||||
|
||||
with allure.step('Put objects into one bucket'):
|
||||
for obj in (file_simple, file_large):
|
||||
s3_gate_object.put_object_s3(self.s3_client, bucket_1, obj)
|
||||
|
||||
with allure.step('Copy object from first bucket into second'):
|
||||
copy_obj_path_b2 = s3_gate_object.copy_object_s3(self.s3_client, bucket_1, file_name_large,
|
||||
bucket_dst=bucket_2)
|
||||
self.check_objects_in_bucket(bucket_1, expected_objects=bucket_1_objects)
|
||||
self.check_objects_in_bucket(bucket_2, expected_objects=[copy_obj_path_b2])
|
||||
|
||||
with allure.step('Check copied object has the same content'):
|
||||
got_copied_file_b2 = s3_gate_object.get_object_s3(self.s3_client, bucket_2, copy_obj_path_b2)
|
||||
assert get_file_hash(file_large) == get_file_hash(got_copied_file_b2), 'Hashes must be the same'
|
||||
|
||||
with allure.step('Delete one object from first bucket'):
|
||||
s3_gate_object.delete_object_s3(self.s3_client, bucket_1, file_name_simple)
|
||||
bucket_1_objects.remove(file_name_simple)
|
||||
|
||||
self.check_objects_in_bucket(bucket_1, expected_objects=bucket_1_objects)
|
||||
self.check_objects_in_bucket(bucket_2, expected_objects=[copy_obj_path_b2])
|
||||
|
||||
with allure.step('Delete one object from second bucket and check it is empty'):
|
||||
s3_gate_object.delete_object_s3(self.s3_client, bucket_2, copy_obj_path_b2)
|
||||
self.check_objects_in_bucket(bucket_2, expected_objects=[])
|
||||
|
||||
def check_object_attributes(self, bucket: str, object_key: str, parts_count: int):
|
||||
if not isinstance(self.s3_client, AwsCliClient):
|
||||
logger.warning('Attributes check is not supported for boto3 implementation')
|
||||
return
|
||||
|
||||
with allure.step("Check object's attributes"):
|
||||
obj_parts = s3_gate_object.get_object_attributes(self.s3_client, bucket, object_key, 'ObjectParts',
|
||||
get_full_resp=False)
|
||||
assert obj_parts.get('TotalPartsCount') == parts_count, f'Expected TotalPartsCount is {parts_count}'
|
||||
assert len(obj_parts.get('Parts')) == parts_count, f'Expected Parts cunt is {parts_count}'
|
||||
|
||||
with allure.step("Check object's attribute max-parts"):
|
||||
max_parts = 2
|
||||
obj_parts = s3_gate_object.get_object_attributes(self.s3_client, bucket, object_key, 'ObjectParts',
|
||||
max_parts=max_parts, get_full_resp=False)
|
||||
assert obj_parts.get('TotalPartsCount') == parts_count, f'Expected TotalPartsCount is {parts_count}'
|
||||
assert obj_parts.get('MaxParts') == max_parts, f'Expected MaxParts is {parts_count}'
|
||||
assert len(obj_parts.get('Parts')) == max_parts, f'Expected Parts count is {parts_count}'
|
||||
|
||||
with allure.step("Check object's attribute part-number-marker"):
|
||||
part_number_marker = 3
|
||||
obj_parts = s3_gate_object.get_object_attributes(self.s3_client, bucket, object_key, 'ObjectParts',
|
||||
part_number=part_number_marker, get_full_resp=False)
|
||||
assert obj_parts.get('TotalPartsCount') == parts_count, f'Expected TotalPartsCount is {parts_count}'
|
||||
assert obj_parts.get(
|
||||
'PartNumberMarker') == part_number_marker, f'Expected PartNumberMarker is {part_number_marker}'
|
||||
assert len(obj_parts.get('Parts')) == 1, f'Expected Parts count is {parts_count}'
|
||||
|
||||
@allure.step('Expected all objects are presented in the bucket')
|
||||
def check_objects_in_bucket(self, bucket, expected_objects: list, unexpected_objects: list = None):
|
||||
unexpected_objects = unexpected_objects or []
|
||||
bucket_objects = s3_gate_object.list_objects_s3(self.s3_client, bucket)
|
||||
assert len(bucket_objects) == len(expected_objects), f'Expected {len(expected_objects)} objects in the bucket'
|
||||
|
||||
for bucket_object in expected_objects:
|
||||
assert bucket_object in bucket_objects, \
|
||||
f'Expected object {bucket_object} in objects list {bucket_objects}'
|
||||
|
||||
for bucket_object in unexpected_objects:
|
||||
assert bucket_object not in bucket_objects, \
|
||||
f'Expected object {bucket_object} not in objects list {bucket_objects}'
|
||||
|
||||
@allure.step('Try to get object and got error')
|
||||
def try_to_get_object_and_got_error(self, bucket: str, unexpected_objects: list):
|
||||
for obj in unexpected_objects:
|
||||
try:
|
||||
s3_gate_object.get_object_s3(self.s3_client, bucket, obj)
|
||||
raise AssertionError(f'Object {obj} found in bucket {bucket}')
|
||||
except Exception as err:
|
||||
assert 'The specified key does not exist' in str(err), f'Expected error in exception {err}'
|
||||
|
||||
@staticmethod
|
||||
def object_key_from_file_path(full_path: str) -> str:
|
||||
return os.path.basename(full_path)
|
||||
1
pytest_tests/wallet_pass.yml
Normal file
1
pytest_tests/wallet_pass.yml
Normal file
|
|
@ -0,0 +1 @@
|
|||
password: ""
|
||||
219
robot/resources/lib/python_keywords/aws_cli_client.py
Normal file
219
robot/resources/lib/python_keywords/aws_cli_client.py
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
import allure
|
||||
from cli_helpers import _cmd_run, _configure_aws_cli
|
||||
from common import ASSETS_DIR, S3_GATE
|
||||
|
||||
logger = logging.getLogger('NeoLogger')
|
||||
|
||||
|
||||
class AwsCliClient:
|
||||
|
||||
def __init__(self, access_key_id: str, secret_access_key: str):
|
||||
self.access_key_id = access_key_id
|
||||
self.secret_access_key = secret_access_key
|
||||
self.config_aws_client()
|
||||
|
||||
def config_aws_client(self):
|
||||
cmd = 'aws configure'
|
||||
logger.info(f'Executing command: {cmd}')
|
||||
_configure_aws_cli(cmd, self.access_key_id, self.secret_access_key)
|
||||
|
||||
def create_bucket(self, Bucket: str):
|
||||
cmd = f'aws --no-verify-ssl s3api create-bucket --bucket {Bucket} --endpoint-url {S3_GATE}'
|
||||
_cmd_run(cmd)
|
||||
|
||||
def list_buckets(self) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api list-buckets --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def get_bucket_versioning(self, Bucket: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api get-bucket-versioning --bucket {Bucket}' \
|
||||
f' --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def put_bucket_versioning(self, Bucket: str, VersioningConfiguration: dict) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api put-bucket-versioning --bucket {Bucket} ' \
|
||||
f'--versioning-configuration Status={VersioningConfiguration.get("Status")}' \
|
||||
f' --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def list_objects(self, Bucket: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api list-objects --bucket {Bucket}' \
|
||||
f' --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def list_objects_v2(self, Bucket: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api list-objects-v2 --bucket {Bucket}' \
|
||||
f' --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def list_object_versions(self, Bucket: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api list-object-versions --bucket {Bucket}' \
|
||||
f' --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def copy_object(self, Bucket: str, CopySource: str, Key: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api copy-object --copy-source {CopySource} --bucket {Bucket} --key {Key}' \
|
||||
f' --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def head_bucket(self, Bucket: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api head-bucket --bucket {Bucket} --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def put_object(self, Body: str, Bucket: str, Key: str) -> dict:
|
||||
cmd = f' aws --no-verify-ssl s3api put-object --bucket {Bucket} --key {Key} --body {Body}' \
|
||||
f' --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def head_object(self, Bucket: str, Key: str, VersionId: str = None) -> dict:
|
||||
version = f' --version-id {VersionId}' if VersionId else ''
|
||||
cmd = f' aws --no-verify-ssl s3api head-object --bucket {Bucket} --key {Key} {version}' \
|
||||
f' --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def get_object(self, Bucket: str, Key: str, file_path: str, VersionId: str = None) -> dict:
|
||||
version = f' --version-id {VersionId}' if VersionId else ''
|
||||
cmd = f' aws --no-verify-ssl s3api get-object --bucket {Bucket} ' \
|
||||
f'--key {Key} {version} {file_path} --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def delete_objects(self, Bucket: str, Delete: dict) -> dict:
|
||||
file_path = f"{os.getcwd()}/{ASSETS_DIR}/delete.json"
|
||||
with open(file_path, 'w') as out_file:
|
||||
out_file.write(json.dumps(Delete))
|
||||
|
||||
cmd = f'aws --no-verify-ssl s3api delete-objects --bucket {Bucket} --delete file://{file_path} ' \
|
||||
f'--endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def delete_object(self, Bucket: str, Key: str, VersionId: str = None) -> dict:
|
||||
version = f' --version-id {VersionId}' if VersionId else ''
|
||||
cmd = f'aws --no-verify-ssl s3api delete-object --bucket {Bucket} --key {Key} {version}' \
|
||||
f' --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def get_object_attributes(self, bucket: str, key: str, *attributes: str, version_id: str = None,
|
||||
max_parts: int = None, part_number: int = None) -> dict:
|
||||
attrs = ','.join(attributes)
|
||||
version = f' --version-id {version_id}' if version_id else ''
|
||||
parts = f'--max-parts {max_parts}' if max_parts else ''
|
||||
part_number = f'--part-number-marker {part_number}' if part_number else ''
|
||||
cmd = f'aws --no-verify-ssl s3api get-object-attributes --bucket {bucket} --key {key} {version}' \
|
||||
f' {parts} {part_number} --object-attributes {attrs} --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def delete_bucket(self, Bucket: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api delete-bucket --bucket {Bucket} --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def get_bucket_tagging(self, Bucket: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api get-bucket-tagging --bucket {Bucket} --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def put_bucket_tagging(self, Bucket: str, Tagging: dict) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api put-bucket-tagging --bucket {Bucket} --tagging \'{json.dumps(Tagging)}\'' \
|
||||
f' --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def delete_bucket_tagging(self, Bucket: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api delete-bucket-tagging --bucket {Bucket} --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def put_object_tagging(self, Bucket: str, Key: str, Tagging: dict) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api put-object-tagging --bucket {Bucket} --key {Key}' \
|
||||
f' --tagging \'{json.dumps(Tagging)}\'' \
|
||||
f' --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def get_object_tagging(self, Bucket: str, Key: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api get-object-tagging --bucket {Bucket} --key {Key} --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def delete_object_tagging(self, Bucket: str, Key: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api delete-object-tagging --bucket {Bucket} --key {Key} --endpoint {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
@allure.step('Sync directory S3')
|
||||
def sync(self, bucket_name: str, dir_path: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3 sync {dir_path} s3://{bucket_name} --endpoint-url {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def create_multipart_upload(self, Bucket: str, Key: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api create-multipart-upload --bucket {Bucket} --key {Key}' \
|
||||
f' --endpoint-url {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def list_multipart_uploads(self, Bucket: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api list-multipart-uploads --bucket {Bucket}' \
|
||||
f' --endpoint-url {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def abort_multipart_upload(self, Bucket: str, Key: str, UploadId: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api abort-multipart-upload --bucket {Bucket} --key {Key}' \
|
||||
f' --upload-id {UploadId} --endpoint-url {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def upload_part(self, UploadId: str, Bucket: str, Key: str, PartNumber: int, Body: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api upload-part --bucket {Bucket} --key {Key} --upload-id {UploadId} ' \
|
||||
f'--part-number {PartNumber} --body {Body} --endpoint-url {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def list_parts(self, UploadId: str, Bucket: str, Key: str) -> dict:
|
||||
cmd = f'aws --no-verify-ssl s3api list-parts --bucket {Bucket} --key {Key} --upload-id {UploadId} ' \
|
||||
f' --endpoint-url {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
def complete_multipart_upload(self, Bucket: str, Key: str, UploadId: str, MultipartUpload: dict) -> dict:
|
||||
file_path = f"{os.getcwd()}/{ASSETS_DIR}/parts.json"
|
||||
with open(file_path, 'w') as out_file:
|
||||
out_file.write(json.dumps(MultipartUpload))
|
||||
|
||||
cmd = f'aws --no-verify-ssl s3api complete-multipart-upload --bucket {Bucket} --key {Key}' \
|
||||
f' --upload-id {UploadId} --multipart-upload file://{file_path}' \
|
||||
f' --endpoint-url {S3_GATE}'
|
||||
output = _cmd_run(cmd)
|
||||
return self._to_json(output)
|
||||
|
||||
@staticmethod
|
||||
def _to_json(output: str) -> dict:
|
||||
json_output = {}
|
||||
try:
|
||||
json_output = json.loads(output)
|
||||
except Exception:
|
||||
if '{' not in output and '}' not in output:
|
||||
logger.warning(f'Could not parse json from output {output}')
|
||||
return json_output
|
||||
json_output = json.loads(output[output.index('{'):])
|
||||
|
||||
return json_output
|
||||
|
|
@ -4,9 +4,15 @@
|
|||
Helper functions to use with `neofs-cli`, `neo-go`
|
||||
and other CLIs.
|
||||
"""
|
||||
|
||||
from typing import Union
|
||||
import subprocess
|
||||
import sys
|
||||
from contextlib import suppress
|
||||
from datetime import datetime
|
||||
from json import dumps
|
||||
from textwrap import shorten
|
||||
|
||||
import allure
|
||||
import pexpect
|
||||
from robot.api import logger
|
||||
|
||||
|
|
@ -20,11 +26,16 @@ def _cmd_run(cmd, timeout=30):
|
|||
"""
|
||||
try:
|
||||
logger.info(f"Executing command: {cmd}")
|
||||
start_time = datetime.now()
|
||||
compl_proc = subprocess.run(cmd, check=True, universal_newlines=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout,
|
||||
shell=True)
|
||||
output = compl_proc.stdout
|
||||
return_code = compl_proc.returncode
|
||||
end_time = datetime.now()
|
||||
logger.info(f"Output: {output}")
|
||||
_attach_allure_log(cmd, output, return_code, start_time, end_time)
|
||||
|
||||
return output
|
||||
except subprocess.CalledProcessError as exc:
|
||||
raise RuntimeError(f"Error:\nreturn code: {exc.returncode} "
|
||||
|
|
@ -38,8 +49,62 @@ def _cmd_run(cmd, timeout=30):
|
|||
|
||||
def _run_with_passwd(cmd):
|
||||
child = pexpect.spawn(cmd)
|
||||
child.delaybeforesend = 1
|
||||
child.expect(".*")
|
||||
child.sendline('\r')
|
||||
if sys.platform == "darwin":
|
||||
child.expect(pexpect.EOF)
|
||||
cmd = child.before
|
||||
else:
|
||||
child.wait()
|
||||
cmd = child.read()
|
||||
return cmd.decode()
|
||||
|
||||
|
||||
def _configure_aws_cli(cmd, key_id, access_key, out_format='json'):
|
||||
child = pexpect.spawn(cmd)
|
||||
child.delaybeforesend = 1
|
||||
|
||||
child.expect("AWS Access Key ID.*")
|
||||
child.sendline(key_id)
|
||||
|
||||
child.expect("AWS Secret Access Key.*")
|
||||
child.sendline(access_key)
|
||||
|
||||
child.expect("Default region name.*")
|
||||
child.sendline('')
|
||||
|
||||
child.expect("Default output format.*")
|
||||
child.sendline(out_format)
|
||||
|
||||
child.wait()
|
||||
cmd = child.read()
|
||||
# child.expect(pexpect.EOF)
|
||||
# cmd = child.before
|
||||
return cmd.decode()
|
||||
|
||||
|
||||
def _attach_allure_log(cmd: str, output: str, return_code: int, start_time: datetime, end_time: datetime):
|
||||
if 'allure' in sys.modules:
|
||||
command_attachment = (
|
||||
f"COMMAND: '{cmd}'\n"
|
||||
f'OUTPUT:\n {output}\n'
|
||||
f'RC: {return_code}\n'
|
||||
f'Start / End / Elapsed\t {start_time.time()} / {end_time.time()} / {end_time - start_time}'
|
||||
)
|
||||
with allure.step(f'COMMAND: {shorten(cmd, width=60, placeholder="...")}'):
|
||||
allure.attach(command_attachment, 'Command execution', allure.attachment_type.TEXT)
|
||||
|
||||
|
||||
def log_command_execution(cmd: str, output: Union[str, dict]):
|
||||
logger.info(f'{cmd}: {output}')
|
||||
if 'allure' in sys.modules:
|
||||
with suppress(Exception):
|
||||
json_output = dumps(output, indent=4, sort_keys=True)
|
||||
output = json_output
|
||||
command_attachment = (
|
||||
f"COMMAND: '{cmd}'\n"
|
||||
f'OUTPUT:\n {output}\n'
|
||||
)
|
||||
with allure.step(f'COMMAND: {shorten(cmd, width=60, placeholder="...")}'):
|
||||
allure.attach(command_attachment, 'Command execution', allure.attachment_type.TEXT)
|
||||
|
|
|
|||
|
|
@ -1,16 +1,28 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import uuid
|
||||
import zipfile
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import allure
|
||||
import requests
|
||||
from cli_helpers import _cmd_run
|
||||
from common import HTTP_GATE
|
||||
from robot.api import logger
|
||||
from robot.api.deco import keyword
|
||||
from robot.libraries.BuiltIn import BuiltIn
|
||||
|
||||
from common import HTTP_GATE
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
ASSETS_DIR = BuiltIn().get_variable_value("${ASSETS_DIR}")
|
||||
|
||||
if "pytest" in sys.modules:
|
||||
import os
|
||||
|
||||
ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/")
|
||||
else:
|
||||
ASSETS_DIR = BuiltIn().get_variable_value("${ASSETS_DIR}")
|
||||
|
||||
|
||||
@keyword('Get via HTTP Gate')
|
||||
|
|
@ -30,7 +42,141 @@ def get_via_http_gate(cid: str, oid: str):
|
|||
status code: {resp.status_code} {resp.reason}""")
|
||||
|
||||
logger.info(f'Request: {request}')
|
||||
_attach_allure_step(request, resp.status_code)
|
||||
|
||||
filename = f"{ASSETS_DIR}/{cid}_{oid}"
|
||||
with open(filename, "wb") as get_file:
|
||||
shutil.copyfileobj(resp.raw, get_file)
|
||||
return filename
|
||||
|
||||
|
||||
@keyword('Get via Zip HTTP Gate')
|
||||
def get_via_zip_http_gate(cid: str, prefix: str):
|
||||
"""
|
||||
This function gets given object from HTTP gate
|
||||
:param cid: CID to get object from
|
||||
:param prefix: common prefix
|
||||
"""
|
||||
request = f'{HTTP_GATE}/zip/{cid}/{prefix}'
|
||||
resp = requests.get(request, stream=True)
|
||||
|
||||
if not resp.ok:
|
||||
raise Exception(f"""Failed to get object via HTTP gate:
|
||||
request: {resp.request.path_url},
|
||||
response: {resp.text},
|
||||
status code: {resp.status_code} {resp.reason}""")
|
||||
|
||||
logger.info(f'Request: {request}')
|
||||
_attach_allure_step(request, resp.status_code)
|
||||
|
||||
filename = f'{ASSETS_DIR}/{cid}_archive.zip'
|
||||
with open(filename, 'wb') as get_file:
|
||||
shutil.copyfileobj(resp.raw, get_file)
|
||||
|
||||
with zipfile.ZipFile(filename, 'r') as zip_ref:
|
||||
zip_ref.extractall(ASSETS_DIR)
|
||||
|
||||
return f'{ASSETS_DIR}/{prefix}'
|
||||
|
||||
|
||||
@keyword('Get via HTTP Gate by attribute')
|
||||
def get_via_http_gate_by_attribute(cid: str, attribute: dict):
|
||||
"""
|
||||
This function gets given object from HTTP gate
|
||||
:param cid: CID to get object from
|
||||
:param attribute: attribute name: attribute value pair
|
||||
"""
|
||||
attr_name = list(attribute.keys())[0]
|
||||
attr_value = quote_plus(str(attribute.get(attr_name)))
|
||||
request = f'{HTTP_GATE}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}'
|
||||
resp = requests.get(request, stream=True)
|
||||
|
||||
if not resp.ok:
|
||||
raise Exception(f"""Failed to get object via HTTP gate:
|
||||
request: {resp.request.path_url},
|
||||
response: {resp.text},
|
||||
status code: {resp.status_code} {resp.reason}""")
|
||||
|
||||
logger.info(f'Request: {request}')
|
||||
_attach_allure_step(request, resp.status_code)
|
||||
|
||||
filename = f"{ASSETS_DIR}/{cid}_{str(uuid.uuid4())}"
|
||||
with open(filename, "wb") as get_file:
|
||||
shutil.copyfileobj(resp.raw, get_file)
|
||||
return filename
|
||||
|
||||
|
||||
@keyword('Upload via HTTP Gate')
|
||||
def upload_via_http_gate(cid: str, path: str, headers: dict = None) -> str:
|
||||
"""
|
||||
This function upload given object through HTTP gate
|
||||
:param cid: CID to get object from
|
||||
:param path: File path to upload
|
||||
:param headers: Object header
|
||||
"""
|
||||
request = f'{HTTP_GATE}/upload/{cid}'
|
||||
files = {'upload_file': open(path, 'rb')}
|
||||
body = {
|
||||
'filename': path
|
||||
}
|
||||
resp = requests.post(request, files=files, data=body, headers=headers)
|
||||
|
||||
if not resp.ok:
|
||||
raise Exception(f"""Failed to get object via HTTP gate:
|
||||
request: {resp.request.path_url},
|
||||
response: {resp.text},
|
||||
status code: {resp.status_code} {resp.reason}""")
|
||||
|
||||
logger.info(f'Request: {request}')
|
||||
_attach_allure_step(request, resp.json(), req_type='POST')
|
||||
|
||||
assert resp.json().get('object_id'), f'OID found in response {resp}'
|
||||
|
||||
return resp.json().get('object_id')
|
||||
|
||||
|
||||
@keyword('Upload via HTTP Gate using Curl')
|
||||
def upload_via_http_gate_curl(cid: str, filepath: str, large_object=False, headers: dict = None) -> str:
|
||||
"""
|
||||
This function upload given object through HTTP gate using curl utility.
|
||||
:param cid: CID to get object from
|
||||
:param filepath: File path to upload
|
||||
:param headers: Object header
|
||||
"""
|
||||
request = f'{HTTP_GATE}/upload/{cid}'
|
||||
files = f'file=@{filepath};filename={os.path.basename(filepath)}'
|
||||
cmd = f'curl -F \'{files}\' {request}'
|
||||
if large_object:
|
||||
files = f'file=@pipe;filename={os.path.basename(filepath)}'
|
||||
cmd = f'mkfifo pipe;cat {filepath} > pipe & curl --no-buffer -F \'{files}\' {request}'
|
||||
output = _cmd_run(cmd)
|
||||
oid_re = re.search(r'"object_id": "(.*)"', output)
|
||||
if not oid_re:
|
||||
raise AssertionError(f'Could not find "object_id" in {output}')
|
||||
return oid_re.group(1)
|
||||
|
||||
|
||||
@keyword('Get via HTTP Gate using Curl')
|
||||
def get_via_http_curl(cid: str, oid: str) -> str:
|
||||
"""
|
||||
This function gets given object from HTTP gate using curl utility.
|
||||
:param cid: CID to get object from
|
||||
:param oid: object OID
|
||||
"""
|
||||
request = f'{HTTP_GATE}/get/{cid}/{oid}'
|
||||
filename = f"{ASSETS_DIR}/{cid}_{oid}_{str(uuid.uuid4())}"
|
||||
cmd = f'curl {request} > {filename}'
|
||||
|
||||
_cmd_run(cmd)
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
def _attach_allure_step(request: str, status_code: int, req_type='GET'):
|
||||
if 'allure' in sys.modules:
|
||||
command_attachment = (
|
||||
f"REQUEST: '{request}'\n"
|
||||
f'RESPONSE:\n {status_code}\n'
|
||||
)
|
||||
with allure.step(f'{req_type} Request'):
|
||||
allure.attach(command_attachment, f'{req_type} Request', allure.attachment_type.TEXT)
|
||||
|
|
|
|||
|
|
@ -1,236 +0,0 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
||||
import boto3
|
||||
import botocore
|
||||
import urllib3
|
||||
from robot.api import logger
|
||||
from robot.api.deco import keyword
|
||||
|
||||
from cli_helpers import _run_with_passwd
|
||||
from common import GATE_PUB_KEY, NEOFS_ENDPOINT, S3_GATE
|
||||
|
||||
##########################################################
|
||||
# Disabling warnings on self-signed certificate which the
|
||||
# boto library produces on requests to S3-gate in dev-env.
|
||||
urllib3.disable_warnings()
|
||||
##########################################################
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
CREDENTIALS_CREATE_TIMEOUT = '30s'
|
||||
|
||||
NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate')
|
||||
|
||||
|
||||
@keyword('Init S3 Credentials')
|
||||
def init_s3_credentials(wallet):
|
||||
bucket = str(uuid.uuid4())
|
||||
s3_bearer_rules = "robot/resources/files/s3_bearer_rules.json"
|
||||
cmd = (
|
||||
f'{NEOFS_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} '
|
||||
f'issue-secret --wallet {wallet} --gate-public-key={GATE_PUB_KEY} '
|
||||
f'--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} '
|
||||
f'--bearer-rules {s3_bearer_rules}'
|
||||
)
|
||||
logger.info(f"Executing command: {cmd}")
|
||||
|
||||
try:
|
||||
output = _run_with_passwd(cmd)
|
||||
logger.info(f"Command completed with output: {output}")
|
||||
# first five string are log output, cutting them off and parse
|
||||
# the rest of the output as JSON
|
||||
output = '\n'.join(output.split('\n')[5:])
|
||||
output_dict = json.loads(output)
|
||||
|
||||
return (output_dict['container_id'],
|
||||
bucket,
|
||||
output_dict['access_key_id'],
|
||||
output_dict['secret_access_key'],
|
||||
output_dict['owner_private_key'])
|
||||
|
||||
except Exception as exc:
|
||||
raise RuntimeError("failed to init s3 credentials") from exc
|
||||
|
||||
|
||||
@keyword('Config S3 client')
|
||||
def config_s3_client(access_key_id, secret_access_key):
|
||||
try:
|
||||
|
||||
session = boto3.session.Session()
|
||||
|
||||
s3_client = session.client(
|
||||
service_name='s3',
|
||||
aws_access_key_id=access_key_id,
|
||||
aws_secret_access_key=secret_access_key,
|
||||
endpoint_url=S3_GATE, verify=False
|
||||
)
|
||||
|
||||
return s3_client
|
||||
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
|
||||
|
||||
@keyword('List objects S3 v2')
|
||||
def list_objects_s3_v2(s3_client, bucket):
|
||||
try:
|
||||
response = s3_client.list_objects_v2(Bucket=bucket)
|
||||
logger.info(f"S3 v2 List objects result: {response['Contents']}")
|
||||
obj_list = []
|
||||
for obj in response['Contents']:
|
||||
obj_list.append(obj['Key'])
|
||||
logger.info(f"Found s3 objects: {obj_list}")
|
||||
return obj_list
|
||||
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
|
||||
|
||||
@keyword('List objects S3')
|
||||
def list_objects_s3(s3_client, bucket):
|
||||
try:
|
||||
response = s3_client.list_objects(Bucket=bucket)
|
||||
logger.info(f"S3 List objects result: {response['Contents']}")
|
||||
obj_list = []
|
||||
for obj in response['Contents']:
|
||||
obj_list.append(obj['Key'])
|
||||
logger.info(f"Found s3 objects: {obj_list}")
|
||||
return obj_list
|
||||
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
|
||||
|
||||
@keyword('Create bucket S3')
|
||||
def create_bucket_s3(s3_client):
|
||||
bucket_name = str(uuid.uuid4())
|
||||
|
||||
try:
|
||||
s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
|
||||
logger.info(f"Created S3 bucket: {s3_bucket}")
|
||||
return bucket_name
|
||||
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
|
||||
|
||||
@keyword('List buckets S3')
|
||||
def list_buckets_s3(s3_client):
|
||||
found_buckets = []
|
||||
try:
|
||||
response = s3_client.list_buckets()
|
||||
logger.info(f"S3 List buckets result: {response}")
|
||||
|
||||
for bucket in response['Buckets']:
|
||||
found_buckets.append(bucket['Name'])
|
||||
|
||||
return found_buckets
|
||||
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
|
||||
|
||||
@keyword('Delete bucket S3')
|
||||
def delete_bucket_s3(s3_client, bucket):
|
||||
try:
|
||||
response = s3_client.delete_bucket(Bucket=bucket)
|
||||
logger.info(f"S3 Delete bucket result: {response}")
|
||||
|
||||
return response
|
||||
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
|
||||
|
||||
@keyword('Head bucket S3')
|
||||
def head_bucket(s3_client, bucket):
|
||||
try:
|
||||
response = s3_client.head_bucket(Bucket=bucket)
|
||||
logger.info(f"S3 Head bucket result: {response}")
|
||||
return response
|
||||
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
|
||||
|
||||
@keyword('Put object S3')
|
||||
def put_object_s3(s3_client, bucket, filepath):
|
||||
filename = os.path.basename(filepath)
|
||||
|
||||
with open(filepath, "rb") as put_file:
|
||||
file_content = put_file.read()
|
||||
|
||||
try:
|
||||
response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename)
|
||||
logger.info(f"S3 Put object result: {response}")
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
|
||||
|
||||
@keyword('Head object S3')
|
||||
def head_object_s3(s3_client, bucket, object_key):
|
||||
try:
|
||||
response = s3_client.head_object(Bucket=bucket, Key=object_key)
|
||||
logger.info(f"S3 Head object result: {response}")
|
||||
return response
|
||||
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
|
||||
|
||||
@keyword('Delete object S3')
|
||||
def delete_object_s3(s3_client, bucket, object_key):
|
||||
try:
|
||||
response = s3_client.delete_object(Bucket=bucket, Key=object_key)
|
||||
logger.info(f"S3 Put object result: {response}")
|
||||
return response
|
||||
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
|
||||
|
||||
@keyword('Copy object S3')
|
||||
def copy_object_s3(s3_client, bucket, object_key):
|
||||
filename = f"{os.getcwd()}/{uuid.uuid4()}"
|
||||
try:
|
||||
response = s3_client.copy_object(Bucket=bucket,
|
||||
CopySource=f"{bucket}/{object_key}",
|
||||
Key=filename)
|
||||
logger.info(f"S3 Copy object result: {response}")
|
||||
return filename
|
||||
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
|
||||
|
||||
@keyword('Get object S3')
|
||||
def get_object_s3(s3_client, bucket, object_key):
|
||||
filename = f"{os.getcwd()}/{uuid.uuid4()}"
|
||||
try:
|
||||
response = s3_client.get_object(Bucket=bucket, Key=object_key)
|
||||
|
||||
with open(f"{filename}", 'wb') as get_file:
|
||||
chunk = response['Body'].read(1024)
|
||||
while chunk:
|
||||
get_file.write(chunk)
|
||||
chunk = response['Body'].read(1024)
|
||||
|
||||
return filename
|
||||
|
||||
except botocore.exceptions.ClientError as err:
|
||||
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
||||
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
||||
194
robot/resources/lib/python_keywords/s3_gate_bucket.py
Normal file
194
robot/resources/lib/python_keywords/s3_gate_bucket.py
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from enum import Enum
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
import urllib3
|
||||
from robot.api import logger
|
||||
from robot.api.deco import keyword
|
||||
|
||||
from cli_helpers import _run_with_passwd, log_command_execution
|
||||
from common import GATE_PUB_KEY, NEOFS_ENDPOINT, S3_GATE
|
||||
|
||||
##########################################################
|
||||
# Disabling warnings on self-signed certificate which the
|
||||
# boto library produces on requests to S3-gate in dev-env.
|
||||
urllib3.disable_warnings()
|
||||
##########################################################
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
CREDENTIALS_CREATE_TIMEOUT = '30s'
|
||||
|
||||
NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate')
|
||||
ASSETS_DIR = os.getenv('ASSETS_DIR', 'TemporaryDir/')
|
||||
|
||||
|
||||
class VersioningStatus(Enum):
|
||||
ENABLED = 'Enabled'
|
||||
SUSPENDED = 'Suspended'
|
||||
|
||||
|
||||
@keyword('Init S3 Credentials')
|
||||
def init_s3_credentials(wallet, s3_bearer_rules_file: str = None):
|
||||
bucket = str(uuid.uuid4())
|
||||
s3_bearer_rules = s3_bearer_rules_file or 'robot/resources/files/s3_bearer_rules.json'
|
||||
cmd = (
|
||||
f'{NEOFS_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} '
|
||||
f'issue-secret --wallet {wallet} --gate-public-key={GATE_PUB_KEY} '
|
||||
f'--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} '
|
||||
f'--bearer-rules {s3_bearer_rules}'
|
||||
)
|
||||
logger.info(f'Executing command: {cmd}')
|
||||
|
||||
try:
|
||||
output = _run_with_passwd(cmd)
|
||||
logger.info(f'Command completed with output: {output}')
|
||||
# first five string are log output, cutting them off and parse
|
||||
# the rest of the output as JSON
|
||||
output = '\n'.join(output.split('\n')[5:])
|
||||
output_dict = json.loads(output)
|
||||
|
||||
return (output_dict['container_id'],
|
||||
bucket,
|
||||
output_dict['access_key_id'],
|
||||
output_dict['secret_access_key'],
|
||||
output_dict['owner_private_key'])
|
||||
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f'Failed to init s3 credentials because of error\n{exc}') from exc
|
||||
|
||||
|
||||
@keyword('Config S3 client')
|
||||
def config_s3_client(access_key_id: str, secret_access_key: str):
|
||||
try:
|
||||
|
||||
session = boto3.session.Session()
|
||||
|
||||
s3_client = session.client(
|
||||
service_name='s3',
|
||||
aws_access_key_id=access_key_id,
|
||||
aws_secret_access_key=secret_access_key,
|
||||
endpoint_url=S3_GATE, verify=False
|
||||
)
|
||||
|
||||
return s3_client
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Create bucket S3')
|
||||
def create_bucket_s3(s3_client):
|
||||
bucket_name = str(uuid.uuid4())
|
||||
|
||||
try:
|
||||
s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
|
||||
log_command_execution(f'Created S3 bucket {bucket_name}', s3_bucket)
|
||||
return bucket_name
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('List buckets S3')
|
||||
def list_buckets_s3(s3_client):
|
||||
found_buckets = []
|
||||
try:
|
||||
response = s3_client.list_buckets()
|
||||
log_command_execution('S3 List buckets result', response)
|
||||
|
||||
for bucket in response['Buckets']:
|
||||
found_buckets.append(bucket['Name'])
|
||||
|
||||
return found_buckets
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Delete bucket S3')
|
||||
def delete_bucket_s3(s3_client, bucket: str):
|
||||
try:
|
||||
response = s3_client.delete_bucket(Bucket=bucket)
|
||||
log_command_execution('S3 Delete bucket result', response)
|
||||
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
log_command_execution('S3 Delete bucket error', str(err))
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Head bucket S3')
|
||||
def head_bucket(s3_client, bucket: str):
|
||||
try:
|
||||
response = s3_client.head_bucket(Bucket=bucket)
|
||||
log_command_execution('S3 Head bucket result', response)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
log_command_execution('S3 Head bucket error', str(err))
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Set bucket versioning status')
|
||||
def set_bucket_versioning(s3_client, bucket_name: str, status: VersioningStatus):
|
||||
try:
|
||||
response = s3_client.put_bucket_versioning(Bucket=bucket_name, VersioningConfiguration={'Status': status.value})
|
||||
log_command_execution('S3 Set bucket versioning to', response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during set bucket versioning: {err}') from err
|
||||
|
||||
|
||||
@keyword('Get bucket versioning status')
|
||||
def get_bucket_versioning_status(s3_client, bucket_name: str) -> str:
|
||||
try:
|
||||
response = s3_client.get_bucket_versioning(Bucket=bucket_name)
|
||||
status = response.get('Status')
|
||||
log_command_execution('S3 Got bucket versioning status', response)
|
||||
return status
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during get bucket versioning status: {err}') from err
|
||||
|
||||
|
||||
@keyword('Put bucket tagging')
|
||||
def put_bucket_tagging(s3_client, bucket_name: str, tags: list):
|
||||
try:
|
||||
tags = [{'Key': tag_key, 'Value': tag_value} for tag_key, tag_value in tags]
|
||||
tagging = {'TagSet': tags}
|
||||
response = s3_client.put_bucket_tagging(Bucket=bucket_name, Tagging=tagging)
|
||||
log_command_execution('S3 Put bucket tagging', response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during put bucket tagging: {err}') from err
|
||||
|
||||
|
||||
@keyword('Get bucket tagging')
|
||||
def get_bucket_tagging(s3_client, bucket_name: str) -> list:
|
||||
try:
|
||||
response = s3_client.get_bucket_tagging(Bucket=bucket_name)
|
||||
log_command_execution('S3 Get bucket tagging', response)
|
||||
return response.get('TagSet')
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during get bucket tagging: {err}') from err
|
||||
|
||||
|
||||
@keyword('Delete bucket tagging')
|
||||
def delete_bucket_tagging(s3_client, bucket_name: str):
|
||||
try:
|
||||
response = s3_client.delete_bucket_tagging(Bucket=bucket_name)
|
||||
log_command_execution('S3 Delete bucket tagging', response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during delete bucket tagging: {err}') from err
|
||||
330
robot/resources/lib/python_keywords/s3_gate_object.py
Normal file
330
robot/resources/lib/python_keywords/s3_gate_object.py
Normal file
|
|
@ -0,0 +1,330 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from enum import Enum
|
||||
from typing import Optional, List
|
||||
|
||||
import urllib3
|
||||
from botocore.exceptions import ClientError
|
||||
from robot.api import logger
|
||||
from robot.api.deco import keyword
|
||||
|
||||
from cli_helpers import log_command_execution
|
||||
from python_keywords.aws_cli_client import AwsCliClient
|
||||
|
||||
##########################################################
|
||||
# Disabling warnings on self-signed certificate which the
|
||||
# boto library produces on requests to S3-gate in dev-env.
|
||||
urllib3.disable_warnings()
|
||||
##########################################################
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
CREDENTIALS_CREATE_TIMEOUT = '30s'
|
||||
|
||||
ASSETS_DIR = os.getenv('ASSETS_DIR', 'TemporaryDir/')
|
||||
|
||||
|
||||
class VersioningStatus(Enum):
|
||||
ENABLED = 'Enabled'
|
||||
SUSPENDED = 'Suspended'
|
||||
|
||||
|
||||
@keyword('List objects S3 v2')
|
||||
def list_objects_s3_v2(s3_client, bucket: str) -> list:
|
||||
try:
|
||||
response = s3_client.list_objects_v2(Bucket=bucket)
|
||||
content = response.get('Contents', [])
|
||||
log_command_execution('S3 v2 List objects result', response)
|
||||
obj_list = []
|
||||
for obj in content:
|
||||
obj_list.append(obj['Key'])
|
||||
logger.info(f'Found s3 objects: {obj_list}')
|
||||
return obj_list
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('List objects S3')
|
||||
def list_objects_s3(s3_client, bucket: str) -> list:
|
||||
try:
|
||||
response = s3_client.list_objects(Bucket=bucket)
|
||||
content = response.get('Contents', [])
|
||||
log_command_execution('S3 List objects result', response)
|
||||
obj_list = []
|
||||
for obj in content:
|
||||
obj_list.append(obj['Key'])
|
||||
logger.info(f'Found s3 objects: {obj_list}')
|
||||
return obj_list
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('List objects versions S3')
|
||||
def list_objects_versions_s3(s3_client, bucket: str) -> list:
|
||||
try:
|
||||
response = s3_client.list_object_versions(Bucket=bucket)
|
||||
versions = response.get('Versions', [])
|
||||
log_command_execution('S3 List objects versions result', response)
|
||||
return versions
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Put object S3')
|
||||
def put_object_s3(s3_client, bucket: str, filepath: str):
|
||||
filename = os.path.basename(filepath)
|
||||
|
||||
if isinstance(s3_client, AwsCliClient):
|
||||
file_content = filepath
|
||||
else:
|
||||
with open(filepath, 'rb') as put_file:
|
||||
file_content = put_file.read()
|
||||
|
||||
try:
|
||||
response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename)
|
||||
log_command_execution('S3 Put object result', response)
|
||||
return response.get('VersionId')
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Head object S3')
|
||||
def head_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None):
|
||||
try:
|
||||
params = {'Bucket': bucket, 'Key': object_key}
|
||||
if version_id:
|
||||
params['VersionId'] = version_id
|
||||
response = s3_client.head_object(**params)
|
||||
log_command_execution('S3 Head object result', response)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Delete object S3')
|
||||
def delete_object_s3(s3_client, bucket, object_key, version_id: str = None):
|
||||
try:
|
||||
params = {'Bucket': bucket, 'Key': object_key}
|
||||
if version_id:
|
||||
params['VersionId'] = version_id
|
||||
response = s3_client.delete_object(**params)
|
||||
log_command_execution('S3 Delete object result', response)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Delete objects S3')
|
||||
def delete_objects_s3(s3_client, bucket: str, object_keys: list):
|
||||
try:
|
||||
response = s3_client.delete_objects(Bucket=bucket, Delete=_make_objs_dict(object_keys))
|
||||
log_command_execution('S3 Delete objects result', response)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Copy object S3')
|
||||
def copy_object_s3(s3_client, bucket, object_key, bucket_dst=None):
|
||||
filename = f'{os.getcwd()}/{uuid.uuid4()}'
|
||||
try:
|
||||
response = s3_client.copy_object(Bucket=bucket_dst or bucket,
|
||||
CopySource=f'{bucket}/{object_key}',
|
||||
Key=filename)
|
||||
log_command_execution('S3 Copy objects result', response)
|
||||
return filename
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Get object S3')
|
||||
def get_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None):
|
||||
filename = f'{ASSETS_DIR}/{uuid.uuid4()}'
|
||||
try:
|
||||
params = {'Bucket': bucket, 'Key': object_key}
|
||||
if version_id:
|
||||
params['VersionId'] = version_id
|
||||
|
||||
if isinstance(s3_client, AwsCliClient):
|
||||
params['file_path'] = filename
|
||||
|
||||
response = s3_client.get_object(**params)
|
||||
log_command_execution('S3 Get objects result', response)
|
||||
|
||||
if not isinstance(s3_client, AwsCliClient):
|
||||
with open(f'{filename}', 'wb') as get_file:
|
||||
chunk = response['Body'].read(1024)
|
||||
while chunk:
|
||||
get_file.write(chunk)
|
||||
chunk = response['Body'].read(1024)
|
||||
|
||||
return filename
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Create multipart upload S3')
|
||||
def create_multipart_upload_s3(s3_client, bucket_name: str, object_key: str) -> str:
|
||||
try:
|
||||
response = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution('S3 Created multipart upload', response)
|
||||
assert response.get('UploadId'), f'Expected UploadId in response:\n{response}'
|
||||
|
||||
return response.get('UploadId')
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('List multipart uploads S3')
|
||||
def list_multipart_uploads_s3(s3_client, bucket_name: str) -> Optional[List[dict]]:
|
||||
try:
|
||||
response = s3_client.list_multipart_uploads(Bucket=bucket_name)
|
||||
log_command_execution('S3 List multipart upload', response)
|
||||
|
||||
return response.get('Uploads')
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Abort multipart upload S3')
|
||||
def abort_multipart_uploads_s3(s3_client, bucket_name: str, object_key: str, upload_id: str):
|
||||
try:
|
||||
response = s3_client.abort_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id)
|
||||
log_command_execution('S3 Abort multipart upload', response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Upload part S3')
|
||||
def upload_part_s3(s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, filepath: str) -> str:
|
||||
if isinstance(s3_client, AwsCliClient):
|
||||
file_content = filepath
|
||||
else:
|
||||
with open(filepath, 'rb') as put_file:
|
||||
file_content = put_file.read()
|
||||
|
||||
try:
|
||||
response = s3_client.upload_part(UploadId=upload_id, Bucket=bucket_name, Key=object_key, PartNumber=part_num,
|
||||
Body=file_content)
|
||||
log_command_execution('S3 Upload part', response)
|
||||
assert response.get('ETag'), f'Expected ETag in response:\n{response}'
|
||||
|
||||
return response.get('ETag')
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('List parts S3')
|
||||
def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> List[dict]:
|
||||
try:
|
||||
response = s3_client.list_parts(UploadId=upload_id, Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution('S3 List part', response)
|
||||
assert response.get('Parts'), f'Expected Parts in response:\n{response}'
|
||||
|
||||
return response.get('Parts')
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Complete multipart upload S3')
|
||||
def complete_multipart_upload_s3(s3_client, bucket_name: str, object_key: str, upload_id: str,
|
||||
parts: list):
|
||||
try:
|
||||
parts = [{'ETag': etag, 'PartNumber': part_num} for part_num, etag in parts]
|
||||
response = s3_client.complete_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id,
|
||||
MultipartUpload={'Parts': parts})
|
||||
log_command_execution('S3 Complete multipart upload', response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Put object tagging')
|
||||
def put_object_tagging(s3_client, bucket_name: str, object_key: str, tags: list):
|
||||
try:
|
||||
tags = [{'Key': tag_key, 'Value': tag_value} for tag_key, tag_value in tags]
|
||||
tagging = {'TagSet': tags}
|
||||
s3_client.put_object_tagging(Bucket=bucket_name, Key=object_key, Tagging=tagging)
|
||||
log_command_execution('S3 Put object tagging', str(tags))
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during put object tagging: {err}') from err
|
||||
|
||||
|
||||
@keyword('Get object tagging')
|
||||
def get_object_tagging(s3_client, bucket_name: str, object_key: str) -> list:
|
||||
try:
|
||||
response = s3_client.get_object_tagging(Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution('S3 Get object tagging', response)
|
||||
return response.get('TagSet')
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during get object tagging: {err}') from err
|
||||
|
||||
|
||||
@keyword('Delete object tagging')
|
||||
def delete_object_tagging(s3_client, bucket_name: str, object_key: str):
|
||||
try:
|
||||
response = s3_client.delete_object_tagging(Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution('S3 Delete object tagging', response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during delete object tagging: {err}') from err
|
||||
|
||||
|
||||
@keyword('Get object tagging')
|
||||
def get_object_attributes(s3_client, bucket_name: str, object_key: str, *attributes: str, version_id: str = None,
|
||||
max_parts: int = None, part_number: int = None, get_full_resp=True) -> dict:
|
||||
try:
|
||||
if not isinstance(s3_client, AwsCliClient):
|
||||
logger.warn('Method get_object_attributes is not supported by boto3 client')
|
||||
return {}
|
||||
response = s3_client.get_object_attributes(bucket_name, object_key, *attributes, version_id=version_id,
|
||||
max_parts=max_parts, part_number=part_number)
|
||||
log_command_execution('S3 Get object attributes', response)
|
||||
for attr in attributes:
|
||||
assert attr in response, f'Expected attribute {attr} in {response}'
|
||||
|
||||
if get_full_resp:
|
||||
return response
|
||||
else:
|
||||
return response.get(attributes[0])
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during get object attributes: {err}') from err
|
||||
|
||||
|
||||
def _make_objs_dict(key_names):
|
||||
objs_list = []
|
||||
for key in key_names:
|
||||
obj_dict = {'Key': key}
|
||||
objs_list.append(obj_dict)
|
||||
objs_dict = {'Objects': objs_list}
|
||||
return objs_dict
|
||||
|
|
@ -126,9 +126,15 @@ def get_nodes_without_object(wallet: str, cid: str, oid: str):
|
|||
"""
|
||||
nodes_list = []
|
||||
for node in NEOFS_NETMAP:
|
||||
try:
|
||||
res = neofs_verbs.head_object(wallet, cid, oid,
|
||||
endpoint=node,
|
||||
is_direct=True)
|
||||
if res is None:
|
||||
nodes_list.append(node)
|
||||
except Exception as err:
|
||||
if 'object not found' in str(err):
|
||||
nodes_list.append(node)
|
||||
else:
|
||||
raise Exception(f'Got error {err} on head object command') from err
|
||||
return nodes_list
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ Library Collections
|
|||
Library OperatingSystem
|
||||
|
||||
Library container.py
|
||||
Library s3_gate.py
|
||||
Library s3_gate_bucket.py
|
||||
Library s3_gate_object.py
|
||||
Library epoch.py
|
||||
Library utility_keywords.py
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ Library Collections
|
|||
Library OperatingSystem
|
||||
|
||||
Library container.py
|
||||
Library s3_gate.py
|
||||
Library s3_gate_bucket.py
|
||||
Library s3_gate_object.py
|
||||
Library utility_keywords.py
|
||||
|
||||
Resource payment_operations.robot
|
||||
|
|
|
|||
8
venv/local-pytest/environment.sh
Normal file
8
venv/local-pytest/environment.sh
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# DevEnv variables
|
||||
export NEOFS_MORPH_DISABLE_CACHE=true
|
||||
export DEVENV_PATH="${DEVENV_PATH:-${VIRTUAL_ENV}/../../neofs-dev-env}"
|
||||
pushd $DEVENV_PATH > /dev/null
|
||||
export `make env`
|
||||
popd > /dev/null
|
||||
|
||||
export PYTHONPATH=${PYTHONPATH}:${VIRTUAL_ENV}/../neofs-keywords/lib:${VIRTUAL_ENV}/../neofs-keywords/robot:${VIRTUAL_ENV}/../robot/resources/lib/:${VIRTUAL_ENV}/../robot/resources/lib/python_keywords:${VIRTUAL_ENV}/../robot/resources/lib/robot:${VIRTUAL_ENV}/../robot/variables:${VIRTUAL_ENV}/../pytest_tests/helpers
|
||||
10
venv/local-pytest/requirements.txt
Normal file
10
venv/local-pytest/requirements.txt
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
robotframework==4.1.2
|
||||
requests==2.25.1
|
||||
pexpect==4.8.0
|
||||
boto3==1.16.33
|
||||
docker==4.4.0
|
||||
botocore==1.19.33
|
||||
urllib3==1.26.3
|
||||
base58==2.1.0
|
||||
allure-pytest==2.9.45
|
||||
pytest==7.1.2
|
||||
Loading…
Reference in a new issue