From 6b1e1ab28d8bb362c8a89b43ec76893134275257 Mon Sep 17 00:00:00 2001 From: Vladimir Domnich Date: Tue, 5 Jul 2022 11:18:37 +0400 Subject: [PATCH] Add pytest tests. Change AWS cli v1 to cli v2. Improve allure attachments. Add tests for S3 API. Signed-off-by: a.y.volkov --- .flake8 | 12 + Makefile | 36 ++ pytest_tests/Dockerfile | 40 ++ pytest_tests/helpers/__init__.py | 0 pytest_tests/helpers/utility.py | 49 ++ pytest_tests/pytest.ini | 17 + pytest_tests/requirements.txt | 63 +++ pytest_tests/testsuites/__init__.py | 0 pytest_tests/testsuites/conftest.py | 148 +++++ .../testsuites/object/test_object_api.py | 93 ++++ .../testsuites/services/test_http_gate.py | 235 ++++++++ .../testsuites/services/test_s3_gate.py | 525 ++++++++++++++++++ pytest_tests/wallet_pass.yml | 1 + .../lib/python_keywords/aws_cli_client.py | 219 ++++++++ .../lib/python_keywords/cli_helpers.py | 67 ++- .../lib/python_keywords/http_gate.py | 152 ++++- .../resources/lib/python_keywords/s3_gate.py | 236 -------- .../lib/python_keywords/s3_gate_bucket.py | 194 +++++++ .../lib/python_keywords/s3_gate_object.py | 330 +++++++++++ .../lib/python_keywords/storage_policy.py | 16 +- .../integration/services/s3_gate_bucket.robot | 3 +- .../integration/services/s3_gate_object.robot | 3 +- venv/local-pytest/environment.sh | 8 + venv/local-pytest/requirements.txt | 10 + 24 files changed, 2210 insertions(+), 247 deletions(-) create mode 100644 .flake8 create mode 100644 pytest_tests/Dockerfile create mode 100644 pytest_tests/helpers/__init__.py create mode 100644 pytest_tests/helpers/utility.py create mode 100644 pytest_tests/pytest.ini create mode 100644 pytest_tests/requirements.txt create mode 100644 pytest_tests/testsuites/__init__.py create mode 100644 pytest_tests/testsuites/conftest.py create mode 100644 pytest_tests/testsuites/object/test_object_api.py create mode 100644 pytest_tests/testsuites/services/test_http_gate.py create mode 100644 pytest_tests/testsuites/services/test_s3_gate.py create mode 100644 pytest_tests/wallet_pass.yml create mode 100644 robot/resources/lib/python_keywords/aws_cli_client.py delete mode 100644 robot/resources/lib/python_keywords/s3_gate.py create mode 100644 robot/resources/lib/python_keywords/s3_gate_bucket.py create mode 100644 robot/resources/lib/python_keywords/s3_gate_object.py create mode 100644 venv/local-pytest/environment.sh create mode 100644 venv/local-pytest/requirements.txt diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..eaa71fb --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +exclude = + .git, + __pycache__, + .idea, + .pytest_cache, + venv +per-file-ignores = + # imported but unused + __init__.py: F401 +max-line-length = 120 +disable-noqa \ No newline at end of file diff --git a/Makefile b/Makefile index bd9dd61..699ba64 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,13 @@ SHELL = bash OUTPUT_DIR = artifacts/ KEYWORDS_REPO = git@github.com:nspcc-dev/neofs-keywords.git VENVS = $(shell ls -1d venv/*/ | sort -u | xargs basename -a) +ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) +DEV_IMAGE_PY ?= registry.spb.yadro.com/tools/pytest-neofs-x86_64:4 + +ifeq ($(shell uname -s),Darwin) + DOCKER_NETWORK = --network bridge -p 389:389 -p 636:636 +endif + .PHONY: all all: venvs @@ -30,5 +37,34 @@ submodules: clean: rm -rf venv.* +pytest-local: + @echo "⇒ Run Pytest" + export PYTHONPATH=$(ROOT_DIR)/neofs-keywords/lib:$(ROOT_DIR)/neofs-keywords/robot:$(ROOT_DIR)/robot/resources/lib:$(ROOT_DIR)/robot/resources/lib/python_keywords:$(ROOT_DIR)/robot/variables && \ + python -m pytest pytest_tests/testsuites/ + help: @echo "⇒ run Run testcases ${R}" + +.PHONY: pytest-docker +pytest-docker: + -docker ps + -docker rm neofs_tests_py + -docker pull $(DEV_IMAGE_PY) + docker run -t --rm \ + -w /tests \ + --name neofs_tests_py \ + -e PYTHONPATH="/tests/neofs-keywords/lib:/tests/neofs-keywords/robot:/tests/robot/resources/lib:/tests/robot/resources/lib/python_keywords:/tests/robot/variables:/tests/pytest_tests/helpers" \ + -v $(CURDIR):/tests \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(NEO_BIN_DIR):/neofs \ + --privileged \ + $(DOCKER_NETWORK) \ + --env-file $(CURDIR)/.env \ + $(DEV_IMAGE_PY) \ + -v \ + -m "$(CI_MARKERS)" \ + --color=no \ + --junitxml=/tests/xunit_results.xml \ + --alluredir=/tests/allure_results \ + --setup-show \ + /tests/pytest_tests/testsuites diff --git a/pytest_tests/Dockerfile b/pytest_tests/Dockerfile new file mode 100644 index 0000000..a4a172d --- /dev/null +++ b/pytest_tests/Dockerfile @@ -0,0 +1,40 @@ +ARG ARCH + +FROM python:3.9-slim + +ARG ARCH + +RUN apt-get -y update && apt-get -y install \ + gcc \ + make \ + git \ + curl \ + wget \ + openssh-client \ + iputils-ping \ + unzip \ + vim \ + dbus \ + lsof \ + tini \ + libssl-dev \ + expect \ + runc \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +COPY requirements.txt ./ + +# install aws cli utility v2 from pip +RUN python3 -m pip install awscliv2 +RUN ln /usr/local/bin/awscliv2 /usr/bin/aws +RUN awsv2 --install +RUN aws --version + +RUN pip install --no-cache-dir -r requirements.txt + +ENV PATH=$PATH:/root:/neofs + +WORKDIR /root + +ENTRYPOINT ["tini", "--", "pytest"] \ No newline at end of file diff --git a/pytest_tests/helpers/__init__.py b/pytest_tests/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pytest_tests/helpers/utility.py b/pytest_tests/helpers/utility.py new file mode 100644 index 0000000..92cff98 --- /dev/null +++ b/pytest_tests/helpers/utility.py @@ -0,0 +1,49 @@ +import os +import uuid +from typing import List + +from common import ASSETS_DIR, SIMPLE_OBJ_SIZE + + +def create_file_with_content(file_path: str = None, content: str = None) -> str: + mode = 'w+' + if not content: + content = os.urandom(SIMPLE_OBJ_SIZE) + mode = 'wb' + + if not file_path: + file_path = f"{os.getcwd()}/{ASSETS_DIR}/{str(uuid.uuid4())}" + else: + if not os.path.exists(os.path.dirname(file_path)): + os.makedirs(os.path.dirname(file_path)) + + with open(file_path, mode) as out_file: + out_file.write(content) + + return file_path + + +def get_file_content(file_path: str) -> str: + with open(file_path, 'r') as out_file: + content = out_file.read() + + return content + + +def split_file(file_path: str, parts: int) -> List[str]: + files = [] + with open(file_path, 'rb') as in_file: + data = in_file.read() + + content_size = len(data) + + chunk_size = int((content_size + parts) / parts) + part_id = 1 + for start_position in range(0, content_size + 1, chunk_size): + part_file_name = f'{file_path}_part_{part_id}' + files.append(part_file_name) + with open(part_file_name, 'wb') as out_file: + out_file.write(data[start_position:start_position + chunk_size]) + part_id += 1 + + return files diff --git a/pytest_tests/pytest.ini b/pytest_tests/pytest.ini new file mode 100644 index 0000000..846a754 --- /dev/null +++ b/pytest_tests/pytest.ini @@ -0,0 +1,17 @@ +[pytest] +log_cli = 1 +log_cli_level = DEBUG +log_cli_format = %(asctime)s [%(levelname)4s] %(message)s +log_format = %(asctime)s [%(levelname)4s] %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S +log_date_format = %H:%M:%S +markers = +# special markers + sanity: small tests subset + staging: test to be excluded from run in verifier/pr-validation/sanity jobs and run test in staging job +# functional markers + grpc_api: standard gRPC API tests + http_gate: HTTP gate contract + s3_gate: S3 gate tests + curl: tests for HTTP gate with curl utility + long: long tests (with long execution time) \ No newline at end of file diff --git a/pytest_tests/requirements.txt b/pytest_tests/requirements.txt new file mode 100644 index 0000000..72a4cea --- /dev/null +++ b/pytest_tests/requirements.txt @@ -0,0 +1,63 @@ +aiodns==3.0.0 +aiohttp==3.7.4.post0 +aioresponses==0.7.2 +allure-pytest==2.9.45 +allure-python-commons==2.9.45 +async-timeout==3.0.1 +asynctest==0.13.0 +attrs==21.4.0 +base58==2.1.0 +bitarray==2.3.4 +boto3==1.16.33 +botocore==1.19.33 +certifi==2022.5.18 +cffi==1.15.0 +chardet==4.0.0 +charset-normalizer==2.0.12 +coverage==6.3.3 +docker==4.4.0 +docutils==0.17.1 +Events==0.4 +flake8==4.0.1 +idna==3.3 +iniconfig==1.1.1 +isort==5.10.1 +jmespath==0.10.0 +jsonschema==4.5.1 +lz4==3.1.3 +mccabe==0.6.1 +mmh3==3.0.0 +multidict==6.0.2 +mypy==0.950 +mypy-extensions==0.4.3 +neo-mamba==0.10.0 +neo3crypto==0.2.1 +neo3vm==0.9.0 +neo3vm-stubs==0.9.0 +netaddr==0.8.0 +orjson==3.6.8 +packaging==21.3 +pexpect==4.8.0 +pluggy==1.0.0 +ptyprocess==0.7.0 +py==1.11.0 +pybiginteger==1.2.6 +pybiginteger-stubs==1.2.6 +pycares==4.1.2 +pycodestyle==2.8.0 +pycparser==2.21 +pycryptodome==3.11.0 +pyflakes==2.4.0 +pyparsing==3.0.9 +pyrsistent==0.18.1 +pytest==7.1.2 +python-dateutil==2.8.2 +requests==2.27.1 +robotframework==4.1.2 +s3transfer==0.3.7 +six==1.16.0 +tomli==2.0.1 +typing-extensions==4.2.0 +urllib3==1.26.9 +websocket-client==1.3.2 +yarl==1.7.2 \ No newline at end of file diff --git a/pytest_tests/testsuites/__init__.py b/pytest_tests/testsuites/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pytest_tests/testsuites/conftest.py b/pytest_tests/testsuites/conftest.py new file mode 100644 index 0000000..cdd2fb0 --- /dev/null +++ b/pytest_tests/testsuites/conftest.py @@ -0,0 +1,148 @@ +import logging +import os +import shutil +from re import search +from time import sleep + +import allure +import pytest +import rpc_client +import wallet +from cli_helpers import _cmd_run +from common import (ASSETS_DIR, COMMON_PLACEMENT_RULE, COMPLEX_OBJ_SIZE, + MAINNET_WALLET_WIF, NEO_MAINNET_ENDPOINT, SIMPLE_OBJ_SIZE) +from python_keywords.container import create_container +from python_keywords.payment_neogo import get_balance +from python_keywords.utility_keywords import generate_file_and_file_hash +from robot.api import deco +from wallet_keywords import neofs_deposit, transfer_mainnet_gas +from wellknown_acl import PUBLIC_ACL + +deco.keyword = allure.step + + +logger = logging.getLogger('NeoLogger') + + +@pytest.fixture(scope='session', autouse=True) +@allure.title('Check binary versions') +def check_binary_versions(request): + environment_dir = request.config.getoption('--alluredir') + binaries = ['neo-go', 'neofs-cli', 'neofs-authmate', 'aws'] + env_out = {} + for binary in binaries: + out = _cmd_run(f'{binary} --version') + version = search(r'(v?\d.*)\s+', out) + version = version.group(1) if version else 'Unknown' + env_out[binary.upper()] = version + + if environment_dir: + with open(f'{environment_dir}/environment.properties', 'w') as out_file: + for env, env_value in env_out.items(): + out_file.write(f'{env}={env_value}\n') + + +@pytest.fixture(scope='session') +@allure.title('Init wallet with address') +def init_wallet_with_address(): + full_path = f'{os.getcwd()}/{ASSETS_DIR}' + os.mkdir(full_path) + + yield wallet.init_wallet_w_addr(ASSETS_DIR) + + shutil.rmtree(full_path) + + +@pytest.fixture(scope='session') +@allure.title('Prepare wallet and deposit') +def prepare_wallet_and_deposit(init_wallet_with_address): + deposit = 30 + wallet, addr, wif = init_wallet_with_address + logger.info(f'Init wallet: {wallet},\naddr: {addr},\nwif: {wif}') + + txid = transfer_mainnet_gas(MAINNET_WALLET_WIF, addr, deposit + 1) + wait_unitl_transaction_accepted_in_block(txid) + deposit_tx = neofs_deposit(wif, deposit) + wait_unitl_transaction_accepted_in_block(deposit_tx) + + return wallet, wif + + +@pytest.fixture() +@allure.title('Create Container') +def prepare_container(prepare_wallet_and_deposit): + wallet, wif = prepare_wallet_and_deposit + return prepare_container_impl(wallet, wif) + + +@pytest.fixture(scope='module') +@allure.title('Create Public Container') +def prepare_public_container(prepare_wallet_and_deposit): + placement_rule = 'REP 1 IN X CBF 1 SELECT 1 FROM * AS X' + wallet, wif = prepare_wallet_and_deposit + return prepare_container_impl(wallet, wif, rule=placement_rule, basic_acl=PUBLIC_ACL) + + +def prepare_container_impl(wallet: str, wif: str, rule=COMMON_PLACEMENT_RULE, basic_acl: str = ''): + balance = get_balance(wif) + assert balance > 0, f'Expected balance is greater than 0. Got {balance}' + + cid = create_container(wallet, rule=rule, basic_acl=basic_acl) + + new_balance = get_balance(wif) + assert new_balance < balance, 'Expected some fee has charged' + + return cid, wallet + + +@allure.step('Wait until transaction accepted in block') +def wait_unitl_transaction_accepted_in_block(tx_id: str): + """ + This function return True in case of accepted TX. + Parameters: + :param tx_id: transaction ID + """ + mainnet_rpc_cli = rpc_client.RPCClient(NEO_MAINNET_ENDPOINT) + + if isinstance(tx_id, bytes): + tx_id = tx_id.decode() + + sleep_interval, attempts = 5, 10 + + for __attempt in range(attempts): + try: + resp = mainnet_rpc_cli.get_transaction_height(tx_id) + if resp is not None: + logger.info(f"got block height: {resp}") + return True + except Exception as e: + logger.info(f"request failed with error: {e}") + raise e + sleep(sleep_interval) + raise TimeoutError(f'Timeout {sleep_interval * attempts} sec. reached on waiting for transaction accepted') + + +@pytest.fixture() +@allure.title('Generate files') +def generate_files(): + file_name_simple, _ = generate_file_and_file_hash(SIMPLE_OBJ_SIZE) + large_file_name, _ = generate_file_and_file_hash(COMPLEX_OBJ_SIZE) + + return file_name_simple, large_file_name + + +@pytest.fixture() +@allure.title('Generate file') +def generate_file(): + file_name_simple, _ = generate_file_and_file_hash(SIMPLE_OBJ_SIZE) + + return file_name_simple + + +@pytest.fixture() +@allure.title('Generate large file') +def generate_large_file(): + obj_size = int(os.getenv('BIG_OBJ_SIZE', COMPLEX_OBJ_SIZE)) + file_path, file_hash = generate_file_and_file_hash(obj_size) + + return file_path, file_hash diff --git a/pytest_tests/testsuites/object/test_object_api.py b/pytest_tests/testsuites/object/test_object_api.py new file mode 100644 index 0000000..9ff2e05 --- /dev/null +++ b/pytest_tests/testsuites/object/test_object_api.py @@ -0,0 +1,93 @@ +import logging +from time import sleep + +import allure +import pytest +from contract_keywords import tick_epoch +from python_keywords.neofs import verify_head_tombstone +from python_keywords.neofs_verbs import (delete_object, get_object, get_range, + get_range_hash, head_object, + put_object, search_object) +from python_keywords.storage_policy import get_simple_object_copies +from python_keywords.utility_keywords import get_file_hash + +logger = logging.getLogger('NeoLogger') + +CLEANUP_TIMEOUT = 10 + + +@allure.title('Test native object API') +@pytest.mark.sanity +@pytest.mark.grpc_api +def test_object_api(prepare_container, generate_file): + cid, wallet = prepare_container + wallet_cid = {'wallet': wallet, 'cid': cid} + file_usr_header = {'key1': 1, 'key2': 'abc'} + file_usr_header_oth = {'key1': 2} + range_cut = '0:10' + oids = [] + + file_name = generate_file + file_hash = get_file_hash(file_name) + + search_object(**wallet_cid, expected_objects_list=oids) + + with allure.step('Put objects'): + oids.append(put_object(wallet=wallet, path=file_name, cid=cid)) + oids.append(put_object(wallet=wallet, path=file_name, cid=cid, user_headers=file_usr_header)) + oids.append(put_object(wallet=wallet, path=file_name, cid=cid, user_headers=file_usr_header_oth)) + + with allure.step('Validate storage policy for objects'): + for oid_to_check in oids: + assert get_simple_object_copies(wallet=wallet, cid=cid, oid=oid_to_check) == 2, 'Expected 2 copies' + + with allure.step('Get objects and compare hashes'): + for oid_to_check in oids: + got_file_path = get_object(wallet=wallet, cid=cid, oid=oid_to_check) + got_file_hash = get_file_hash(got_file_path) + assert file_hash == got_file_hash + + with allure.step('Get range/range hash'): + get_range_hash(**wallet_cid, oid=oids[0], bearer_token='', range_cut=range_cut) + get_range_hash(**wallet_cid, oid=oids[1], bearer_token='', range_cut=range_cut) + get_range(**wallet_cid, oid=oids[1], file_path='s_get_range', bearer='', range_cut=range_cut) + + with allure.step('Search objects'): + search_object(**wallet_cid, expected_objects_list=oids) + search_object(**wallet_cid, filters=file_usr_header, expected_objects_list=oids[1:2]) + search_object(**wallet_cid, filters=file_usr_header_oth, expected_objects_list=oids[2:3]) + + with allure.step('Head object and validate'): + head_object(**wallet_cid, oid=oids[0]) + head_info = head_object(**wallet_cid, oid=oids[1]) + check_header_is_presented(head_info, file_usr_header) + + with allure.step('Delete objects'): + tombstone_s = delete_object(**wallet_cid, oid=oids[0]) + tombstone_h = delete_object(**wallet_cid, oid=oids[1]) + + verify_head_tombstone(wallet_path=wallet, cid=cid, oid_ts=tombstone_s, oid=oids[0]) + verify_head_tombstone(wallet_path=wallet, cid=cid, oid_ts=tombstone_h, oid=oids[1]) + + tick_epoch() + sleep(CLEANUP_TIMEOUT) + + with allure.step('Get objects and check errors'): + get_object_and_check_error(**wallet_cid, oid=oids[0], err_msg='object already removed') + get_object_and_check_error(**wallet_cid, oid=oids[1], err_msg='object already removed') + + +def get_object_and_check_error(wallet: str, cid: str, oid: str, err_msg: str): + try: + get_object(wallet=wallet, cid=cid, oid=oid) + raise AssertionError(f'Expected object {oid} removed, but it is not') + except Exception as err: + logger.info(f'Error is {err}') + assert err_msg in str(err), f'Expected message {err_msg} in error: {err}' + + +def check_header_is_presented(head_info: dict, object_header: dict): + for key_to_check, val_to_check in object_header.items(): + assert key_to_check in head_info['header']['attributes'], f'Key {key_to_check} is found in {head_object}' + assert head_info['header']['attributes'].get(key_to_check) == str( + val_to_check), f'Value {val_to_check} is equal' diff --git a/pytest_tests/testsuites/services/test_http_gate.py b/pytest_tests/testsuites/services/test_http_gate.py new file mode 100644 index 0000000..a4ce4dc --- /dev/null +++ b/pytest_tests/testsuites/services/test_http_gate.py @@ -0,0 +1,235 @@ +import logging +from random import choice +from time import sleep + +import allure +import pytest +from contract_keywords import get_epoch, tick_epoch +from python_keywords.http_gate import (get_via_http_curl, get_via_http_gate, + get_via_http_gate_by_attribute, + get_via_zip_http_gate, + upload_via_http_gate, + upload_via_http_gate_curl) +from python_keywords.neofs_verbs import get_object, put_object +from python_keywords.storage_policy import get_nodes_without_object +from python_keywords.utility_keywords import get_file_hash + +logger = logging.getLogger('NeoLogger') + +CLEANUP_TIMEOUT = 10 + + +@allure.link('https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway', name='neofs-http-gateway') +@allure.link('https://github.com/nspcc-dev/neofs-http-gw#uploading', name='uploading') +@allure.link('https://github.com/nspcc-dev/neofs-http-gw#downloading', name='downloading') +@pytest.mark.http_gate +class TestHttpGate: + + @allure.title('Test Put over gRPC, Get over HTTP') + def test_put_grpc_get_http(self, prepare_public_container, generate_files): + """ + Test that object can be put using gRPC interface and get using HTTP. + + Steps: + 1. Create simple and large objects. + 2. Put objects using gRPC (neofs-cli). + 3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading). + 4. Get objects using gRPC (neofs-cli). + 5. Compare hashes for got objects. + 6. Compare hashes for got and original objects. + + Expected result: + Hashes must be the same. + """ + cid, wallet = prepare_public_container + file_name_simple, large_file_name = generate_files + + with allure.step('Put objects using gRPC'): + oid_simple = put_object(wallet=wallet, path=file_name_simple, cid=cid) + oid_large = put_object(wallet=wallet, path=large_file_name, cid=cid) + + for oid, file_name in ((oid_simple, file_name_simple), (oid_large, large_file_name)): + self.get_object_and_verify_hashes(oid, file_name, wallet, cid) + + @allure.link('https://github.com/nspcc-dev/neofs-http-gw#uploading', name='uploading') + @allure.link('https://github.com/nspcc-dev/neofs-http-gw#downloading', name='downloading') + @pytest.mark.sanity + @allure.title('Test Put over HTTP, Get over HTTP') + def test_put_http_get_http(self, prepare_public_container, generate_files): + """ + Test that object can be put and get using HTTP interface. + + Steps: + 1. Create simple and large objects. + 2. Upload objects using HTTP (https://github.com/nspcc-dev/neofs-http-gw#uploading). + 3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading). + 4. Compare hashes for got and original objects. + + Expected result: + Hashes must be the same. + """ + cid, wallet = prepare_public_container + file_name_simple, large_file_name = generate_files + + with allure.step('Put objects using HTTP'): + oid_simple = upload_via_http_gate(cid=cid, path=file_name_simple) + oid_large = upload_via_http_gate(cid=cid, path=large_file_name) + + for oid, file_name in ((oid_simple, file_name_simple), (oid_large, large_file_name)): + self.get_object_and_verify_hashes(oid, file_name, wallet, cid) + + @allure.link('https://github.com/nspcc-dev/neofs-http-gw#by-attributes', name='download by attributes') + @allure.title('Test Put over HTTP, Get over HTTP with headers') + @pytest.mark.parametrize('attributes', + [ + {'fileName': 'simple_obj_filename'}, + {'file-Name': 'simple obj filename'}, + {'cat%jpeg': 'cat%jpeg'} + ], ids=['simple', 'hyphen', 'percent'] + ) + def test_put_http_get_http_with_headers(self, prepare_public_container, generate_files, attributes): + """ + Test that object can be downloaded using different attributes in HTTP header. + + Steps: + 1. Create simple and large objects. + 2. Upload objects using HTTP with particular attributes in the header. + 3. Download objects by attributes using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#by-attributes). + 4. Compare hashes for got and original objects. + + Expected result: + Hashes must be the same. + """ + cid, wallet = prepare_public_container + file_name_simple, _ = generate_files + + with allure.step('Put objects using HTTP with attribute'): + oid_simple = upload_via_http_gate(cid=cid, path=file_name_simple, + headers=self._attr_into_header(attributes)) + + self.get_object_by_attr_and_verify_hashes(oid_simple, file_name_simple, cid, attributes) + + @allure.title('Test Expiration-Epoch in HTTP header') + def test_expiration_epoch_in_http(self, prepare_public_container, generate_file): + cid, wallet = prepare_public_container + file_name_simple = generate_file + object_not_found_err = 'object not found' + oids = [] + + curr_epoch = get_epoch() + epochs = (curr_epoch, curr_epoch + 1, curr_epoch + 2, curr_epoch + 100) + + for epoch in epochs: + headers = {'X-Attribute-Neofs-Expiration-Epoch': str(epoch)} + + with allure.step('Put objects using HTTP with attribute Expiration-Epoch'): + oids.append(upload_via_http_gate(cid=cid, path=file_name_simple, headers=headers)) + + assert len(oids) == len(epochs), 'Expected all objects has been put successfully' + + with allure.step('All objects can be get'): + for oid in oids: + get_via_http_gate(cid=cid, oid=oid) + + for expired_objects, not_expired_objects in [(oids[:1], oids[1:]), (oids[:2], oids[2:])]: + tick_epoch() + sleep(CLEANUP_TIMEOUT) + + for oid in expired_objects: + self.try_to_get_object_and_expect_error(cid=cid, oid=oid, expected_err=object_not_found_err) + + with allure.step('Other objects can be get'): + for oid in not_expired_objects: + get_via_http_gate(cid=cid, oid=oid) + + @allure.title('Test Zip in HTTP header') + def test_zip_in_http(self, prepare_public_container, generate_files): + cid, wallet = prepare_public_container + file_name_simple, file_name_complex = generate_files + common_prefix = 'my_files' + + headers1 = {'X-Attribute-FilePath': f'{common_prefix}/file1'} + headers2 = {'X-Attribute-FilePath': f'{common_prefix}/file2'} + + upload_via_http_gate(cid=cid, path=file_name_simple, headers=headers1) + upload_via_http_gate(cid=cid, path=file_name_complex, headers=headers2) + + dir_path = get_via_zip_http_gate(cid=cid, prefix=common_prefix) + + with allure.step('Verify hashes'): + assert get_file_hash(f'{dir_path}/file1') == get_file_hash(file_name_simple) + assert get_file_hash(f'{dir_path}/file2') == get_file_hash(file_name_complex) + + @pytest.mark.curl + @pytest.mark.long + @allure.title('Test Put over HTTP/Curl, Get over HTTP/Curl for large object') + def test_put_http_get_http_large_file(self, prepare_public_container, generate_large_file): + """ + This test checks upload and download using curl with 'large' object. Large is object with size up to 20Mb. + """ + cid, wallet = prepare_public_container + file_path, file_hash = generate_large_file + + with allure.step('Put objects using HTTP'): + oid_simple = upload_via_http_gate(cid=cid, path=file_path) + oid_curl = upload_via_http_gate_curl(cid=cid, filepath=file_path, large_object=True) + + self.get_object_and_verify_hashes(oid_simple, file_path, wallet, cid) + self.get_object_and_verify_hashes(oid_curl, file_path, wallet, cid, object_getter=get_via_http_curl) + + @pytest.mark.curl + @allure.title('Test Put/Get over HTTP using Curl utility') + def test_put_http_get_http_curl(self, prepare_public_container, generate_files): + """ + Test checks upload and download over HTTP using curl utility. + """ + cid, wallet = prepare_public_container + file_name_simple, large_file_name = generate_files + + with allure.step('Put objects using curl utility'): + oid_simple = upload_via_http_gate_curl(cid=cid, filepath=file_name_simple) + oid_large = upload_via_http_gate_curl(cid=cid, filepath=large_file_name) + + for oid, file_name in ((oid_simple, file_name_simple), (oid_large, large_file_name)): + self.get_object_and_verify_hashes(oid, file_name, wallet, cid, object_getter=get_via_http_curl) + + @staticmethod + @allure.step('Try to get object and expect error') + def try_to_get_object_and_expect_error(cid: str, oid: str, expected_err: str): + try: + get_via_http_gate(cid=cid, oid=oid) + raise AssertionError(f'Expected error on getting object with cid: {cid}') + except Exception as err: + assert expected_err in str(err), f'Expected error {expected_err} in {err}' + + @staticmethod + @allure.step('Verify object can be get using HTTP header attribute') + def get_object_by_attr_and_verify_hashes(oid: str, file_name: str, cid: str, attrs: dict): + + got_file_path_http = get_via_http_gate(cid=cid, oid=oid) + got_file_path_http_attr = get_via_http_gate_by_attribute(cid=cid, attribute=attrs) + + TestHttpGate._assert_hashes_the_same(file_name, got_file_path_http, got_file_path_http_attr) + + @staticmethod + @allure.step('Verify object can be get using HTTP') + def get_object_and_verify_hashes(oid: str, file_name: str, wallet: str, cid: str, object_getter=None): + nodes = get_nodes_without_object(wallet=wallet, cid=cid, oid=oid) + random_node = choice(nodes) + object_getter = object_getter or get_via_http_gate + + got_file_path = get_object(wallet=wallet, cid=cid, oid=oid, endpoint=random_node) + got_file_path_http = object_getter(cid=cid, oid=oid) + + TestHttpGate._assert_hashes_the_same(file_name, got_file_path, got_file_path_http) + + @staticmethod + def _assert_hashes_the_same(orig_file_name: str, got_file_1: str, got_file_2: str): + msg = 'Expected hashes are equal for files {f1} and {f2}' + got_file_hash_http = get_file_hash(got_file_1) + assert get_file_hash(got_file_2) == got_file_hash_http, msg.format(f1=got_file_2, f2=got_file_1) + assert get_file_hash(orig_file_name) == got_file_hash_http, msg.format(f1=orig_file_name, f2=got_file_1) + + @staticmethod + def _attr_into_header(attrs: dict) -> dict: + return {f'X-Attribute-{_key}': _value for _key, _value in attrs.items()} diff --git a/pytest_tests/testsuites/services/test_s3_gate.py b/pytest_tests/testsuites/services/test_s3_gate.py new file mode 100644 index 0000000..ad34e82 --- /dev/null +++ b/pytest_tests/testsuites/services/test_s3_gate.py @@ -0,0 +1,525 @@ +import logging +import os +from random import choice, choices + +import allure +import pytest +from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE +from contract_keywords import tick_epoch +from python_keywords import s3_gate_bucket, s3_gate_object +from python_keywords.aws_cli_client import AwsCliClient +from python_keywords.container import list_containers +from python_keywords.utility_keywords import (generate_file_and_file_hash, + get_file_hash) +from utility import create_file_with_content, get_file_content, split_file + +logger = logging.getLogger('NeoLogger') + + +def pytest_generate_tests(metafunc): + if "s3_client" in metafunc.fixturenames: + metafunc.parametrize("s3_client", ['aws cli', 'boto3'], indirect=True) + + +@allure.link('https://github.com/nspcc-dev/neofs-s3-gw#neofs-s3-gateway', name='neofs-s3-gateway') +@pytest.mark.s3_gate +class TestS3Gate: + s3_client = None + + @pytest.fixture(scope='class', autouse=True) + @allure.title('[Class/Autouse]: Create S3 client') + def s3_client(self, prepare_wallet_and_deposit, request): + wallet, wif = prepare_wallet_and_deposit + s3_bearer_rules_file = f"{os.getcwd()}/robot/resources/files/s3_bearer_rules.json" + + cid, bucket, access_key_id, secret_access_key, owner_private_key = \ + s3_gate_bucket.init_s3_credentials(wallet, s3_bearer_rules_file=s3_bearer_rules_file) + containers_list = list_containers(wallet) + assert cid in containers_list, f'Expected cid {cid} in {containers_list}' + + if request.param == 'aws cli': + try: + client = AwsCliClient(access_key_id, secret_access_key) + except Exception as err: + if 'command was not found or was not executable' in str(err): + pytest.skip('AWS CLI was not found') + else: + raise RuntimeError('Error on creating instance for AwsCliClient') from err + else: + client = s3_gate_bucket.config_s3_client(access_key_id, secret_access_key) + TestS3Gate.s3_client = client + + @pytest.fixture + @allure.title('Create two buckets') + def create_buckets(self): + bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client) + bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client) + return bucket_1, bucket_2 + + @pytest.fixture + @allure.title('Create/delete bucket') + def bucket(self): + bucket = s3_gate_bucket.create_bucket_s3(self.s3_client) + yield bucket + objects = s3_gate_object.list_objects_s3(self.s3_client, bucket) + if objects: + s3_gate_object.delete_objects_s3(self.s3_client, bucket, objects) + s3_gate_bucket.delete_bucket_s3(self.s3_client, bucket) + + @allure.title('Test S3 Bucket API') + def test_s3_buckets(self, generate_files): + """ + Test base S3 Bucket API (Create/List/Head/Delete). + """ + + file_name_simple, file_name_large = generate_files + file_name = self.object_key_from_file_path(file_name_simple) + + with allure.step('Create buckets'): + bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client) + bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client) + + with allure.step('Check buckets are presented in the system'): + buckets = s3_gate_bucket.list_buckets_s3(self.s3_client) + assert bucket_1 in buckets, f'Expected bucket {bucket_1} is in the list' + assert bucket_2 in buckets, f'Expected bucket {bucket_2} is in the list' + + with allure.step('Bucket must be empty'): + for bucket in (bucket_1, bucket_2): + objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket) + assert not objects_list, f'Expected empty bucket, got {objects_list}' + + with allure.step('Check buckets are visible with S3 head command'): + s3_gate_bucket.head_bucket(self.s3_client, bucket_1) + s3_gate_bucket.head_bucket(self.s3_client, bucket_2) + + with allure.step('Check we can put/list object with S3 commands'): + s3_gate_object.put_object_s3(self.s3_client, bucket_1, file_name_simple) + s3_gate_object.head_object_s3(self.s3_client, bucket_1, file_name) + + bucket_objects = s3_gate_object.list_objects_s3(self.s3_client, bucket_1) + assert file_name in bucket_objects, \ + f'Expected file {file_name} in objects list {bucket_objects}' + + with allure.step('Try to delete not empty bucket and get error'): + with pytest.raises(Exception, match=r'.*The bucket you tried to delete is not empty.*'): + s3_gate_bucket.delete_bucket_s3(self.s3_client, bucket_1) + + s3_gate_bucket.head_bucket(self.s3_client, bucket_1) + + with allure.step(f'Delete empty bucket {bucket_2}'): + s3_gate_bucket.delete_bucket_s3(self.s3_client, bucket_2) + tick_epoch() + + with allure.step(f'Check bucket {bucket_2} deleted'): + with pytest.raises(Exception, match=r'.*Not Found.*'): + s3_gate_bucket.head_bucket(self.s3_client, bucket_2) + + buckets = s3_gate_bucket.list_buckets_s3(self.s3_client) + assert bucket_1 in buckets, f'Expected bucket {bucket_1} is in the list' + assert bucket_2 not in buckets, f'Expected bucket {bucket_2} is not in the list' + + @allure.title('Test S3 Object API') + @pytest.mark.sanity + @pytest.mark.parametrize('file_type', ['simple', 'large'], ids=['Simple object', 'Large object']) + def test_s3_api_object(self, generate_files, file_type): + """ + Test base S3 Object API (Put/Head/List) for simple and large objects. + """ + file_name_simple, file_name_large = generate_files + file_name_path = file_name_simple if file_type == 'simple' else file_name_large + file_name = self.object_key_from_file_path(file_name_path) + + bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client) + bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client) + + for bucket in (bucket_1, bucket_2): + with allure.step('Bucket must be empty'): + objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket) + assert not objects_list, f'Expected empty bucket, got {objects_list}' + + s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_path) + s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_large) + s3_gate_object.head_object_s3(self.s3_client, bucket, file_name) + + bucket_objects = s3_gate_object.list_objects_s3(self.s3_client, bucket) + assert file_name in bucket_objects, \ + f'Expected file {file_name} in objects list {bucket_objects}' + + with allure.step("Check object's attributes"): + for attrs in (['ETag'], ['ObjectSize', 'StorageClass']): + s3_gate_object.get_object_attributes(self.s3_client, bucket, file_name, *attrs) + + @allure.title('Test S3 Sync directory') + def test_s3_sync_dir(self, bucket): + """ + Test checks sync directory with AWS CLI utility. + """ + file_path_1 = f"{os.getcwd()}/{ASSETS_DIR}/test_sync/test_file_1" + file_path_2 = f"{os.getcwd()}/{ASSETS_DIR}/test_sync/test_file_2" + key_to_path = {'test_file_1': file_path_1, 'test_file_2': file_path_2} + + if not isinstance(self.s3_client, AwsCliClient): + pytest.skip('This test is not supported with boto3 client') + + create_file_with_content(file_path=file_path_1) + create_file_with_content(file_path=file_path_2) + + self.s3_client.sync(bucket_name=bucket, dir_path=os.path.dirname(file_path_1)) + + with allure.step('Check objects are synced'): + objects = s3_gate_object.list_objects_s3(self.s3_client, bucket) + + with allure.step('Check these are the same objects'): + assert set(key_to_path.keys()) == set(objects), f'Expected all abjects saved. Got {objects}' + for obj_key in objects: + got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, obj_key) + assert get_file_hash(got_object) == get_file_hash(key_to_path.get(obj_key)), \ + 'Expected hashes are the same' + + @allure.title('Test S3 Object versioning') + def test_s3_api_versioning(self, bucket): + """ + Test checks basic versioning functionality for S3 bucket. + """ + version_1_content = 'Version 1' + version_2_content = 'Version 2' + file_name_simple = create_file_with_content(content=version_1_content) + obj_key = os.path.basename(file_name_simple) + + with allure.step('Set versioning enable for bucket'): + s3_gate_bucket.get_bucket_versioning_status(self.s3_client, bucket) + + s3_gate_bucket.set_bucket_versioning(self.s3_client, bucket, status=s3_gate_bucket.VersioningStatus.ENABLED) + status = s3_gate_bucket.get_bucket_versioning_status(self.s3_client, bucket) + assert status == s3_gate_bucket.VersioningStatus.ENABLED.value, f'Expected enabled status. Got {status}' + + with allure.step('Put several versions of object into bucket'): + version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) + create_file_with_content(file_path=file_name_simple, content=version_2_content) + version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) + + with allure.step('Check bucket shows all versions'): + versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket) + obj_versions = {version.get('VersionId') for version in versions if version.get('Key') == obj_key} + assert obj_versions == {version_id_1, version_id_2}, \ + f'Expected object has versions: {version_id_1, version_id_2}' + + with allure.step('Show information about particular version'): + for version_id in (version_id_1, version_id_2): + response = s3_gate_object.head_object_s3(self.s3_client, bucket, obj_key, version_id=version_id) + assert 'LastModified' in response, 'Expected LastModified field' + assert 'ETag' in response, 'Expected ETag field' + assert response.get('VersionId') == version_id, f'Expected VersionId is {version_id}' + assert response.get('ContentLength') != 0, 'Expected ContentLength is not zero' + + with allure.step("Check object's attributes"): + for version_id in (version_id_1, version_id_2): + got_attrs = s3_gate_object.get_object_attributes(self.s3_client, bucket, obj_key, 'ETag', + version_id=version_id) + if got_attrs: + assert got_attrs.get('VersionId') == version_id, f'Expected VersionId is {version_id}' + + with allure.step('Delete object and check it was deleted'): + response = s3_gate_object.delete_object_s3(self.s3_client, bucket, obj_key) + version_id_delete = response.get('VersionId') + + with pytest.raises(Exception, match=r'.*Not Found.*'): + s3_gate_object.head_object_s3(self.s3_client, bucket, obj_key) + + with allure.step('Get content for all versions and check it is correct'): + for version, content in ((version_id_2, version_2_content), (version_id_1, version_1_content)): + file_name = s3_gate_object.get_object_s3(self.s3_client, bucket, obj_key, version_id=version) + got_content = get_file_content(file_name) + assert got_content == content, f'Expected object content is\n{content}\nGot\n{got_content}' + + with allure.step('Restore previous object version'): + s3_gate_object.delete_object_s3(self.s3_client, bucket, obj_key, version_id=version_id_delete) + + file_name = s3_gate_object.get_object_s3(self.s3_client, bucket, obj_key) + got_content = get_file_content(file_name) + assert got_content == version_2_content, \ + f'Expected object content is\n{version_2_content}\nGot\n{got_content}' + + @allure.title('Test S3 Object Multipart API') + def test_s3_api_multipart(self, bucket): + """ + Test checks S3 Multipart API (Create multipart upload/Abort multipart upload/List multipart upload/ + Upload part/List parts/Complete multipart upload). + """ + parts_count = 3 + file_name_large, _ = generate_file_and_file_hash(SIMPLE_OBJ_SIZE * 1024 * 6 * parts_count) # 5Mb - min part + # file_name_large, _ = generate_file_and_file_hash(SIMPLE_OBJ_SIZE * 1024 * 30 * parts_count) # 5Mb - min part + object_key = self.object_key_from_file_path(file_name_large) + part_files = split_file(file_name_large, parts_count) + parts = [] + + uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket) + assert not uploads, f'Expected there is no uploads in bucket {bucket}' + + with allure.step('Create and abort multipart upload'): + upload_id = s3_gate_object.create_multipart_upload_s3(self.s3_client, bucket, object_key) + uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket) + assert uploads, f'Expected there one upload in bucket {bucket}' + assert uploads[0].get('Key') == object_key, f'Expected correct key {object_key} in upload {uploads}' + assert uploads[0].get('UploadId') == upload_id, f'Expected correct UploadId {upload_id} in upload {uploads}' + + s3_gate_object.abort_multipart_uploads_s3(self.s3_client, bucket, object_key, upload_id) + uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket) + assert not uploads, f'Expected there is no uploads in bucket {bucket}' + + with allure.step('Create new multipart upload and upload several parts'): + upload_id = s3_gate_object.create_multipart_upload_s3(self.s3_client, bucket, object_key) + for part_id, file_path in enumerate(part_files, start=1): + etag = s3_gate_object.upload_part_s3(self.s3_client, bucket, object_key, upload_id, part_id, file_path) + parts.append((part_id, etag)) + + with allure.step('Check all parts are visible in bucket'): + got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id) + assert len(got_parts) == len(part_files), f'Expected {parts_count} parts, got\n{got_parts}' + + s3_gate_object.complete_multipart_upload_s3(self.s3_client, bucket, object_key, upload_id, parts) + + uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket) + assert not uploads, f'Expected there is no uploads in bucket {bucket}' + + with allure.step('Check we can get whole object from bucket'): + got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, object_key) + assert get_file_hash(got_object) == get_file_hash(file_name_large) + + self.check_object_attributes(bucket, object_key, parts_count) + + @allure.title('Test S3 Bucket tagging API') + def test_s3_api_bucket_tagging(self, bucket): + """ + Test checks S3 Bucket tagging API (Put tag/Get tag). + """ + key_value_pair = [('some-key', 'some-value'), ('some-key-2', 'some-value-2')] + + s3_gate_bucket.put_bucket_tagging(self.s3_client, bucket, key_value_pair) + got_tags = s3_gate_bucket.get_bucket_tagging(self.s3_client, bucket) + + with allure.step('Check all tags are presented'): + assert got_tags, f'Expected tags, got {got_tags}' + expected_tags = [{'Key': key, 'Value': value} for key, value in key_value_pair] + for tag in expected_tags: + assert tag in got_tags + + s3_gate_bucket.delete_bucket_tagging(self.s3_client, bucket) + tags = s3_gate_bucket.get_bucket_tagging(self.s3_client, bucket) + assert not tags, f'Expected there is no tags for bucket {bucket}, got {tags}' + + @allure.title('Test S3 Object tagging API') + def test_s3_api_object_tagging(self, bucket): + """ + Test checks S3 Object tagging API (Put tag/Get tag/Update tag). + """ + key_value_pair_bucket = [('some-key', 'some-value'), ('some-key-2', 'some-value-2')] + key_value_pair_obj = [('some-key-obj', 'some-value-obj'), ('some-key--obj2', 'some-value--obj2')] + key_value_pair_obj_new = [('some-key-obj-new', 'some-value-obj-new')] + file_name_simple, _ = generate_file_and_file_hash(SIMPLE_OBJ_SIZE) + obj_key = self.object_key_from_file_path(file_name_simple) + + s3_gate_bucket.put_bucket_tagging(self.s3_client, bucket, key_value_pair_bucket) + + s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) + + for tags in (key_value_pair_obj, key_value_pair_obj_new): + s3_gate_object.put_object_tagging(self.s3_client, bucket, obj_key, tags) + + got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, obj_key) + assert got_tags, f'Expected tags, got {got_tags}' + expected_tags = [{'Key': key, 'Value': value} for key, value in tags] + for tag in expected_tags: + assert tag in got_tags + + s3_gate_object.delete_object_tagging(self.s3_client, bucket, obj_key) + got_tags = s3_gate_object.get_object_tagging(self.s3_client, bucket, obj_key) + assert not got_tags, f'Expected there is no tags for bucket {bucket}, got {got_tags}' + + @allure.title('Test S3: Delete object & delete objects S3 API') + def test_s3_api_delete(self, create_buckets): + """ + Check delete_object and delete_objects S3 API operation. From first bucket some objects deleted one by one. + From second bucket some objects deleted all at once. + """ + max_obj_count = 20 + max_delete_objects = 17 + put_objects = [] + file_paths = [] + obj_sizes = [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE] + + bucket_1, bucket_2 = create_buckets + + with allure.step(f'Generate {max_obj_count} files'): + for _ in range(max_obj_count): + file_paths.append(generate_file_and_file_hash(choice(obj_sizes))[0]) + + for bucket in (bucket_1, bucket_2): + with allure.step(f'Bucket {bucket} must be empty as it just created'): + objects_list = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket) + assert not objects_list, f'Expected empty bucket, got {objects_list}' + + for file_path in file_paths: + s3_gate_object.put_object_s3(self.s3_client, bucket, file_path) + put_objects.append(self.object_key_from_file_path(file_path)) + + with allure.step(f'Check all objects put in bucket {bucket} successfully'): + bucket_objects = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket) + assert set(put_objects) == set(bucket_objects), \ + f'Expected all objects {put_objects} in objects list {bucket_objects}' + + with allure.step('Delete some objects from bucket_1 one by one'): + objects_to_delete_b1 = choices(put_objects, k=max_delete_objects) + for obj in objects_to_delete_b1: + s3_gate_object.delete_object_s3(self.s3_client, bucket_1, obj) + + with allure.step('Check deleted objects are not visible in bucket bucket_1'): + bucket_objects = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket_1) + assert set(put_objects).difference(set(objects_to_delete_b1)) == set(bucket_objects), \ + f'Expected all objects {put_objects} in objects list {bucket_objects}' + self.try_to_get_object_and_got_error(bucket_1, objects_to_delete_b1) + + with allure.step('Delete some objects from bucket_2 at once'): + objects_to_delete_b2 = choices(put_objects, k=max_delete_objects) + s3_gate_object.delete_objects_s3(self.s3_client, bucket_2, objects_to_delete_b2) + + with allure.step('Check deleted objects are not visible in bucket bucket_2'): + objects_list = s3_gate_object.list_objects_s3_v2(self.s3_client, bucket_2) + assert set(put_objects).difference(set(objects_to_delete_b2)) == set(objects_list), \ + f'Expected all objects {put_objects} in objects list {bucket_objects}' + self.try_to_get_object_and_got_error(bucket_2, objects_to_delete_b2) + + @allure.title('Test S3: Copy object to the same bucket') + def test_s3_copy_same_bucket(self, generate_files): + """ + Test object can be copied to the same bucket. + """ + file_simple, file_large = generate_files + file_name_simple = self.object_key_from_file_path(file_simple) + file_name_large = self.object_key_from_file_path(file_large) + bucket_objects = [file_name_simple, file_name_large] + + bucket = s3_gate_bucket.create_bucket_s3(self.s3_client) + + with allure.step('Bucket must be empty'): + objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket) + assert not objects_list, f'Expected empty bucket, got {objects_list}' + + with allure.step('Put objects into bucket'): + for obj in (file_simple, file_large): + s3_gate_object.put_object_s3(self.s3_client, bucket, obj) + + with allure.step('Copy one object into the same bucket'): + copy_obj_path = s3_gate_object.copy_object_s3(self.s3_client, bucket, file_name_simple) + bucket_objects.append(copy_obj_path) + + self.check_objects_in_bucket(bucket, bucket_objects) + + with allure.step('Check copied object has the same content'): + got_copied_file = s3_gate_object.get_object_s3(self.s3_client, bucket, copy_obj_path) + assert get_file_hash(file_simple) == get_file_hash(got_copied_file), 'Hashes must be the same' + + with allure.step('Delete one object from bucket'): + s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name_simple) + bucket_objects.remove(file_name_simple) + + self.check_objects_in_bucket(bucket, expected_objects=bucket_objects, unexpected_objects=[file_name_simple]) + + @allure.title('Test S3: Copy object to another bucket') + def test_s3_copy_to_another_bucket(self, generate_files): + """ + Test object can be copied to another bucket. + """ + file_simple, file_large = generate_files + file_name_simple = self.object_key_from_file_path(file_simple) + file_name_large = self.object_key_from_file_path(file_large) + bucket_1_objects = [file_name_simple, file_name_large] + + bucket_1 = s3_gate_bucket.create_bucket_s3(self.s3_client) + bucket_2 = s3_gate_bucket.create_bucket_s3(self.s3_client) + + with allure.step('Buckets must be empty'): + for bucket in (bucket_1, bucket_2): + objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket) + assert not objects_list, f'Expected empty bucket, got {objects_list}' + + with allure.step('Put objects into one bucket'): + for obj in (file_simple, file_large): + s3_gate_object.put_object_s3(self.s3_client, bucket_1, obj) + + with allure.step('Copy object from first bucket into second'): + copy_obj_path_b2 = s3_gate_object.copy_object_s3(self.s3_client, bucket_1, file_name_large, + bucket_dst=bucket_2) + self.check_objects_in_bucket(bucket_1, expected_objects=bucket_1_objects) + self.check_objects_in_bucket(bucket_2, expected_objects=[copy_obj_path_b2]) + + with allure.step('Check copied object has the same content'): + got_copied_file_b2 = s3_gate_object.get_object_s3(self.s3_client, bucket_2, copy_obj_path_b2) + assert get_file_hash(file_large) == get_file_hash(got_copied_file_b2), 'Hashes must be the same' + + with allure.step('Delete one object from first bucket'): + s3_gate_object.delete_object_s3(self.s3_client, bucket_1, file_name_simple) + bucket_1_objects.remove(file_name_simple) + + self.check_objects_in_bucket(bucket_1, expected_objects=bucket_1_objects) + self.check_objects_in_bucket(bucket_2, expected_objects=[copy_obj_path_b2]) + + with allure.step('Delete one object from second bucket and check it is empty'): + s3_gate_object.delete_object_s3(self.s3_client, bucket_2, copy_obj_path_b2) + self.check_objects_in_bucket(bucket_2, expected_objects=[]) + + def check_object_attributes(self, bucket: str, object_key: str, parts_count: int): + if not isinstance(self.s3_client, AwsCliClient): + logger.warning('Attributes check is not supported for boto3 implementation') + return + + with allure.step("Check object's attributes"): + obj_parts = s3_gate_object.get_object_attributes(self.s3_client, bucket, object_key, 'ObjectParts', + get_full_resp=False) + assert obj_parts.get('TotalPartsCount') == parts_count, f'Expected TotalPartsCount is {parts_count}' + assert len(obj_parts.get('Parts')) == parts_count, f'Expected Parts cunt is {parts_count}' + + with allure.step("Check object's attribute max-parts"): + max_parts = 2 + obj_parts = s3_gate_object.get_object_attributes(self.s3_client, bucket, object_key, 'ObjectParts', + max_parts=max_parts, get_full_resp=False) + assert obj_parts.get('TotalPartsCount') == parts_count, f'Expected TotalPartsCount is {parts_count}' + assert obj_parts.get('MaxParts') == max_parts, f'Expected MaxParts is {parts_count}' + assert len(obj_parts.get('Parts')) == max_parts, f'Expected Parts count is {parts_count}' + + with allure.step("Check object's attribute part-number-marker"): + part_number_marker = 3 + obj_parts = s3_gate_object.get_object_attributes(self.s3_client, bucket, object_key, 'ObjectParts', + part_number=part_number_marker, get_full_resp=False) + assert obj_parts.get('TotalPartsCount') == parts_count, f'Expected TotalPartsCount is {parts_count}' + assert obj_parts.get( + 'PartNumberMarker') == part_number_marker, f'Expected PartNumberMarker is {part_number_marker}' + assert len(obj_parts.get('Parts')) == 1, f'Expected Parts count is {parts_count}' + + @allure.step('Expected all objects are presented in the bucket') + def check_objects_in_bucket(self, bucket, expected_objects: list, unexpected_objects: list = None): + unexpected_objects = unexpected_objects or [] + bucket_objects = s3_gate_object.list_objects_s3(self.s3_client, bucket) + assert len(bucket_objects) == len(expected_objects), f'Expected {len(expected_objects)} objects in the bucket' + + for bucket_object in expected_objects: + assert bucket_object in bucket_objects, \ + f'Expected object {bucket_object} in objects list {bucket_objects}' + + for bucket_object in unexpected_objects: + assert bucket_object not in bucket_objects, \ + f'Expected object {bucket_object} not in objects list {bucket_objects}' + + @allure.step('Try to get object and got error') + def try_to_get_object_and_got_error(self, bucket: str, unexpected_objects: list): + for obj in unexpected_objects: + try: + s3_gate_object.get_object_s3(self.s3_client, bucket, obj) + raise AssertionError(f'Object {obj} found in bucket {bucket}') + except Exception as err: + assert 'The specified key does not exist' in str(err), f'Expected error in exception {err}' + + @staticmethod + def object_key_from_file_path(full_path: str) -> str: + return os.path.basename(full_path) diff --git a/pytest_tests/wallet_pass.yml b/pytest_tests/wallet_pass.yml new file mode 100644 index 0000000..1986cf9 --- /dev/null +++ b/pytest_tests/wallet_pass.yml @@ -0,0 +1 @@ +password: "" diff --git a/robot/resources/lib/python_keywords/aws_cli_client.py b/robot/resources/lib/python_keywords/aws_cli_client.py new file mode 100644 index 0000000..104e1d6 --- /dev/null +++ b/robot/resources/lib/python_keywords/aws_cli_client.py @@ -0,0 +1,219 @@ +import json +import logging +import os + +import allure +from cli_helpers import _cmd_run, _configure_aws_cli +from common import ASSETS_DIR, S3_GATE + +logger = logging.getLogger('NeoLogger') + + +class AwsCliClient: + + def __init__(self, access_key_id: str, secret_access_key: str): + self.access_key_id = access_key_id + self.secret_access_key = secret_access_key + self.config_aws_client() + + def config_aws_client(self): + cmd = 'aws configure' + logger.info(f'Executing command: {cmd}') + _configure_aws_cli(cmd, self.access_key_id, self.secret_access_key) + + def create_bucket(self, Bucket: str): + cmd = f'aws --no-verify-ssl s3api create-bucket --bucket {Bucket} --endpoint-url {S3_GATE}' + _cmd_run(cmd) + + def list_buckets(self) -> dict: + cmd = f'aws --no-verify-ssl s3api list-buckets --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def get_bucket_versioning(self, Bucket: str) -> dict: + cmd = f'aws --no-verify-ssl s3api get-bucket-versioning --bucket {Bucket}' \ + f' --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def put_bucket_versioning(self, Bucket: str, VersioningConfiguration: dict) -> dict: + cmd = f'aws --no-verify-ssl s3api put-bucket-versioning --bucket {Bucket} ' \ + f'--versioning-configuration Status={VersioningConfiguration.get("Status")}' \ + f' --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def list_objects(self, Bucket: str) -> dict: + cmd = f'aws --no-verify-ssl s3api list-objects --bucket {Bucket}' \ + f' --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def list_objects_v2(self, Bucket: str) -> dict: + cmd = f'aws --no-verify-ssl s3api list-objects-v2 --bucket {Bucket}' \ + f' --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def list_object_versions(self, Bucket: str) -> dict: + cmd = f'aws --no-verify-ssl s3api list-object-versions --bucket {Bucket}' \ + f' --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def copy_object(self, Bucket: str, CopySource: str, Key: str) -> dict: + cmd = f'aws --no-verify-ssl s3api copy-object --copy-source {CopySource} --bucket {Bucket} --key {Key}' \ + f' --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def head_bucket(self, Bucket: str) -> dict: + cmd = f'aws --no-verify-ssl s3api head-bucket --bucket {Bucket} --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def put_object(self, Body: str, Bucket: str, Key: str) -> dict: + cmd = f' aws --no-verify-ssl s3api put-object --bucket {Bucket} --key {Key} --body {Body}' \ + f' --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def head_object(self, Bucket: str, Key: str, VersionId: str = None) -> dict: + version = f' --version-id {VersionId}' if VersionId else '' + cmd = f' aws --no-verify-ssl s3api head-object --bucket {Bucket} --key {Key} {version}' \ + f' --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def get_object(self, Bucket: str, Key: str, file_path: str, VersionId: str = None) -> dict: + version = f' --version-id {VersionId}' if VersionId else '' + cmd = f' aws --no-verify-ssl s3api get-object --bucket {Bucket} ' \ + f'--key {Key} {version} {file_path} --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def delete_objects(self, Bucket: str, Delete: dict) -> dict: + file_path = f"{os.getcwd()}/{ASSETS_DIR}/delete.json" + with open(file_path, 'w') as out_file: + out_file.write(json.dumps(Delete)) + + cmd = f'aws --no-verify-ssl s3api delete-objects --bucket {Bucket} --delete file://{file_path} ' \ + f'--endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def delete_object(self, Bucket: str, Key: str, VersionId: str = None) -> dict: + version = f' --version-id {VersionId}' if VersionId else '' + cmd = f'aws --no-verify-ssl s3api delete-object --bucket {Bucket} --key {Key} {version}' \ + f' --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def get_object_attributes(self, bucket: str, key: str, *attributes: str, version_id: str = None, + max_parts: int = None, part_number: int = None) -> dict: + attrs = ','.join(attributes) + version = f' --version-id {version_id}' if version_id else '' + parts = f'--max-parts {max_parts}' if max_parts else '' + part_number = f'--part-number-marker {part_number}' if part_number else '' + cmd = f'aws --no-verify-ssl s3api get-object-attributes --bucket {bucket} --key {key} {version}' \ + f' {parts} {part_number} --object-attributes {attrs} --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def delete_bucket(self, Bucket: str) -> dict: + cmd = f'aws --no-verify-ssl s3api delete-bucket --bucket {Bucket} --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def get_bucket_tagging(self, Bucket: str) -> dict: + cmd = f'aws --no-verify-ssl s3api get-bucket-tagging --bucket {Bucket} --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def put_bucket_tagging(self, Bucket: str, Tagging: dict) -> dict: + cmd = f'aws --no-verify-ssl s3api put-bucket-tagging --bucket {Bucket} --tagging \'{json.dumps(Tagging)}\'' \ + f' --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def delete_bucket_tagging(self, Bucket: str) -> dict: + cmd = f'aws --no-verify-ssl s3api delete-bucket-tagging --bucket {Bucket} --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def put_object_tagging(self, Bucket: str, Key: str, Tagging: dict) -> dict: + cmd = f'aws --no-verify-ssl s3api put-object-tagging --bucket {Bucket} --key {Key}' \ + f' --tagging \'{json.dumps(Tagging)}\'' \ + f' --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def get_object_tagging(self, Bucket: str, Key: str) -> dict: + cmd = f'aws --no-verify-ssl s3api get-object-tagging --bucket {Bucket} --key {Key} --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def delete_object_tagging(self, Bucket: str, Key: str) -> dict: + cmd = f'aws --no-verify-ssl s3api delete-object-tagging --bucket {Bucket} --key {Key} --endpoint {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + @allure.step('Sync directory S3') + def sync(self, bucket_name: str, dir_path: str) -> dict: + cmd = f'aws --no-verify-ssl s3 sync {dir_path} s3://{bucket_name} --endpoint-url {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def create_multipart_upload(self, Bucket: str, Key: str) -> dict: + cmd = f'aws --no-verify-ssl s3api create-multipart-upload --bucket {Bucket} --key {Key}' \ + f' --endpoint-url {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def list_multipart_uploads(self, Bucket: str) -> dict: + cmd = f'aws --no-verify-ssl s3api list-multipart-uploads --bucket {Bucket}' \ + f' --endpoint-url {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def abort_multipart_upload(self, Bucket: str, Key: str, UploadId: str) -> dict: + cmd = f'aws --no-verify-ssl s3api abort-multipart-upload --bucket {Bucket} --key {Key}' \ + f' --upload-id {UploadId} --endpoint-url {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def upload_part(self, UploadId: str, Bucket: str, Key: str, PartNumber: int, Body: str) -> dict: + cmd = f'aws --no-verify-ssl s3api upload-part --bucket {Bucket} --key {Key} --upload-id {UploadId} ' \ + f'--part-number {PartNumber} --body {Body} --endpoint-url {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def list_parts(self, UploadId: str, Bucket: str, Key: str) -> dict: + cmd = f'aws --no-verify-ssl s3api list-parts --bucket {Bucket} --key {Key} --upload-id {UploadId} ' \ + f' --endpoint-url {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + def complete_multipart_upload(self, Bucket: str, Key: str, UploadId: str, MultipartUpload: dict) -> dict: + file_path = f"{os.getcwd()}/{ASSETS_DIR}/parts.json" + with open(file_path, 'w') as out_file: + out_file.write(json.dumps(MultipartUpload)) + + cmd = f'aws --no-verify-ssl s3api complete-multipart-upload --bucket {Bucket} --key {Key}' \ + f' --upload-id {UploadId} --multipart-upload file://{file_path}' \ + f' --endpoint-url {S3_GATE}' + output = _cmd_run(cmd) + return self._to_json(output) + + @staticmethod + def _to_json(output: str) -> dict: + json_output = {} + try: + json_output = json.loads(output) + except Exception: + if '{' not in output and '}' not in output: + logger.warning(f'Could not parse json from output {output}') + return json_output + json_output = json.loads(output[output.index('{'):]) + + return json_output diff --git a/robot/resources/lib/python_keywords/cli_helpers.py b/robot/resources/lib/python_keywords/cli_helpers.py index e0f0d15..692db06 100644 --- a/robot/resources/lib/python_keywords/cli_helpers.py +++ b/robot/resources/lib/python_keywords/cli_helpers.py @@ -4,9 +4,15 @@ Helper functions to use with `neofs-cli`, `neo-go` and other CLIs. """ - +from typing import Union import subprocess +import sys +from contextlib import suppress +from datetime import datetime +from json import dumps +from textwrap import shorten +import allure import pexpect from robot.api import logger @@ -20,11 +26,16 @@ def _cmd_run(cmd, timeout=30): """ try: logger.info(f"Executing command: {cmd}") + start_time = datetime.now() compl_proc = subprocess.run(cmd, check=True, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout, shell=True) output = compl_proc.stdout + return_code = compl_proc.returncode + end_time = datetime.now() logger.info(f"Output: {output}") + _attach_allure_log(cmd, output, return_code, start_time, end_time) + return output except subprocess.CalledProcessError as exc: raise RuntimeError(f"Error:\nreturn code: {exc.returncode} " @@ -38,8 +49,62 @@ def _cmd_run(cmd, timeout=30): def _run_with_passwd(cmd): child = pexpect.spawn(cmd) + child.delaybeforesend = 1 child.expect(".*") child.sendline('\r') + if sys.platform == "darwin": + child.expect(pexpect.EOF) + cmd = child.before + else: + child.wait() + cmd = child.read() + return cmd.decode() + + +def _configure_aws_cli(cmd, key_id, access_key, out_format='json'): + child = pexpect.spawn(cmd) + child.delaybeforesend = 1 + + child.expect("AWS Access Key ID.*") + child.sendline(key_id) + + child.expect("AWS Secret Access Key.*") + child.sendline(access_key) + + child.expect("Default region name.*") + child.sendline('') + + child.expect("Default output format.*") + child.sendline(out_format) + child.wait() cmd = child.read() + # child.expect(pexpect.EOF) + # cmd = child.before return cmd.decode() + + +def _attach_allure_log(cmd: str, output: str, return_code: int, start_time: datetime, end_time: datetime): + if 'allure' in sys.modules: + command_attachment = ( + f"COMMAND: '{cmd}'\n" + f'OUTPUT:\n {output}\n' + f'RC: {return_code}\n' + f'Start / End / Elapsed\t {start_time.time()} / {end_time.time()} / {end_time - start_time}' + ) + with allure.step(f'COMMAND: {shorten(cmd, width=60, placeholder="...")}'): + allure.attach(command_attachment, 'Command execution', allure.attachment_type.TEXT) + + +def log_command_execution(cmd: str, output: Union[str, dict]): + logger.info(f'{cmd}: {output}') + if 'allure' in sys.modules: + with suppress(Exception): + json_output = dumps(output, indent=4, sort_keys=True) + output = json_output + command_attachment = ( + f"COMMAND: '{cmd}'\n" + f'OUTPUT:\n {output}\n' + ) + with allure.step(f'COMMAND: {shorten(cmd, width=60, placeholder="...")}'): + allure.attach(command_attachment, 'Command execution', allure.attachment_type.TEXT) diff --git a/robot/resources/lib/python_keywords/http_gate.py b/robot/resources/lib/python_keywords/http_gate.py index 26eecd7..ded0470 100644 --- a/robot/resources/lib/python_keywords/http_gate.py +++ b/robot/resources/lib/python_keywords/http_gate.py @@ -1,16 +1,28 @@ #!/usr/bin/python3 +import re import shutil +import sys +import uuid +import zipfile +from urllib.parse import quote_plus +import allure import requests +from cli_helpers import _cmd_run +from common import HTTP_GATE from robot.api import logger from robot.api.deco import keyword from robot.libraries.BuiltIn import BuiltIn -from common import HTTP_GATE - ROBOT_AUTO_KEYWORDS = False -ASSETS_DIR = BuiltIn().get_variable_value("${ASSETS_DIR}") + +if "pytest" in sys.modules: + import os + + ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/") +else: + ASSETS_DIR = BuiltIn().get_variable_value("${ASSETS_DIR}") @keyword('Get via HTTP Gate') @@ -30,7 +42,141 @@ def get_via_http_gate(cid: str, oid: str): status code: {resp.status_code} {resp.reason}""") logger.info(f'Request: {request}') + _attach_allure_step(request, resp.status_code) + filename = f"{ASSETS_DIR}/{cid}_{oid}" with open(filename, "wb") as get_file: shutil.copyfileobj(resp.raw, get_file) return filename + + +@keyword('Get via Zip HTTP Gate') +def get_via_zip_http_gate(cid: str, prefix: str): + """ + This function gets given object from HTTP gate + :param cid: CID to get object from + :param prefix: common prefix + """ + request = f'{HTTP_GATE}/zip/{cid}/{prefix}' + resp = requests.get(request, stream=True) + + if not resp.ok: + raise Exception(f"""Failed to get object via HTTP gate: + request: {resp.request.path_url}, + response: {resp.text}, + status code: {resp.status_code} {resp.reason}""") + + logger.info(f'Request: {request}') + _attach_allure_step(request, resp.status_code) + + filename = f'{ASSETS_DIR}/{cid}_archive.zip' + with open(filename, 'wb') as get_file: + shutil.copyfileobj(resp.raw, get_file) + + with zipfile.ZipFile(filename, 'r') as zip_ref: + zip_ref.extractall(ASSETS_DIR) + + return f'{ASSETS_DIR}/{prefix}' + + +@keyword('Get via HTTP Gate by attribute') +def get_via_http_gate_by_attribute(cid: str, attribute: dict): + """ + This function gets given object from HTTP gate + :param cid: CID to get object from + :param attribute: attribute name: attribute value pair + """ + attr_name = list(attribute.keys())[0] + attr_value = quote_plus(str(attribute.get(attr_name))) + request = f'{HTTP_GATE}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}' + resp = requests.get(request, stream=True) + + if not resp.ok: + raise Exception(f"""Failed to get object via HTTP gate: + request: {resp.request.path_url}, + response: {resp.text}, + status code: {resp.status_code} {resp.reason}""") + + logger.info(f'Request: {request}') + _attach_allure_step(request, resp.status_code) + + filename = f"{ASSETS_DIR}/{cid}_{str(uuid.uuid4())}" + with open(filename, "wb") as get_file: + shutil.copyfileobj(resp.raw, get_file) + return filename + + +@keyword('Upload via HTTP Gate') +def upload_via_http_gate(cid: str, path: str, headers: dict = None) -> str: + """ + This function upload given object through HTTP gate + :param cid: CID to get object from + :param path: File path to upload + :param headers: Object header + """ + request = f'{HTTP_GATE}/upload/{cid}' + files = {'upload_file': open(path, 'rb')} + body = { + 'filename': path + } + resp = requests.post(request, files=files, data=body, headers=headers) + + if not resp.ok: + raise Exception(f"""Failed to get object via HTTP gate: + request: {resp.request.path_url}, + response: {resp.text}, + status code: {resp.status_code} {resp.reason}""") + + logger.info(f'Request: {request}') + _attach_allure_step(request, resp.json(), req_type='POST') + + assert resp.json().get('object_id'), f'OID found in response {resp}' + + return resp.json().get('object_id') + + +@keyword('Upload via HTTP Gate using Curl') +def upload_via_http_gate_curl(cid: str, filepath: str, large_object=False, headers: dict = None) -> str: + """ + This function upload given object through HTTP gate using curl utility. + :param cid: CID to get object from + :param filepath: File path to upload + :param headers: Object header + """ + request = f'{HTTP_GATE}/upload/{cid}' + files = f'file=@{filepath};filename={os.path.basename(filepath)}' + cmd = f'curl -F \'{files}\' {request}' + if large_object: + files = f'file=@pipe;filename={os.path.basename(filepath)}' + cmd = f'mkfifo pipe;cat {filepath} > pipe & curl --no-buffer -F \'{files}\' {request}' + output = _cmd_run(cmd) + oid_re = re.search(r'"object_id": "(.*)"', output) + if not oid_re: + raise AssertionError(f'Could not find "object_id" in {output}') + return oid_re.group(1) + + +@keyword('Get via HTTP Gate using Curl') +def get_via_http_curl(cid: str, oid: str) -> str: + """ + This function gets given object from HTTP gate using curl utility. + :param cid: CID to get object from + :param oid: object OID + """ + request = f'{HTTP_GATE}/get/{cid}/{oid}' + filename = f"{ASSETS_DIR}/{cid}_{oid}_{str(uuid.uuid4())}" + cmd = f'curl {request} > {filename}' + + _cmd_run(cmd) + + return filename + + +def _attach_allure_step(request: str, status_code: int, req_type='GET'): + if 'allure' in sys.modules: + command_attachment = ( + f"REQUEST: '{request}'\n" + f'RESPONSE:\n {status_code}\n' + ) + with allure.step(f'{req_type} Request'): + allure.attach(command_attachment, f'{req_type} Request', allure.attachment_type.TEXT) diff --git a/robot/resources/lib/python_keywords/s3_gate.py b/robot/resources/lib/python_keywords/s3_gate.py deleted file mode 100644 index a59bf1f..0000000 --- a/robot/resources/lib/python_keywords/s3_gate.py +++ /dev/null @@ -1,236 +0,0 @@ -#!/usr/bin/python3 - -import json -import os -import uuid - -import boto3 -import botocore -import urllib3 -from robot.api import logger -from robot.api.deco import keyword - -from cli_helpers import _run_with_passwd -from common import GATE_PUB_KEY, NEOFS_ENDPOINT, S3_GATE - -########################################################## -# Disabling warnings on self-signed certificate which the -# boto library produces on requests to S3-gate in dev-env. -urllib3.disable_warnings() -########################################################## - -ROBOT_AUTO_KEYWORDS = False -CREDENTIALS_CREATE_TIMEOUT = '30s' - -NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate') - - -@keyword('Init S3 Credentials') -def init_s3_credentials(wallet): - bucket = str(uuid.uuid4()) - s3_bearer_rules = "robot/resources/files/s3_bearer_rules.json" - cmd = ( - f'{NEOFS_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} ' - f'issue-secret --wallet {wallet} --gate-public-key={GATE_PUB_KEY} ' - f'--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} ' - f'--bearer-rules {s3_bearer_rules}' - ) - logger.info(f"Executing command: {cmd}") - - try: - output = _run_with_passwd(cmd) - logger.info(f"Command completed with output: {output}") - # first five string are log output, cutting them off and parse - # the rest of the output as JSON - output = '\n'.join(output.split('\n')[5:]) - output_dict = json.loads(output) - - return (output_dict['container_id'], - bucket, - output_dict['access_key_id'], - output_dict['secret_access_key'], - output_dict['owner_private_key']) - - except Exception as exc: - raise RuntimeError("failed to init s3 credentials") from exc - - -@keyword('Config S3 client') -def config_s3_client(access_key_id, secret_access_key): - try: - - session = boto3.session.Session() - - s3_client = session.client( - service_name='s3', - aws_access_key_id=access_key_id, - aws_secret_access_key=secret_access_key, - endpoint_url=S3_GATE, verify=False - ) - - return s3_client - - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('List objects S3 v2') -def list_objects_s3_v2(s3_client, bucket): - try: - response = s3_client.list_objects_v2(Bucket=bucket) - logger.info(f"S3 v2 List objects result: {response['Contents']}") - obj_list = [] - for obj in response['Contents']: - obj_list.append(obj['Key']) - logger.info(f"Found s3 objects: {obj_list}") - return obj_list - - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('List objects S3') -def list_objects_s3(s3_client, bucket): - try: - response = s3_client.list_objects(Bucket=bucket) - logger.info(f"S3 List objects result: {response['Contents']}") - obj_list = [] - for obj in response['Contents']: - obj_list.append(obj['Key']) - logger.info(f"Found s3 objects: {obj_list}") - return obj_list - - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('Create bucket S3') -def create_bucket_s3(s3_client): - bucket_name = str(uuid.uuid4()) - - try: - s3_bucket = s3_client.create_bucket(Bucket=bucket_name) - logger.info(f"Created S3 bucket: {s3_bucket}") - return bucket_name - - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('List buckets S3') -def list_buckets_s3(s3_client): - found_buckets = [] - try: - response = s3_client.list_buckets() - logger.info(f"S3 List buckets result: {response}") - - for bucket in response['Buckets']: - found_buckets.append(bucket['Name']) - - return found_buckets - - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('Delete bucket S3') -def delete_bucket_s3(s3_client, bucket): - try: - response = s3_client.delete_bucket(Bucket=bucket) - logger.info(f"S3 Delete bucket result: {response}") - - return response - - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('Head bucket S3') -def head_bucket(s3_client, bucket): - try: - response = s3_client.head_bucket(Bucket=bucket) - logger.info(f"S3 Head bucket result: {response}") - return response - - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('Put object S3') -def put_object_s3(s3_client, bucket, filepath): - filename = os.path.basename(filepath) - - with open(filepath, "rb") as put_file: - file_content = put_file.read() - - try: - response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename) - logger.info(f"S3 Put object result: {response}") - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('Head object S3') -def head_object_s3(s3_client, bucket, object_key): - try: - response = s3_client.head_object(Bucket=bucket, Key=object_key) - logger.info(f"S3 Head object result: {response}") - return response - - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('Delete object S3') -def delete_object_s3(s3_client, bucket, object_key): - try: - response = s3_client.delete_object(Bucket=bucket, Key=object_key) - logger.info(f"S3 Put object result: {response}") - return response - - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('Copy object S3') -def copy_object_s3(s3_client, bucket, object_key): - filename = f"{os.getcwd()}/{uuid.uuid4()}" - try: - response = s3_client.copy_object(Bucket=bucket, - CopySource=f"{bucket}/{object_key}", - Key=filename) - logger.info(f"S3 Copy object result: {response}") - return filename - - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err - - -@keyword('Get object S3') -def get_object_s3(s3_client, bucket, object_key): - filename = f"{os.getcwd()}/{uuid.uuid4()}" - try: - response = s3_client.get_object(Bucket=bucket, Key=object_key) - - with open(f"{filename}", 'wb') as get_file: - chunk = response['Body'].read(1024) - while chunk: - get_file.write(chunk) - chunk = response['Body'].read(1024) - - return filename - - except botocore.exceptions.ClientError as err: - raise Exception(f"Error Message: {err.response['Error']['Message']}\n" - f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err diff --git a/robot/resources/lib/python_keywords/s3_gate_bucket.py b/robot/resources/lib/python_keywords/s3_gate_bucket.py new file mode 100644 index 0000000..9c40879 --- /dev/null +++ b/robot/resources/lib/python_keywords/s3_gate_bucket.py @@ -0,0 +1,194 @@ +#!/usr/bin/python3 + +import json +import os +import uuid +from enum import Enum + +import boto3 +from botocore.exceptions import ClientError +import urllib3 +from robot.api import logger +from robot.api.deco import keyword + +from cli_helpers import _run_with_passwd, log_command_execution +from common import GATE_PUB_KEY, NEOFS_ENDPOINT, S3_GATE + +########################################################## +# Disabling warnings on self-signed certificate which the +# boto library produces on requests to S3-gate in dev-env. +urllib3.disable_warnings() +########################################################## + +ROBOT_AUTO_KEYWORDS = False +CREDENTIALS_CREATE_TIMEOUT = '30s' + +NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate') +ASSETS_DIR = os.getenv('ASSETS_DIR', 'TemporaryDir/') + + +class VersioningStatus(Enum): + ENABLED = 'Enabled' + SUSPENDED = 'Suspended' + + +@keyword('Init S3 Credentials') +def init_s3_credentials(wallet, s3_bearer_rules_file: str = None): + bucket = str(uuid.uuid4()) + s3_bearer_rules = s3_bearer_rules_file or 'robot/resources/files/s3_bearer_rules.json' + cmd = ( + f'{NEOFS_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} ' + f'issue-secret --wallet {wallet} --gate-public-key={GATE_PUB_KEY} ' + f'--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} ' + f'--bearer-rules {s3_bearer_rules}' + ) + logger.info(f'Executing command: {cmd}') + + try: + output = _run_with_passwd(cmd) + logger.info(f'Command completed with output: {output}') + # first five string are log output, cutting them off and parse + # the rest of the output as JSON + output = '\n'.join(output.split('\n')[5:]) + output_dict = json.loads(output) + + return (output_dict['container_id'], + bucket, + output_dict['access_key_id'], + output_dict['secret_access_key'], + output_dict['owner_private_key']) + + except Exception as exc: + raise RuntimeError(f'Failed to init s3 credentials because of error\n{exc}') from exc + + +@keyword('Config S3 client') +def config_s3_client(access_key_id: str, secret_access_key: str): + try: + + session = boto3.session.Session() + + s3_client = session.client( + service_name='s3', + aws_access_key_id=access_key_id, + aws_secret_access_key=secret_access_key, + endpoint_url=S3_GATE, verify=False + ) + + return s3_client + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Create bucket S3') +def create_bucket_s3(s3_client): + bucket_name = str(uuid.uuid4()) + + try: + s3_bucket = s3_client.create_bucket(Bucket=bucket_name) + log_command_execution(f'Created S3 bucket {bucket_name}', s3_bucket) + return bucket_name + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('List buckets S3') +def list_buckets_s3(s3_client): + found_buckets = [] + try: + response = s3_client.list_buckets() + log_command_execution('S3 List buckets result', response) + + for bucket in response['Buckets']: + found_buckets.append(bucket['Name']) + + return found_buckets + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Delete bucket S3') +def delete_bucket_s3(s3_client, bucket: str): + try: + response = s3_client.delete_bucket(Bucket=bucket) + log_command_execution('S3 Delete bucket result', response) + + return response + + except ClientError as err: + log_command_execution('S3 Delete bucket error', str(err)) + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Head bucket S3') +def head_bucket(s3_client, bucket: str): + try: + response = s3_client.head_bucket(Bucket=bucket) + log_command_execution('S3 Head bucket result', response) + return response + + except ClientError as err: + log_command_execution('S3 Head bucket error', str(err)) + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Set bucket versioning status') +def set_bucket_versioning(s3_client, bucket_name: str, status: VersioningStatus): + try: + response = s3_client.put_bucket_versioning(Bucket=bucket_name, VersioningConfiguration={'Status': status.value}) + log_command_execution('S3 Set bucket versioning to', response) + + except ClientError as err: + raise Exception(f'Got error during set bucket versioning: {err}') from err + + +@keyword('Get bucket versioning status') +def get_bucket_versioning_status(s3_client, bucket_name: str) -> str: + try: + response = s3_client.get_bucket_versioning(Bucket=bucket_name) + status = response.get('Status') + log_command_execution('S3 Got bucket versioning status', response) + return status + except ClientError as err: + raise Exception(f'Got error during get bucket versioning status: {err}') from err + + +@keyword('Put bucket tagging') +def put_bucket_tagging(s3_client, bucket_name: str, tags: list): + try: + tags = [{'Key': tag_key, 'Value': tag_value} for tag_key, tag_value in tags] + tagging = {'TagSet': tags} + response = s3_client.put_bucket_tagging(Bucket=bucket_name, Tagging=tagging) + log_command_execution('S3 Put bucket tagging', response) + + except ClientError as err: + raise Exception(f'Got error during put bucket tagging: {err}') from err + + +@keyword('Get bucket tagging') +def get_bucket_tagging(s3_client, bucket_name: str) -> list: + try: + response = s3_client.get_bucket_tagging(Bucket=bucket_name) + log_command_execution('S3 Get bucket tagging', response) + return response.get('TagSet') + + except ClientError as err: + raise Exception(f'Got error during get bucket tagging: {err}') from err + + +@keyword('Delete bucket tagging') +def delete_bucket_tagging(s3_client, bucket_name: str): + try: + response = s3_client.delete_bucket_tagging(Bucket=bucket_name) + log_command_execution('S3 Delete bucket tagging', response) + + except ClientError as err: + raise Exception(f'Got error during delete bucket tagging: {err}') from err diff --git a/robot/resources/lib/python_keywords/s3_gate_object.py b/robot/resources/lib/python_keywords/s3_gate_object.py new file mode 100644 index 0000000..17ac301 --- /dev/null +++ b/robot/resources/lib/python_keywords/s3_gate_object.py @@ -0,0 +1,330 @@ +#!/usr/bin/python3 + +import os +import uuid +from enum import Enum +from typing import Optional, List + +import urllib3 +from botocore.exceptions import ClientError +from robot.api import logger +from robot.api.deco import keyword + +from cli_helpers import log_command_execution +from python_keywords.aws_cli_client import AwsCliClient + +########################################################## +# Disabling warnings on self-signed certificate which the +# boto library produces on requests to S3-gate in dev-env. +urllib3.disable_warnings() +########################################################## + +ROBOT_AUTO_KEYWORDS = False +CREDENTIALS_CREATE_TIMEOUT = '30s' + +ASSETS_DIR = os.getenv('ASSETS_DIR', 'TemporaryDir/') + + +class VersioningStatus(Enum): + ENABLED = 'Enabled' + SUSPENDED = 'Suspended' + + +@keyword('List objects S3 v2') +def list_objects_s3_v2(s3_client, bucket: str) -> list: + try: + response = s3_client.list_objects_v2(Bucket=bucket) + content = response.get('Contents', []) + log_command_execution('S3 v2 List objects result', response) + obj_list = [] + for obj in content: + obj_list.append(obj['Key']) + logger.info(f'Found s3 objects: {obj_list}') + return obj_list + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('List objects S3') +def list_objects_s3(s3_client, bucket: str) -> list: + try: + response = s3_client.list_objects(Bucket=bucket) + content = response.get('Contents', []) + log_command_execution('S3 List objects result', response) + obj_list = [] + for obj in content: + obj_list.append(obj['Key']) + logger.info(f'Found s3 objects: {obj_list}') + return obj_list + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('List objects versions S3') +def list_objects_versions_s3(s3_client, bucket: str) -> list: + try: + response = s3_client.list_object_versions(Bucket=bucket) + versions = response.get('Versions', []) + log_command_execution('S3 List objects versions result', response) + return versions + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Put object S3') +def put_object_s3(s3_client, bucket: str, filepath: str): + filename = os.path.basename(filepath) + + if isinstance(s3_client, AwsCliClient): + file_content = filepath + else: + with open(filepath, 'rb') as put_file: + file_content = put_file.read() + + try: + response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename) + log_command_execution('S3 Put object result', response) + return response.get('VersionId') + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Head object S3') +def head_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None): + try: + params = {'Bucket': bucket, 'Key': object_key} + if version_id: + params['VersionId'] = version_id + response = s3_client.head_object(**params) + log_command_execution('S3 Head object result', response) + return response + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Delete object S3') +def delete_object_s3(s3_client, bucket, object_key, version_id: str = None): + try: + params = {'Bucket': bucket, 'Key': object_key} + if version_id: + params['VersionId'] = version_id + response = s3_client.delete_object(**params) + log_command_execution('S3 Delete object result', response) + return response + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Delete objects S3') +def delete_objects_s3(s3_client, bucket: str, object_keys: list): + try: + response = s3_client.delete_objects(Bucket=bucket, Delete=_make_objs_dict(object_keys)) + log_command_execution('S3 Delete objects result', response) + return response + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Copy object S3') +def copy_object_s3(s3_client, bucket, object_key, bucket_dst=None): + filename = f'{os.getcwd()}/{uuid.uuid4()}' + try: + response = s3_client.copy_object(Bucket=bucket_dst or bucket, + CopySource=f'{bucket}/{object_key}', + Key=filename) + log_command_execution('S3 Copy objects result', response) + return filename + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Get object S3') +def get_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None): + filename = f'{ASSETS_DIR}/{uuid.uuid4()}' + try: + params = {'Bucket': bucket, 'Key': object_key} + if version_id: + params['VersionId'] = version_id + + if isinstance(s3_client, AwsCliClient): + params['file_path'] = filename + + response = s3_client.get_object(**params) + log_command_execution('S3 Get objects result', response) + + if not isinstance(s3_client, AwsCliClient): + with open(f'{filename}', 'wb') as get_file: + chunk = response['Body'].read(1024) + while chunk: + get_file.write(chunk) + chunk = response['Body'].read(1024) + + return filename + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Create multipart upload S3') +def create_multipart_upload_s3(s3_client, bucket_name: str, object_key: str) -> str: + try: + response = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_key) + log_command_execution('S3 Created multipart upload', response) + assert response.get('UploadId'), f'Expected UploadId in response:\n{response}' + + return response.get('UploadId') + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('List multipart uploads S3') +def list_multipart_uploads_s3(s3_client, bucket_name: str) -> Optional[List[dict]]: + try: + response = s3_client.list_multipart_uploads(Bucket=bucket_name) + log_command_execution('S3 List multipart upload', response) + + return response.get('Uploads') + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Abort multipart upload S3') +def abort_multipart_uploads_s3(s3_client, bucket_name: str, object_key: str, upload_id: str): + try: + response = s3_client.abort_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id) + log_command_execution('S3 Abort multipart upload', response) + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Upload part S3') +def upload_part_s3(s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, filepath: str) -> str: + if isinstance(s3_client, AwsCliClient): + file_content = filepath + else: + with open(filepath, 'rb') as put_file: + file_content = put_file.read() + + try: + response = s3_client.upload_part(UploadId=upload_id, Bucket=bucket_name, Key=object_key, PartNumber=part_num, + Body=file_content) + log_command_execution('S3 Upload part', response) + assert response.get('ETag'), f'Expected ETag in response:\n{response}' + + return response.get('ETag') + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('List parts S3') +def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> List[dict]: + try: + response = s3_client.list_parts(UploadId=upload_id, Bucket=bucket_name, Key=object_key) + log_command_execution('S3 List part', response) + assert response.get('Parts'), f'Expected Parts in response:\n{response}' + + return response.get('Parts') + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Complete multipart upload S3') +def complete_multipart_upload_s3(s3_client, bucket_name: str, object_key: str, upload_id: str, + parts: list): + try: + parts = [{'ETag': etag, 'PartNumber': part_num} for part_num, etag in parts] + response = s3_client.complete_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id, + MultipartUpload={'Parts': parts}) + log_command_execution('S3 Complete multipart upload', response) + + except ClientError as err: + raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err + + +@keyword('Put object tagging') +def put_object_tagging(s3_client, bucket_name: str, object_key: str, tags: list): + try: + tags = [{'Key': tag_key, 'Value': tag_value} for tag_key, tag_value in tags] + tagging = {'TagSet': tags} + s3_client.put_object_tagging(Bucket=bucket_name, Key=object_key, Tagging=tagging) + log_command_execution('S3 Put object tagging', str(tags)) + + except ClientError as err: + raise Exception(f'Got error during put object tagging: {err}') from err + + +@keyword('Get object tagging') +def get_object_tagging(s3_client, bucket_name: str, object_key: str) -> list: + try: + response = s3_client.get_object_tagging(Bucket=bucket_name, Key=object_key) + log_command_execution('S3 Get object tagging', response) + return response.get('TagSet') + + except ClientError as err: + raise Exception(f'Got error during get object tagging: {err}') from err + + +@keyword('Delete object tagging') +def delete_object_tagging(s3_client, bucket_name: str, object_key: str): + try: + response = s3_client.delete_object_tagging(Bucket=bucket_name, Key=object_key) + log_command_execution('S3 Delete object tagging', response) + + except ClientError as err: + raise Exception(f'Got error during delete object tagging: {err}') from err + + +@keyword('Get object tagging') +def get_object_attributes(s3_client, bucket_name: str, object_key: str, *attributes: str, version_id: str = None, + max_parts: int = None, part_number: int = None, get_full_resp=True) -> dict: + try: + if not isinstance(s3_client, AwsCliClient): + logger.warn('Method get_object_attributes is not supported by boto3 client') + return {} + response = s3_client.get_object_attributes(bucket_name, object_key, *attributes, version_id=version_id, + max_parts=max_parts, part_number=part_number) + log_command_execution('S3 Get object attributes', response) + for attr in attributes: + assert attr in response, f'Expected attribute {attr} in {response}' + + if get_full_resp: + return response + else: + return response.get(attributes[0]) + + except ClientError as err: + raise Exception(f'Got error during get object attributes: {err}') from err + + +def _make_objs_dict(key_names): + objs_list = [] + for key in key_names: + obj_dict = {'Key': key} + objs_list.append(obj_dict) + objs_dict = {'Objects': objs_list} + return objs_dict diff --git a/robot/resources/lib/python_keywords/storage_policy.py b/robot/resources/lib/python_keywords/storage_policy.py index a2d91ca..15edf37 100644 --- a/robot/resources/lib/python_keywords/storage_policy.py +++ b/robot/resources/lib/python_keywords/storage_policy.py @@ -126,9 +126,15 @@ def get_nodes_without_object(wallet: str, cid: str, oid: str): """ nodes_list = [] for node in NEOFS_NETMAP: - res = neofs_verbs.head_object(wallet, cid, oid, - endpoint=node, - is_direct=True) - if res is None: - nodes_list.append(node) + try: + res = neofs_verbs.head_object(wallet, cid, oid, + endpoint=node, + is_direct=True) + if res is None: + nodes_list.append(node) + except Exception as err: + if 'object not found' in str(err): + nodes_list.append(node) + else: + raise Exception(f'Got error {err} on head object command') from err return nodes_list diff --git a/robot/testsuites/integration/services/s3_gate_bucket.robot b/robot/testsuites/integration/services/s3_gate_bucket.robot index 0e719ed..d283a7d 100644 --- a/robot/testsuites/integration/services/s3_gate_bucket.robot +++ b/robot/testsuites/integration/services/s3_gate_bucket.robot @@ -5,7 +5,8 @@ Library Collections Library OperatingSystem Library container.py -Library s3_gate.py +Library s3_gate_bucket.py +Library s3_gate_object.py Library epoch.py Library utility_keywords.py diff --git a/robot/testsuites/integration/services/s3_gate_object.robot b/robot/testsuites/integration/services/s3_gate_object.robot index 57e9308..4ca64f0 100644 --- a/robot/testsuites/integration/services/s3_gate_object.robot +++ b/robot/testsuites/integration/services/s3_gate_object.robot @@ -5,7 +5,8 @@ Library Collections Library OperatingSystem Library container.py -Library s3_gate.py +Library s3_gate_bucket.py +Library s3_gate_object.py Library utility_keywords.py Resource payment_operations.robot diff --git a/venv/local-pytest/environment.sh b/venv/local-pytest/environment.sh new file mode 100644 index 0000000..b9b985c --- /dev/null +++ b/venv/local-pytest/environment.sh @@ -0,0 +1,8 @@ +# DevEnv variables +export NEOFS_MORPH_DISABLE_CACHE=true +export DEVENV_PATH="${DEVENV_PATH:-${VIRTUAL_ENV}/../../neofs-dev-env}" +pushd $DEVENV_PATH > /dev/null +export `make env` +popd > /dev/null + +export PYTHONPATH=${PYTHONPATH}:${VIRTUAL_ENV}/../neofs-keywords/lib:${VIRTUAL_ENV}/../neofs-keywords/robot:${VIRTUAL_ENV}/../robot/resources/lib/:${VIRTUAL_ENV}/../robot/resources/lib/python_keywords:${VIRTUAL_ENV}/../robot/resources/lib/robot:${VIRTUAL_ENV}/../robot/variables:${VIRTUAL_ENV}/../pytest_tests/helpers diff --git a/venv/local-pytest/requirements.txt b/venv/local-pytest/requirements.txt new file mode 100644 index 0000000..947e5fe --- /dev/null +++ b/venv/local-pytest/requirements.txt @@ -0,0 +1,10 @@ +robotframework==4.1.2 +requests==2.25.1 +pexpect==4.8.0 +boto3==1.16.33 +docker==4.4.0 +botocore==1.19.33 +urllib3==1.26.3 +base58==2.1.0 +allure-pytest==2.9.45 +pytest==7.1.2