From 6230d2244eadf31525eeb923651bf13e3a8863a3 Mon Sep 17 00:00:00 2001 From: Vladislav Karakozov Date: Thu, 8 Dec 2022 16:03:27 +0300 Subject: [PATCH] create http folder, and adding a new test for http attributes Signed-off-by: Vladislav Karakozov --- .../{ => http_gate}/test_http_gate.py | 158 +++++++----------- .../services/http_gate/test_http_object.py | 121 ++++++++++++++ .../lib/python_keywords/http_gate.py | 137 +++++++++++++-- 3 files changed, 309 insertions(+), 107 deletions(-) rename pytest_tests/testsuites/services/{ => http_gate}/test_http_gate.py (75%) create mode 100644 pytest_tests/testsuites/services/http_gate/test_http_object.py diff --git a/pytest_tests/testsuites/services/test_http_gate.py b/pytest_tests/testsuites/services/http_gate/test_http_gate.py similarity index 75% rename from pytest_tests/testsuites/services/test_http_gate.py rename to pytest_tests/testsuites/services/http_gate/test_http_gate.py index 4efcc645..c7a3cf54 100644 --- a/pytest_tests/testsuites/services/test_http_gate.py +++ b/pytest_tests/testsuites/services/http_gate/test_http_gate.py @@ -1,7 +1,5 @@ import logging import os -import random -from time import sleep import allure import pytest @@ -9,15 +7,17 @@ from epoch import get_epoch, tick_epoch from file_helper import generate_file, get_file_hash from python_keywords.container import create_container from python_keywords.http_gate import ( + attr_into_header, + get_object_and_verify_hashes, + get_object_by_attr_and_verify_hashes, get_via_http_curl, get_via_http_gate, - get_via_http_gate_by_attribute, get_via_zip_http_gate, + try_to_get_object_and_expect_error, upload_via_http_gate, upload_via_http_gate_curl, ) -from python_keywords.neofs_verbs import get_object, put_object_to_random_node -from python_keywords.storage_policy import get_nodes_without_object +from python_keywords.neofs_verbs import put_object_to_random_node from utility import wait_for_gc_pass_on_storage_nodes from wellknown_acl import PUBLIC_ACL @@ -26,11 +26,6 @@ from steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") OBJECT_NOT_FOUND_ERROR = "not found" -# For some reason object uploaded via http gateway is not immediately available for downloading -# Until this issue is resolved we are waiting for some time before attempting to read an object -# TODO: remove after https://github.com/nspcc-dev/neofs-http-gw/issues/176 is fixed -OBJECT_UPLOAD_DELAY = 10 - @allure.link( "https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway", name="neofs-http-gateway" @@ -92,7 +87,15 @@ class TestHttpGate(ClusterTestBase): ) for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): - self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid) + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet, + cid=cid, + shell=self.shell, + nodes=self.cluster.storage_nodes, + endpoint=self.cluster.default_http_gate_endpoint, + ) @allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading") @allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading") @@ -131,7 +134,15 @@ class TestHttpGate(ClusterTestBase): ) for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): - self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid) + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet, + cid=cid, + shell=self.shell, + nodes=self.cluster.storage_nodes, + endpoint=self.cluster.default_http_gate_endpoint, + ) @allure.link( "https://github.com/nspcc-dev/neofs-http-gw#by-attributes", name="download by attributes" @@ -169,7 +180,7 @@ class TestHttpGate(ClusterTestBase): file_path = generate_file(simple_object_size) with allure.step("Put objects using HTTP with attribute"): - headers = self._attr_into_header(attributes) + headers = attr_into_header(attributes) oid = upload_via_http_gate( cid=cid, path=file_path, @@ -177,9 +188,13 @@ class TestHttpGate(ClusterTestBase): endpoint=self.cluster.default_http_gate_endpoint, ) - sleep(OBJECT_UPLOAD_DELAY) - - self.get_object_by_attr_and_verify_hashes(oid, file_path, cid, attributes) + get_object_by_attr_and_verify_hashes( + oid=oid, + file_name=file_path, + cid=cid, + attrs=attributes, + endpoint=self.cluster.default_http_gate_endpoint, + ) @allure.title("Test Expiration-Epoch in HTTP header") def test_expiration_epoch_in_http(self, simple_object_size): @@ -222,8 +237,11 @@ class TestHttpGate(ClusterTestBase): wait_for_gc_pass_on_storage_nodes() for oid in expired_objects: - self.try_to_get_object_and_expect_error( - cid=cid, oid=oid, error_pattern=OBJECT_NOT_FOUND_ERROR + try_to_get_object_and_expect_error( + cid=cid, + oid=oid, + error_pattern=OBJECT_NOT_FOUND_ERROR, + endpoint=self.cluster.default_http_gate_endpoint, ) with allure.step("Other objects can be get"): @@ -260,8 +278,6 @@ class TestHttpGate(ClusterTestBase): endpoint=self.cluster.default_http_gate_endpoint, ) - sleep(OBJECT_UPLOAD_DELAY) - dir_path = get_via_zip_http_gate( cid=cid, prefix=common_prefix, endpoint=self.cluster.default_http_gate_endpoint ) @@ -299,12 +315,23 @@ class TestHttpGate(ClusterTestBase): endpoint=self.cluster.default_http_gate_endpoint, ) - self.get_object_and_verify_hashes(oid_gate, file_path, self.wallet, cid) - self.get_object_and_verify_hashes( - oid_curl, - file_path, - self.wallet, - cid, + get_object_and_verify_hashes( + oid=oid_gate, + file_name=file_path, + wallet=self.wallet, + cid=cid, + shell=self.shell, + nodes=self.cluster.storage_nodes, + endpoint=self.cluster.default_http_gate_endpoint, + ) + get_object_and_verify_hashes( + oid=oid_curl, + file_name=file_path, + wallet=self.wallet, + cid=cid, + shell=self.shell, + nodes=self.cluster.storage_nodes, + endpoint=self.cluster.default_http_gate_endpoint, object_getter=get_via_http_curl, ) @@ -333,76 +360,13 @@ class TestHttpGate(ClusterTestBase): ) for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): - self.get_object_and_verify_hashes( - oid, - file_path, - self.wallet, - cid, + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet, + cid=cid, + shell=self.shell, + nodes=self.cluster.storage_nodes, + endpoint=self.cluster.default_http_gate_endpoint, object_getter=get_via_http_curl, ) - - @allure.step("Try to get object and expect error") - def try_to_get_object_and_expect_error(self, cid: str, oid: str, error_pattern: str) -> None: - try: - get_via_http_gate(cid=cid, oid=oid, endpoint=self.cluster.default_http_gate_endpoint) - raise AssertionError(f"Expected error on getting object with cid: {cid}") - except Exception as err: - match = error_pattern.casefold() in str(err).casefold() - assert match, f"Expected {err} to match {error_pattern}" - - @allure.step("Verify object can be get using HTTP header attribute") - def get_object_by_attr_and_verify_hashes( - self, oid: str, file_name: str, cid: str, attrs: dict - ) -> None: - got_file_path_http = get_via_http_gate( - cid=cid, oid=oid, endpoint=self.cluster.default_http_gate_endpoint - ) - got_file_path_http_attr = get_via_http_gate_by_attribute( - cid=cid, attribute=attrs, endpoint=self.cluster.default_http_gate_endpoint - ) - - TestHttpGate._assert_hashes_are_equal( - file_name, got_file_path_http, got_file_path_http_attr - ) - - @allure.step("Verify object can be get using HTTP") - def get_object_and_verify_hashes( - self, oid: str, file_name: str, wallet: str, cid: str, object_getter=None - ) -> None: - nodes = get_nodes_without_object( - wallet=wallet, - cid=cid, - oid=oid, - shell=self.shell, - nodes=self.cluster.storage_nodes, - ) - random_node = random.choice(nodes) - object_getter = object_getter or get_via_http_gate - - got_file_path = get_object( - wallet=wallet, - cid=cid, - oid=oid, - shell=self.shell, - endpoint=random_node.get_rpc_endpoint(), - ) - got_file_path_http = object_getter( - cid=cid, oid=oid, endpoint=self.cluster.default_http_gate_endpoint - ) - - TestHttpGate._assert_hashes_are_equal(file_name, got_file_path, got_file_path_http) - - @staticmethod - def _assert_hashes_are_equal(orig_file_name: str, got_file_1: str, got_file_2: str) -> None: - msg = "Expected hashes are equal for files {f1} and {f2}" - got_file_hash_http = get_file_hash(got_file_1) - assert get_file_hash(got_file_2) == got_file_hash_http, msg.format( - f1=got_file_2, f2=got_file_1 - ) - assert get_file_hash(orig_file_name) == got_file_hash_http, msg.format( - f1=orig_file_name, f2=got_file_1 - ) - - @staticmethod - def _attr_into_header(attrs: dict) -> dict: - return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()} diff --git a/pytest_tests/testsuites/services/http_gate/test_http_object.py b/pytest_tests/testsuites/services/http_gate/test_http_object.py new file mode 100644 index 00000000..468e7cd4 --- /dev/null +++ b/pytest_tests/testsuites/services/http_gate/test_http_object.py @@ -0,0 +1,121 @@ +import logging + +import allure +import pytest +from container import create_container +from file_helper import generate_file +from python_keywords.http_gate import ( + get_object_and_verify_hashes, + get_object_by_attr_and_verify_hashes, + try_to_get_object_via_passed_request_and_expect_error, +) +from python_keywords.neofs_verbs import put_object_to_random_node +from wellknown_acl import PUBLIC_ACL + +from steps.cluster_test_base import ClusterTestBase + +logger = logging.getLogger("NeoLogger") + + +@pytest.mark.sanity +@pytest.mark.http_gate +class Test_http_object(ClusterTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_object.wallet = default_wallet + + @allure.title("Test Put over gRPC, Get over HTTP") + def test_object_put_get_attributes(self): + """ + Test that object can be put using gRPC interface and get using HTTP. + + Steps: + 1. Create object; + 2. Put objects using gRPC (neofs-cli) with attributes [--attributes chapter1=peace,chapter2=war]; + 3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading); + 4. Compare hashes between original and downloaded object; + 5. [Negative] Try to the get object with specified attributes and `get` request: [get/$CID/chapter1/peace]; + 6. Download the object with specified attributes and `get_by_attribute` request: [get_by_attribute/$CID/chapter1/peace]; + 7. Compare hashes between original and downloaded object; + 8. [Negative] Try to the get object via `get_by_attribute` request: [get_by_attribute/$CID/$OID]; + + + Expected result: + Hashes must be the same. + """ + with allure.step("Create public container"): + cid = create_container( + self.wallet, + shell=self.shell, + endpoint=self.cluster.default_rpc_endpoint, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + + # Generate file + file_path = generate_file() + + # List of Key=Value attributes + obj_key1 = "chapter1" + obj_value1 = "peace" + obj_key2 = "chapter2" + obj_value2 = "war" + + # Prepare for grpc PUT request + key_value1 = obj_key1 + "=" + obj_value1 + key_value2 = obj_key2 + "=" + obj_value2 + + with allure.step("Put objects using gRPC [--attributes chapter1=peace,chapter2=war]"): + oid = put_object_to_random_node( + wallet=self.wallet, + path=file_path, + cid=cid, + shell=self.shell, + cluster=self.cluster, + attributes=f"{key_value1},{key_value2}", + ) + with allure.step("Get object and verify hashes [ get/$CID/$OID ]"): + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet, + cid=cid, + shell=self.shell, + nodes=self.cluster.storage_nodes, + endpoint=self.cluster.default_http_gate_endpoint, + ) + with allure.step("[Negative] try to get object: [get/$CID/chapter1/peace]"): + attrs = {obj_key1: obj_value1, obj_key2: obj_value2} + request = f"/get/{cid}/{obj_key1}/{obj_value1}" + expected_err_msg = "Failed to get object via HTTP gate:" + try_to_get_object_via_passed_request_and_expect_error( + cid=cid, + oid=oid, + error_pattern=expected_err_msg, + http_request_path=request, + attrs=attrs, + endpoint=self.cluster.default_http_gate_endpoint, + ) + + with allure.step( + "Download the object with attribute [get_by_attribute/$CID/chapter1/peace]" + ): + get_object_by_attr_and_verify_hashes( + oid=oid, + file_name=file_path, + cid=cid, + attrs=attrs, + endpoint=self.cluster.default_http_gate_endpoint, + ) + with allure.step("[Negative] try to get object: get_by_attribute/$CID/$OID"): + request = f"/get_by_attribute/{cid}/{oid}" + try_to_get_object_via_passed_request_and_expect_error( + cid=cid, + oid=oid, + error_pattern=expected_err_msg, + http_request_path=request, + endpoint=self.cluster.default_http_gate_endpoint, + ) diff --git a/robot/resources/lib/python_keywords/http_gate.py b/robot/resources/lib/python_keywords/http_gate.py index f376afef..1a81cd81 100644 --- a/robot/resources/lib/python_keywords/http_gate.py +++ b/robot/resources/lib/python_keywords/http_gate.py @@ -1,14 +1,23 @@ import logging import os +import random import re import shutil import uuid import zipfile +from typing import Optional from urllib.parse import quote_plus import allure import requests from cli_helpers import _cmd_run +from cluster import StorageNode +from file_helper import get_file_hash +from neofs_testlib.shell import Shell +from python_keywords.neofs_verbs import get_object +from python_keywords.storage_policy import get_nodes_without_object + +from pytest_tests.steps.cluster_test_base import ClusterTestBase logger = logging.getLogger("NeoLogger") @@ -16,14 +25,21 @@ ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/") @allure.step("Get via HTTP Gate") -def get_via_http_gate(cid: str, oid: str, endpoint: str): +def get_via_http_gate(cid: str, oid: str, endpoint: str, request_path: Optional[str] = None): """ This function gets given object from HTTP gate - cid: container id to get object from - oid: object ID - endpoint: http gate endpoint + cid: container id to get object from + oid: object ID + endpoint: http gate endpoint + request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}] """ - request = f"{endpoint}/get/{cid}/{oid}" + + # if `request_path` parameter ommited, use default + if request_path is None: + request = f"{endpoint}/get/{cid}/{oid}" + else: + request = f"{endpoint}{request_path}" + resp = requests.get(request, stream=True) if not resp.ok: @@ -76,16 +92,24 @@ def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str): @allure.step("Get via HTTP Gate by attribute") -def get_via_http_gate_by_attribute(cid: str, attribute: dict, endpoint: str): +def get_via_http_gate_by_attribute( + cid: str, attribute: dict, endpoint: str, request_path: Optional[str] = None +): """ This function gets given object from HTTP gate - cid: CID to get object from - attribute: attribute {name: attribute} value pair - endpoint: http gate endpoint + cid: CID to get object from + attribute: attribute {name: attribute} value pair + endpoint: http gate endpoint + request_path: (optional) http request path, if ommited - use default [{endpoint}/get_by_attribute/{Key}/{Value}] """ attr_name = list(attribute.keys())[0] attr_value = quote_plus(str(attribute.get(attr_name))) - request = f"{endpoint}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}" + # if `request_path` parameter ommited, use default + if request_path is None: + request = f"{endpoint}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}" + else: + request = f"{endpoint}{request_path}" + resp = requests.get(request, stream=True) if not resp.ok: @@ -180,3 +204,96 @@ def _attach_allure_step(request: str, status_code: int, req_type="GET"): command_attachment = f"REQUEST: '{request}'\n" f"RESPONSE:\n {status_code}\n" with allure.step(f"{req_type} Request"): allure.attach(command_attachment, f"{req_type} Request", allure.attachment_type.TEXT) + + +@allure.step("Try to get object and expect error") +def try_to_get_object_and_expect_error( + cid: str, oid: str, error_pattern: str, endpoint: str +) -> None: + try: + get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint) + raise AssertionError(f"Expected error on getting object with cid: {cid}") + except Exception as err: + match = error_pattern.casefold() in str(err).casefold() + assert match, f"Expected {err} to match {error_pattern}" + + +@allure.step("Verify object can be get using HTTP header attribute") +def get_object_by_attr_and_verify_hashes( + oid: str, file_name: str, cid: str, attrs: dict, endpoint: str +) -> None: + got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint) + got_file_path_http_attr = get_via_http_gate_by_attribute( + cid=cid, attribute=attrs, endpoint=endpoint + ) + + assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr) + + +def get_object_and_verify_hashes( + oid: str, + file_name: str, + wallet: str, + cid: str, + shell: Shell, + nodes: list[StorageNode], + endpoint: str, + object_getter=None, +) -> None: + nodes = get_nodes_without_object( + wallet=wallet, + cid=cid, + oid=oid, + shell=shell, + nodes=nodes, + ) + random_node = random.choice(nodes) + object_getter = object_getter or get_via_http_gate + + got_file_path = get_object( + wallet=wallet, + cid=cid, + oid=oid, + shell=shell, + endpoint=random_node.get_rpc_endpoint(), + ) + got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint) + + assert_hashes_are_equal(file_name, got_file_path, got_file_path_http) + + +def assert_hashes_are_equal(orig_file_name: str, got_file_1: str, got_file_2: str) -> None: + msg = "Expected hashes are equal for files {f1} and {f2}" + got_file_hash_http = get_file_hash(got_file_1) + assert get_file_hash(got_file_2) == got_file_hash_http, msg.format(f1=got_file_2, f2=got_file_1) + assert get_file_hash(orig_file_name) == got_file_hash_http, msg.format( + f1=orig_file_name, f2=got_file_1 + ) + + +def attr_into_header(attrs: dict) -> dict: + return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()} + + +@allure.step( + "Try to get object via http (pass http_request and optional attributes) and expect error" +) +def try_to_get_object_via_passed_request_and_expect_error( + cid: str, + oid: str, + error_pattern: str, + endpoint: str, + http_request_path: str, + attrs: dict = None, +) -> None: + try: + if attrs is None: + get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path) + else: + get_via_http_gate_by_attribute( + cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path + ) + raise AssertionError(f"Expected error on getting object with cid: {cid}") + except Exception as err: + match = error_pattern.casefold() in str(err).casefold() + assert match, f"Expected {err} to match {error_pattern}"