From 5c4f6b6a7dce9f49ddde8a7b19960119d48c9848 Mon Sep 17 00:00:00 2001 From: Vladislav Karakozov Date: Mon, 19 Dec 2022 20:45:26 +0300 Subject: [PATCH] new http test Signed-off-by: Vladislav Karakozov --- .../services/http_gate/test_http_gate.py | 5 +- .../services/http_gate/test_http_headers.py | 229 ++++++++++++++++++ .../services/http_gate/test_http_object.py | 12 +- .../services/http_gate/test_http_streaming.py | 70 ++++++ .../lib/python_keywords/http_gate.py | 70 +++++- 5 files changed, 373 insertions(+), 13 deletions(-) create mode 100644 pytest_tests/testsuites/services/http_gate/test_http_headers.py create mode 100644 pytest_tests/testsuites/services/http_gate/test_http_streaming.py diff --git a/pytest_tests/testsuites/services/http_gate/test_http_gate.py b/pytest_tests/testsuites/services/http_gate/test_http_gate.py index c7a3cf54..58429a81 100644 --- a/pytest_tests/testsuites/services/http_gate/test_http_gate.py +++ b/pytest_tests/testsuites/services/http_gate/test_http_gate.py @@ -311,7 +311,6 @@ class TestHttpGate(ClusterTestBase): oid_curl = upload_via_http_gate_curl( cid=cid, filepath=file_path, - large_object=True, endpoint=self.cluster.default_http_gate_endpoint, ) @@ -356,7 +355,9 @@ class TestHttpGate(ClusterTestBase): cid=cid, filepath=file_path_simple, endpoint=self.cluster.default_http_gate_endpoint ) oid_large = upload_via_http_gate_curl( - cid=cid, filepath=file_path_large, endpoint=self.cluster.default_http_gate_endpoint + cid=cid, + filepath=file_path_large, + endpoint=self.cluster.default_http_gate_endpoint, ) for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): diff --git a/pytest_tests/testsuites/services/http_gate/test_http_headers.py b/pytest_tests/testsuites/services/http_gate/test_http_headers.py new file mode 100644 index 00000000..c450a7ac --- /dev/null +++ b/pytest_tests/testsuites/services/http_gate/test_http_headers.py @@ -0,0 +1,229 @@ +import logging +import os + +import allure +import pytest +from container import ( + create_container, + delete_container, + list_containers, + wait_for_container_deletion, +) +from epoch import tick_epoch +from file_helper import generate_file +from http_gate import ( + attr_into_str_header_curl, + get_object_by_attr_and_verify_hashes, + try_to_get_object_and_expect_error, + try_to_get_object_via_passed_request_and_expect_error, + upload_via_http_gate_curl, +) +from pytest import FixtureRequest +from python_keywords.neofs_verbs import delete_object +from wellknown_acl import PUBLIC_ACL + +from helpers.storage_object_info import StorageObjectInfo +from steps.cluster_test_base import ClusterTestBase + +OBJECT_ALREADY_REMOVED_ERROR = "object already removed" +logger = logging.getLogger("NeoLogger") + + +@pytest.mark.sanity +@pytest.mark.http_gate +class Test_http_headers(ClusterTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" + obj1_keys = ["Writer", "Chapter1", "Chapter2"] + obj2_keys = ["Writer", "Ch@pter1", "chapter2"] + values = ["Leo Tolstoy", "peace", "w@r"] + OBJECT_ATTRIBUTES = [ + {obj1_keys[0]: values[0], obj1_keys[1]: values[1], obj1_keys[2]: values[2]}, + {obj2_keys[0]: values[0], obj2_keys[1]: values[1], obj2_keys[2]: values[2]}, + ] + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_headers.wallet = default_wallet + + @pytest.fixture( + params=[ + pytest.lazy_fixture("simple_object_size"), + pytest.lazy_fixture("complex_object_size"), + ], + ids=["simple object", "complex object"], + scope="class", + ) + def storage_objects_with_attributes(self, request: FixtureRequest) -> list[StorageObjectInfo]: + storage_objects = [] + wallet = self.wallet + cid = create_container( + wallet=self.wallet, + shell=self.shell, + endpoint=self.cluster.default_rpc_endpoint, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + file_path = generate_file(request.param) + for attributes in self.OBJECT_ATTRIBUTES: + storage_object_id = upload_via_http_gate_curl( + cid=cid, + filepath=file_path, + endpoint=self.cluster.default_http_gate_endpoint, + headers=attr_into_str_header_curl(attributes), + ) + storage_object = StorageObjectInfo(cid, storage_object_id) + storage_object.size = os.path.getsize(file_path) + storage_object.wallet_file_path = wallet + storage_object.file_path = file_path + storage_object.attributes = attributes + + storage_objects.append(storage_object) + + yield storage_objects + + @allure.title("Get object1 by attribute") + def test_object1_can_be_get_by_attr( + self, storage_objects_with_attributes: list[StorageObjectInfo] + ): + """ + Test to get object#1 by attribute and comapre hashes + + Steps: + 1. Download object#1 with attributes [Chapter2=w@r] and compare hashes + """ + + storage_object_1 = storage_objects_with_attributes[0] + + with allure.step( + f'Download object#1 via wget with attributes Chapter2: {storage_object_1.attributes["Chapter2"]} and compare hashes' + ): + get_object_by_attr_and_verify_hashes( + oid=storage_object_1.oid, + file_name=storage_object_1.file_path, + cid=storage_object_1.cid, + attrs={"Chapter2": storage_object_1.attributes["Chapter2"]}, + endpoint=self.cluster.default_http_gate_endpoint, + ) + + @allure.title("Test get object2 with different attributes, then delete object2 and get object1") + def test_object2_can_be_get_by_attr( + self, storage_objects_with_attributes: list[StorageObjectInfo] + ): + """ + Test to get object2 with different attributes, then delete object2 and get object1 using 1st attribute. Note: obj1 and obj2 have the same attribute#1, + and when obj2 is deleted you can get obj1 by 1st attribute + + Steps: + 1. Download object#2 with attributes [chapter2=w@r] and compare hashes + 2. Download object#2 with attributes [Ch@pter1=peace] and compare hashes + 3. Delete object#2 + 4. Download object#1 with attributes [Writer=Leo Tolstoy] and compare hashes + """ + storage_object_1 = storage_objects_with_attributes[0] + storage_object_2 = storage_objects_with_attributes[1] + + with allure.step( + f'Download object#2 via wget with attributes [chapter2={storage_object_2.attributes["chapter2"]}] / [Ch@pter1={storage_object_2.attributes["Ch@pter1"]}] and compare hashes' + ): + selected_attributes_object2 = [ + {"chapter2": storage_object_2.attributes["chapter2"]}, + {"Ch@pter1": storage_object_2.attributes["Ch@pter1"]}, + ] + for attributes in selected_attributes_object2: + get_object_by_attr_and_verify_hashes( + oid=storage_object_2.oid, + file_name=storage_object_2.file_path, + cid=storage_object_2.cid, + attrs=attributes, + endpoint=self.cluster.default_http_gate_endpoint, + ) + with allure.step("Delete object#2 and verify is the container deleted"): + delete_object( + wallet=self.wallet, + cid=storage_object_2.cid, + oid=storage_object_2.oid, + shell=self.shell, + endpoint=self.cluster.default_rpc_endpoint, + ) + try_to_get_object_and_expect_error( + cid=storage_object_2.cid, + oid=storage_object_2.oid, + error_pattern=OBJECT_ALREADY_REMOVED_ERROR, + endpoint=self.cluster.default_http_gate_endpoint, + ) + storage_objects_with_attributes.remove(storage_object_2) + + with allure.step( + f'Download object#1 with attributes [Writer={storage_object_1.attributes["Writer"]}] and compare hashes' + ): + key_value_pair = {"Writer": storage_object_1.attributes["Writer"]} + get_object_by_attr_and_verify_hashes( + oid=storage_object_1.oid, + file_name=storage_object_1.file_path, + cid=storage_object_1.cid, + attrs=key_value_pair, + endpoint=self.cluster.default_http_gate_endpoint, + ) + + @allure.title("[Negative] Try to put object and get right after container is deleted") + def test_negative_put_and_get_object3( + self, storage_objects_with_attributes: list[StorageObjectInfo] + ): + """ + Test to attempt to put object and try to download it right after the container has been deleted + + Steps: + 1. [Negative] Allocate and attempt to put object#3 via http with attributes: [Writer=Leo Tolstoy, Writer=peace, peace=peace] + Expected: "Error duplication of attributes detected" + 2. Delete container + 3. [Negative] Try to download object with attributes [peace=peace] + Expected: "HTTP request sent, awaiting response... 404 Not Found" + """ + storage_object_1 = storage_objects_with_attributes[0] + + with allure.step( + "[Negative] Allocate and attemt to put object#3 via http with attributes: [Writer=Leo Tolstoy, Writer=peace, peace=peace]" + ): + file_path_3 = generate_file(storage_object_1.size) + attrs_obj3 = {"Writer": "Leo Tolstoy", "peace": "peace"} + headers = attr_into_str_header_curl(attrs_obj3) + headers.append(" ".join(attr_into_str_header_curl({"Writer": "peace"}))) + error_pattern = f"key duplication error: X-Attribute-Writer" + upload_via_http_gate_curl( + cid=storage_object_1.cid, + filepath=file_path_3, + endpoint=self.cluster.default_http_gate_endpoint, + headers=headers, + error_pattern=error_pattern, + ) + with allure.step("Delete container and verify container deletion"): + delete_container( + wallet=self.wallet, + cid=storage_object_1.cid, + shell=self.shell, + endpoint=self.cluster.default_rpc_endpoint, + ) + tick_epoch(self.shell, self.cluster) + wait_for_container_deletion( + self.wallet, + storage_object_1.cid, + shell=self.shell, + endpoint=self.cluster.default_rpc_endpoint, + ) + assert storage_object_1.cid not in list_containers( + self.wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint + ) + with allure.step( + "[Negative] Try to download (wget) object via wget with attributes [peace=peace]" + ): + request = f"/get/{storage_object_1.cid}/peace/peace" + error_pattern = "404 Not Found" + try_to_get_object_via_passed_request_and_expect_error( + cid=storage_object_1.cid, + oid="", + error_pattern=error_pattern, + attrs=attrs_obj3, + http_request_path=request, + endpoint=self.cluster.default_http_gate_endpoint, + ) diff --git a/pytest_tests/testsuites/services/http_gate/test_http_object.py b/pytest_tests/testsuites/services/http_gate/test_http_object.py index ee312048..50ad72ac 100644 --- a/pytest_tests/testsuites/services/http_gate/test_http_object.py +++ b/pytest_tests/testsuites/services/http_gate/test_http_object.py @@ -1,10 +1,11 @@ import logging +import os import allure import pytest from container import create_container from file_helper import generate_file -from python_keywords.http_gate import ( +from http_gate import ( get_object_and_verify_hashes, get_object_by_attr_and_verify_hashes, try_to_get_object_via_passed_request_and_expect_error, @@ -28,7 +29,12 @@ class Test_http_object(ClusterTestBase): Test_http_object.wallet = default_wallet @allure.title("Test Put over gRPC, Get over HTTP") - def test_object_put_get_attributes(self, simple_object_size): + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], + ) + def test_object_put_get_attributes(self, object_size: int): """ Test that object can be put using gRPC interface and get using HTTP. @@ -56,7 +62,7 @@ class Test_http_object(ClusterTestBase): ) # Generate file - file_path = generate_file(simple_object_size) + file_path = generate_file(object_size) # List of Key=Value attributes obj_key1 = "chapter1" diff --git a/pytest_tests/testsuites/services/http_gate/test_http_streaming.py b/pytest_tests/testsuites/services/http_gate/test_http_streaming.py new file mode 100644 index 00000000..851b2bf9 --- /dev/null +++ b/pytest_tests/testsuites/services/http_gate/test_http_streaming.py @@ -0,0 +1,70 @@ +import logging + +import allure +import pytest +from container import create_container +from file_helper import generate_file +from http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl +from wellknown_acl import PUBLIC_ACL + +from steps.cluster_test_base import ClusterTestBase + +logger = logging.getLogger("NeoLogger") + + +@pytest.mark.sanity +@pytest.mark.http_gate +class Test_http_streaming(ClusterTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X" + + @pytest.fixture(scope="class", autouse=True) + @allure.title("[Class/Autouse]: Prepare wallet and deposit") + def prepare_wallet(self, default_wallet): + Test_http_streaming.wallet = default_wallet + + @allure.title("Test Put via pipe (steaming), Get over HTTP and verify hashes") + @pytest.mark.parametrize( + "object_size", + [pytest.lazy_fixture("complex_object_size")], + ids=["complex object"], + ) + def test_object_can_be_put_get_by_streaming(self, object_size: int): + """ + Test that object can be put using gRPC interface and get using HTTP. + + Steps: + 1. Create big object; + 2. Put object using curl with pipe (streaming); + 3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading); + 4. Compare hashes between original and downloaded object; + + Expected result: + Hashes must be the same. + """ + with allure.step("Create public container and verify container creation"): + cid = create_container( + self.wallet, + shell=self.shell, + endpoint=self.cluster.default_rpc_endpoint, + rule=self.PLACEMENT_RULE, + basic_acl=PUBLIC_ACL, + ) + with allure.step("Allocate big object"): + # Generate file + file_path = generate_file(object_size) + + with allure.step( + "Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]" + ): + oid = upload_via_http_gate_curl( + cid=cid, filepath=file_path, endpoint=self.cluster.default_http_gate_endpoint + ) + get_object_and_verify_hashes( + oid=oid, + file_name=file_path, + wallet=self.wallet, + cid=cid, + shell=self.shell, + nodes=self.cluster.storage_nodes, + endpoint=self.cluster.default_http_gate_endpoint, + ) diff --git a/robot/resources/lib/python_keywords/http_gate.py b/robot/resources/lib/python_keywords/http_gate.py index 1a81cd81..86f55e4f 100644 --- a/robot/resources/lib/python_keywords/http_gate.py +++ b/robot/resources/lib/python_keywords/http_gate.py @@ -10,8 +10,10 @@ from urllib.parse import quote_plus import allure import requests +from aws_cli_client import LONG_TIMEOUT from cli_helpers import _cmd_run from cluster import StorageNode +from common import SIMPLE_OBJECT_SIZE from file_helper import get_file_hash from neofs_testlib.shell import Shell from python_keywords.neofs_verbs import get_object @@ -159,9 +161,27 @@ def upload_via_http_gate(cid: str, path: str, endpoint: str, headers: dict = Non return resp.json().get("object_id") +@allure.step("Check is the passed object large") +def is_object_large(filepath: str) -> bool: + """ + This function check passed file size and return True if file_size > SIMPLE_OBJECT_SIZE + filepath: File path to check + """ + file_size = os.path.getsize(filepath) + logger.info(f"Size= {file_size}") + if file_size > int(SIMPLE_OBJECT_SIZE): + return True + else: + return False + + @allure.step("Upload via HTTP Gate using Curl") def upload_via_http_gate_curl( - cid: str, filepath: str, endpoint: str, large_object=False, headers: dict = None + cid: str, + filepath: str, + endpoint: str, + headers: list = None, + error_pattern: Optional[str] = None, ) -> str: """ This function upload given object through HTTP gate using curl utility. @@ -169,14 +189,33 @@ def upload_via_http_gate_curl( filepath: File path to upload headers: Object header endpoint: http gate endpoint + error_pattern: [optional] expected error message from the command """ request = f"{endpoint}/upload/{cid}" - files = f"file=@{filepath};filename={os.path.basename(filepath)}" - cmd = f"curl -F '{files}' {request}" + attributes = "" + if headers: + # parse attributes + attributes = " ".join(headers) + + large_object = is_object_large(filepath) if large_object: + # pre-clean + _cmd_run("rm pipe -f") files = f"file=@pipe;filename={os.path.basename(filepath)}" - cmd = f"mkfifo pipe;cat {filepath} > pipe & curl --no-buffer -F '{files}' {request}" - output = _cmd_run(cmd) + cmd = f"mkfifo pipe;cat {filepath} > pipe & curl --no-buffer -F '{files}' {attributes} {request}" + output = _cmd_run(cmd, LONG_TIMEOUT) + # clean up pipe + _cmd_run("rm pipe") + else: + files = f"file=@{filepath};filename={os.path.basename(filepath)}" + cmd = f"curl -F '{files}' {attributes} {request}" + output = _cmd_run(cmd) + + if error_pattern: + match = error_pattern.casefold() in str(output).casefold() + assert match, f"Expected {output} to match {error_pattern}" + return "" + oid_re = re.search(r'"object_id": "(.*)"', output) if not oid_re: raise AssertionError(f'Could not find "object_id" in {output}') @@ -226,7 +265,6 @@ def get_object_by_attr_and_verify_hashes( got_file_path_http_attr = get_via_http_gate_by_attribute( cid=cid, attribute=attrs, endpoint=endpoint ) - assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr) @@ -240,14 +278,19 @@ def get_object_and_verify_hashes( endpoint: str, object_getter=None, ) -> None: - nodes = get_nodes_without_object( + nodes_list = get_nodes_without_object( wallet=wallet, cid=cid, oid=oid, shell=shell, nodes=nodes, ) - random_node = random.choice(nodes) + # for some reason we can face with case when nodes_list is empty due to object resides in all nodes + if nodes_list is None: + random_node = random.choice(nodes) + else: + random_node = random.choice(nodes_list) + object_getter = object_getter or get_via_http_gate got_file_path = get_object( @@ -275,6 +318,17 @@ def attr_into_header(attrs: dict) -> dict: return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()} +@allure.step( + "Convert each attribute (Key=Value) to the following format: -H 'X-Attribute-Key: Value'" +) +def attr_into_str_header_curl(attrs: dict) -> list: + headers = [] + for k, v in attrs.items(): + headers.append(f"-H 'X-Attribute-{k}: {v}'") + logger.info(f"[List of Attrs for curl:] {headers}") + return headers + + @allure.step( "Try to get object via http (pass http_request and optional attributes) and expect error" )