new http test

Signed-off-by: Vladislav Karakozov <v.karakozov@yadro.com>

pytest_tests/testsuites/services/http_gate/test_http_gate.py

@@ -311,7 +311,6 @@ class TestHttpGate(ClusterTestBase):
             oid_curl = upload_via_http_gate_curl(
                 cid=cid,
                 filepath=file_path,
-                large_object=True,
                 endpoint=self.cluster.default_http_gate_endpoint,
             )
 
@@ -356,7 +355,9 @@ class TestHttpGate(ClusterTestBase):
                 cid=cid, filepath=file_path_simple, endpoint=self.cluster.default_http_gate_endpoint
             )
             oid_large = upload_via_http_gate_curl(
-                cid=cid, filepath=file_path_large, endpoint=self.cluster.default_http_gate_endpoint
+                cid=cid,
+                filepath=file_path_large,
+                endpoint=self.cluster.default_http_gate_endpoint,
             )
 
         for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):

pytest_tests/testsuites/services/http_gate/test_http_headers.py

@@ -0,0 +1,229 @@
+import logging
+import os
+
+import allure
+import pytest
+from container import (
+    create_container,
+    delete_container,
+    list_containers,
+    wait_for_container_deletion,
+)
+from epoch import tick_epoch
+from file_helper import generate_file
+from http_gate import (
+    attr_into_str_header_curl,
+    get_object_by_attr_and_verify_hashes,
+    try_to_get_object_and_expect_error,
+    try_to_get_object_via_passed_request_and_expect_error,
+    upload_via_http_gate_curl,
+)
+from pytest import FixtureRequest
+from python_keywords.neofs_verbs import delete_object
+from wellknown_acl import PUBLIC_ACL
+
+from helpers.storage_object_info import StorageObjectInfo
+from steps.cluster_test_base import ClusterTestBase
+
+OBJECT_ALREADY_REMOVED_ERROR = "object already removed"
+logger = logging.getLogger("NeoLogger")
+
+
+@pytest.mark.sanity
+@pytest.mark.http_gate
+class Test_http_headers(ClusterTestBase):
+    PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
+    obj1_keys = ["Writer", "Chapter1", "Chapter2"]
+    obj2_keys = ["Writer", "Ch@pter1", "chapter2"]
+    values = ["Leo Tolstoy", "peace", "w@r"]
+    OBJECT_ATTRIBUTES = [
+        {obj1_keys[0]: values[0], obj1_keys[1]: values[1], obj1_keys[2]: values[2]},
+        {obj2_keys[0]: values[0], obj2_keys[1]: values[1], obj2_keys[2]: values[2]},
+    ]
+
+    @pytest.fixture(scope="class", autouse=True)
+    @allure.title("[Class/Autouse]: Prepare wallet and deposit")
+    def prepare_wallet(self, default_wallet):
+        Test_http_headers.wallet = default_wallet
+
+    @pytest.fixture(
+        params=[
+            pytest.lazy_fixture("simple_object_size"),
+            pytest.lazy_fixture("complex_object_size"),
+        ],
+        ids=["simple object", "complex object"],
+        scope="class",
+    )
+    def storage_objects_with_attributes(self, request: FixtureRequest) -> list[StorageObjectInfo]:
+        storage_objects = []
+        wallet = self.wallet
+        cid = create_container(
+            wallet=self.wallet,
+            shell=self.shell,
+            endpoint=self.cluster.default_rpc_endpoint,
+            rule=self.PLACEMENT_RULE,
+            basic_acl=PUBLIC_ACL,
+        )
+        file_path = generate_file(request.param)
+        for attributes in self.OBJECT_ATTRIBUTES:
+            storage_object_id = upload_via_http_gate_curl(
+                cid=cid,
+                filepath=file_path,
+                endpoint=self.cluster.default_http_gate_endpoint,
+                headers=attr_into_str_header_curl(attributes),
+            )
+            storage_object = StorageObjectInfo(cid, storage_object_id)
+            storage_object.size = os.path.getsize(file_path)
+            storage_object.wallet_file_path = wallet
+            storage_object.file_path = file_path
+            storage_object.attributes = attributes
+
+            storage_objects.append(storage_object)
+
+        yield storage_objects
+
+    @allure.title("Get object1 by attribute")
+    def test_object1_can_be_get_by_attr(
+        self, storage_objects_with_attributes: list[StorageObjectInfo]
+    ):
+        """
+        Test to get object#1 by attribute and comapre hashes
+
+        Steps:
+        1. Download object#1 with attributes [Chapter2=w@r] and compare hashes
+        """
+
+        storage_object_1 = storage_objects_with_attributes[0]
+
+        with allure.step(
+            f'Download object#1 via wget with attributes Chapter2: {storage_object_1.attributes["Chapter2"]} and compare hashes'
+        ):
+            get_object_by_attr_and_verify_hashes(
+                oid=storage_object_1.oid,
+                file_name=storage_object_1.file_path,
+                cid=storage_object_1.cid,
+                attrs={"Chapter2": storage_object_1.attributes["Chapter2"]},
+                endpoint=self.cluster.default_http_gate_endpoint,
+            )
+
+    @allure.title("Test get object2 with different attributes, then delete object2 and get object1")
+    def test_object2_can_be_get_by_attr(
+        self, storage_objects_with_attributes: list[StorageObjectInfo]
+    ):
+        """
+        Test to get object2 with different attributes, then delete object2 and get object1 using 1st attribute. Note: obj1 and obj2 have the same attribute#1,
+        and when obj2 is deleted you can get obj1 by 1st attribute
+
+        Steps:
+        1. Download object#2 with attributes [chapter2=w@r] and compare hashes
+        2. Download object#2 with attributes [Ch@pter1=peace] and compare hashes
+        3. Delete object#2
+        4. Download object#1 with attributes [Writer=Leo Tolstoy] and compare hashes
+        """
+        storage_object_1 = storage_objects_with_attributes[0]
+        storage_object_2 = storage_objects_with_attributes[1]
+
+        with allure.step(
+            f'Download object#2 via wget with attributes [chapter2={storage_object_2.attributes["chapter2"]}] / [Ch@pter1={storage_object_2.attributes["Ch@pter1"]}]  and compare hashes'
+        ):
+            selected_attributes_object2 = [
+                {"chapter2": storage_object_2.attributes["chapter2"]},
+                {"Ch@pter1": storage_object_2.attributes["Ch@pter1"]},
+            ]
+            for attributes in selected_attributes_object2:
+                get_object_by_attr_and_verify_hashes(
+                    oid=storage_object_2.oid,
+                    file_name=storage_object_2.file_path,
+                    cid=storage_object_2.cid,
+                    attrs=attributes,
+                    endpoint=self.cluster.default_http_gate_endpoint,
+                )
+        with allure.step("Delete object#2 and verify is the container deleted"):
+            delete_object(
+                wallet=self.wallet,
+                cid=storage_object_2.cid,
+                oid=storage_object_2.oid,
+                shell=self.shell,
+                endpoint=self.cluster.default_rpc_endpoint,
+            )
+            try_to_get_object_and_expect_error(
+                cid=storage_object_2.cid,
+                oid=storage_object_2.oid,
+                error_pattern=OBJECT_ALREADY_REMOVED_ERROR,
+                endpoint=self.cluster.default_http_gate_endpoint,
+            )
+            storage_objects_with_attributes.remove(storage_object_2)
+
+        with allure.step(
+            f'Download object#1 with attributes [Writer={storage_object_1.attributes["Writer"]}] and compare hashes'
+        ):
+            key_value_pair = {"Writer": storage_object_1.attributes["Writer"]}
+            get_object_by_attr_and_verify_hashes(
+                oid=storage_object_1.oid,
+                file_name=storage_object_1.file_path,
+                cid=storage_object_1.cid,
+                attrs=key_value_pair,
+                endpoint=self.cluster.default_http_gate_endpoint,
+            )
+
+    @allure.title("[Negative] Try to put object and get right after container is deleted")
+    def test_negative_put_and_get_object3(
+        self, storage_objects_with_attributes: list[StorageObjectInfo]
+    ):
+        """
+        Test to attempt to put object and try to download it right after the container has been deleted
+
+        Steps:
+        1. [Negative] Allocate and attempt to put object#3 via http with attributes: [Writer=Leo Tolstoy, Writer=peace, peace=peace]
+            Expected: "Error duplication of attributes detected"
+        2. Delete container
+        3. [Negative] Try to download object with attributes [peace=peace]
+            Expected: "HTTP request sent, awaiting response... 404 Not Found"
+        """
+        storage_object_1 = storage_objects_with_attributes[0]
+
+        with allure.step(
+            "[Negative] Allocate and attemt to put object#3 via http with attributes: [Writer=Leo Tolstoy, Writer=peace, peace=peace]"
+        ):
+            file_path_3 = generate_file(storage_object_1.size)
+            attrs_obj3 = {"Writer": "Leo Tolstoy", "peace": "peace"}
+            headers = attr_into_str_header_curl(attrs_obj3)
+            headers.append(" ".join(attr_into_str_header_curl({"Writer": "peace"})))
+            error_pattern = f"key duplication error: X-Attribute-Writer"
+            upload_via_http_gate_curl(
+                cid=storage_object_1.cid,
+                filepath=file_path_3,
+                endpoint=self.cluster.default_http_gate_endpoint,
+                headers=headers,
+                error_pattern=error_pattern,
+            )
+        with allure.step("Delete container and verify container deletion"):
+            delete_container(
+                wallet=self.wallet,
+                cid=storage_object_1.cid,
+                shell=self.shell,
+                endpoint=self.cluster.default_rpc_endpoint,
+            )
+            tick_epoch(self.shell, self.cluster)
+            wait_for_container_deletion(
+                self.wallet,
+                storage_object_1.cid,
+                shell=self.shell,
+                endpoint=self.cluster.default_rpc_endpoint,
+            )
+            assert storage_object_1.cid not in list_containers(
+                self.wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
+            )
+        with allure.step(
+            "[Negative] Try to download (wget) object via wget with attributes [peace=peace]"
+        ):
+            request = f"/get/{storage_object_1.cid}/peace/peace"
+            error_pattern = "404 Not Found"
+            try_to_get_object_via_passed_request_and_expect_error(
+                cid=storage_object_1.cid,
+                oid="",
+                error_pattern=error_pattern,
+                attrs=attrs_obj3,
+                http_request_path=request,
+                endpoint=self.cluster.default_http_gate_endpoint,
+            )

pytest_tests/testsuites/services/http_gate/test_http_object.py

@@ -1,10 +1,11 @@
 import logging
+import os
 
 import allure
 import pytest
 from container import create_container
 from file_helper import generate_file
-from python_keywords.http_gate import (
+from http_gate import (
     get_object_and_verify_hashes,
     get_object_by_attr_and_verify_hashes,
     try_to_get_object_via_passed_request_and_expect_error,
@@ -28,7 +29,12 @@ class Test_http_object(ClusterTestBase):
         Test_http_object.wallet = default_wallet
 
     @allure.title("Test Put over gRPC, Get over HTTP")
-    def test_object_put_get_attributes(self, simple_object_size):
+    @pytest.mark.parametrize(
+        "object_size",
+        [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
+        ids=["simple object", "complex object"],
+    )
+    def test_object_put_get_attributes(self, object_size: int):
         """
         Test that object can be put using gRPC interface and get using HTTP.
 
@@ -56,7 +62,7 @@ class Test_http_object(ClusterTestBase):
             )
 
         # Generate file
-        file_path = generate_file(simple_object_size)
+        file_path = generate_file(object_size)
 
         # List of Key=Value attributes
         obj_key1 = "chapter1"

pytest_tests/testsuites/services/http_gate/test_http_streaming.py

@@ -0,0 +1,70 @@
+import logging
+
+import allure
+import pytest
+from container import create_container
+from file_helper import generate_file
+from http_gate import get_object_and_verify_hashes, upload_via_http_gate_curl
+from wellknown_acl import PUBLIC_ACL
+
+from steps.cluster_test_base import ClusterTestBase
+
+logger = logging.getLogger("NeoLogger")
+
+
+@pytest.mark.sanity
+@pytest.mark.http_gate
+class Test_http_streaming(ClusterTestBase):
+    PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
+
+    @pytest.fixture(scope="class", autouse=True)
+    @allure.title("[Class/Autouse]: Prepare wallet and deposit")
+    def prepare_wallet(self, default_wallet):
+        Test_http_streaming.wallet = default_wallet
+
+    @allure.title("Test Put via pipe (steaming), Get over HTTP and verify hashes")
+    @pytest.mark.parametrize(
+        "object_size",
+        [pytest.lazy_fixture("complex_object_size")],
+        ids=["complex object"],
+    )
+    def test_object_can_be_put_get_by_streaming(self, object_size: int):
+        """
+        Test that object can be put using gRPC interface and get using HTTP.
+
+        Steps:
+        1. Create big object;
+        2. Put object using curl with pipe (streaming);
+        3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading);
+        4. Compare hashes between original and downloaded object;
+
+        Expected result:
+        Hashes must be the same.
+        """
+        with allure.step("Create public container and verify container creation"):
+            cid = create_container(
+                self.wallet,
+                shell=self.shell,
+                endpoint=self.cluster.default_rpc_endpoint,
+                rule=self.PLACEMENT_RULE,
+                basic_acl=PUBLIC_ACL,
+            )
+        with allure.step("Allocate big object"):
+            # Generate file
+            file_path = generate_file(object_size)
+
+        with allure.step(
+            "Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]"
+        ):
+            oid = upload_via_http_gate_curl(
+                cid=cid, filepath=file_path, endpoint=self.cluster.default_http_gate_endpoint
+            )
+            get_object_and_verify_hashes(
+                oid=oid,
+                file_name=file_path,
+                wallet=self.wallet,
+                cid=cid,
+                shell=self.shell,
+                nodes=self.cluster.storage_nodes,
+                endpoint=self.cluster.default_http_gate_endpoint,
+            )

robot/resources/lib/python_keywords/http_gate.py

@@ -10,8 +10,10 @@ from urllib.parse import quote_plus
 
 import allure
 import requests
+from aws_cli_client import LONG_TIMEOUT
 from cli_helpers import _cmd_run
 from cluster import StorageNode
+from common import SIMPLE_OBJECT_SIZE
 from file_helper import get_file_hash
 from neofs_testlib.shell import Shell
 from python_keywords.neofs_verbs import get_object
@@ -159,9 +161,27 @@ def upload_via_http_gate(cid: str, path: str, endpoint: str, headers: dict = Non
     return resp.json().get("object_id")
 
 
+@allure.step("Check is the passed object large")
+def is_object_large(filepath: str) -> bool:
+    """
+    This function check passed file size and return True if file_size > SIMPLE_OBJECT_SIZE
+    filepath: File path to check
+    """
+    file_size = os.path.getsize(filepath)
+    logger.info(f"Size= {file_size}")
+    if file_size > int(SIMPLE_OBJECT_SIZE):
+        return True
+    else:
+        return False
+
+
 @allure.step("Upload via HTTP Gate using Curl")
 def upload_via_http_gate_curl(
-    cid: str, filepath: str, endpoint: str, large_object=False, headers: dict = None
+    cid: str,
+    filepath: str,
+    endpoint: str,
+    headers: list = None,
+    error_pattern: Optional[str] = None,
 ) -> str:
     """
     This function upload given object through HTTP gate using curl utility.
@@ -169,14 +189,33 @@ def upload_via_http_gate_curl(
     filepath: File path to upload
     headers: Object header
     endpoint: http gate endpoint
+    error_pattern: [optional] expected error message from the command
     """
     request = f"{endpoint}/upload/{cid}"
-    files = f"file=@{filepath};filename={os.path.basename(filepath)}"
-    cmd = f"curl -F '{files}' {request}"
+    attributes = ""
+    if headers:
+        # parse attributes
+        attributes = " ".join(headers)
+
+    large_object = is_object_large(filepath)
     if large_object:
+        # pre-clean
+        _cmd_run("rm pipe -f")
         files = f"file=@pipe;filename={os.path.basename(filepath)}"
-        cmd = f"mkfifo pipe;cat {filepath} > pipe & curl --no-buffer -F '{files}' {request}"
-    output = _cmd_run(cmd)
+        cmd = f"mkfifo pipe;cat {filepath} > pipe & curl --no-buffer -F '{files}' {attributes} {request}"
+        output = _cmd_run(cmd, LONG_TIMEOUT)
+        # clean up pipe
+        _cmd_run("rm pipe")
+    else:
+        files = f"file=@{filepath};filename={os.path.basename(filepath)}"
+        cmd = f"curl -F '{files}' {attributes} {request}"
+        output = _cmd_run(cmd)
+
+    if error_pattern:
+        match = error_pattern.casefold() in str(output).casefold()
+        assert match, f"Expected {output} to match {error_pattern}"
+        return ""
+
     oid_re = re.search(r'"object_id": "(.*)"', output)
     if not oid_re:
         raise AssertionError(f'Could not find "object_id" in {output}')
@@ -226,7 +265,6 @@ def get_object_by_attr_and_verify_hashes(
     got_file_path_http_attr = get_via_http_gate_by_attribute(
         cid=cid, attribute=attrs, endpoint=endpoint
     )
-
     assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr)
 
 
@@ -240,14 +278,19 @@ def get_object_and_verify_hashes(
     endpoint: str,
     object_getter=None,
 ) -> None:
-    nodes = get_nodes_without_object(
+    nodes_list = get_nodes_without_object(
         wallet=wallet,
         cid=cid,
         oid=oid,
         shell=shell,
         nodes=nodes,
     )
-    random_node = random.choice(nodes)
+    # for some reason we can face with case when nodes_list is empty due to object resides in all nodes
+    if nodes_list is None:
+        random_node = random.choice(nodes)
+    else:
+        random_node = random.choice(nodes_list)
+
     object_getter = object_getter or get_via_http_gate
 
     got_file_path = get_object(
@@ -275,6 +318,17 @@ def attr_into_header(attrs: dict) -> dict:
     return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()}
 
 
+@allure.step(
+    "Convert each attribute (Key=Value) to the following format: -H 'X-Attribute-Key: Value'"
+)
+def attr_into_str_header_curl(attrs: dict) -> list:
+    headers = []
+    for k, v in attrs.items():
+        headers.append(f"-H 'X-Attribute-{k}: {v}'")
+    logger.info(f"[List of Attrs for curl:] {headers}")
+    return headers
+
+
 @allure.step(
     "Try to get object via http (pass http_request and optional attributes) and expect error"
 )