create http folder, and adding a new test for http attributes

Signed-off-by: Vladislav Karakozov <v.karakozov@yadro.com>

pytest_tests/testsuites/services/http_gate/test_http_gate.py

@@ -1,7 +1,5 @@
 import logging
 import os
-import random
-from time import sleep
 
 import allure
 import pytest
@@ -9,15 +7,17 @@ from epoch import get_epoch, tick_epoch
 from file_helper import generate_file, get_file_hash
 from python_keywords.container import create_container
 from python_keywords.http_gate import (
+    attr_into_header,
+    get_object_and_verify_hashes,
+    get_object_by_attr_and_verify_hashes,
     get_via_http_curl,
     get_via_http_gate,
-    get_via_http_gate_by_attribute,
     get_via_zip_http_gate,
+    try_to_get_object_and_expect_error,
     upload_via_http_gate,
     upload_via_http_gate_curl,
 )
-from python_keywords.neofs_verbs import get_object, put_object_to_random_node
+from python_keywords.neofs_verbs import put_object_to_random_node
-from python_keywords.storage_policy import get_nodes_without_object
 from utility import wait_for_gc_pass_on_storage_nodes
 from wellknown_acl import PUBLIC_ACL
 
@@ -26,11 +26,6 @@ from steps.cluster_test_base import ClusterTestBase
 logger = logging.getLogger("NeoLogger")
 OBJECT_NOT_FOUND_ERROR = "not found"
 
-# For some reason object uploaded via http gateway is not immediately available for downloading
-# Until this issue is resolved we are waiting for some time before attempting to read an object
-# TODO: remove after https://github.com/nspcc-dev/neofs-http-gw/issues/176 is fixed
-OBJECT_UPLOAD_DELAY = 10
-
 
 @allure.link(
     "https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway", name="neofs-http-gateway"
@@ -92,7 +87,15 @@ class TestHttpGate(ClusterTestBase):
             )
 
         for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
-            self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid)
+            get_object_and_verify_hashes(
+                oid=oid,
+                file_name=file_path,
+                wallet=self.wallet,
+                cid=cid,
+                shell=self.shell,
+                nodes=self.cluster.storage_nodes,
+                endpoint=self.cluster.default_http_gate_endpoint,
+            )
 
     @allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading")
     @allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading")
@@ -131,7 +134,15 @@ class TestHttpGate(ClusterTestBase):
             )
 
         for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
-            self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid)
+            get_object_and_verify_hashes(
+                oid=oid,
+                file_name=file_path,
+                wallet=self.wallet,
+                cid=cid,
+                shell=self.shell,
+                nodes=self.cluster.storage_nodes,
+                endpoint=self.cluster.default_http_gate_endpoint,
+            )
 
     @allure.link(
         "https://github.com/nspcc-dev/neofs-http-gw#by-attributes", name="download by attributes"
@@ -169,7 +180,7 @@ class TestHttpGate(ClusterTestBase):
         file_path = generate_file(simple_object_size)
 
         with allure.step("Put objects using HTTP with attribute"):
-            headers = self._attr_into_header(attributes)
+            headers = attr_into_header(attributes)
             oid = upload_via_http_gate(
                 cid=cid,
                 path=file_path,
@@ -177,9 +188,13 @@ class TestHttpGate(ClusterTestBase):
                 endpoint=self.cluster.default_http_gate_endpoint,
             )
 
-        sleep(OBJECT_UPLOAD_DELAY)
+        get_object_by_attr_and_verify_hashes(
-
+            oid=oid,
-        self.get_object_by_attr_and_verify_hashes(oid, file_path, cid, attributes)
+            file_name=file_path,
+            cid=cid,
+            attrs=attributes,
+            endpoint=self.cluster.default_http_gate_endpoint,
+        )
 
     @allure.title("Test Expiration-Epoch in HTTP header")
     def test_expiration_epoch_in_http(self, simple_object_size):
@@ -222,8 +237,11 @@ class TestHttpGate(ClusterTestBase):
             wait_for_gc_pass_on_storage_nodes()
 
             for oid in expired_objects:
-                self.try_to_get_object_and_expect_error(
+                try_to_get_object_and_expect_error(
-                    cid=cid, oid=oid, error_pattern=OBJECT_NOT_FOUND_ERROR
+                    cid=cid,
+                    oid=oid,
+                    error_pattern=OBJECT_NOT_FOUND_ERROR,
+                    endpoint=self.cluster.default_http_gate_endpoint,
                 )
 
             with allure.step("Other objects can be get"):
@@ -260,8 +278,6 @@ class TestHttpGate(ClusterTestBase):
             endpoint=self.cluster.default_http_gate_endpoint,
         )
 
-        sleep(OBJECT_UPLOAD_DELAY)
-
         dir_path = get_via_zip_http_gate(
             cid=cid, prefix=common_prefix, endpoint=self.cluster.default_http_gate_endpoint
         )
@@ -299,12 +315,23 @@ class TestHttpGate(ClusterTestBase):
                 endpoint=self.cluster.default_http_gate_endpoint,
             )
 
-        self.get_object_and_verify_hashes(oid_gate, file_path, self.wallet, cid)
+        get_object_and_verify_hashes(
-        self.get_object_and_verify_hashes(
+            oid=oid_gate,
-            oid_curl,
+            file_name=file_path,
-            file_path,
+            wallet=self.wallet,
-            self.wallet,
+            cid=cid,
-            cid,
+            shell=self.shell,
+            nodes=self.cluster.storage_nodes,
+            endpoint=self.cluster.default_http_gate_endpoint,
+        )
+        get_object_and_verify_hashes(
+            oid=oid_curl,
+            file_name=file_path,
+            wallet=self.wallet,
+            cid=cid,
+            shell=self.shell,
+            nodes=self.cluster.storage_nodes,
+            endpoint=self.cluster.default_http_gate_endpoint,
             object_getter=get_via_http_curl,
         )
 
@@ -333,76 +360,13 @@ class TestHttpGate(ClusterTestBase):
             )
 
         for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
-            self.get_object_and_verify_hashes(
+            get_object_and_verify_hashes(
-                oid,
+                oid=oid,
-                file_path,
+                file_name=file_path,
-                self.wallet,
+                wallet=self.wallet,
-                cid,
+                cid=cid,
+                shell=self.shell,
+                nodes=self.cluster.storage_nodes,
+                endpoint=self.cluster.default_http_gate_endpoint,
                 object_getter=get_via_http_curl,
             )
-
-    @allure.step("Try to get object and expect error")
-    def try_to_get_object_and_expect_error(self, cid: str, oid: str, error_pattern: str) -> None:
-        try:
-            get_via_http_gate(cid=cid, oid=oid, endpoint=self.cluster.default_http_gate_endpoint)
-            raise AssertionError(f"Expected error on getting object with cid: {cid}")
-        except Exception as err:
-            match = error_pattern.casefold() in str(err).casefold()
-            assert match, f"Expected {err} to match {error_pattern}"
-
-    @allure.step("Verify object can be get using HTTP header attribute")
-    def get_object_by_attr_and_verify_hashes(
-        self, oid: str, file_name: str, cid: str, attrs: dict
-    ) -> None:
-        got_file_path_http = get_via_http_gate(
-            cid=cid, oid=oid, endpoint=self.cluster.default_http_gate_endpoint
-        )
-        got_file_path_http_attr = get_via_http_gate_by_attribute(
-            cid=cid, attribute=attrs, endpoint=self.cluster.default_http_gate_endpoint
-        )
-
-        TestHttpGate._assert_hashes_are_equal(
-            file_name, got_file_path_http, got_file_path_http_attr
-        )
-
-    @allure.step("Verify object can be get using HTTP")
-    def get_object_and_verify_hashes(
-        self, oid: str, file_name: str, wallet: str, cid: str, object_getter=None
-    ) -> None:
-        nodes = get_nodes_without_object(
-            wallet=wallet,
-            cid=cid,
-            oid=oid,
-            shell=self.shell,
-            nodes=self.cluster.storage_nodes,
-        )
-        random_node = random.choice(nodes)
-        object_getter = object_getter or get_via_http_gate
-
-        got_file_path = get_object(
-            wallet=wallet,
-            cid=cid,
-            oid=oid,
-            shell=self.shell,
-            endpoint=random_node.get_rpc_endpoint(),
-        )
-        got_file_path_http = object_getter(
-            cid=cid, oid=oid, endpoint=self.cluster.default_http_gate_endpoint
-        )
-
-        TestHttpGate._assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
-
-    @staticmethod
-    def _assert_hashes_are_equal(orig_file_name: str, got_file_1: str, got_file_2: str) -> None:
-        msg = "Expected hashes are equal for files {f1} and {f2}"
-        got_file_hash_http = get_file_hash(got_file_1)
-        assert get_file_hash(got_file_2) == got_file_hash_http, msg.format(
-            f1=got_file_2, f2=got_file_1
-        )
-        assert get_file_hash(orig_file_name) == got_file_hash_http, msg.format(
-            f1=orig_file_name, f2=got_file_1
-        )
-
-    @staticmethod
-    def _attr_into_header(attrs: dict) -> dict:
-        return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()}

pytest_tests/testsuites/services/http_gate/test_http_object.py

@@ -0,0 +1,121 @@
+import logging
+
+import allure
+import pytest
+from container import create_container
+from file_helper import generate_file
+from python_keywords.http_gate import (
+    get_object_and_verify_hashes,
+    get_object_by_attr_and_verify_hashes,
+    try_to_get_object_via_passed_request_and_expect_error,
+)
+from python_keywords.neofs_verbs import put_object_to_random_node
+from wellknown_acl import PUBLIC_ACL
+
+from steps.cluster_test_base import ClusterTestBase
+
+logger = logging.getLogger("NeoLogger")
+
+
+@pytest.mark.sanity
+@pytest.mark.http_gate
+class Test_http_object(ClusterTestBase):
+    PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
+
+    @pytest.fixture(scope="class", autouse=True)
+    @allure.title("[Class/Autouse]: Prepare wallet and deposit")
+    def prepare_wallet(self, default_wallet):
+        Test_http_object.wallet = default_wallet
+
+    @allure.title("Test Put over gRPC, Get over HTTP")
+    def test_object_put_get_attributes(self):
+        """
+        Test that object can be put using gRPC interface and get using HTTP.
+
+        Steps:
+        1. Create object;
+        2. Put objects using gRPC (neofs-cli) with attributes [--attributes chapter1=peace,chapter2=war];
+        3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading);
+        4. Compare hashes between original and downloaded object;
+        5. [Negative] Try to the get object with specified attributes and `get` request: [get/$CID/chapter1/peace];
+        6. Download the object with specified attributes and `get_by_attribute` request: [get_by_attribute/$CID/chapter1/peace];
+        7. Compare hashes between original and downloaded object;
+        8. [Negative] Try to the get object via `get_by_attribute` request: [get_by_attribute/$CID/$OID];
+
+
+        Expected result:
+        Hashes must be the same.
+        """
+        with allure.step("Create public container"):
+            cid = create_container(
+                self.wallet,
+                shell=self.shell,
+                endpoint=self.cluster.default_rpc_endpoint,
+                rule=self.PLACEMENT_RULE,
+                basic_acl=PUBLIC_ACL,
+            )
+
+        # Generate file
+        file_path = generate_file()
+
+        # List of Key=Value attributes
+        obj_key1 = "chapter1"
+        obj_value1 = "peace"
+        obj_key2 = "chapter2"
+        obj_value2 = "war"
+
+        # Prepare for grpc PUT request
+        key_value1 = obj_key1 + "=" + obj_value1
+        key_value2 = obj_key2 + "=" + obj_value2
+
+        with allure.step("Put objects using gRPC [--attributes chapter1=peace,chapter2=war]"):
+            oid = put_object_to_random_node(
+                wallet=self.wallet,
+                path=file_path,
+                cid=cid,
+                shell=self.shell,
+                cluster=self.cluster,
+                attributes=f"{key_value1},{key_value2}",
+            )
+        with allure.step("Get object and verify hashes [ get/$CID/$OID ]"):
+            get_object_and_verify_hashes(
+                oid=oid,
+                file_name=file_path,
+                wallet=self.wallet,
+                cid=cid,
+                shell=self.shell,
+                nodes=self.cluster.storage_nodes,
+                endpoint=self.cluster.default_http_gate_endpoint,
+            )
+        with allure.step("[Negative] try to get object: [get/$CID/chapter1/peace]"):
+            attrs = {obj_key1: obj_value1, obj_key2: obj_value2}
+            request = f"/get/{cid}/{obj_key1}/{obj_value1}"
+            expected_err_msg = "Failed to get object via HTTP gate:"
+            try_to_get_object_via_passed_request_and_expect_error(
+                cid=cid,
+                oid=oid,
+                error_pattern=expected_err_msg,
+                http_request_path=request,
+                attrs=attrs,
+                endpoint=self.cluster.default_http_gate_endpoint,
+            )
+
+        with allure.step(
+            "Download the object with attribute [get_by_attribute/$CID/chapter1/peace]"
+        ):
+            get_object_by_attr_and_verify_hashes(
+                oid=oid,
+                file_name=file_path,
+                cid=cid,
+                attrs=attrs,
+                endpoint=self.cluster.default_http_gate_endpoint,
+            )
+        with allure.step("[Negative] try to get object: get_by_attribute/$CID/$OID"):
+            request = f"/get_by_attribute/{cid}/{oid}"
+            try_to_get_object_via_passed_request_and_expect_error(
+                cid=cid,
+                oid=oid,
+                error_pattern=expected_err_msg,
+                http_request_path=request,
+                endpoint=self.cluster.default_http_gate_endpoint,
+            )

robot/resources/lib/python_keywords/http_gate.py

@@ -1,14 +1,23 @@
 import logging
 import os
+import random
 import re
 import shutil
 import uuid
 import zipfile
+from typing import Optional
 from urllib.parse import quote_plus
 
 import allure
 import requests
 from cli_helpers import _cmd_run
+from cluster import StorageNode
+from file_helper import get_file_hash
+from neofs_testlib.shell import Shell
+from python_keywords.neofs_verbs import get_object
+from python_keywords.storage_policy import get_nodes_without_object
+
+from pytest_tests.steps.cluster_test_base import ClusterTestBase
 
 logger = logging.getLogger("NeoLogger")
 
@@ -16,14 +25,21 @@ ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/")
 
 
 @allure.step("Get via HTTP Gate")
-def get_via_http_gate(cid: str, oid: str, endpoint: str):
+def get_via_http_gate(cid: str, oid: str, endpoint: str, request_path: Optional[str] = None):
     """
     This function gets given object from HTTP gate
-    cid:      container id to get object from
+    cid:          container id to get object from
-    oid:      object ID
+    oid:          object ID
-    endpoint: http gate endpoint
+    endpoint:     http gate endpoint
+    request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}]
     """
-    request = f"{endpoint}/get/{cid}/{oid}"
+
+    # if `request_path` parameter ommited, use default
+    if request_path is None:
+        request = f"{endpoint}/get/{cid}/{oid}"
+    else:
+        request = f"{endpoint}{request_path}"
+
     resp = requests.get(request, stream=True)
 
     if not resp.ok:
@@ -76,16 +92,24 @@ def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str):
 
 
 @allure.step("Get via HTTP Gate by attribute")
-def get_via_http_gate_by_attribute(cid: str, attribute: dict, endpoint: str):
+def get_via_http_gate_by_attribute(
+    cid: str, attribute: dict, endpoint: str, request_path: Optional[str] = None
+):
     """
     This function gets given object from HTTP gate
-    cid:         CID to get object from
+    cid:          CID to get object from
-    attribute:   attribute {name: attribute} value pair
+    attribute:    attribute {name: attribute} value pair
-    endpoint: http gate endpoint
+    endpoint:     http gate endpoint
+    request_path: (optional) http request path, if ommited - use default [{endpoint}/get_by_attribute/{Key}/{Value}]
     """
     attr_name = list(attribute.keys())[0]
     attr_value = quote_plus(str(attribute.get(attr_name)))
-    request = f"{endpoint}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}"
+    # if `request_path` parameter ommited, use default
+    if request_path is None:
+        request = f"{endpoint}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}"
+    else:
+        request = f"{endpoint}{request_path}"
+
     resp = requests.get(request, stream=True)
 
     if not resp.ok:
@@ -180,3 +204,96 @@ def _attach_allure_step(request: str, status_code: int, req_type="GET"):
     command_attachment = f"REQUEST: '{request}'\n" f"RESPONSE:\n {status_code}\n"
     with allure.step(f"{req_type} Request"):
         allure.attach(command_attachment, f"{req_type} Request", allure.attachment_type.TEXT)
+
+
+@allure.step("Try to get object and expect error")
+def try_to_get_object_and_expect_error(
+    cid: str, oid: str, error_pattern: str, endpoint: str
+) -> None:
+    try:
+        get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint)
+        raise AssertionError(f"Expected error on getting object with cid: {cid}")
+    except Exception as err:
+        match = error_pattern.casefold() in str(err).casefold()
+        assert match, f"Expected {err} to match {error_pattern}"
+
+
+@allure.step("Verify object can be get using HTTP header attribute")
+def get_object_by_attr_and_verify_hashes(
+    oid: str, file_name: str, cid: str, attrs: dict, endpoint: str
+) -> None:
+    got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint)
+    got_file_path_http_attr = get_via_http_gate_by_attribute(
+        cid=cid, attribute=attrs, endpoint=endpoint
+    )
+
+    assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr)
+
+
+def get_object_and_verify_hashes(
+    oid: str,
+    file_name: str,
+    wallet: str,
+    cid: str,
+    shell: Shell,
+    nodes: list[StorageNode],
+    endpoint: str,
+    object_getter=None,
+) -> None:
+    nodes = get_nodes_without_object(
+        wallet=wallet,
+        cid=cid,
+        oid=oid,
+        shell=shell,
+        nodes=nodes,
+    )
+    random_node = random.choice(nodes)
+    object_getter = object_getter or get_via_http_gate
+
+    got_file_path = get_object(
+        wallet=wallet,
+        cid=cid,
+        oid=oid,
+        shell=shell,
+        endpoint=random_node.get_rpc_endpoint(),
+    )
+    got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint)
+
+    assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
+
+
+def assert_hashes_are_equal(orig_file_name: str, got_file_1: str, got_file_2: str) -> None:
+    msg = "Expected hashes are equal for files {f1} and {f2}"
+    got_file_hash_http = get_file_hash(got_file_1)
+    assert get_file_hash(got_file_2) == got_file_hash_http, msg.format(f1=got_file_2, f2=got_file_1)
+    assert get_file_hash(orig_file_name) == got_file_hash_http, msg.format(
+        f1=orig_file_name, f2=got_file_1
+    )
+
+
+def attr_into_header(attrs: dict) -> dict:
+    return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()}
+
+
+@allure.step(
+    "Try to get object via http (pass http_request and optional attributes) and expect error"
+)
+def try_to_get_object_via_passed_request_and_expect_error(
+    cid: str,
+    oid: str,
+    error_pattern: str,
+    endpoint: str,
+    http_request_path: str,
+    attrs: dict = None,
+) -> None:
+    try:
+        if attrs is None:
+            get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path)
+        else:
+            get_via_http_gate_by_attribute(
+                cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path
+            )
+        raise AssertionError(f"Expected error on getting object with cid: {cid}")
+    except Exception as err:
+        match = error_pattern.casefold() in str(err).casefold()
+        assert match, f"Expected {err} to match {error_pattern}"