create http folder, and adding a new test for http attributes

Signed-off-by: Vladislav Karakozov <v.karakozov@yadro.com>
This commit is contained in:
Vladislav Karakozov 2022-12-08 16:03:27 +03:00 committed by Julia Kovshova
parent e99cc7d05d
commit b9b8c4e1f8
3 changed files with 309 additions and 107 deletions

View file

@ -1,7 +1,5 @@
import logging import logging
import os import os
import random
from time import sleep
import allure import allure
import pytest import pytest
@ -9,15 +7,17 @@ from epoch import get_epoch, tick_epoch
from file_helper import generate_file, get_file_hash from file_helper import generate_file, get_file_hash
from python_keywords.container import create_container from python_keywords.container import create_container
from python_keywords.http_gate import ( from python_keywords.http_gate import (
attr_into_header,
get_object_and_verify_hashes,
get_object_by_attr_and_verify_hashes,
get_via_http_curl, get_via_http_curl,
get_via_http_gate, get_via_http_gate,
get_via_http_gate_by_attribute,
get_via_zip_http_gate, get_via_zip_http_gate,
try_to_get_object_and_expect_error,
upload_via_http_gate, upload_via_http_gate,
upload_via_http_gate_curl, upload_via_http_gate_curl,
) )
from python_keywords.neofs_verbs import get_object, put_object_to_random_node from python_keywords.neofs_verbs import put_object_to_random_node
from python_keywords.storage_policy import get_nodes_without_object
from utility import wait_for_gc_pass_on_storage_nodes from utility import wait_for_gc_pass_on_storage_nodes
from wellknown_acl import PUBLIC_ACL from wellknown_acl import PUBLIC_ACL
@ -26,11 +26,6 @@ from steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
OBJECT_NOT_FOUND_ERROR = "not found" OBJECT_NOT_FOUND_ERROR = "not found"
# For some reason object uploaded via http gateway is not immediately available for downloading
# Until this issue is resolved we are waiting for some time before attempting to read an object
# TODO: remove after https://github.com/nspcc-dev/neofs-http-gw/issues/176 is fixed
OBJECT_UPLOAD_DELAY = 10
@allure.link( @allure.link(
"https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway", name="neofs-http-gateway" "https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway", name="neofs-http-gateway"
@ -92,7 +87,15 @@ class TestHttpGate(ClusterTestBase):
) )
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid) get_object_and_verify_hashes(
oid=oid,
file_name=file_path,
wallet=self.wallet,
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
endpoint=self.cluster.default_http_gate_endpoint,
)
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading") @allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading")
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading") @allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading")
@ -131,7 +134,15 @@ class TestHttpGate(ClusterTestBase):
) )
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid) get_object_and_verify_hashes(
oid=oid,
file_name=file_path,
wallet=self.wallet,
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
endpoint=self.cluster.default_http_gate_endpoint,
)
@allure.link( @allure.link(
"https://github.com/nspcc-dev/neofs-http-gw#by-attributes", name="download by attributes" "https://github.com/nspcc-dev/neofs-http-gw#by-attributes", name="download by attributes"
@ -169,7 +180,7 @@ class TestHttpGate(ClusterTestBase):
file_path = generate_file(simple_object_size) file_path = generate_file(simple_object_size)
with allure.step("Put objects using HTTP with attribute"): with allure.step("Put objects using HTTP with attribute"):
headers = self._attr_into_header(attributes) headers = attr_into_header(attributes)
oid = upload_via_http_gate( oid = upload_via_http_gate(
cid=cid, cid=cid,
path=file_path, path=file_path,
@ -177,9 +188,13 @@ class TestHttpGate(ClusterTestBase):
endpoint=self.cluster.default_http_gate_endpoint, endpoint=self.cluster.default_http_gate_endpoint,
) )
sleep(OBJECT_UPLOAD_DELAY) get_object_by_attr_and_verify_hashes(
oid=oid,
self.get_object_by_attr_and_verify_hashes(oid, file_path, cid, attributes) file_name=file_path,
cid=cid,
attrs=attributes,
endpoint=self.cluster.default_http_gate_endpoint,
)
@allure.title("Test Expiration-Epoch in HTTP header") @allure.title("Test Expiration-Epoch in HTTP header")
def test_expiration_epoch_in_http(self, simple_object_size): def test_expiration_epoch_in_http(self, simple_object_size):
@ -222,8 +237,11 @@ class TestHttpGate(ClusterTestBase):
wait_for_gc_pass_on_storage_nodes() wait_for_gc_pass_on_storage_nodes()
for oid in expired_objects: for oid in expired_objects:
self.try_to_get_object_and_expect_error( try_to_get_object_and_expect_error(
cid=cid, oid=oid, error_pattern=OBJECT_NOT_FOUND_ERROR cid=cid,
oid=oid,
error_pattern=OBJECT_NOT_FOUND_ERROR,
endpoint=self.cluster.default_http_gate_endpoint,
) )
with allure.step("Other objects can be get"): with allure.step("Other objects can be get"):
@ -260,8 +278,6 @@ class TestHttpGate(ClusterTestBase):
endpoint=self.cluster.default_http_gate_endpoint, endpoint=self.cluster.default_http_gate_endpoint,
) )
sleep(OBJECT_UPLOAD_DELAY)
dir_path = get_via_zip_http_gate( dir_path = get_via_zip_http_gate(
cid=cid, prefix=common_prefix, endpoint=self.cluster.default_http_gate_endpoint cid=cid, prefix=common_prefix, endpoint=self.cluster.default_http_gate_endpoint
) )
@ -299,12 +315,23 @@ class TestHttpGate(ClusterTestBase):
endpoint=self.cluster.default_http_gate_endpoint, endpoint=self.cluster.default_http_gate_endpoint,
) )
self.get_object_and_verify_hashes(oid_gate, file_path, self.wallet, cid) get_object_and_verify_hashes(
self.get_object_and_verify_hashes( oid=oid_gate,
oid_curl, file_name=file_path,
file_path, wallet=self.wallet,
self.wallet, cid=cid,
cid, shell=self.shell,
nodes=self.cluster.storage_nodes,
endpoint=self.cluster.default_http_gate_endpoint,
)
get_object_and_verify_hashes(
oid=oid_curl,
file_name=file_path,
wallet=self.wallet,
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
endpoint=self.cluster.default_http_gate_endpoint,
object_getter=get_via_http_curl, object_getter=get_via_http_curl,
) )
@ -333,76 +360,13 @@ class TestHttpGate(ClusterTestBase):
) )
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
self.get_object_and_verify_hashes( get_object_and_verify_hashes(
oid, oid=oid,
file_path, file_name=file_path,
self.wallet, wallet=self.wallet,
cid, cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
endpoint=self.cluster.default_http_gate_endpoint,
object_getter=get_via_http_curl, object_getter=get_via_http_curl,
) )
@allure.step("Try to get object and expect error")
def try_to_get_object_and_expect_error(self, cid: str, oid: str, error_pattern: str) -> None:
try:
get_via_http_gate(cid=cid, oid=oid, endpoint=self.cluster.default_http_gate_endpoint)
raise AssertionError(f"Expected error on getting object with cid: {cid}")
except Exception as err:
match = error_pattern.casefold() in str(err).casefold()
assert match, f"Expected {err} to match {error_pattern}"
@allure.step("Verify object can be get using HTTP header attribute")
def get_object_by_attr_and_verify_hashes(
self, oid: str, file_name: str, cid: str, attrs: dict
) -> None:
got_file_path_http = get_via_http_gate(
cid=cid, oid=oid, endpoint=self.cluster.default_http_gate_endpoint
)
got_file_path_http_attr = get_via_http_gate_by_attribute(
cid=cid, attribute=attrs, endpoint=self.cluster.default_http_gate_endpoint
)
TestHttpGate._assert_hashes_are_equal(
file_name, got_file_path_http, got_file_path_http_attr
)
@allure.step("Verify object can be get using HTTP")
def get_object_and_verify_hashes(
self, oid: str, file_name: str, wallet: str, cid: str, object_getter=None
) -> None:
nodes = get_nodes_without_object(
wallet=wallet,
cid=cid,
oid=oid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
)
random_node = random.choice(nodes)
object_getter = object_getter or get_via_http_gate
got_file_path = get_object(
wallet=wallet,
cid=cid,
oid=oid,
shell=self.shell,
endpoint=random_node.get_rpc_endpoint(),
)
got_file_path_http = object_getter(
cid=cid, oid=oid, endpoint=self.cluster.default_http_gate_endpoint
)
TestHttpGate._assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
@staticmethod
def _assert_hashes_are_equal(orig_file_name: str, got_file_1: str, got_file_2: str) -> None:
msg = "Expected hashes are equal for files {f1} and {f2}"
got_file_hash_http = get_file_hash(got_file_1)
assert get_file_hash(got_file_2) == got_file_hash_http, msg.format(
f1=got_file_2, f2=got_file_1
)
assert get_file_hash(orig_file_name) == got_file_hash_http, msg.format(
f1=orig_file_name, f2=got_file_1
)
@staticmethod
def _attr_into_header(attrs: dict) -> dict:
return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()}

View file

@ -0,0 +1,121 @@
import logging
import allure
import pytest
from container import create_container
from file_helper import generate_file
from python_keywords.http_gate import (
get_object_and_verify_hashes,
get_object_by_attr_and_verify_hashes,
try_to_get_object_via_passed_request_and_expect_error,
)
from python_keywords.neofs_verbs import put_object_to_random_node
from wellknown_acl import PUBLIC_ACL
from steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger")
@pytest.mark.sanity
@pytest.mark.http_gate
class Test_http_object(ClusterTestBase):
PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
@pytest.fixture(scope="class", autouse=True)
@allure.title("[Class/Autouse]: Prepare wallet and deposit")
def prepare_wallet(self, default_wallet):
Test_http_object.wallet = default_wallet
@allure.title("Test Put over gRPC, Get over HTTP")
def test_object_put_get_attributes(self):
"""
Test that object can be put using gRPC interface and get using HTTP.
Steps:
1. Create object;
2. Put objects using gRPC (neofs-cli) with attributes [--attributes chapter1=peace,chapter2=war];
3. Download object using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading);
4. Compare hashes between original and downloaded object;
5. [Negative] Try to the get object with specified attributes and `get` request: [get/$CID/chapter1/peace];
6. Download the object with specified attributes and `get_by_attribute` request: [get_by_attribute/$CID/chapter1/peace];
7. Compare hashes between original and downloaded object;
8. [Negative] Try to the get object via `get_by_attribute` request: [get_by_attribute/$CID/$OID];
Expected result:
Hashes must be the same.
"""
with allure.step("Create public container"):
cid = create_container(
self.wallet,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
rule=self.PLACEMENT_RULE,
basic_acl=PUBLIC_ACL,
)
# Generate file
file_path = generate_file()
# List of Key=Value attributes
obj_key1 = "chapter1"
obj_value1 = "peace"
obj_key2 = "chapter2"
obj_value2 = "war"
# Prepare for grpc PUT request
key_value1 = obj_key1 + "=" + obj_value1
key_value2 = obj_key2 + "=" + obj_value2
with allure.step("Put objects using gRPC [--attributes chapter1=peace,chapter2=war]"):
oid = put_object_to_random_node(
wallet=self.wallet,
path=file_path,
cid=cid,
shell=self.shell,
cluster=self.cluster,
attributes=f"{key_value1},{key_value2}",
)
with allure.step("Get object and verify hashes [ get/$CID/$OID ]"):
get_object_and_verify_hashes(
oid=oid,
file_name=file_path,
wallet=self.wallet,
cid=cid,
shell=self.shell,
nodes=self.cluster.storage_nodes,
endpoint=self.cluster.default_http_gate_endpoint,
)
with allure.step("[Negative] try to get object: [get/$CID/chapter1/peace]"):
attrs = {obj_key1: obj_value1, obj_key2: obj_value2}
request = f"/get/{cid}/{obj_key1}/{obj_value1}"
expected_err_msg = "Failed to get object via HTTP gate:"
try_to_get_object_via_passed_request_and_expect_error(
cid=cid,
oid=oid,
error_pattern=expected_err_msg,
http_request_path=request,
attrs=attrs,
endpoint=self.cluster.default_http_gate_endpoint,
)
with allure.step(
"Download the object with attribute [get_by_attribute/$CID/chapter1/peace]"
):
get_object_by_attr_and_verify_hashes(
oid=oid,
file_name=file_path,
cid=cid,
attrs=attrs,
endpoint=self.cluster.default_http_gate_endpoint,
)
with allure.step("[Negative] try to get object: get_by_attribute/$CID/$OID"):
request = f"/get_by_attribute/{cid}/{oid}"
try_to_get_object_via_passed_request_and_expect_error(
cid=cid,
oid=oid,
error_pattern=expected_err_msg,
http_request_path=request,
endpoint=self.cluster.default_http_gate_endpoint,
)

View file

@ -1,14 +1,23 @@
import logging import logging
import os import os
import random
import re import re
import shutil import shutil
import uuid import uuid
import zipfile import zipfile
from typing import Optional
from urllib.parse import quote_plus from urllib.parse import quote_plus
import allure import allure
import requests import requests
from cli_helpers import _cmd_run from cli_helpers import _cmd_run
from cluster import StorageNode
from file_helper import get_file_hash
from neofs_testlib.shell import Shell
from python_keywords.neofs_verbs import get_object
from python_keywords.storage_policy import get_nodes_without_object
from pytest_tests.steps.cluster_test_base import ClusterTestBase
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -16,14 +25,21 @@ ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/")
@allure.step("Get via HTTP Gate") @allure.step("Get via HTTP Gate")
def get_via_http_gate(cid: str, oid: str, endpoint: str): def get_via_http_gate(cid: str, oid: str, endpoint: str, request_path: Optional[str] = None):
""" """
This function gets given object from HTTP gate This function gets given object from HTTP gate
cid: container id to get object from cid: container id to get object from
oid: object ID oid: object ID
endpoint: http gate endpoint endpoint: http gate endpoint
request_path: (optional) http request, if ommited - use default [{endpoint}/get/{cid}/{oid}]
""" """
request = f"{endpoint}/get/{cid}/{oid}"
# if `request_path` parameter ommited, use default
if request_path is None:
request = f"{endpoint}/get/{cid}/{oid}"
else:
request = f"{endpoint}{request_path}"
resp = requests.get(request, stream=True) resp = requests.get(request, stream=True)
if not resp.ok: if not resp.ok:
@ -76,16 +92,24 @@ def get_via_zip_http_gate(cid: str, prefix: str, endpoint: str):
@allure.step("Get via HTTP Gate by attribute") @allure.step("Get via HTTP Gate by attribute")
def get_via_http_gate_by_attribute(cid: str, attribute: dict, endpoint: str): def get_via_http_gate_by_attribute(
cid: str, attribute: dict, endpoint: str, request_path: Optional[str] = None
):
""" """
This function gets given object from HTTP gate This function gets given object from HTTP gate
cid: CID to get object from cid: CID to get object from
attribute: attribute {name: attribute} value pair attribute: attribute {name: attribute} value pair
endpoint: http gate endpoint endpoint: http gate endpoint
request_path: (optional) http request path, if ommited - use default [{endpoint}/get_by_attribute/{Key}/{Value}]
""" """
attr_name = list(attribute.keys())[0] attr_name = list(attribute.keys())[0]
attr_value = quote_plus(str(attribute.get(attr_name))) attr_value = quote_plus(str(attribute.get(attr_name)))
request = f"{endpoint}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}" # if `request_path` parameter ommited, use default
if request_path is None:
request = f"{endpoint}/get_by_attribute/{cid}/{quote_plus(str(attr_name))}/{attr_value}"
else:
request = f"{endpoint}{request_path}"
resp = requests.get(request, stream=True) resp = requests.get(request, stream=True)
if not resp.ok: if not resp.ok:
@ -180,3 +204,96 @@ def _attach_allure_step(request: str, status_code: int, req_type="GET"):
command_attachment = f"REQUEST: '{request}'\n" f"RESPONSE:\n {status_code}\n" command_attachment = f"REQUEST: '{request}'\n" f"RESPONSE:\n {status_code}\n"
with allure.step(f"{req_type} Request"): with allure.step(f"{req_type} Request"):
allure.attach(command_attachment, f"{req_type} Request", allure.attachment_type.TEXT) allure.attach(command_attachment, f"{req_type} Request", allure.attachment_type.TEXT)
@allure.step("Try to get object and expect error")
def try_to_get_object_and_expect_error(
cid: str, oid: str, error_pattern: str, endpoint: str
) -> None:
try:
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint)
raise AssertionError(f"Expected error on getting object with cid: {cid}")
except Exception as err:
match = error_pattern.casefold() in str(err).casefold()
assert match, f"Expected {err} to match {error_pattern}"
@allure.step("Verify object can be get using HTTP header attribute")
def get_object_by_attr_and_verify_hashes(
oid: str, file_name: str, cid: str, attrs: dict, endpoint: str
) -> None:
got_file_path_http = get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint)
got_file_path_http_attr = get_via_http_gate_by_attribute(
cid=cid, attribute=attrs, endpoint=endpoint
)
assert_hashes_are_equal(file_name, got_file_path_http, got_file_path_http_attr)
def get_object_and_verify_hashes(
oid: str,
file_name: str,
wallet: str,
cid: str,
shell: Shell,
nodes: list[StorageNode],
endpoint: str,
object_getter=None,
) -> None:
nodes = get_nodes_without_object(
wallet=wallet,
cid=cid,
oid=oid,
shell=shell,
nodes=nodes,
)
random_node = random.choice(nodes)
object_getter = object_getter or get_via_http_gate
got_file_path = get_object(
wallet=wallet,
cid=cid,
oid=oid,
shell=shell,
endpoint=random_node.get_rpc_endpoint(),
)
got_file_path_http = object_getter(cid=cid, oid=oid, endpoint=endpoint)
assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
def assert_hashes_are_equal(orig_file_name: str, got_file_1: str, got_file_2: str) -> None:
msg = "Expected hashes are equal for files {f1} and {f2}"
got_file_hash_http = get_file_hash(got_file_1)
assert get_file_hash(got_file_2) == got_file_hash_http, msg.format(f1=got_file_2, f2=got_file_1)
assert get_file_hash(orig_file_name) == got_file_hash_http, msg.format(
f1=orig_file_name, f2=got_file_1
)
def attr_into_header(attrs: dict) -> dict:
return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()}
@allure.step(
"Try to get object via http (pass http_request and optional attributes) and expect error"
)
def try_to_get_object_via_passed_request_and_expect_error(
cid: str,
oid: str,
error_pattern: str,
endpoint: str,
http_request_path: str,
attrs: dict = None,
) -> None:
try:
if attrs is None:
get_via_http_gate(cid=cid, oid=oid, endpoint=endpoint, request_path=http_request_path)
else:
get_via_http_gate_by_attribute(
cid=cid, attribute=attrs, endpoint=endpoint, request_path=http_request_path
)
raise AssertionError(f"Expected error on getting object with cid: {cid}")
except Exception as err:
match = error_pattern.casefold() in str(err).casefold()
assert match, f"Expected {err} to match {error_pattern}"