frostfs-testcases/pytest_tests/testsuites/services/test_http_gate.py
2022-11-22 16:06:23 +03:00

314 lines
13 KiB
Python

import logging
import os
from random import choice
from time import sleep
import allure
import pytest
from common import COMPLEX_OBJ_SIZE
from container import create_container
from epoch import get_epoch, tick_epoch
from file_helper import generate_file, get_file_hash
from neofs_testlib.shell import Shell
from python_keywords.http_gate import (
get_via_http_curl,
get_via_http_gate,
get_via_http_gate_by_attribute,
get_via_zip_http_gate,
upload_via_http_gate,
upload_via_http_gate_curl,
)
from python_keywords.neofs_verbs import get_object, put_object
from python_keywords.storage_policy import get_nodes_without_object
from utility import wait_for_gc_pass_on_storage_nodes
from wellknown_acl import PUBLIC_ACL
logger = logging.getLogger("NeoLogger")
OBJECT_NOT_FOUND_ERROR = "not found"
# For some reason object uploaded via http gateway is not immediately available for downloading
# Until this issue is resolved we are waiting for some time before attempting to read an object
# TODO: remove after https://github.com/nspcc-dev/neofs-http-gw/issues/176 is fixed
OBJECT_UPLOAD_DELAY = 10
@allure.link(
"https://github.com/nspcc-dev/neofs-http-gw#neofs-http-gateway", name="neofs-http-gateway"
)
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading")
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading")
@pytest.mark.sanity
@pytest.mark.http_gate
class TestHttpGate:
PLACEMENT_RULE_1 = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
PLACEMENT_RULE_2 = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
@pytest.fixture(scope="class", autouse=True)
@allure.title("[Class/Autouse]: Prepare wallet and deposit")
def prepare_wallet(self, prepare_wallet_and_deposit):
TestHttpGate.wallet = prepare_wallet_and_deposit
@allure.title("Test Put over gRPC, Get over HTTP")
def test_put_grpc_get_http(self, client_shell):
"""
Test that object can be put using gRPC interface and get using HTTP.
Steps:
1. Create simple and large objects.
2. Put objects using gRPC (neofs-cli).
3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading).
4. Get objects using gRPC (neofs-cli).
5. Compare hashes for got objects.
6. Compare hashes for got and original objects.
Expected result:
Hashes must be the same.
"""
cid = create_container(
self.wallet, shell=client_shell, rule=self.PLACEMENT_RULE_1, basic_acl=PUBLIC_ACL
)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
with allure.step("Put objects using gRPC"):
oid_simple = put_object(
wallet=self.wallet, path=file_path_simple, cid=cid, shell=client_shell
)
oid_large = put_object(
wallet=self.wallet, path=file_path_large, cid=cid, shell=client_shell
)
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid, shell=client_shell)
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#uploading", name="uploading")
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading")
@allure.title("Test Put over HTTP, Get over HTTP")
@pytest.mark.smoke
def test_put_http_get_http(self, client_shell):
"""
Test that object can be put and get using HTTP interface.
Steps:
1. Create simple and large objects.
2. Upload objects using HTTP (https://github.com/nspcc-dev/neofs-http-gw#uploading).
3. Download objects using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#downloading).
4. Compare hashes for got and original objects.
Expected result:
Hashes must be the same.
"""
cid = create_container(
self.wallet, shell=client_shell, rule=self.PLACEMENT_RULE_2, basic_acl=PUBLIC_ACL
)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
with allure.step("Put objects using HTTP"):
oid_simple = upload_via_http_gate(cid=cid, path=file_path_simple)
oid_large = upload_via_http_gate(cid=cid, path=file_path_large)
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
self.get_object_and_verify_hashes(oid, file_path, self.wallet, cid, shell=client_shell)
@allure.link(
"https://github.com/nspcc-dev/neofs-http-gw#by-attributes", name="download by attributes"
)
@allure.title("Test Put over HTTP, Get over HTTP with headers")
@pytest.mark.parametrize(
"attributes",
[
{"fileName": "simple_obj_filename"},
{"file-Name": "simple obj filename"},
{"cat%jpeg": "cat%jpeg"},
],
ids=["simple", "hyphen", "percent"],
)
def test_put_http_get_http_with_headers(self, client_shell, attributes: dict):
"""
Test that object can be downloaded using different attributes in HTTP header.
Steps:
1. Create simple and large objects.
2. Upload objects using HTTP with particular attributes in the header.
3. Download objects by attributes using HTTP gate (https://github.com/nspcc-dev/neofs-http-gw#by-attributes).
4. Compare hashes for got and original objects.
Expected result:
Hashes must be the same.
"""
cid = create_container(
self.wallet, shell=client_shell, rule=self.PLACEMENT_RULE_2, basic_acl=PUBLIC_ACL
)
file_path = generate_file()
with allure.step("Put objects using HTTP with attribute"):
headers = self._attr_into_header(attributes)
oid = upload_via_http_gate(cid=cid, path=file_path, headers=headers)
sleep(OBJECT_UPLOAD_DELAY)
self.get_object_by_attr_and_verify_hashes(oid, file_path, cid, attributes)
@allure.title("Test Expiration-Epoch in HTTP header")
def test_expiration_epoch_in_http(self, client_shell):
cid = create_container(
self.wallet, shell=client_shell, rule=self.PLACEMENT_RULE_2, basic_acl=PUBLIC_ACL
)
file_path = generate_file()
oids = []
curr_epoch = get_epoch(client_shell)
epochs = (curr_epoch, curr_epoch + 1, curr_epoch + 2, curr_epoch + 100)
for epoch in epochs:
headers = {"X-Attribute-Neofs-Expiration-Epoch": str(epoch)}
with allure.step("Put objects using HTTP with attribute Expiration-Epoch"):
oids.append(upload_via_http_gate(cid=cid, path=file_path, headers=headers))
assert len(oids) == len(epochs), "Expected all objects have been put successfully"
with allure.step("All objects can be get"):
for oid in oids:
get_via_http_gate(cid=cid, oid=oid)
for expired_objects, not_expired_objects in [(oids[:1], oids[1:]), (oids[:2], oids[2:])]:
tick_epoch(shell=client_shell)
# Wait for GC, because object with expiration is counted as alive until GC removes it
wait_for_gc_pass_on_storage_nodes()
for oid in expired_objects:
self.try_to_get_object_and_expect_error(
cid=cid, oid=oid, error_pattern=OBJECT_NOT_FOUND_ERROR
)
with allure.step("Other objects can be get"):
for oid in not_expired_objects:
get_via_http_gate(cid=cid, oid=oid)
@allure.title("Test Zip in HTTP header")
def test_zip_in_http(self, client_shell):
cid = create_container(
self.wallet, shell=client_shell, rule=self.PLACEMENT_RULE_2, basic_acl=PUBLIC_ACL
)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
common_prefix = "my_files"
headers1 = {"X-Attribute-FilePath": f"{common_prefix}/file1"}
headers2 = {"X-Attribute-FilePath": f"{common_prefix}/file2"}
upload_via_http_gate(cid=cid, path=file_path_simple, headers=headers1)
upload_via_http_gate(cid=cid, path=file_path_large, headers=headers2)
sleep(OBJECT_UPLOAD_DELAY)
dir_path = get_via_zip_http_gate(cid=cid, prefix=common_prefix)
with allure.step("Verify hashes"):
assert get_file_hash(f"{dir_path}/file1") == get_file_hash(file_path_simple)
assert get_file_hash(f"{dir_path}/file2") == get_file_hash(file_path_large)
@pytest.mark.long
@allure.title("Test Put over HTTP/Curl, Get over HTTP/Curl for large object")
def test_put_http_get_http_large_file(self, client_shell):
"""
This test checks upload and download using curl with 'large' object.
Large is object with size up to 20Mb.
"""
cid = create_container(
self.wallet, shell=client_shell, rule=self.PLACEMENT_RULE_2, basic_acl=PUBLIC_ACL
)
obj_size = int(os.getenv("BIG_OBJ_SIZE", COMPLEX_OBJ_SIZE))
file_path = generate_file(obj_size)
with allure.step("Put objects using HTTP"):
oid_gate = upload_via_http_gate(cid=cid, path=file_path)
oid_curl = upload_via_http_gate_curl(cid=cid, filepath=file_path, large_object=True)
self.get_object_and_verify_hashes(oid_gate, file_path, self.wallet, cid, shell=client_shell)
self.get_object_and_verify_hashes(
oid_curl,
file_path,
self.wallet,
cid,
shell=client_shell,
object_getter=get_via_http_curl,
)
@allure.title("Test Put/Get over HTTP using Curl utility")
def test_put_http_get_http_curl(self, client_shell):
"""
Test checks upload and download over HTTP using curl utility.
"""
cid = create_container(
self.wallet, shell=client_shell, rule=self.PLACEMENT_RULE_2, basic_acl=PUBLIC_ACL
)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
with allure.step("Put objects using curl utility"):
oid_simple = upload_via_http_gate_curl(cid=cid, filepath=file_path_simple)
oid_large = upload_via_http_gate_curl(cid=cid, filepath=file_path_large)
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
self.get_object_and_verify_hashes(
oid,
file_path,
self.wallet,
cid,
shell=client_shell,
object_getter=get_via_http_curl,
)
@staticmethod
@allure.step("Try to get object and expect error")
def try_to_get_object_and_expect_error(cid: str, oid: str, error_pattern: str) -> None:
try:
get_via_http_gate(cid=cid, oid=oid)
raise AssertionError(f"Expected error on getting object with cid: {cid}")
except Exception as err:
match = error_pattern.casefold() in str(err).casefold()
assert match, f"Expected {err} to match {error_pattern}"
@staticmethod
@allure.step("Verify object can be get using HTTP header attribute")
def get_object_by_attr_and_verify_hashes(
oid: str, file_name: str, cid: str, attrs: dict
) -> None:
got_file_path_http = get_via_http_gate(cid=cid, oid=oid)
got_file_path_http_attr = get_via_http_gate_by_attribute(cid=cid, attribute=attrs)
TestHttpGate._assert_hashes_are_equal(
file_name, got_file_path_http, got_file_path_http_attr
)
@staticmethod
@allure.step("Verify object can be get using HTTP")
def get_object_and_verify_hashes(
oid: str, file_name: str, wallet: str, cid: str, shell: Shell, object_getter=None
) -> None:
nodes = get_nodes_without_object(wallet=wallet, cid=cid, oid=oid, shell=shell)
random_node = choice(nodes)
object_getter = object_getter or get_via_http_gate
got_file_path = get_object(
wallet=wallet, cid=cid, oid=oid, shell=shell, endpoint=random_node
)
got_file_path_http = object_getter(cid=cid, oid=oid)
TestHttpGate._assert_hashes_are_equal(file_name, got_file_path, got_file_path_http)
@staticmethod
def _assert_hashes_are_equal(orig_file_name: str, got_file_1: str, got_file_2: str) -> None:
msg = "Expected hashes are equal for files {f1} and {f2}"
got_file_hash_http = get_file_hash(got_file_1)
assert get_file_hash(got_file_2) == got_file_hash_http, msg.format(
f1=got_file_2, f2=got_file_1
)
assert get_file_hash(orig_file_name) == got_file_hash_http, msg.format(
f1=orig_file_name, f2=got_file_1
)
@staticmethod
def _attr_into_header(attrs: dict) -> dict:
return {f"X-Attribute-{_key}": _value for _key, _value in attrs.items()}