TrueCloudLab/frostfs-testcases
25
0
Fork 21
Code Issues 2 Pull requests 1 Releases Activity Actions

Backport sanity labels rework to v0.37 #159

Merged
abereziny merged 6 commits from abereziny/frostfs-testcases:sanity-rework-for-0-37 into support/v0.37 2023-11-28 08:20:51 +00:00
Conversation 0 Commits 6 Files changed 28 +315 -910
28 changed files with 315 additions and 910 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
pytest_tests/testsuites/acl/test_eacl.py
View file

@ -9,10 +9,7 @@ from frostfs_testlib.storage.dataclasses.acl import EACLAccess, EACLOperation, E
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.failover_utils import wait_object_replication
from pytest_tests.helpers.container_access import (
check_full_access_to_container,
check_no_access_to_container,
)
from pytest_tests.helpers.container_access import check_full_access_to_container, check_no_access_to_container
from pytest_tests.helpers.object_access import (
can_delete_object,
can_get_head_object,
@ -25,7 +22,6 @@ from pytest_tests.helpers.object_access import (
from pytest_tests.testsuites.acl.conftest import Wallets
@pytest.mark.sanity
@pytest.mark.acl
@pytest.mark.acl_extended
class TestEACLContainer(ClusterTestBase):
@ -58,6 +54,7 @@ class TestEACLContainer(ClusterTestBase):
yield cid, oid, file_path
@pytest.mark.sanity
@allure.title("Deny operations (role={deny_role.value}, obj_size={object_size})")
@pytest.mark.parametrize("deny_role", [EACLRole.USER, EACLRole.OTHERS])
def test_extended_acl_deny_all_operations(
@ -75,10 +72,7 @@ class TestEACLContainer(ClusterTestBase):
cid, object_oids, file_path = eacl_container_with_objects
with allure.step(f"Deny all operations for {deny_role_str} via eACL"):
eacl_deny = [
EACLRule(access=EACLAccess.DENY, role=deny_role, operation=op)
for op in EACLOperation
]
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=deny_role, operation=op) for op in EACLOperation]
set_eacl(
user_wallet.wallet_path,
cid,
@ -89,9 +83,7 @@ class TestEACLContainer(ClusterTestBase):
wait_for_cache_expired()
with allure.step(f"Check only {not_deny_role_str} has full access to container"):
with allure.step(
f"Check {deny_role_str} has not access to any operations with container"
):
with allure.step(f"Check {deny_role_str} has not access to any operations with container"):
check_no_access_to_container(
deny_role_wallet.wallet_path,
cid,
@ -101,9 +93,7 @@ class TestEACLContainer(ClusterTestBase):
cluster=self.cluster,
)
with allure.step(
f"Check {not_deny_role_wallet} has full access to eACL public container"
):
with allure.step(f"Check {not_deny_role_wallet} has full access to eACL public container"):
check_full_access_to_container(
not_deny_role_wallet.wallet_path,
cid,
@ -114,10 +104,7 @@ class TestEACLContainer(ClusterTestBase):
)
with allure.step(f"Allow all operations for {deny_role_str} via eACL"):
eacl_deny = [
EACLRule(access=EACLAccess.ALLOW, role=deny_role, operation=op)
for op in EACLOperation
]
eacl_deny = [EACLRule(access=EACLAccess.ALLOW, role=deny_role, operation=op) for op in EACLOperation]
set_eacl(
user_wallet.wallet_path,
cid,
@ -162,10 +149,7 @@ class TestEACLContainer(ClusterTestBase):
)
for op in EACLOperation
]
eacl += [
EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op)
for op in EACLOperation
]
eacl += [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
set_eacl(
user_wallet.wallet_path,
cid,
@ -218,14 +202,8 @@ class TestEACLContainer(ClusterTestBase):
storage_node = self.cluster.storage_nodes[0]
with allure.step("Deny all operations for user via eACL"):
eacl_deny = [
EACLRule(access=EACLAccess.DENY, role=EACLRole.USER, operation=op)
for op in EACLOperation
]
eacl_deny += [
EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op)
for op in EACLOperation
]
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=EACLRole.USER, operation=op) for op in EACLOperation]
eacl_deny += [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
set_eacl(
user_wallet.wallet_path,
cid,
@ -249,9 +227,7 @@ class TestEACLContainer(ClusterTestBase):
)
@allure.title("Operations with extended ACL for SYSTEM (obj_size={object_size})")
def test_extended_actions_system(
self, wallets: Wallets, eacl_container_with_objects: tuple[str, list[str], str]
):
def test_extended_actions_system(self, wallets: Wallets, eacl_container_with_objects: tuple[str, list[str], str]):
user_wallet = wallets.get_wallet()
ir_wallet, storage_wallet = wallets.get_wallets_list(role=EACLRole.SYSTEM)[:2]
@ -392,8 +368,7 @@ class TestEACLContainer(ClusterTestBase):
create_eacl(
cid=cid,
rules_list=[
EACLRule(access=EACLAccess.DENY, role=EACLRole.SYSTEM, operation=op)
for op in EACLOperation
EACLRule(access=EACLAccess.DENY, role=EACLRole.SYSTEM, operation=op) for op in EACLOperation
],
shell=self.shell,
),
@ -543,8 +518,7 @@ class TestEACLContainer(ClusterTestBase):
create_eacl(
cid=cid,
rules_list=[
EACLRule(access=EACLAccess.ALLOW, role=EACLRole.SYSTEM, operation=op)
for op in EACLOperation
EACLRule(access=EACLAccess.ALLOW, role=EACLRole.SYSTEM, operation=op) for op in EACLOperation
],
shell=self.shell,
),

113
pytest_tests/testsuites/acl/test_eacl_filters.py
View file

@ -1,12 +1,7 @@
import allure
import pytest
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.acl import (
create_eacl,
form_bearertoken_file,
set_eacl,
wait_for_cache_expired,
)
from frostfs_testlib.steps.acl import create_eacl, form_bearertoken_file, set_eacl, wait_for_cache_expired
from frostfs_testlib.steps.cli.container import create_container, delete_container
from frostfs_testlib.steps.cli.object import put_object_to_random_node
from frostfs_testlib.storage.dataclasses.acl import (
@ -21,15 +16,11 @@ from frostfs_testlib.storage.dataclasses.acl import (
)
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.container_access import (
check_full_access_to_container,
check_no_access_to_container,
)
from pytest_tests.helpers.container_access import check_full_access_to_container, check_no_access_to_container
from pytest_tests.helpers.object_access import can_get_head_object, can_get_object, can_put_object
from pytest_tests.testsuites.acl.conftest import Wallets
@pytest.mark.sanity
@pytest.mark.acl
@pytest.mark.acl_filters
class TestEACLFilters(ClusterTestBase):
@ -46,18 +37,14 @@ class TestEACLFilters(ClusterTestBase):
"x_key": "other_value",
"check_key": "other_value",
}
REQ_EQUAL_FILTER = EACLFilter(
key="check_key", value="check_value", header_type=EACLHeaderType.REQUEST
)
REQ_EQUAL_FILTER = EACLFilter(key="check_key", value="check_value", header_type=EACLHeaderType.REQUEST)
NOT_REQ_EQUAL_FILTER = EACLFilter(
key="check_key",
value="other_value",
match_type=EACLMatchType.STRING_NOT_EQUAL,
header_type=EACLHeaderType.REQUEST,
)
OBJ_EQUAL_FILTER = EACLFilter(
key="check_key", value="check_value", header_type=EACLHeaderType.OBJECT
)
OBJ_EQUAL_FILTER = EACLFilter(key="check_key", value="check_value", header_type=EACLHeaderType.OBJECT)
NOT_OBJ_EQUAL_FILTER = EACLFilter(
key="check_key",
value="other_value",
@ -128,12 +115,9 @@ class TestEACLFilters(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint,
)
@allure.title(
"Operations with request filter (match_type={match_type}, obj_size={object_size})"
)
@pytest.mark.parametrize(
"match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL]
)
@pytest.mark.sanity
@allure.title("Operations with request filter (match_type={match_type}, obj_size={object_size})")
@pytest.mark.parametrize("match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL])
def test_extended_acl_filters_request(
self,
wallets: Wallets,
@ -174,12 +158,8 @@ class TestEACLFilters(ClusterTestBase):
# Filter denies requests where "check_key {match_type} ATTRIBUTE", so when match_type
# is STRING_EQUAL, then requests with "check_key=OTHER_ATTRIBUTE" will be allowed while
# requests with "check_key=ATTRIBUTE" will be denied, and vice versa
allow_headers = (
self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
)
deny_headers = (
self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
)
allow_headers = self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
deny_headers = self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
# We test on 3 groups of objects with various headers,
# but eACL rule should ignore object headers and
# work only based on request headers
@ -198,9 +178,7 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster,
)
with allure.step(
"Check other has full access when sending request with allowed headers"
):
with allure.step("Check other has full access when sending request with allowed headers"):
check_full_access_to_container(
other_wallet.wallet_path,
cid,
@ -223,16 +201,12 @@ class TestEACLFilters(ClusterTestBase):
)
with allure.step(
"Check other has full access when sending request "
"with denied headers and using bearer token"
"Check other has full access when sending request " "with denied headers and using bearer token"
):
bearer_other = form_bearertoken_file(
user_wallet.wallet_path,
cid,
[
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS)
for op in EACLOperation
],
[EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) for op in EACLOperation],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
@ -247,12 +221,8 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other,
)
@allure.title(
"Operations with deny user headers filter (match_type={match_type}, obj_size={object_size})"
)
@pytest.mark.parametrize(
"match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL]
)
@allure.title("Operations with deny user headers filter (match_type={match_type}, obj_size={object_size})")
@pytest.mark.parametrize("match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL])
def test_extended_acl_deny_filters_object(
self,
wallets: Wallets,
@ -290,16 +260,8 @@ class TestEACLFilters(ClusterTestBase):
)
wait_for_cache_expired()
allow_objects = (
objects_with_other_header
if match_type == EACLMatchType.STRING_EQUAL
else objects_with_header
)
deny_objects = (
objects_with_header
if match_type == EACLMatchType.STRING_EQUAL
else objects_with_other_header
)
allow_objects = objects_with_other_header if match_type == EACLMatchType.STRING_EQUAL else objects_with_header
deny_objects = objects_with_header if match_type == EACLMatchType.STRING_EQUAL else objects_with_other_header
# We will attempt requests with various headers,
# but eACL rule should ignore request headers and validate
@ -348,9 +310,7 @@ class TestEACLFilters(ClusterTestBase):
xhdr=xhdr,
)
with allure.step(
"Check other have access to objects with deny attribute and using bearer token"
):
with allure.step("Check other have access to objects with deny attribute and using bearer token"):
bearer_other = form_bearertoken_file(
user_wallet.wallet_path,
cid,
@ -376,9 +336,7 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other,
)
allow_attribute = (
self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
)
allow_attribute = self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
with allure.step("Check other can PUT objects without denied attribute"):
assert can_put_object(
other_wallet.wallet_path,
@ -388,13 +346,9 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster,
attributes=allow_attribute,
)
assert can_put_object(
other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster
)
assert can_put_object(other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster)
deny_attribute = (
self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
)
deny_attribute = self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
with allure.step("Check other can not PUT objects with denied attribute"):
with pytest.raises(AssertionError):
assert can_put_object(
@ -406,9 +360,7 @@ class TestEACLFilters(ClusterTestBase):
attributes=deny_attribute,
)
with allure.step(
"Check other can PUT objects with denied attribute and using bearer token"
):
with allure.step("Check other can PUT objects with denied attribute and using bearer token"):
bearer_other_for_put = form_bearertoken_file(
user_wallet.wallet_path,
cid,
@ -432,12 +384,8 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other_for_put,
)
@allure.title(
"Operations with allow eACL user headers filters (match_type={match_type}, obj_size={object_size})"
)
@pytest.mark.parametrize(
"match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL]
)
@allure.title("Operations with allow eACL user headers filters (match_type={match_type}, obj_size={object_size})")
@pytest.mark.parametrize("match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL])
def test_extended_acl_allow_filters_object(
self,
wallets: Wallets,
@ -454,9 +402,7 @@ class TestEACLFilters(ClusterTestBase):
file_path,
) = eacl_container_with_objects
with allure.step(
"Deny all operations for others except few operations allowed by object filter"
):
with allure.step("Deny all operations for others except few operations allowed by object filter"):
equal_filter = EACLFilter(**self.OBJ_EQUAL_FILTER.__dict__)
equal_filter.match_type = match_type
eacl = [
@ -511,13 +457,9 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster,
)
with pytest.raises(AssertionError):
assert can_put_object(
other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster
)
assert can_put_object(other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster)
with allure.step(
"Check other can get and put objects without attributes and using bearer token"
):
with allure.step("Check other can get and put objects without attributes and using bearer token"):
bearer_other = form_bearertoken_file(
user_wallet.wallet_path,
cid,
@ -613,8 +555,7 @@ class TestEACLFilters(ClusterTestBase):
)
with allure.step(
"Check other can get objects without attributes matching the filter "
"and using bearer token"
"Check other can get objects without attributes matching the filter " "and using bearer token"
):
oid = deny_objects.pop()
assert can_get_head_object(

30
pytest_tests/testsuites/conftest.py
View file

@ -19,7 +19,7 @@ from frostfs_testlib.resources.common import (
DEFAULT_WALLET_PASS,
SIMPLE_OBJECT_SIZE,
)
from frostfs_testlib.s3.interfaces import S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.shell import LocalShell, Shell
from frostfs_testlib.steps.cli.container import list_containers
from frostfs_testlib.steps.cli.object import get_netmap_netinfo
@ -146,7 +146,10 @@ def complex_object_size(max_object_size: int) -> ObjectSize:
# By default we want all tests to be executed with both object sizes
# This can be overriden in choosen tests if needed
@pytest.fixture(scope="session", params=["simple", "complex"])
@pytest.fixture(
scope="session",
params=[pytest.param("simple", marks=pytest.mark.simple), pytest.param("complex", marks=pytest.mark.complex)],
)
def object_size(
simple_object_size: ObjectSize, complex_object_size: ObjectSize, request: pytest.FixtureRequest
) -> ObjectSize:
@ -200,7 +203,13 @@ def cluster_state_controller(client_shell: Shell, cluster: Cluster, healthcheck:
@allure.step("[Class]: Create S3 client")
@pytest.fixture(scope="class")
@pytest.fixture(
scope="class",
params=[
pytest.param(AwsCliClient, marks=pytest.mark.aws),
pytest.param(Boto3ClientWrapper, marks=pytest.mark.boto3),
],
)
def s3_client(
default_wallet: str,
client_shell: Shell,
@ -237,24 +246,29 @@ def versioning_status(request: pytest.FixtureRequest) -> VersioningStatus:
@allure.step("Create/delete bucket")
@pytest.fixture
def bucket(s3_client: S3ClientWrapper, versioning_status: VersioningStatus):
def bucket(s3_client: S3ClientWrapper, versioning_status: VersioningStatus, request: pytest.FixtureRequest):
bucket_name = s3_client.create_bucket()
if versioning_status:
s3_helper.set_bucket_versioning(s3_client, bucket_name, versioning_status)
yield bucket_name
s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
if "sanity" not in request.config.option.markexpr:
s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
@allure.step("Create two buckets")
@pytest.fixture
def two_buckets(s3_client: S3ClientWrapper):
def two_buckets(s3_client: S3ClientWrapper, request: pytest.FixtureRequest):
bucket_1 = s3_client.create_bucket()
bucket_2 = s3_client.create_bucket()
yield bucket_1, bucket_2
for bucket_name in [bucket_1, bucket_2]:
s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
if "sanity" not in request.config.option.markexpr:
for bucket_name in [bucket_1, bucket_2]:
s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
@allure.step("[Autouse/Session] Check binary versions")

9
pytest_tests/testsuites/container/test_policy.py
View file

@ -92,6 +92,7 @@ class TestPolicy(ClusterTestBase):
endpoint=endpoint,
)
@pytest.mark.sanity
@allure.title("Simple policy results with one node")
def test_simple_policy_results_with_one_node(
self,
@ -166,6 +167,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Policy with SELECT and FILTER results with one node")
def test_policy_with_select_and_filter_results_with_one_node(
self,
@ -262,6 +264,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Policy with Multi SELECTs and FILTERs results with one node")
def test_policy_with_multi_selects_and_filters_results_with_one_node(
self,
@ -312,6 +315,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Policy with SELECT and FILTER results with UNIQUE nodes")
def test_policy_with_select_and_filter_results_with_unique_nodes(
self,
@ -818,6 +822,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Simple policy results with 75% of available nodes")
def test_simple_policy_results_with_75_of_available_nodes(
self,
@ -1049,6 +1054,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Simple policy results with 100% of available nodes")
def test_simple_policy_results_with_100_of_available_nodes(
self,
@ -1267,6 +1273,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Simple policy results with UNIQUE nodes")
def test_simple_policy_results_with_unique_nodes(
self,
@ -1348,6 +1355,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Policy with SELECT and Complex FILTER results with UNIQUE nodes")
def test_policy_with_select_and_complex_filter_results_with_unique_nodes(
self,
@ -1399,6 +1407,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Policy with Multi SELECTs and FILTERs results with UNIQUE nodes")
def test_policy_with_multi_selects_and_filters_results_with_unique_nodes(
self,

6
pytest_tests/testsuites/failovers/test_failover_network.py
View file

@ -6,7 +6,6 @@ from time import sleep
import allure
import pytest
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.s3 import AwsCliClient
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import (
get_object,
@ -38,11 +37,6 @@ OBJECT_ATTRIBUTES = [
]
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient], indirect=True)
@pytest.mark.failover
@pytest.mark.failover_network
class TestFailoverNetwork(ClusterTestBase):

8
pytest_tests/testsuites/failovers/test_failover_storage.py
View file

@ -7,7 +7,8 @@ import allure
import pytest
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.shell import CommandOptions
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
from frostfs_testlib.steps.cli.object import get_object, put_object_to_random_node
from frostfs_testlib.steps.node_management import (
@ -35,11 +36,6 @@ logger = logging.getLogger("NeoLogger")
stopped_nodes: list[StorageNode] = []
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.fixture(scope="function")
@allure.title("Provide File Keeper")
def file_keeper():

59
pytest_tests/testsuites/object/test_object_api.py
View file

@ -79,9 +79,7 @@ def generate_ranges(
range_length = random.randint(RANGE_MIN_LEN, RANGE_MAX_LEN)
range_start = random.randint(offset, offset + length)
file_ranges_to_test.append(
(range_start, min(range_length, storage_object.size - range_start))
)
file_ranges_to_test.append((range_start, min(range_length, storage_object.size - range_start)))
file_ranges_to_test.extend(STATIC_RANGES.get(storage_object.size, []))
@ -250,9 +248,7 @@ class TestObjectApi(ClusterTestBase):
assert sorted(expected_oids) == sorted(result)
@allure.title("Search objects with removed items (obj_size={object_size})")
def test_object_search_should_return_tombstone_items(
self, default_wallet: str, object_size: ObjectSize
):
def test_object_search_should_return_tombstone_items(self, default_wallet: str, object_size: ObjectSize):
"""
Validate object search with removed items
"""
@ -275,9 +271,7 @@ class TestObjectApi(ClusterTestBase):
with allure.step("Search object"):
# Root Search object should return root object oid
result = search_object(
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True
)
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True)
assert result == [storage_object.oid]
with allure.step("Delete file"):
@ -285,22 +279,14 @@ class TestObjectApi(ClusterTestBase):
with allure.step("Search deleted object with --root"):
# Root Search object should return nothing
result = search_object(
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True
)
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True)
assert len(result) == 0
with allure.step("Search deleted object with --phy should return only tombstones"):
# Physical Search object should return only tombstones
result = search_object(
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, phy=True
)
assert (
storage_object.tombstone in result
), "Search result should contain tombstone of removed object"
assert (
storage_object.oid not in result
), "Search result should not contain ObjectId of removed object"
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, phy=True)
assert storage_object.tombstone in result, "Search result should contain tombstone of removed object"
assert storage_object.oid not in result, "Search result should not contain ObjectId of removed object"
for tombstone_oid in result:
header = head_object(
wallet,
@ -315,7 +301,6 @@ class TestObjectApi(ClusterTestBase):
), f"Object wasn't deleted properly. Found object {tombstone_oid} with type {object_type}"
@allure.title("Get range hash by native API (obj_size={object_size})")
@pytest.mark.sanity
@pytest.mark.grpc_api
def test_object_get_range_hash(self, storage_objects: list[StorageObjectInfo], max_object_size):
"""
@ -327,9 +312,7 @@ class TestObjectApi(ClusterTestBase):
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path
file_ranges_to_test = generate_ranges(
storage_objects[0], max_object_size, self.shell, self.cluster
)
file_ranges_to_test = generate_ranges(storage_objects[0], max_object_size, self.shell, self.cluster)
logging.info(f"Ranges used in test {file_ranges_to_test}")
for range_start, range_len in file_ranges_to_test:
@ -349,7 +332,6 @@ class TestObjectApi(ClusterTestBase):
), f"Expected range hash to match {range_cut} slice of file payload"
@allure.title("Get range by native API (obj_size={object_size})")
@pytest.mark.sanity
@pytest.mark.grpc_api
def test_object_get_range(self, storage_objects: list[StorageObjectInfo], max_object_size):
"""
@ -361,9 +343,7 @@ class TestObjectApi(ClusterTestBase):
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path
file_ranges_to_test = generate_ranges(
storage_objects[0], max_object_size, self.shell, self.cluster
)
file_ranges_to_test = generate_ranges(storage_objects[0], max_object_size, self.shell, self.cluster)
logging.info(f"Ranges used in test {file_ranges_to_test}")
for range_start, range_len in file_ranges_to_test:
@ -379,14 +359,11 @@ class TestObjectApi(ClusterTestBase):
range_cut=range_cut,
)
assert (
get_file_content(
file_path, content_len=range_len, mode="rb", offset=range_start
)
get_file_content(file_path, content_len=range_len, mode="rb", offset=range_start)
== range_content
), f"Expected range content to match {range_cut} slice of file payload"
@allure.title("[NEGATIVE] Get invalid range by native API (obj_size={object_size})")
@pytest.mark.sanity
@pytest.mark.grpc_api
def test_object_get_range_negatives(
self,
@ -421,11 +398,7 @@ class TestObjectApi(ClusterTestBase):
for range_start, range_len, expected_error in file_ranges_to_test:
range_cut = f"{range_start}:{range_len}"
expected_error = (
expected_error.format(range=range_cut)
if "{range}" in expected_error
else expected_error
)
expected_error = expected_error.format(range=range_cut) if "{range}" in expected_error else expected_error
with allure.step(f"Get range ({range_cut})"):
for oid in oids:
with pytest.raises(Exception, match=expected_error):
@ -472,11 +445,7 @@ class TestObjectApi(ClusterTestBase):
for range_start, range_len, expected_error in file_ranges_to_test:
range_cut = f"{range_start}:{range_len}"
expected_error = (
expected_error.format(range=range_cut)
if "{range}" in expected_error
else expected_error
)
expected_error = expected_error.format(range=range_cut) if "{range}" in expected_error else expected_error
with allure.step(f"Get range hash ({range_cut})"):
for oid in oids:
with pytest.raises(Exception, match=expected_error):
@ -491,9 +460,7 @@ class TestObjectApi(ClusterTestBase):
def check_header_is_presented(self, head_info: dict, object_header: dict) -> None:
for key_to_check, val_to_check in object_header.items():
assert (
key_to_check in head_info["header"]["attributes"]
), f"Key {key_to_check} is found in {head_object}"
assert key_to_check in head_info["header"]["attributes"], f"Key {key_to_check} is found in {head_object}"
assert head_info["header"]["attributes"].get(key_to_check) == str(
val_to_check
), f"Value {val_to_check} is equal"

13
pytest_tests/testsuites/object/test_object_api_bearer.py
View file

@ -28,10 +28,7 @@ def bearer_token_file_all_allow(default_wallet: str, client_shell: Shell, cluste
bearer = form_bearertoken_file(
default_wallet,
"",
[
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS)
for op in EACLOperation
],
[EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) for op in EACLOperation],
shell=client_shell,
endpoint=cluster.default_rpc_endpoint,
)
@ -85,9 +82,7 @@ def storage_objects(
@pytest.mark.smoke
@pytest.mark.bearer
class TestObjectApiWithBearerToken(ClusterTestBase):
@allure.title(
"Object can be deleted from any node using s3gate wallet with bearer token (obj_size={object_size})"
)
@allure.title("Object can be deleted from any node using s3gate wallet with bearer token (obj_size={object_size})")
@pytest.mark.parametrize(
"user_container",
[SINGLE_PLACEMENT_RULE],
@ -112,9 +107,7 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
wallet_config=s3_gate_wallet.get_wallet_config_path(),
)
@allure.title(
"Object can be fetched from any node using s3gate wallet with bearer token (obj_size={object_size})"
)
@allure.title("Object can be fetched from any node using s3gate wallet with bearer token (obj_size={object_size})")
@pytest.mark.parametrize(
"user_container",
[REP_2_FOR_3_NODES_PLACEMENT_RULE],

91
pytest_tests/testsuites/object/test_object_lock.py
View file

@ -14,11 +14,7 @@ from frostfs_testlib.resources.error_patterns import (
OBJECT_NOT_FOUND,
)
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import (
StorageContainer,
StorageContainerInfo,
create_container,
)
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
from frostfs_testlib.steps.cli.object import delete_object, head_object, lock_object
from frostfs_testlib.steps.complex_object_actions import get_link_object, get_storage_object_chunks
from frostfs_testlib.steps.epoch import ensure_fresh_epoch, get_epoch, tick_epoch
@ -27,10 +23,7 @@ from frostfs_testlib.steps.storage_object import delete_objects
from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import (
LockObjectInfo,
StorageObjectInfo,
)
from frostfs_testlib.storage.dataclasses.storage_object_info import LockObjectInfo, StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success
@ -57,9 +50,7 @@ def user_wallet(wallet_factory: WalletFactory):
scope="module",
)
def user_container(user_wallet: WalletInfo, client_shell: Shell, cluster: Cluster):
container_id = create_container(
user_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint
)
container_id = create_container(user_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
return StorageContainer(StorageContainerInfo(container_id, user_wallet), client_shell, cluster)
@ -91,9 +82,7 @@ def locked_storage_object(
lifetime=FIXTURE_LOCK_LIFETIME,
)
storage_object.locks = [
LockObjectInfo(
storage_object.cid, lock_object_id, FIXTURE_LOCK_LIFETIME, expiration_epoch
)
LockObjectInfo(storage_object.cid, lock_object_id, FIXTURE_LOCK_LIFETIME, expiration_epoch)
]
yield storage_object
@ -117,17 +106,13 @@ def locked_storage_object(
except Exception as ex:
ex_message = str(ex)
# It's okay if object already removed
if not re.search(OBJECT_NOT_FOUND, ex_message) and not re.search(
OBJECT_ALREADY_REMOVED, ex_message
):
if not re.search(OBJECT_NOT_FOUND, ex_message) and not re.search(OBJECT_ALREADY_REMOVED, ex_message):
raise ex
logger.debug(ex_message)
@wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME))
def check_object_not_found(
wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str
):
def check_object_not_found(wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object(
wallet_file_path,
@ -138,9 +123,7 @@ def check_object_not_found(
)
def verify_object_available(
wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str
):
def verify_object_available(wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
with expect_not_raises():
head_object(
wallet_file_path,
@ -151,13 +134,10 @@ def verify_object_available(
)
@pytest.mark.sanity
@pytest.mark.grpc_object_lock
class TestObjectLockWithGrpc(ClusterTestBase):
@pytest.fixture()
def new_locked_storage_object(
self, user_container: StorageContainer, object_size: ObjectSize
) -> StorageObjectInfo:
def new_locked_storage_object(self, user_container: StorageContainer, object_size: ObjectSize) -> StorageObjectInfo:
"""
Intention of this fixture is to provide new storage object for tests which may delete or corrupt the object or it's complementary objects
So we need a new one each time we ask for it
@ -284,6 +264,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
expire_at=wrong_expire_at,
)
@pytest.mark.sanity
@allure.title("Expired object is deleted when locks are expired (obj_size={object_size})")
def test_expired_object_should_be_deleted_after_locks_are_expired(
self,
@ -295,9 +276,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object(
object_size.value, expire_at=current_epoch + 1
)
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 1)
with allure.step("Lock object for couple epochs"):
lock_object(
@ -355,9 +334,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with allure.step("Generate three objects"):
for _ in range(3):
storage_objects.append(
user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
)
storage_objects.append(user_container.generate_object(object_size.value, expire_at=current_epoch + 5))
lock_object(
storage_objects[0].wallet_file_path,
@ -398,16 +375,12 @@ class TestObjectLockWithGrpc(ClusterTestBase):
current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object(
object_size.value, expire_at=current_epoch + 1
)
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 1)
expiration_epoch = current_epoch - 1
with pytest.raises(
Exception,
match=LOCK_OBJECT_EXPIRATION.format(
expiration_epoch=expiration_epoch, current_epoch=current_epoch
),
match=LOCK_OBJECT_EXPIRATION.format(expiration_epoch=expiration_epoch, current_epoch=current_epoch),
):
lock_object(
storage_object.wallet_file_path,
@ -418,6 +391,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
expire_at=expiration_epoch,
)
@pytest.mark.sanity
@allure.title("Delete object when lock is expired by lifetime (obj_size={object_size})")
@expect_not_raises()
def test_after_lock_expiration_with_lifetime_user_should_be_able_to_delete_object(
@ -430,9 +404,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
"""
current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object(
object_size.value, expire_at=current_epoch + 5
)
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
lock_object(
storage_object.wallet_file_path,
@ -466,9 +438,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object(
object_size.value, expire_at=current_epoch + 5
)
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
lock_object(
storage_object.wallet_file_path,
@ -505,9 +475,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
Complex object chunks should also be protected from deletion
"""
chunk_object_ids = get_storage_object_chunks(
locked_storage_object, self.shell, self.cluster
)
chunk_object_ids = get_storage_object_chunks(locked_storage_object, self.shell, self.cluster)
for chunk_object_id in chunk_object_ids:
with allure.step(f"Try to delete chunk object {chunk_object_id}"):
with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
@ -527,9 +495,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
["complex"],
indirect=True,
)
def test_link_object_of_locked_complex_object_can_be_dropped(
self, new_locked_storage_object: StorageObjectInfo
):
def test_link_object_of_locked_complex_object_can_be_dropped(self, new_locked_storage_object: StorageObjectInfo):
link_object_id = get_link_object(
new_locked_storage_object.wallet_file_path,
new_locked_storage_object.cid,
@ -557,12 +523,8 @@ class TestObjectLockWithGrpc(ClusterTestBase):
["complex"],
indirect=True,
)
def test_chunks_of_locked_complex_object_can_be_dropped(
self, new_locked_storage_object: StorageObjectInfo
):
chunk_objects = get_storage_object_chunks(
new_locked_storage_object, self.shell, self.cluster
)
def test_chunks_of_locked_complex_object_can_be_dropped(self, new_locked_storage_object: StorageObjectInfo):
chunk_objects = get_storage_object_chunks(new_locked_storage_object, self.shell, self.cluster)
for chunk_object_id in chunk_objects:
with allure.step(f"Drop chunk object with id {chunk_object_id} from nodes"):
@ -630,9 +592,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
object_size: ObjectSize,
):
current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object(
object_size.value, expire_at=current_epoch + 1
)
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 1)
with allure.step("Apply first lock to object for 3 epochs"):
lock_object_id_0 = lock_object(
@ -705,9 +665,8 @@ class TestObjectLockWithGrpc(ClusterTestBase):
self.cluster.default_rpc_endpoint,
)
@allure.title(
"Two expired objects with one lock are deleted after lock expiration (obj_size={object_size})"
)
@pytest.mark.sanity
@allure.title("Two expired objects with one lock are deleted after lock expiration (obj_size={object_size})")
def test_two_objects_expiration_with_one_lock(
self,
user_container: StorageContainer,
@ -720,9 +679,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with allure.step("Generate two objects"):
for epoch_i in range(2):
storage_objects.append(
user_container.generate_object(
object_size.value, expire_at=current_epoch + epoch_i + 3
)
user_container.generate_object(object_size.value, expire_at=current_epoch + epoch_i + 3)
)
self.tick_epoch()

1
pytest_tests/testsuites/replication/test_replication.py
View file

@ -20,6 +20,7 @@ OBJECT_ATTRIBUTES = {"common_key": "common_value"}
WAIT_FOR_REPLICATION = 60
# Adding failover mark because it may make cluster unhealthy
@pytest.mark.sanity
@pytest.mark.failover
@pytest.mark.replication
class TestReplication(ClusterTestBase):

14
pytest_tests/testsuites/services/http_gate/test_http_bearer.py
View file

@ -21,7 +21,6 @@ from frostfs_testlib.utils.file_utils import generate_file
logger = logging.getLogger("NeoLogger")
@pytest.mark.sanity
@pytest.mark.http_gate
@pytest.mark.skip("Skipped due to deprecated PUT via http")
@pytest.mark.http_put
@ -46,9 +45,7 @@ class Test_http_bearer(ClusterTestBase):
@pytest.fixture(scope="class")
def eacl_deny_for_others(self, user_container: str) -> None:
with allure.step(f"Set deny all operations for {EACLRole.OTHERS} via eACL"):
eacl = EACLRule(
access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=EACLOperation.PUT
)
eacl = EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=EACLOperation.PUT)
set_eacl(
self.wallet,
user_container,
@ -64,10 +61,7 @@ class Test_http_bearer(ClusterTestBase):
bearer = form_bearertoken_file(
self.wallet,
user_container,
[
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS)
for op in EACLOperation
],
[EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) for op in EACLOperation],
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
sign=False,
@ -105,9 +99,7 @@ class Test_http_bearer(ClusterTestBase):
eacl_deny_for_others
bearer = bearer_token_no_limit_for_others
file_path = generate_file(object_size.value)
with allure.step(
f"Put object with bearer token for {EACLRole.OTHERS}, then get and verify hashes"
):
with allure.step(f"Put object with bearer token for {EACLRole.OTHERS}, then get and verify hashes"):
headers = [f" -H 'Authorization: Bearer {bearer}'"]
oid = upload_via_http_gate_curl(
cid=user_container,

17
pytest_tests/testsuites/services/http_gate/test_http_gate.py
View file

@ -44,9 +44,7 @@ class TestHttpGate(ClusterTestBase):
TestHttpGate.wallet = default_wallet
@allure.title("Put over gRPC, Get over HTTP")
def test_put_grpc_get_http(
self, complex_object_size: ObjectSize, simple_object_size: ObjectSize
):
def test_put_grpc_get_http(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
"""
Test that object can be put using gRPC interface and get using HTTP.
@ -106,7 +104,6 @@ class TestHttpGate(ClusterTestBase):
)
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@pytest.mark.sanity
@pytest.mark.http_gate
@pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http")
@ -115,9 +112,7 @@ class TestHttpPut(ClusterTestBase):
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@allure.title("Put over HTTP, Get over HTTP")
@pytest.mark.smoke
def test_put_http_get_http(
self, complex_object_size: ObjectSize, simple_object_size: ObjectSize
):
def test_put_http_get_http(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
"""
Test that object can be put and get using HTTP interface.
@ -338,9 +333,7 @@ class TestHttpPut(ClusterTestBase):
file_path = generate_file(complex_object_size.value)
with allure.step("Put objects using HTTP"):
oid_gate = upload_via_http_gate(
cid=cid, path=file_path, endpoint=self.cluster.default_http_gate_endpoint
)
oid_gate = upload_via_http_gate(cid=cid, path=file_path, endpoint=self.cluster.default_http_gate_endpoint)
oid_curl = upload_via_http_gate_curl(
cid=cid,
filepath=file_path,
@ -371,9 +364,7 @@ class TestHttpPut(ClusterTestBase):
@pytest.mark.skip("Skipped due to deprecated PUT via http")
@allure.title("Put/Get over HTTP using Curl utility")
def test_put_http_get_http_curl(
self, complex_object_size: ObjectSize, simple_object_size: ObjectSize
):
def test_put_http_get_http_curl(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
"""
Test checks upload and download over HTTP using curl utility.
"""

17
pytest_tests/testsuites/services/http_gate/test_http_headers.py
View file

@ -27,7 +27,6 @@ OBJECT_ALREADY_REMOVED_ERROR = "object already removed"
logger = logging.getLogger("NeoLogger")
@pytest.mark.sanity
@pytest.mark.http_gate
@pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http")
@ -79,9 +78,7 @@ class Test_http_headers(ClusterTestBase):
yield storage_objects
@allure.title("Get object1 by attribute")
def test_object1_can_be_get_by_attr(
self, storage_objects_with_attributes: list[StorageObjectInfo]
):
def test_object1_can_be_get_by_attr(self, storage_objects_with_attributes: list[StorageObjectInfo]):
"""
Test to get object#1 by attribute and comapre hashes
@ -104,9 +101,7 @@ class Test_http_headers(ClusterTestBase):
)
@allure.title("Get object2 with different attributes, then delete object2 and get object1")
def test_object2_can_be_get_by_attr(
self, storage_objects_with_attributes: list[StorageObjectInfo]
):
def test_object2_can_be_get_by_attr(self, storage_objects_with_attributes: list[StorageObjectInfo]):
"""
Test to get object2 with different attributes, then delete object2 and get object1 using 1st attribute. Note: obj1 and obj2 have the same attribute#1,
and when obj2 is deleted you can get obj1 by 1st attribute
@ -167,9 +162,7 @@ class Test_http_headers(ClusterTestBase):
)
@allure.title("[NEGATIVE] Put object and get right after container is deleted")
def test_negative_put_and_get_object3(
self, storage_objects_with_attributes: list[StorageObjectInfo]
):
def test_negative_put_and_get_object3(self, storage_objects_with_attributes: list[StorageObjectInfo]):
"""
Test to attempt to put object and try to download it right after the container has been deleted
@ -215,9 +208,7 @@ class Test_http_headers(ClusterTestBase):
assert storage_object_1.cid not in list_containers(
self.wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
with allure.step(
"[Negative] Try to download (wget) object via wget with attributes [peace=peace]"
):
with allure.step("[Negative] Try to download (wget) object via wget with attributes [peace=peace]"):
request = f"/get/{storage_object_1.cid}/peace/peace"
error_pattern = "404 Not Found"
try_to_get_object_via_passed_request_and_expect_error(

5
pytest_tests/testsuites/services/http_gate/test_http_streaming.py
View file

@ -12,7 +12,6 @@ from frostfs_testlib.utils.file_utils import generate_file
logger = logging.getLogger("NeoLogger")
@pytest.mark.sanity
@pytest.mark.http_gate
@pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http")
@ -50,9 +49,7 @@ class Test_http_streaming(ClusterTestBase):
# Generate file
file_path = generate_file(complex_object_size.value)
with allure.step(
"Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]"
):
with allure.step("Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]"):
oid = upload_via_http_gate_curl(
cid=cid, filepath=file_path, endpoint=self.cluster.default_http_gate_endpoint
)

75
pytest_tests/testsuites/services/http_gate/test_http_system_header.py
View file

@ -8,11 +8,7 @@ import pytest
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import (
get_netmap_netinfo,
get_object_from_random_node,
head_object,
)
from frostfs_testlib.steps.cli.object import get_netmap_netinfo, get_object_from_random_node, head_object
from frostfs_testlib.steps.epoch import get_epoch, wait_for_epochs_align
from frostfs_testlib.steps.http.http_gate import (
attr_into_str_header_curl,
@ -35,7 +31,6 @@ SYSTEM_EXPIRATION_TIMESTAMP = "System-Expiration-Timestamp"
SYSTEM_EXPIRATION_RFC3339 = "System-Expiration-RFC3339"
@pytest.mark.sanity
@pytest.mark.http_gate
@pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http")
@ -76,9 +71,7 @@ class Test_http_system_header(ClusterTestBase):
return f"{mins}m"
@allure.title("Return future timestamp after N epochs are passed")
def epoch_count_into_timestamp(
self, epoch_duration: int, epoch: int, rfc3339: Optional[bool] = False
) -> str:
def epoch_count_into_timestamp(self, epoch_duration: int, epoch: int, rfc3339: Optional[bool] = False) -> str:
current_datetime = datetime.datetime.utcnow()
epoch_count_in_seconds = epoch_duration * epoch
future_datetime = current_datetime + datetime.timedelta(seconds=epoch_count_in_seconds)
@ -96,9 +89,7 @@ class Test_http_system_header(ClusterTestBase):
return False
return True
@allure.title(
f"Validate that only {EXPIRATION_EPOCH_HEADER} exists in header and other headers are abesent"
)
@allure.title(f"Validate that only {EXPIRATION_EPOCH_HEADER} exists in header and other headers are abesent")
def validation_for_http_header_attr(self, head_info: dict, expected_epoch: int) -> None:
# check that __SYSTEM__EXPIRATION_EPOCH attribute has corresponding epoch
assert self.check_key_value_presented_header(
@ -146,13 +137,9 @@ class Test_http_system_header(ClusterTestBase):
@allure.title("[NEGATIVE] Put object with expired epoch")
def test_unable_put_expired_epoch(self, user_container: str, simple_object_size: ObjectSize):
headers = attr_into_str_header_curl(
{"System-Expiration-Epoch": str(get_epoch(self.shell, self.cluster) - 1)}
)
headers = attr_into_str_header_curl({"System-Expiration-Epoch": str(get_epoch(self.shell, self.cluster) - 1)})
file_path = generate_file(simple_object_size.value)
with allure.step(
"Put object using HTTP with attribute Expiration-Epoch where epoch is expired"
):
with allure.step("Put object using HTTP with attribute Expiration-Epoch where epoch is expired"):
upload_via_http_gate_curl(
cid=user_container,
filepath=file_path,
@ -162,14 +149,10 @@ class Test_http_system_header(ClusterTestBase):
)
@allure.title("[NEGATIVE] Put object with negative System-Expiration-Duration")
def test_unable_put_negative_duration(
self, user_container: str, simple_object_size: ObjectSize
):
def test_unable_put_negative_duration(self, user_container: str, simple_object_size: ObjectSize):
headers = attr_into_str_header_curl({"System-Expiration-Duration": "-1h"})
file_path = generate_file(simple_object_size.value)
with allure.step(
"Put object using HTTP with attribute System-Expiration-Duration where duration is negative"
):
with allure.step("Put object using HTTP with attribute System-Expiration-Duration where duration is negative"):
upload_via_http_gate_curl(
cid=user_container,
filepath=file_path,
@ -179,9 +162,7 @@ class Test_http_system_header(ClusterTestBase):
)
@allure.title("[NEGATIVE] Put object with System-Expiration-Timestamp value in the past")
def test_unable_put_expired_timestamp(
self, user_container: str, simple_object_size: ObjectSize
):
def test_unable_put_expired_timestamp(self, user_container: str, simple_object_size: ObjectSize):
headers = attr_into_str_header_curl({"System-Expiration-Timestamp": "1635075727"})
file_path = generate_file(simple_object_size.value)
with allure.step(
@ -211,9 +192,7 @@ class Test_http_system_header(ClusterTestBase):
@allure.title("Priority of attributes epoch>duration (obj_size={object_size})")
@pytest.mark.skip("Temp disable for v0.37")
def test_http_attr_priority_epoch_duration(
self, user_container: str, object_size: ObjectSize, epoch_duration: int
):
def test_http_attr_priority_epoch_duration(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
self.tick_epoch()
epoch_count = 1
expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count
@ -247,15 +226,11 @@ class Test_http_system_header(ClusterTestBase):
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node(
self.wallet, user_container, oid, self.shell, self.cluster
)
get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
@allure.title("Priority of attributes duration>timestamp (obj_size={object_size})")
@pytest.mark.skip("Temp disable for v0.37")
def test_http_attr_priority_dur_timestamp(
self, user_container: str, object_size: ObjectSize, epoch_duration: int
):
def test_http_attr_priority_dur_timestamp(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
self.tick_epoch()
epoch_count = 2
expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count
@ -263,12 +238,8 @@ class Test_http_system_header(ClusterTestBase):
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
)
attributes = {
SYSTEM_EXPIRATION_DURATION: self.epoch_count_into_mins(
epoch_duration=epoch_duration, epoch=2
),
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(
epoch_duration=epoch_duration, epoch=1
),
SYSTEM_EXPIRATION_DURATION: self.epoch_count_into_mins(epoch_duration=epoch_duration, epoch=2),
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(epoch_duration=epoch_duration, epoch=1),
}
file_path = generate_file(object_size.value)
with allure.step(
@ -296,15 +267,11 @@ class Test_http_system_header(ClusterTestBase):
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node(
self.wallet, user_container, oid, self.shell, self.cluster
)
get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
@allure.title("Priority of attributes timestamp>Expiration-RFC (obj_size={object_size})")
@pytest.mark.skip("Temp disable for v0.37")
def test_http_attr_priority_timestamp_rfc(
self, user_container: str, object_size: ObjectSize, epoch_duration: int
):
def test_http_attr_priority_timestamp_rfc(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
self.tick_epoch()
epoch_count = 2
expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count
@ -312,9 +279,7 @@ class Test_http_system_header(ClusterTestBase):
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
)
attributes = {
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(
epoch_duration=epoch_duration, epoch=2
),
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(epoch_duration=epoch_duration, epoch=2),
SYSTEM_EXPIRATION_RFC3339: self.epoch_count_into_timestamp(
epoch_duration=epoch_duration, epoch=1, rfc3339=True
),
@ -345,9 +310,7 @@ class Test_http_system_header(ClusterTestBase):
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node(
self.wallet, user_container, oid, self.shell, self.cluster
)
get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
@allure.title("Object should be deleted when expiration passed (obj_size={object_size})")
@pytest.mark.parametrize(
@ -399,6 +362,4 @@ class Test_http_system_header(ClusterTestBase):
)
# check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node(
self.wallet, user_container, oid, self.shell, self.cluster
)
get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)

17
pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py
View file

@ -6,15 +6,12 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.sanity
@pytest.mark.acl
@pytest.mark.s3_gate
class TestS3GateACL:
@allure.title("Object ACL (s3_client={s3_client})")
@pytest.mark.parametrize("s3_client", [AwsCliClient], indirect=True)
def test_s3_object_ACL(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_object_ACL(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
@ -33,9 +30,7 @@ class TestS3GateACL:
obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
with allure.step(
"Put object with grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
):
with allure.step("Put object with grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"):
s3_client.put_object_acl(
bucket,
file_name,
@ -48,9 +43,7 @@ class TestS3GateACL:
@pytest.mark.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
def test_s3_bucket_ACL(self, s3_client: S3ClientWrapper):
with allure.step("Create bucket with ACL = public-read-write"):
bucket = s3_client.create_bucket(
object_lock_enabled_for_bucket=True, acl="public-read-write"
)
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read-write")
bucket_acl = s3_client.get_bucket_acl(bucket)
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers")
@ -59,9 +52,7 @@ class TestS3GateACL:
bucket_acl = s3_client.get_bucket_acl(bucket)
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser")
with allure.step(
"Change bucket acl to --grant-write uri=http://acs.amazonaws.com/groups/global/AllUsers"
):
with allure.step("Change bucket acl to --grant-write uri=http://acs.amazonaws.com/groups/global/AllUsers"):
s3_client.put_bucket_acl(
bucket,
grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers",

32
pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py
View file

@ -2,18 +2,12 @@ from datetime import datetime, timedelta
import allure
import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper
from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate
@pytest.mark.s3_gate_bucket
class TestS3GateBucket:
@ -26,23 +20,17 @@ class TestS3GateBucket:
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser")
with allure.step("Create bucket with ACL = public-read"):
bucket_1 = s3_client.create_bucket(
object_lock_enabled_for_bucket=True, acl="public-read"
)
bucket_1 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read")
bucket_acl_1 = s3_client.get_bucket_acl(bucket_1)
s3_helper.assert_s3_acl(acl_grants=bucket_acl_1, permitted_users="AllUsers")
with allure.step("Create bucket with ACL public-read-write"):
bucket_2 = s3_client.create_bucket(
object_lock_enabled_for_bucket=True, acl="public-read-write"
)
bucket_2 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read-write")
bucket_acl_2 = s3_client.get_bucket_acl(bucket_2)
s3_helper.assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers")
with allure.step("Create bucket with ACL = authenticated-read"):
bucket_3 = s3_client.create_bucket(
object_lock_enabled_for_bucket=True, acl="authenticated-read"
)
bucket_3 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="authenticated-read")
bucket_acl_3 = s3_client.get_bucket_acl(bucket_3)
s3_helper.assert_s3_acl(acl_grants=bucket_acl_3, permitted_users="AllUsers")
@ -74,18 +62,14 @@ class TestS3GateBucket:
s3_helper.assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers")
@allure.title("Create bucket with object lock (s3_client={s3_client})")
def test_s3_bucket_object_lock(
self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize
):
def test_s3_bucket_object_lock(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
with allure.step("Create bucket with --no-object-lock-enabled-for-bucket"):
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=False)
date_obj = datetime.utcnow() + timedelta(days=1)
with pytest.raises(
Exception, match=r".*Object Lock configuration does not exist for this bucket.*"
):
with pytest.raises(Exception, match=r".*Object Lock configuration does not exist for this bucket.*"):
# An error occurred (ObjectLockConfigurationNotFoundError) when calling the PutObject operation (reached max retries: 0):
# Object Lock configuration does not exist for this bucket
s3_client.put_object(
@ -104,9 +88,7 @@ class TestS3GateBucket:
object_lock_retain_until_date=date_obj_1.strftime("%Y-%m-%dT%H:%M:%S"),
object_lock_legal_hold_status="ON",
)
s3_helper.assert_object_lock_mode(
s3_client, bucket_1, file_name, "COMPLIANCE", date_obj_1, "ON"
)
s3_helper.assert_object_lock_mode(s3_client, bucket_1, file_name, "COMPLIANCE", date_obj_1, "ON")
@allure.title("Delete bucket (s3_client={s3_client})")
def test_s3_delete_bucket(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):

117
pytest_tests/testsuites/services/s3_gate/test_s3_gate.py
View file

@ -22,14 +22,7 @@ from frostfs_testlib.utils.file_utils import (
logger = logging.getLogger("NeoLogger")
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@allure.link(
"https://github.com/TrueCloudLab/frostfs-s3-gw#frostfs-s3-gw", name="frostfs-s3-gateway"
)
@allure.link("https://github.com/TrueCloudLab/frostfs-s3-gw#frostfs-s3-gw", name="frostfs-s3-gateway")
@pytest.mark.sanity
@pytest.mark.s3_gate
@pytest.mark.s3_gate_base
@ -73,9 +66,7 @@ class TestS3Gate:
s3_client.head_object(bucket_1, file_name)
bucket_objects = s3_client.list_objects(bucket_1)
assert (
file_name in bucket_objects
), f"Expected file {file_name} in objects list {bucket_objects}"
assert file_name in bucket_objects, f"Expected file {file_name} in objects list {bucket_objects}"
with allure.step("Try to delete not empty bucket and get error"):
with pytest.raises(Exception, match=r".*The bucket you tried to delete is not empty.*"):
@ -136,18 +127,14 @@ class TestS3Gate:
s3_client.head_object(bucket, file_name)
bucket_objects = s3_client.list_objects(bucket)
assert (
file_name in bucket_objects
), f"Expected file {file_name} in objects list {bucket_objects}"
assert file_name in bucket_objects, f"Expected file {file_name} in objects list {bucket_objects}"
with allure.step("Check object's attributes"):
for attrs in (["ETag"], ["ObjectSize", "StorageClass"]):
s3_client.get_object_attributes(bucket, file_name, attrs)
@allure.title("Sync directory (s3_client={s3_client})")
def test_s3_sync_dir(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_sync_dir(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
"""
Test checks sync directory with AWS CLI utility.
"""
@ -167,9 +154,7 @@ class TestS3Gate:
objects = s3_client.list_objects(bucket)
with allure.step("Check these are the same objects"):
assert set(key_to_path.keys()) == set(
objects
), f"Expected all objects saved. Got {objects}"
assert set(key_to_path.keys()) == set(objects), f"Expected all objects saved. Got {objects}"
for obj_key in objects:
got_object = s3_client.get_object(bucket, obj_key)
assert get_file_hash(got_object) == get_file_hash(
@ -177,32 +162,24 @@ class TestS3Gate:
), "Expected hashes are the same"
@allure.title("Object versioning (s3_client={s3_client})")
def test_s3_api_versioning(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_api_versioning(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
"""
Test checks basic versioning functionality for S3 bucket.
"""
version_1_content = "Version 1"
version_2_content = "Version 2"
file_name_simple = generate_file_with_content(
simple_object_size.value, content=version_1_content
)
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_name_simple)
generate_file_with_content(
simple_object_size.value, file_path=file_name_simple, content=version_2_content
)
generate_file_with_content(simple_object_size.value, file_path=file_name_simple, content=version_2_content)
version_id_2 = s3_client.put_object(bucket, file_name_simple)
with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket)
obj_versions = {
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
assert obj_versions == {
version_id_1,
version_id_2,
@ -213,20 +190,14 @@ class TestS3Gate:
response = s3_client.head_object(bucket, obj_key, version_id=version_id)
assert "LastModified" in response, "Expected LastModified field"
assert "ETag" in response, "Expected ETag field"
assert (
response.get("VersionId") == version_id
), f"Expected VersionId is {version_id}"
assert response.get("VersionId") == version_id, f"Expected VersionId is {version_id}"
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
with allure.step("Check object's attributes"):
for version_id in (version_id_1, version_id_2):
got_attrs = s3_client.get_object_attributes(
bucket, obj_key, ["ETag"], version_id=version_id
)
got_attrs = s3_client.get_object_attributes(bucket, obj_key, ["ETag"], version_id=version_id)
if got_attrs:
assert (
got_attrs.get("VersionId") == version_id
), f"Expected VersionId is {version_id}"
assert got_attrs.get("VersionId") == version_id, f"Expected VersionId is {version_id}"
with allure.step("Delete object and check it was deleted"):
response = s3_client.delete_object(bucket, obj_key)
@ -242,9 +213,7 @@ class TestS3Gate:
):
file_name = s3_client.get_object(bucket, obj_key, version_id=version)
got_content = get_file_content(file_name)
assert (
got_content == content
), f"Expected object content is\n{content}\nGot\n{got_content}"
assert got_content == content, f"Expected object content is\n{content}\nGot\n{got_content}"
with allure.step("Restore previous object version"):
s3_client.delete_object(bucket, obj_key, version_id=version_id_delete)
@ -257,17 +226,13 @@ class TestS3Gate:
@pytest.mark.s3_gate_multipart
@allure.title("Object Multipart API (s3_client={s3_client})")
def test_s3_api_multipart(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_api_multipart(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
"""
Test checks S3 Multipart API (Create multipart upload/Abort multipart upload/List multipart upload/
Upload part/List parts/Complete multipart upload).
"""
parts_count = 3
file_name_large = generate_file(
simple_object_size.value * 1024 * 6 * parts_count
) # 5Mb - min part
file_name_large = generate_file(simple_object_size.value * 1024 * 6 * parts_count) # 5Mb - min part
object_key = s3_helper.object_key_from_file_path(file_name_large)
part_files = split_file(file_name_large, parts_count)
parts = []
@ -279,12 +244,8 @@ class TestS3Gate:
upload_id = s3_client.create_multipart_upload(bucket, object_key)
uploads = s3_client.list_multipart_uploads(bucket)
assert uploads, f"Expected there one upload in bucket {bucket}"
assert (
uploads[0].get("Key") == object_key
), f"Expected correct key {object_key} in upload {uploads}"
assert (
uploads[0].get("UploadId") == upload_id
), f"Expected correct UploadId {upload_id} in upload {uploads}"
assert uploads[0].get("Key") == object_key, f"Expected correct key {object_key} in upload {uploads}"
assert uploads[0].get("UploadId") == upload_id, f"Expected correct UploadId {upload_id} in upload {uploads}"
s3_client.abort_multipart_upload(bucket, object_key, upload_id)
uploads = s3_client.list_multipart_uploads(bucket)
@ -298,9 +259,7 @@ class TestS3Gate:
with allure.step("Check all parts are visible in bucket"):
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
assert len(got_parts) == len(
part_files
), f"Expected {parts_count} parts, got\n{got_parts}"
assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
@ -327,9 +286,7 @@ class TestS3Gate:
s3_helper.check_tags_by_bucket(s3_client, bucket, [])
@allure.title("Object tagging API (s3_client={s3_client})")
def test_s3_api_object_tagging(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_api_object_tagging(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
"""
Test checks S3 Object tagging API (Put tag/Get tag/Update tag).
"""
@ -458,9 +415,7 @@ class TestS3Gate:
with allure.step("Check copied object has the same content"):
got_copied_file = s3_client.get_object(bucket, copy_obj_path)
assert get_file_hash(file_path_simple) == get_file_hash(
got_copied_file
), "Hashes must be the same"
assert get_file_hash(file_path_simple) == get_file_hash(got_copied_file), "Hashes must be the same"
with allure.step("Delete one object from bucket"):
s3_client.delete_object(bucket, file_name_simple)
@ -509,9 +464,7 @@ class TestS3Gate:
with allure.step("Check copied object has the same content"):
got_copied_file_b2 = s3_client.get_object(bucket_2, copy_obj_path_b2)
assert get_file_hash(file_path_large) == get_file_hash(
got_copied_file_b2
), "Hashes must be the same"
assert get_file_hash(file_path_large) == get_file_hash(got_copied_file_b2), "Hashes must be the same"
with allure.step("Delete one object from first bucket"):
s3_client.delete_object(bucket_1, file_name_simple)
@ -524,23 +477,15 @@ class TestS3Gate:
s3_client.delete_object(bucket_2, copy_obj_path_b2)
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[])
def check_object_attributes(
self, s3_client: S3ClientWrapper, bucket: str, object_key: str, parts_count: int
):
def check_object_attributes(self, s3_client: S3ClientWrapper, bucket: str, object_key: str, parts_count: int):
if not isinstance(s3_client, AwsCliClient):
logger.warning("Attributes check is not supported for boto3 implementation")
return
with allure.step("Check object's attributes"):
obj_parts = s3_client.get_object_attributes(
bucket, object_key, ["ObjectParts"], full_output=False
)
assert (
obj_parts.get("TotalPartsCount") == parts_count
), f"Expected TotalPartsCount is {parts_count}"
assert (
len(obj_parts.get("Parts")) == parts_count
), f"Expected Parts cunt is {parts_count}"
obj_parts = s3_client.get_object_attributes(bucket, object_key, ["ObjectParts"], full_output=False)
assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
assert len(obj_parts.get("Parts")) == parts_count, f"Expected Parts cunt is {parts_count}"
with allure.step("Check object's attribute max-parts"):
max_parts = 2
@ -551,13 +496,9 @@ class TestS3Gate:
max_parts=max_parts,
full_output=False,
)
assert (
obj_parts.get("TotalPartsCount") == parts_count
), f"Expected TotalPartsCount is {parts_count}"
assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
assert obj_parts.get("MaxParts") == max_parts, f"Expected MaxParts is {parts_count}"
assert (
len(obj_parts.get("Parts")) == max_parts
), f"Expected Parts count is {parts_count}"
assert len(obj_parts.get("Parts")) == max_parts, f"Expected Parts count is {parts_count}"
with allure.step("Check object's attribute part-number-marker"):
part_number_marker = 3
@ -568,9 +509,7 @@ class TestS3Gate:
part_number=part_number_marker,
full_output=False,
)
assert (
obj_parts.get("TotalPartsCount") == parts_count
), f"Expected TotalPartsCount is {parts_count}"
assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
assert (
obj_parts.get("PartNumberMarker") == part_number_marker
), f"Expected PartNumberMarker is {part_number_marker}"

76
pytest_tests/testsuites/services/s3_gate/test_s3_locking.py
View file

@ -3,28 +3,18 @@ from datetime import datetime, timedelta
import allure
import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper
from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate
@pytest.mark.s3_gate_locking
@pytest.mark.parametrize("version_id", [None, "second"])
class TestS3GateLocking:
@allure.title(
"Retention period and legal lock on object (version_id={version_id}, s3_client={s3_client})"
)
def test_s3_object_locking(
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
):
@allure.title("Retention period and legal lock on object (version_id={version_id}, s3_client={s3_client})")
def test_s3_object_locking(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 2
@ -46,15 +36,11 @@ class TestS3GateLocking:
"RetainUntilDate": date_obj,
}
s3_client.put_object_retention(bucket, file_name, retention, version_id)
s3_helper.assert_object_lock_mode(
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF"
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
with allure.step(f"Put legal hold to object {file_name}"):
s3_client.put_object_legal_hold(bucket, file_name, "ON", version_id)
s3_helper.assert_object_lock_mode(
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON"
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
with allure.step("Fail with deleting object with legal hold and retention period"):
if version_id:
@ -64,9 +50,7 @@ class TestS3GateLocking:
with allure.step("Check retention period is no longer set on the uploaded object"):
time.sleep((retention_period + 1) * 60)
s3_helper.assert_object_lock_mode(
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON"
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
with allure.step("Fail with deleting object with legal hold and retention period"):
if version_id:
@ -76,12 +60,8 @@ class TestS3GateLocking:
else:
s3_client.delete_object(bucket, file_name, version_id)
@allure.title(
"Impossible to change retention mode COMPLIANCE (version_id={version_id}, s3_client={s3_client})"
)
def test_s3_mode_compliance(
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
):
@allure.title("Impossible to change retention mode COMPLIANCE (version_id={version_id}, s3_client={s3_client})")
def test_s3_mode_compliance(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 2
@ -102,13 +82,9 @@ class TestS3GateLocking:
"RetainUntilDate": date_obj,
}
s3_client.put_object_retention(bucket, file_name, retention, version_id)
s3_helper.assert_object_lock_mode(
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF"
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
with allure.step(
f"Try to change retention period {retention_period_1}min to object {file_name}"
):
with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
retention = {
"Mode": "COMPLIANCE",
@ -117,12 +93,8 @@ class TestS3GateLocking:
with pytest.raises(Exception):
s3_client.put_object_retention(bucket, file_name, retention, version_id)
@allure.title(
"Change retention mode GOVERNANCE (version_id={version_id}, s3_client={s3_client})"
)
def test_s3_mode_governance(
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
):
@allure.title("Change retention mode GOVERNANCE (version_id={version_id}, s3_client={s3_client})")
def test_s3_mode_governance(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 3
@ -144,13 +116,9 @@ class TestS3GateLocking:
"RetainUntilDate": date_obj,
}
s3_client.put_object_retention(bucket, file_name, retention, version_id)
s3_helper.assert_object_lock_mode(
s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF"
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
with allure.step(
f"Try to change retention period {retention_period_1}min to object {file_name}"
):
with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
retention = {
"Mode": "GOVERNANCE",
@ -159,9 +127,7 @@ class TestS3GateLocking:
with pytest.raises(Exception):
s3_client.put_object_retention(bucket, file_name, retention, version_id)
with allure.step(
f"Try to change retention period {retention_period_1}min to object {file_name}"
):
with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
retention = {
"Mode": "GOVERNANCE",
@ -177,16 +143,12 @@ class TestS3GateLocking:
"RetainUntilDate": date_obj,
}
s3_client.put_object_retention(bucket, file_name, retention, version_id, True)
s3_helper.assert_object_lock_mode(
s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF"
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
@allure.title(
"[NEGATIVE] Lock object in bucket with disabled locking (version_id={version_id}, s3_client={s3_client})"
)
def test_s3_legal_hold(
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
):
def test_s3_legal_hold(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
@ -227,6 +189,4 @@ class TestS3GateLockingBucket:
with allure.step("Put object into bucket"):
s3_client.put_object(bucket, file_path)
s3_helper.assert_object_lock_mode(
s3_client, bucket, file_name, "COMPLIANCE", None, "OFF", 1
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", None, "OFF", 1)

36
pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py
View file

@ -1,6 +1,6 @@
import allure
import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.cli.container import list_objects, search_container_by_name
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
@ -10,18 +10,10 @@ from frostfs_testlib.utils.file_utils import generate_file, get_file_hash, split
PART_SIZE = 5 * 1024 * 1024
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate
@pytest.mark.s3_gate_multipart
class TestS3GateMultipart(ClusterTestBase):
NO_SUCH_UPLOAD = (
"The upload ID may be invalid, or the upload may have been aborted or completed."
)
NO_SUCH_UPLOAD = "The upload ID may be invalid, or the upload may have been aborted or completed."
@allure.title("Object Multipart API (s3_client={s3_client})")
@pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED], indirect=True)
@ -46,9 +38,7 @@ class TestS3GateMultipart(ClusterTestBase):
parts.append((part_id, etag))
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
assert len(got_parts) == len(
part_files
), f"Expected {parts_count} parts, got\n{got_parts}"
assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
with allure.step("Check upload list is empty"):
uploads = s3_client.list_multipart_uploads(bucket)
@ -91,12 +81,8 @@ class TestS3GateMultipart(ClusterTestBase):
assert len(parts) == files_count, f"Expected {files_count} parts, got\n{parts}"
with allure.step(f"Check that we have {files_count} files in container '{container_id}'"):
objects = list_objects(
default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint
)
assert (
len(objects) == files_count
), f"Expected {files_count} objects in container, got\n{objects}"
objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
assert len(objects) == files_count, f"Expected {files_count} objects in container, got\n{objects}"
with allure.step("Abort multipart upload"):
s3_client.abort_multipart_upload(bucket, upload_key, upload_id)
@ -108,9 +94,7 @@ class TestS3GateMultipart(ClusterTestBase):
s3_client.list_parts(bucket, upload_key, upload_id)
with allure.step("Check that we have no files in container since upload was aborted"):
objects = list_objects(
default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint
)
objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
assert len(objects) == 0, f"Expected no objects in container, got\n{objects}"
@allure.title("Upload Part Copy (s3_client={s3_client})")
@ -136,17 +120,13 @@ class TestS3GateMultipart(ClusterTestBase):
with allure.step("Upload parts to multipart upload"):
for part_id, obj_key in enumerate(objs, start=1):
etag = s3_client.upload_part_copy(
bucket, object_key, upload_id, part_id, f"{bucket}/{obj_key}"
)
etag = s3_client.upload_part_copy(bucket, object_key, upload_id, part_id, f"{bucket}/{obj_key}")
parts.append((part_id, etag))
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
with allure.step("Complete multipart upload"):
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
assert len(got_parts) == len(
part_files
), f"Expected {parts_count} parts, got\n{got_parts}"
assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
with allure.step("Check we can get whole object from bucket"):
got_object = s3_client.get_object(bucket, object_key)

261
pytest_tests/testsuites/services/s3_gate/test_s3_object.py
View file

@ -9,25 +9,14 @@ import allure
import pytest
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_PASS
from frostfs_testlib.resources.error_patterns import S3_MALFORMED_XML_REQUEST
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3 import AwsCliClient, S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils import wallet_utils
from frostfs_testlib.utils.file_utils import (
concat_files,
generate_file,
generate_file_with_content,
get_file_hash,
)
from frostfs_testlib.utils.file_utils import concat_files, generate_file, generate_file_with_content, get_file_hash
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate
@pytest.mark.s3_gate_object
class TestS3GateObject:
@ -67,28 +56,18 @@ class TestS3GateObject:
with allure.step("Copy object from first bucket into second"):
copy_obj_path_b2 = s3_client.copy_object(bucket_1, file_name, bucket=bucket_2)
s3_helper.check_objects_in_bucket(
s3_client, bucket_1, expected_objects=bucket_1_objects
)
s3_helper.check_objects_in_bucket(
s3_client, bucket_2, expected_objects=[copy_obj_path_b2]
)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
with allure.step("Check copied object has the same content"):
got_copied_file_b2 = s3_client.get_object(bucket_2, copy_obj_path_b2)
assert get_file_hash(file_path) == get_file_hash(
got_copied_file_b2
), "Hashes must be the same"
assert get_file_hash(file_path) == get_file_hash(got_copied_file_b2), "Hashes must be the same"
with allure.step("Delete one object from first bucket"):
s3_client.delete_object(bucket_1, file_name)
bucket_1_objects.remove(file_name)
s3_helper.check_objects_in_bucket(
s3_client, bucket_1, expected_objects=bucket_1_objects
)
s3_helper.check_objects_in_bucket(
s3_client, bucket_2, expected_objects=[copy_obj_path_b2]
)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
with allure.step("Copy one object into the same bucket"):
with pytest.raises(Exception):
@ -102,9 +81,7 @@ class TestS3GateObject:
simple_object_size: ObjectSize,
):
version_1_content = "Version 1"
file_name_simple = generate_file_with_content(
simple_object_size.value, content=version_1_content
)
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
bucket_1, bucket_2 = two_buckets
@ -123,32 +100,22 @@ class TestS3GateObject:
s3_helper.set_bucket_versioning(s3_client, bucket_2, VersioningStatus.ENABLED)
with allure.step("Copy object from first bucket into second"):
copy_obj_path_b2 = s3_client.copy_object(bucket_1, obj_key, bucket=bucket_2)
s3_helper.check_objects_in_bucket(
s3_client, bucket_1, expected_objects=bucket_1_objects
)
s3_helper.check_objects_in_bucket(
s3_client, bucket_2, expected_objects=[copy_obj_path_b2]
)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
with allure.step("Delete one object from first bucket and check object in bucket"):
s3_client.delete_object(bucket_1, obj_key)
bucket_1_objects.remove(obj_key)
s3_helper.check_objects_in_bucket(
s3_client, bucket_1, expected_objects=bucket_1_objects
)
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
with allure.step("Copy one object into the same bucket"):
with pytest.raises(Exception):
s3_client.copy_object(bucket_1, obj_key)
@allure.title("Copy with acl (s3_client={s3_client})")
def test_s3_copy_acl(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_copy_acl(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
version_1_content = "Version 1"
file_name_simple = generate_file_with_content(
simple_object_size.value, content=version_1_content
)
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
@ -163,9 +130,7 @@ class TestS3GateObject:
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
@allure.title("Copy object with metadata (s3_client={s3_client})")
def test_s3_copy_metadate(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_copy_metadate(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
@ -183,17 +148,13 @@ class TestS3GateObject:
bucket_1_objects.append(copy_obj_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_1_objects)
obj_head = s3_client.head_object(bucket, copy_obj_path)
assert (
obj_head.get("Metadata") == object_metadata
), f"Metadata must be {object_metadata}"
assert obj_head.get("Metadata") == object_metadata, f"Metadata must be {object_metadata}"
with allure.step("Copy one object with metadata"):
copy_obj_path = s3_client.copy_object(bucket, file_name, metadata_directive="COPY")
bucket_1_objects.append(copy_obj_path)
obj_head = s3_client.head_object(bucket, copy_obj_path)
assert (
obj_head.get("Metadata") == object_metadata
), f"Metadata must be {object_metadata}"
assert obj_head.get("Metadata") == object_metadata, f"Metadata must be {object_metadata}"
with allure.step("Copy one object with new metadata"):
object_metadata_1 = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
@ -205,14 +166,10 @@ class TestS3GateObject:
)
bucket_1_objects.append(copy_obj_path)
obj_head = s3_client.head_object(bucket, copy_obj_path)
assert (
obj_head.get("Metadata") == object_metadata_1
), f"Metadata must be {object_metadata_1}"
assert obj_head.get("Metadata") == object_metadata_1, f"Metadata must be {object_metadata_1}"
@allure.title("Copy object with tagging (s3_client={s3_client})")
def test_s3_copy_tagging(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_copy_tagging(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
object_tagging = [(f"{uuid.uuid4()}", f"{uuid.uuid4()}")]
file_path = generate_file(simple_object_size.value)
file_name_simple = s3_helper.object_key_from_file_path(file_path)
@ -235,9 +192,7 @@ class TestS3GateObject:
assert tag in got_tags, f"Expected tag {tag} in {got_tags}"
with allure.step("Copy one object with tag"):
copy_obj_path_1 = s3_client.copy_object(
bucket, file_name_simple, tagging_directive="COPY"
)
copy_obj_path_1 = s3_client.copy_object(bucket, file_name_simple, tagging_directive="COPY")
got_tags = s3_client.get_object_tagging(bucket, copy_obj_path_1)
assert got_tags, f"Expected tags, got {got_tags}"
expected_tags = [{"Key": key, "Value": value} for key, value in object_tagging]
@ -270,9 +225,7 @@ class TestS3GateObject:
):
version_1_content = "Version 1"
version_2_content = "Version 2"
file_name_simple = generate_file_with_content(
simple_object_size.value, content=version_1_content
)
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
@ -286,9 +239,7 @@ class TestS3GateObject:
with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket)
obj_versions = {
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
assert obj_versions == {
version_id_1,
version_id_2,
@ -297,18 +248,14 @@ class TestS3GateObject:
with allure.step("Delete 1 version of object"):
delete_obj = s3_client.delete_object(bucket, obj_key, version_id=version_id_1)
versions = s3_client.list_objects_versions(bucket)
obj_versions = {
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
assert obj_versions == {version_id_2}, f"Object should have versions: {version_id_2}"
assert "DeleteMarker" not in delete_obj.keys(), "Delete markers should not be created"
with allure.step("Delete second version of object"):
delete_obj = s3_client.delete_object(bucket, obj_key, version_id=version_id_2)
versions = s3_client.list_objects_versions(bucket)
obj_versions = {
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
assert not obj_versions, "Expected object not found"
assert "DeleteMarker" not in delete_obj.keys(), "Delete markers should not be created"
@ -324,16 +271,12 @@ class TestS3GateObject:
assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
@allure.title("Bulk delete version of object (s3_client={s3_client})")
def test_s3_bulk_delete_versioning(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_bulk_delete_versioning(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
version_1_content = "Version 1"
version_2_content = "Version 2"
version_3_content = "Version 3"
version_4_content = "Version 4"
file_name_1 = generate_file_with_content(
simple_object_size.value, content=version_1_content
)
file_name_1 = generate_file_with_content(simple_object_size.value, content=version_1_content)
obj_key = os.path.basename(file_name_1)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
@ -356,37 +299,25 @@ class TestS3GateObject:
with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket)
obj_versions = {
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
assert obj_versions == version_ids, f"Object should have versions: {version_ids}"
with allure.step("Delete two objects from bucket one by one"):
version_to_delete_b1 = sample(
[version_id_1, version_id_2, version_id_3, version_id_4], k=2
)
version_to_delete_b1 = sample([version_id_1, version_id_2, version_id_3, version_id_4], k=2)
version_to_save = list(set(version_ids) - set(version_to_delete_b1))
for ver in version_to_delete_b1:
s3_client.delete_object(bucket, obj_key, ver)
with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket)
obj_versions = [
version.get("VersionId") for version in versions if version.get("Key") == obj_key
]
assert (
obj_versions.sort() == version_to_save.sort()
), f"Object should have versions: {version_to_save}"
obj_versions = [version.get("VersionId") for version in versions if version.get("Key") == obj_key]
assert obj_versions.sort() == version_to_save.sort(), f"Object should have versions: {version_to_save}"
@allure.title("Get versions of object (s3_client={s3_client})")
def test_s3_get_versioning(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_get_versioning(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
version_1_content = "Version 1"
version_2_content = "Version 2"
file_name_simple = generate_file_with_content(
simple_object_size.value, content=version_1_content
)
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
@ -399,21 +330,15 @@ class TestS3GateObject:
with allure.step("Get first version of object"):
object_1 = s3_client.get_object(bucket, obj_key, version_id_1, full_output=True)
assert (
object_1.get("VersionId") == version_id_1
), f"Get object with version {version_id_1}"
assert object_1.get("VersionId") == version_id_1, f"Get object with version {version_id_1}"
with allure.step("Get second version of object"):
object_2 = s3_client.get_object(bucket, obj_key, version_id_2, full_output=True)
assert (
object_2.get("VersionId") == version_id_2
), f"Get object with version {version_id_2}"
assert object_2.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
with allure.step("Get object"):
object_3 = s3_client.get_object(bucket, obj_key, full_output=True)
assert (
object_3.get("VersionId") == version_id_2
), f"Get object with version {version_id_2}"
assert object_3.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
@allure.title("Get range (s3_client={s3_client})")
def test_s3_get_range(
@ -483,9 +408,7 @@ class TestS3GateObject:
object_range=[2 * int(simple_object_size.value / 3) + 1, simple_object_size.value],
)
con_file_1 = concat_files([object_2_part_1, object_2_part_2, object_2_part_3])
assert get_file_hash(con_file_1) == get_file_hash(
file_name_1
), "Hashes must be the same"
assert get_file_hash(con_file_1) == get_file_hash(file_name_1), "Hashes must be the same"
with allure.step("Get object"):
object_3_part_1 = s3_client.get_object(
@ -568,26 +491,18 @@ class TestS3GateObject:
assert "LastModified" in response, "Expected LastModified field"
assert "ETag" in response, "Expected ETag field"
assert response.get("Metadata") == {}, "Expected Metadata empty"
assert (
response.get("VersionId") == version_id_2
), f"Expected VersionId is {version_id_2}"
assert response.get("VersionId") == version_id_2, f"Expected VersionId is {version_id_2}"
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
with allure.step("Get head ob first version of object"):
response = s3_client.head_object(bucket, file_name, version_id=version_id_1)
assert "LastModified" in response, "Expected LastModified field"
assert "ETag" in response, "Expected ETag field"
assert (
response.get("Metadata") == object_metadata
), f"Expected Metadata is {object_metadata}"
assert (
response.get("VersionId") == version_id_1
), f"Expected VersionId is {version_id_1}"
assert response.get("Metadata") == object_metadata, f"Expected Metadata is {object_metadata}"
assert response.get("VersionId") == version_id_1, f"Expected VersionId is {version_id_1}"
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
@allure.title(
"List of objects with version (method_version={list_type}, s3_client={s3_client})"
)
@allure.title("List of objects with version (method_version={list_type}, s3_client={s3_client})")
@pytest.mark.parametrize("list_type", ["v1", "v2"])
def test_s3_list_object(
self,
@ -624,9 +539,7 @@ class TestS3GateObject:
list_obj_1 = s3_client.list_objects_v2(bucket, full_output=True)
contents = list_obj_1.get("Contents", [])
assert len(contents) == 1, "bucket should have only 1 object"
assert (
contents[0].get("Key") == file_name_2
), f"bucket should have object key {file_name_2}"
assert contents[0].get("Key") == file_name_2, f"bucket should have object key {file_name_2}"
assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
@allure.title("Put object (s3_client={s3_client})")
@ -655,22 +568,16 @@ class TestS3GateObject:
assert obj_head.get("Metadata") == object_1_metadata, "Metadata must be the same"
got_tags = s3_client.get_object_tagging(bucket, file_name)
assert got_tags, f"Expected tags, got {got_tags}"
assert got_tags == [
{"Key": tag_key_1, "Value": str(tag_value_1)}
], "Tags must be the same"
assert got_tags == [{"Key": tag_key_1, "Value": str(tag_value_1)}], "Tags must be the same"
with allure.step("Rewrite file into bucket"):
file_path_2 = generate_file_with_content(
simple_object_size.value, file_path=file_path_1
)
file_path_2 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
s3_client.put_object(bucket, file_path_2, metadata=object_2_metadata, tagging=tag_2)
obj_head = s3_client.head_object(bucket, file_name)
assert obj_head.get("Metadata") == object_2_metadata, "Metadata must be the same"
got_tags_1 = s3_client.get_object_tagging(bucket, file_name)
assert got_tags_1, f"Expected tags, got {got_tags_1}"
assert got_tags_1 == [
{"Key": tag_key_2, "Value": str(tag_value_2)}
], "Tags must be the same"
assert got_tags_1 == [{"Key": tag_key_2, "Value": str(tag_value_2)}], "Tags must be the same"
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
@ -683,28 +590,18 @@ class TestS3GateObject:
tag_3 = f"{tag_key_3}={tag_value_3}"
with allure.step("Put third object into bucket"):
version_id_1 = s3_client.put_object(
bucket, file_path_3, metadata=object_3_metadata, tagging=tag_3
)
version_id_1 = s3_client.put_object(bucket, file_path_3, metadata=object_3_metadata, tagging=tag_3)
obj_head_3 = s3_client.head_object(bucket, file_name_3)
assert obj_head_3.get("Metadata") == object_3_metadata, "Matadata must be the same"
got_tags_3 = s3_client.get_object_tagging(bucket, file_name_3)
assert got_tags_3, f"Expected tags, got {got_tags_3}"
assert got_tags_3 == [
{"Key": tag_key_3, "Value": str(tag_value_3)}
], "Tags must be the same"
assert got_tags_3 == [{"Key": tag_key_3, "Value": str(tag_value_3)}], "Tags must be the same"
with allure.step("Put new version of file into bucket"):
file_path_4 = generate_file_with_content(
simple_object_size.value, file_path=file_path_3
)
file_path_4 = generate_file_with_content(simple_object_size.value, file_path=file_path_3)
version_id_2 = s3_client.put_object(bucket, file_path_4)
versions = s3_client.list_objects_versions(bucket)
obj_versions = {
version.get("VersionId")
for version in versions
if version.get("Key") == file_name_3
}
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == file_name_3}
assert obj_versions == {
version_id_1,
version_id_2,
@ -714,26 +611,20 @@ class TestS3GateObject:
with allure.step("Get object"):
object_3 = s3_client.get_object(bucket, file_name_3, full_output=True)
assert (
object_3.get("VersionId") == version_id_2
), f"get object with version {version_id_2}"
assert object_3.get("VersionId") == version_id_2, f"get object with version {version_id_2}"
object_3 = s3_client.get_object(bucket, file_name_3)
assert get_file_hash(file_path_4) == get_file_hash(object_3), "Hashes must be the same"
with allure.step("Get first version of object"):
object_4 = s3_client.get_object(bucket, file_name_3, version_id_1, full_output=True)
assert (
object_4.get("VersionId") == version_id_1
), f"get object with version {version_id_1}"
assert object_4.get("VersionId") == version_id_1, f"get object with version {version_id_1}"
object_4 = s3_client.get_object(bucket, file_name_3, version_id_1)
assert file_hash == get_file_hash(object_4), "Hashes must be the same"
obj_head_3 = s3_client.head_object(bucket, file_name_3, version_id_1)
assert obj_head_3.get("Metadata") == object_3_metadata, "Metadata must be the same"
got_tags_3 = s3_client.get_object_tagging(bucket, file_name_3, version_id_1)
assert got_tags_3, f"Expected tags, got {got_tags_3}"
assert got_tags_3 == [
{"Key": tag_key_3, "Value": str(tag_value_3)}
], "Tags must be the same"
assert got_tags_3 == [{"Key": tag_key_3, "Value": str(tag_value_3)}], "Tags must be the same"
@allure.title("Put object with ACL (versioning={bucket_versioning}, s3_client={s3_client})")
@pytest.mark.parametrize("bucket_versioning", ["ENABLED", "SUSPENDED"])
@ -762,9 +653,7 @@ class TestS3GateObject:
assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same"
with allure.step("Put object with acl public-read"):
file_path_2 = generate_file_with_content(
simple_object_size.value, file_path=file_path_1
)
file_path_2 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
s3_client.put_object(bucket, file_path_2, acl="public-read")
obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
@ -772,9 +661,7 @@ class TestS3GateObject:
assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same"
with allure.step("Put object with acl public-read-write"):
file_path_3 = generate_file_with_content(
simple_object_size.value, file_path=file_path_1
)
file_path_3 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
s3_client.put_object(bucket, file_path_3, acl="public-read-write")
obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
@ -782,9 +669,7 @@ class TestS3GateObject:
assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same"
with allure.step("Put object with acl authenticated-read"):
file_path_4 = generate_file_with_content(
simple_object_size.value, file_path=file_path_1
)
file_path_4 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
s3_client.put_object(bucket, file_path_4, acl="authenticated-read")
obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
@ -806,9 +691,7 @@ class TestS3GateObject:
object_5 = s3_client.get_object(bucket, file_name_5)
assert get_file_hash(file_path_5) == get_file_hash(object_5), "Hashes must be the same"
with allure.step(
"Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
):
with allure.step("Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"):
generate_file_with_content(simple_object_size.value, file_path=file_path_5)
s3_client.put_object(
bucket,
@ -833,9 +716,7 @@ class TestS3GateObject:
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with allure.step(
"Put object with lock-mode GOVERNANCE lock-retain-until-date +1day, lock-legal-hold-status"
):
with allure.step("Put object with lock-mode GOVERNANCE lock-retain-until-date +1day, lock-legal-hold-status"):
date_obj = datetime.utcnow() + timedelta(days=1)
s3_client.put_object(
bucket,
@ -844,9 +725,7 @@ class TestS3GateObject:
object_lock_retain_until_date=date_obj.strftime("%Y-%m-%dT%H:%M:%S"),
object_lock_legal_hold_status="OFF",
)
s3_helper.assert_object_lock_mode(
s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF"
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
with allure.step(
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"
@ -859,9 +738,7 @@ class TestS3GateObject:
object_lock_mode="COMPLIANCE",
object_lock_retain_until_date=date_obj,
)
s3_helper.assert_object_lock_mode(
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF"
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
with allure.step(
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"
@ -875,9 +752,7 @@ class TestS3GateObject:
object_lock_retain_until_date=date_obj,
object_lock_legal_hold_status="ON",
)
s3_helper.assert_object_lock_mode(
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON"
)
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
with allure.step("Put object with lock-mode"):
with pytest.raises(
@ -939,9 +814,7 @@ class TestS3GateObject:
with allure.step("Check objects are synced"):
objects = s3_client.list_objects(bucket)
assert set(key_to_path.keys()) == set(
objects
), f"Expected all abjects saved. Got {objects}"
assert set(key_to_path.keys()) == set(objects), f"Expected all abjects saved. Got {objects}"
with allure.step("Check these are the same objects"):
for obj_key in objects:
@ -950,9 +823,7 @@ class TestS3GateObject:
key_to_path.get(obj_key)
), "Expected hashes are the same"
obj_head = s3_client.head_object(bucket, obj_key)
assert (
obj_head.get("Metadata") == object_metadata
), f"Metadata of object is {object_metadata}"
assert obj_head.get("Metadata") == object_metadata, f"Metadata of object is {object_metadata}"
# Uncomment after https://github.com/nspcc-dev/neofs-s3-gw/issues/685 is solved
# obj_acl = s3_client.get_object_acl(bucket, obj_key)
# s3_helper.assert_s3_acl(acl_grants = obj_acl, permitted_users = "AllUsers")
@ -994,9 +865,7 @@ class TestS3GateObject:
assert not objects_list, f"Expected empty bucket, got {objects_list}"
@allure.title("Delete the same object twice (s3_client={s3_client})")
def test_s3_delete_twice(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_delete_twice(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
objects_list = s3_client.list_objects(bucket)
with allure.step("Check that bucket is empty"):
@ -1012,9 +881,7 @@ class TestS3GateObject:
delete_object = s3_client.delete_object(bucket, file_name)
versions = s3_client.list_objects_versions(bucket)
obj_versions = {
version.get("VersionId") for version in versions if version.get("Key") == file_name
}
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == file_name}
assert obj_versions, f"Object versions were not found {objects_list}"
assert "DeleteMarker" in delete_object.keys(), "Delete markers not found"
@ -1022,9 +889,7 @@ class TestS3GateObject:
delete_object_2nd_attempt = s3_client.delete_object(bucket, file_name)
versions_2nd_attempt = s3_client.list_objects_versions(bucket)
assert (
delete_object.keys() == delete_object_2nd_attempt.keys()
), "Delete markers are not the same"
assert delete_object.keys() == delete_object_2nd_attempt.keys(), "Delete markers are not the same"
# check that nothing was changed
# checking here not VersionId only, but all data (for example LastModified)
assert versions == versions_2nd_attempt, "Versions are not the same"

22
pytest_tests/testsuites/services/s3_gate/test_s3_policy.py
View file

@ -2,7 +2,7 @@ import os
import allure
import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.cli.container import search_container_by_name
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.steps.storage_policy import get_simple_object_copies
@ -12,23 +12,11 @@ from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils.file_utils import generate_file
def pytest_generate_tests(metafunc: pytest.Metafunc):
policy = f"{os.getcwd()}/pytest_tests/resources/files/policy.json"
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize(
"s3_client, s3_policy",
[(AwsCliClient, policy), (Boto3ClientWrapper, policy)],
indirect=True,
ids=["aws cli", "boto3"],
)
@pytest.mark.s3_gate
@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True)
class TestS3GatePolicy(ClusterTestBase):
@allure.title("Bucket creation with retention policy applied (s3_client={s3_client})")
def test_s3_bucket_location(
self, default_wallet: str, s3_client: S3ClientWrapper, simple_object_size: ObjectSize
):
def test_s3_bucket_location(self, default_wallet: str, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
file_path_1 = generate_file(simple_object_size.value)
file_name_1 = s3_helper.object_key_from_file_path(file_path_1)
file_path_2 = generate_file(simple_object_size.value)
@ -156,9 +144,7 @@ class TestS3GatePolicy(ClusterTestBase):
}
s3_client.put_bucket_cors(bucket, cors)
bucket_cors = s3_client.get_bucket_cors(bucket)
assert bucket_cors == cors.get(
"CORSRules"
), f"Expected CORSRules must be {cors.get('CORSRules')}"
assert bucket_cors == cors.get("CORSRules"), f"Expected CORSRules must be {cors.get('CORSRules')}"
with allure.step("delete bucket cors"):
s3_client.delete_bucket_cors(bucket)

16
pytest_tests/testsuites/services/s3_gate/test_s3_tagging.py
View file

@ -4,18 +4,12 @@ from typing import Tuple
import allure
import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper
from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate
@pytest.mark.s3_gate_tagging
class TestS3GateTagging:
@ -29,9 +23,7 @@ class TestS3GateTagging:
return tags
@allure.title("Object tagging (s3_client={s3_client})")
def test_s3_object_tagging(
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
def test_s3_object_tagging(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path)
@ -45,9 +37,7 @@ class TestS3GateTagging:
with allure.step("Put 10 new tags for object"):
tags_2 = self.create_tags(10)
s3_client.put_object_tagging(bucket, file_name, tags=tags_2)
s3_helper.check_tags_by_object(
s3_client, bucket, file_name, tags_2, [("Tag1", "Value1")]
)
s3_helper.check_tags_by_object(s3_client, bucket, file_name, tags_2, [("Tag1", "Value1")])
with allure.step("Put 10 extra new tags for object"):
tags_3 = self.create_tags(10)

40
pytest_tests/testsuites/services/s3_gate/test_s3_versioning.py
View file

@ -1,19 +1,11 @@
import os
import allure
import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate
@pytest.mark.s3_gate_versioning
class TestS3GateVersioning:
@ -34,18 +26,10 @@ class TestS3GateVersioning:
with allure.step("Put object into bucket"):
s3_client.put_object(bucket, file_path)
objects_list = s3_client.list_objects(bucket)
assert (
objects_list == bucket_objects
), f"Expected list with single objects in bucket, got {objects_list}"
assert objects_list == bucket_objects, f"Expected list with single objects in bucket, got {objects_list}"
object_version = s3_client.list_objects_versions(bucket)
actual_version = [
version.get("VersionId")
for version in object_version
if version.get("Key") == file_name
]
assert actual_version == [
"null"
], f"Expected version is null in list-object-versions, got {object_version}"
actual_version = [version.get("VersionId") for version in object_version if version.get("Key") == file_name]
assert actual_version == ["null"], f"Expected version is null in list-object-versions, got {object_version}"
object_0 = s3_client.head_object(bucket, file_name)
assert (
object_0.get("VersionId") == "null"
@ -60,27 +44,19 @@ class TestS3GateVersioning:
with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket)
obj_versions = [
version.get("VersionId") for version in versions if version.get("Key") == file_name
]
obj_versions = [version.get("VersionId") for version in versions if version.get("Key") == file_name]
assert (
obj_versions.sort() == [version_id_1, version_id_2, "null"].sort()
), f"Expected object has versions: {version_id_1, version_id_2, 'null'}"
with allure.step("Get object"):
object_1 = s3_client.get_object(bucket, file_name, full_output=True)
assert (
object_1.get("VersionId") == version_id_2
), f"Get object with version {version_id_2}"
assert object_1.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
with allure.step("Get first version of object"):
object_2 = s3_client.get_object(bucket, file_name, version_id_1, full_output=True)
assert (
object_2.get("VersionId") == version_id_1
), f"Get object with version {version_id_1}"
assert object_2.get("VersionId") == version_id_1, f"Get object with version {version_id_1}"
with allure.step("Get second version of object"):
object_3 = s3_client.get_object(bucket, file_name, version_id_2, full_output=True)
assert (
object_3.get("VersionId") == version_id_2
), f"Get object with version {version_id_2}"
assert object_3.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"

12
pytest_tests/testsuites/services/test_binaries.py
View file

@ -1,7 +1,7 @@
import logging
import os
from http import HTTPStatus
from re import match, fullmatch
from re import fullmatch, match
import allure
import pytest
@ -14,8 +14,8 @@ from pytest import FixtureRequest
logger = logging.getLogger("NeoLogger")
@allure.title("Check binaries versions")
@pytest.mark.sanity
@pytest.mark.check_binaries
def test_binaries_versions(request: FixtureRequest, hosting: Hosting):
"""
@ -31,8 +31,8 @@ def test_binaries_versions(request: FixtureRequest, hosting: Hosting):
# compare versions from servers and file
exeptions = []
additional_env_properties = {}
for binary, version in got_versions.items():
for binary, version in got_versions.items():
if not fullmatch(r"^\d+\.\d+\.\d+(-.*)?(?<!dirty)", version):
exeptions.append(f"{binary}: Actual version doesn't conform to format '0.0.0-000-aaaaaaa': {version}")
@ -56,9 +56,7 @@ def download_versions_info(url: str) -> dict:
response = requests.get(url)
assert (
response.status_code == HTTPStatus.OK
), f"Got {response.status_code} code. Content {response.json()}"
assert response.status_code == HTTPStatus.OK, f"Got {response.status_code} code. Content {response.json()}"
content = response.text
assert content, f"Expected file with content, got {response}"

54
pytest_tests/testsuites/session_token/test_static_object_session_token.py
View file

@ -48,15 +48,9 @@ RANGE_OFFSET_FOR_COMPLEX_OBJECT = 200
@pytest.fixture(scope="module")
def storage_containers(
owner_wallet: WalletInfo, client_shell: Shell, cluster: Cluster
) -> list[str]:
cid = create_container(
owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint
)
other_cid = create_container(
owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint
)
def storage_containers(owner_wallet: WalletInfo, client_shell: Shell, cluster: Cluster) -> list[str]:
cid = create_container(owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
other_cid = create_container(owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
yield [cid, other_cid]
@ -99,9 +93,7 @@ def storage_objects(
@allure.step("Get ranges for test")
def get_ranges(
storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, endpoint: str
) -> list[str]:
def get_ranges(storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, endpoint: str) -> list[str]:
"""
Returns ranges to test range/hash methods via static session
"""
@ -112,8 +104,7 @@ def get_ranges(
return [
"0:10",
f"{object_size-10}:10",
f"{max_object_size - RANGE_OFFSET_FOR_COMPLEX_OBJECT}:"
f"{RANGE_OFFSET_FOR_COMPLEX_OBJECT * 2}",
f"{max_object_size - RANGE_OFFSET_FOR_COMPLEX_OBJECT}:" f"{RANGE_OFFSET_FOR_COMPLEX_OBJECT * 2}",
]
else:
return ["0:10", f"{object_size-10}:10"]
@ -148,9 +139,7 @@ def static_sessions(
@pytest.mark.static_session
class TestObjectStaticSession(ClusterTestBase):
@allure.title(
"Read operations with static session (method={method_under_test.__name__}, obj_size={object_size})"
)
@allure.title("Read operations with static session (method={method_under_test.__name__}, obj_size={object_size})")
@pytest.mark.parametrize(
"method_under_test,verb",
[
@ -181,13 +170,12 @@ class TestObjectStaticSession(ClusterTestBase):
session=static_sessions[verb],
)
@allure.title(
"Range operations with static session (method={method_under_test.__name__}, obj_size={object_size})"
)
@allure.title("Range operations with static session (method={method_under_test.__name__}, obj_size={object_size})")
@pytest.mark.parametrize(
"method_under_test,verb",
[(get_range, ObjectVerb.RANGE), (get_range_hash, ObjectVerb.RANGEHASH)],
)
@pytest.mark.sanity
def test_static_session_range(
self,
user_wallet: WalletInfo,
@ -201,9 +189,7 @@ class TestObjectStaticSession(ClusterTestBase):
Validate static session with range operations
"""
storage_object = storage_objects[0]
ranges_to_test = get_ranges(
storage_object, max_object_size, self.shell, self.cluster.default_rpc_endpoint
)
ranges_to_test = get_ranges(storage_object, max_object_size, self.shell, self.cluster.default_rpc_endpoint)
for range_to_test in ranges_to_test:
with allure.step(f"Check range {range_to_test}"):
@ -241,9 +227,7 @@ class TestObjectStaticSession(ClusterTestBase):
)
assert sorted(expected_object_ids) == sorted(actual_object_ids)
@allure.title(
"[NEGATIVE] Static session with object id not in session (obj_size={object_size})"
)
@allure.title("[NEGATIVE] Static session with object id not in session (obj_size={object_size})")
def test_static_session_unrelated_object(
self,
user_wallet: WalletInfo,
@ -307,9 +291,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=static_sessions[ObjectVerb.HEAD],
)
@allure.title(
"[NEGATIVE] Static session with container id not in session (obj_size={object_size})"
)
@allure.title("[NEGATIVE] Static session with container id not in session (obj_size={object_size})")
def test_static_session_unrelated_container(
self,
user_wallet: WalletInfo,
@ -473,9 +455,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_expire_at_next_epoch,
)
with allure.step(
"Object should be available at last epoch before session token expiration"
):
with allure.step("Object should be available at last epoch before session token expiration"):
self.tick_epoch()
with expect_not_raises():
head_object(
@ -499,6 +479,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_expire_at_next_epoch,
)
@pytest.mark.sanity
@allure.title("Static session which is valid since next epoch (obj_size={object_size})")
def test_static_session_start_at_next(
self,
@ -540,9 +521,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_start_at_next_epoch,
)
with allure.step(
"Object should be available with session token starting from token nbf epoch"
):
with allure.step("Object should be available with session token starting from token nbf epoch"):
self.tick_epoch()
with expect_not_raises():
head_object(
@ -554,9 +533,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_start_at_next_epoch,
)
with allure.step(
"Object should be available at last epoch before session token expiration"
):
with allure.step("Object should be available at last epoch before session token expiration"):
self.tick_epoch()
with expect_not_raises():
head_object(
@ -640,6 +617,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=static_sessions[ObjectVerb.DELETE],
)
@pytest.mark.sanity
@allure.title("Put verb is restricted for static session (obj_size={object_size})")
def test_static_session_put_verb(
self,

21
pytest_tests/testsuites/session_token/test_static_session_token_container.py
View file

@ -3,12 +3,7 @@ import pytest
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.acl import create_eacl, set_eacl, wait_for_cache_expired
from frostfs_testlib.steps.cli.container import (
create_container,
delete_container,
get_container,
list_containers,
)
from frostfs_testlib.steps.cli.container import create_container, delete_container, get_container, list_containers
from frostfs_testlib.steps.session_token import ContainerVerb, get_container_signed_token
from frostfs_testlib.storage.dataclasses.acl import EACLAccess, EACLOperation, EACLRole, EACLRule
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
@ -33,9 +28,7 @@ class TestSessionTokenContainer(ClusterTestBase):
Returns dict with static session token file paths for all verbs with default lifetime
"""
return {
verb: get_container_signed_token(
owner_wallet, user_wallet, verb, client_shell, temp_directory
)
verb: get_container_signed_token(owner_wallet, user_wallet, verb, client_shell, temp_directory)
for verb in ContainerVerb
}
@ -65,9 +58,7 @@ class TestSessionTokenContainer(ClusterTestBase):
assert cid not in list_containers(
user_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
assert cid in list_containers(
owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
assert cid in list_containers(owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
def test_static_session_token_container_create_with_other_verb(
self,
@ -136,6 +127,7 @@ class TestSessionTokenContainer(ClusterTestBase):
owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
@pytest.mark.sanity
def test_static_session_token_container_set_eacl(
self,
owner_wallet: WalletInfo,
@ -158,10 +150,7 @@ class TestSessionTokenContainer(ClusterTestBase):
assert can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster)
with allure.step("Deny all operations for other via eACL"):
eacl_deny = [
EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op)
for op in EACLOperation
]
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
set_eacl(
user_wallet.path,
cid,

13
pytest_tests/testsuites/shard/test_control_shard.py
View file

@ -66,8 +66,7 @@ class Shard:
blobstor_count = Shard._get_blobstor_count_from_section(config_object, shard_id)
blobstors = [
Blobstor.from_config_object(config_object, shard_id, blobstor_id)
for blobstor_id in range(blobstor_count)
Blobstor.from_config_object(config_object, shard_id, blobstor_id) for blobstor_id in range(blobstor_count)
]
write_cache_enabled = config_object.as_bool(f"{var_prefix}_WRITECACHE_ENABLED")
@ -81,15 +80,10 @@ class Shard:
@staticmethod
def from_object(shard):
metabase = shard["metabase"]["path"] if "path" in shard["metabase"] else shard["metabase"]
writecache = (
shard["writecache"]["path"] if "path" in shard["writecache"] else shard["writecache"]
)
writecache = shard["writecache"]["path"] if "path" in shard["writecache"] else shard["writecache"]
return Shard(
blobstor=[
Blobstor(path=blobstor["path"], path_type=blobstor["type"])
for blobstor in shard["blobstor"]
],
blobstor=[Blobstor(path=blobstor["path"], path_type=blobstor["type"]) for blobstor in shard["blobstor"]],
metabase=metabase,
writecache=writecache,
)
@ -111,7 +105,6 @@ def shards_from_env(contents: str) -> list[Shard]:
return [Shard.from_config_object(configObj, shard_id) for shard_id in range(num_shards)]
@pytest.mark.sanity
@pytest.mark.shard
class TestControlShard:
@staticmethod