Compare commits

...

17 commits

Author SHA1 Message Date
c51e4bd44f [#224] Add multipart test cases with bucket without versioning 2024-04-24 11:15:47 +03:00
048f713f3f [#219] Add bucket/container listing check in multipart test case 2024-04-22 17:47:45 +03:00
370c1059f1 update-allure 2023-12-20 17:12:03 +00:00
6980cd15bd [#172] Executive command changed 1.3
Added exception of error 'Too many requests' in log analyzer

Signed-off-by: Mikhail Kadilov m.kadilov@yadro.com
2023-12-11 15:19:58 +03:00
8115d28bcd [#168] Added exception in log analyzer and fixed arguments order in get_filtered_logs
Added exception in log analyzer and fixed arguments order in get_filtered_logs

Signed-off-by: Mikhail Kadilov <m.kadilov@yadro.com>
2023-12-07 08:45:33 +00:00
8d2b3aee0e update-allure-title 2023-12-04 13:52:47 +03:00
fdd5fd55d4 Reduction of sanity tests
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-11-28 08:20:48 +00:00
7f9517075b Change sanity makrs first step
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-11-28 08:20:48 +00:00
849e052ad2 Remove sanity from awsclient
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-11-28 08:20:48 +00:00
354c054a8a Skip teardown if sanity in markexpr 2023-11-28 08:20:48 +00:00
6d32372d93 Remove pytest hooks 2023-11-28 08:20:48 +00:00
b8dbf086be Add sanity marks
Signed-off-by: anikeev-yadro <a.anikeev@yadro.com>
2023-11-28 08:20:48 +00:00
e2e4a0c667 [#160] Add try\except
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
2023-11-28 08:14:35 +00:00
8f339ecbcd Add await
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
2023-11-16 09:06:17 +03:00
9f8485f5eb Update check Policy: REP 1 IN SPB REP 1 IN MSK REP 3
Signed-off-by: Ekaterina Chernitsyna <e.chernitsyna@yadro.com>
2023-11-15 07:39:17 +00:00
1629caddec discard_dynamic_title
Signed-off-by: Ekaterina Chernitsyna <e.chernitsyna@yadro.com>
2023-11-15 07:38:30 +00:00
cd06a073a2 [#129] Updates for failover
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
(cherry picked from commit ae57672c7d)
2023-11-14 10:31:35 +03:00
30 changed files with 585 additions and 1161 deletions

View file

@ -9,10 +9,7 @@ from frostfs_testlib.storage.dataclasses.acl import EACLAccess, EACLOperation, E
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.failover_utils import wait_object_replication from frostfs_testlib.utils.failover_utils import wait_object_replication
from pytest_tests.helpers.container_access import ( from pytest_tests.helpers.container_access import check_full_access_to_container, check_no_access_to_container
check_full_access_to_container,
check_no_access_to_container,
)
from pytest_tests.helpers.object_access import ( from pytest_tests.helpers.object_access import (
can_delete_object, can_delete_object,
can_get_head_object, can_get_head_object,
@ -25,7 +22,6 @@ from pytest_tests.helpers.object_access import (
from pytest_tests.testsuites.acl.conftest import Wallets from pytest_tests.testsuites.acl.conftest import Wallets
@pytest.mark.sanity
@pytest.mark.acl @pytest.mark.acl
@pytest.mark.acl_extended @pytest.mark.acl_extended
class TestEACLContainer(ClusterTestBase): class TestEACLContainer(ClusterTestBase):
@ -58,6 +54,7 @@ class TestEACLContainer(ClusterTestBase):
yield cid, oid, file_path yield cid, oid, file_path
@pytest.mark.sanity
@allure.title("Deny operations (role={deny_role.value}, obj_size={object_size})") @allure.title("Deny operations (role={deny_role.value}, obj_size={object_size})")
@pytest.mark.parametrize("deny_role", [EACLRole.USER, EACLRole.OTHERS]) @pytest.mark.parametrize("deny_role", [EACLRole.USER, EACLRole.OTHERS])
def test_extended_acl_deny_all_operations( def test_extended_acl_deny_all_operations(
@ -75,10 +72,7 @@ class TestEACLContainer(ClusterTestBase):
cid, object_oids, file_path = eacl_container_with_objects cid, object_oids, file_path = eacl_container_with_objects
with allure.step(f"Deny all operations for {deny_role_str} via eACL"): with allure.step(f"Deny all operations for {deny_role_str} via eACL"):
eacl_deny = [ eacl_deny = [EACLRule(access=EACLAccess.DENY, role=deny_role, operation=op) for op in EACLOperation]
EACLRule(access=EACLAccess.DENY, role=deny_role, operation=op)
for op in EACLOperation
]
set_eacl( set_eacl(
user_wallet.wallet_path, user_wallet.wallet_path,
cid, cid,
@ -89,9 +83,7 @@ class TestEACLContainer(ClusterTestBase):
wait_for_cache_expired() wait_for_cache_expired()
with allure.step(f"Check only {not_deny_role_str} has full access to container"): with allure.step(f"Check only {not_deny_role_str} has full access to container"):
with allure.step( with allure.step(f"Check {deny_role_str} has not access to any operations with container"):
f"Check {deny_role_str} has not access to any operations with container"
):
check_no_access_to_container( check_no_access_to_container(
deny_role_wallet.wallet_path, deny_role_wallet.wallet_path,
cid, cid,
@ -101,9 +93,7 @@ class TestEACLContainer(ClusterTestBase):
cluster=self.cluster, cluster=self.cluster,
) )
with allure.step( with allure.step(f"Check {not_deny_role_wallet} has full access to eACL public container"):
f"Check {not_deny_role_wallet} has full access to eACL public container"
):
check_full_access_to_container( check_full_access_to_container(
not_deny_role_wallet.wallet_path, not_deny_role_wallet.wallet_path,
cid, cid,
@ -114,10 +104,7 @@ class TestEACLContainer(ClusterTestBase):
) )
with allure.step(f"Allow all operations for {deny_role_str} via eACL"): with allure.step(f"Allow all operations for {deny_role_str} via eACL"):
eacl_deny = [ eacl_deny = [EACLRule(access=EACLAccess.ALLOW, role=deny_role, operation=op) for op in EACLOperation]
EACLRule(access=EACLAccess.ALLOW, role=deny_role, operation=op)
for op in EACLOperation
]
set_eacl( set_eacl(
user_wallet.wallet_path, user_wallet.wallet_path,
cid, cid,
@ -162,10 +149,7 @@ class TestEACLContainer(ClusterTestBase):
) )
for op in EACLOperation for op in EACLOperation
] ]
eacl += [ eacl += [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op)
for op in EACLOperation
]
set_eacl( set_eacl(
user_wallet.wallet_path, user_wallet.wallet_path,
cid, cid,
@ -218,14 +202,8 @@ class TestEACLContainer(ClusterTestBase):
storage_node = self.cluster.storage_nodes[0] storage_node = self.cluster.storage_nodes[0]
with allure.step("Deny all operations for user via eACL"): with allure.step("Deny all operations for user via eACL"):
eacl_deny = [ eacl_deny = [EACLRule(access=EACLAccess.DENY, role=EACLRole.USER, operation=op) for op in EACLOperation]
EACLRule(access=EACLAccess.DENY, role=EACLRole.USER, operation=op) eacl_deny += [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
for op in EACLOperation
]
eacl_deny += [
EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op)
for op in EACLOperation
]
set_eacl( set_eacl(
user_wallet.wallet_path, user_wallet.wallet_path,
cid, cid,
@ -249,9 +227,7 @@ class TestEACLContainer(ClusterTestBase):
) )
@allure.title("Operations with extended ACL for SYSTEM (obj_size={object_size})") @allure.title("Operations with extended ACL for SYSTEM (obj_size={object_size})")
def test_extended_actions_system( def test_extended_actions_system(self, wallets: Wallets, eacl_container_with_objects: tuple[str, list[str], str]):
self, wallets: Wallets, eacl_container_with_objects: tuple[str, list[str], str]
):
user_wallet = wallets.get_wallet() user_wallet = wallets.get_wallet()
ir_wallet, storage_wallet = wallets.get_wallets_list(role=EACLRole.SYSTEM)[:2] ir_wallet, storage_wallet = wallets.get_wallets_list(role=EACLRole.SYSTEM)[:2]
@ -392,8 +368,7 @@ class TestEACLContainer(ClusterTestBase):
create_eacl( create_eacl(
cid=cid, cid=cid,
rules_list=[ rules_list=[
EACLRule(access=EACLAccess.DENY, role=EACLRole.SYSTEM, operation=op) EACLRule(access=EACLAccess.DENY, role=EACLRole.SYSTEM, operation=op) for op in EACLOperation
for op in EACLOperation
], ],
shell=self.shell, shell=self.shell,
), ),
@ -543,8 +518,7 @@ class TestEACLContainer(ClusterTestBase):
create_eacl( create_eacl(
cid=cid, cid=cid,
rules_list=[ rules_list=[
EACLRule(access=EACLAccess.ALLOW, role=EACLRole.SYSTEM, operation=op) EACLRule(access=EACLAccess.ALLOW, role=EACLRole.SYSTEM, operation=op) for op in EACLOperation
for op in EACLOperation
], ],
shell=self.shell, shell=self.shell,
), ),

View file

@ -1,12 +1,7 @@
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.acl import ( from frostfs_testlib.steps.acl import create_eacl, form_bearertoken_file, set_eacl, wait_for_cache_expired
create_eacl,
form_bearertoken_file,
set_eacl,
wait_for_cache_expired,
)
from frostfs_testlib.steps.cli.container import create_container, delete_container from frostfs_testlib.steps.cli.container import create_container, delete_container
from frostfs_testlib.steps.cli.object import put_object_to_random_node from frostfs_testlib.steps.cli.object import put_object_to_random_node
from frostfs_testlib.storage.dataclasses.acl import ( from frostfs_testlib.storage.dataclasses.acl import (
@ -21,15 +16,11 @@ from frostfs_testlib.storage.dataclasses.acl import (
) )
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.container_access import ( from pytest_tests.helpers.container_access import check_full_access_to_container, check_no_access_to_container
check_full_access_to_container,
check_no_access_to_container,
)
from pytest_tests.helpers.object_access import can_get_head_object, can_get_object, can_put_object from pytest_tests.helpers.object_access import can_get_head_object, can_get_object, can_put_object
from pytest_tests.testsuites.acl.conftest import Wallets from pytest_tests.testsuites.acl.conftest import Wallets
@pytest.mark.sanity
@pytest.mark.acl @pytest.mark.acl
@pytest.mark.acl_filters @pytest.mark.acl_filters
class TestEACLFilters(ClusterTestBase): class TestEACLFilters(ClusterTestBase):
@ -46,18 +37,14 @@ class TestEACLFilters(ClusterTestBase):
"x_key": "other_value", "x_key": "other_value",
"check_key": "other_value", "check_key": "other_value",
} }
REQ_EQUAL_FILTER = EACLFilter( REQ_EQUAL_FILTER = EACLFilter(key="check_key", value="check_value", header_type=EACLHeaderType.REQUEST)
key="check_key", value="check_value", header_type=EACLHeaderType.REQUEST
)
NOT_REQ_EQUAL_FILTER = EACLFilter( NOT_REQ_EQUAL_FILTER = EACLFilter(
key="check_key", key="check_key",
value="other_value", value="other_value",
match_type=EACLMatchType.STRING_NOT_EQUAL, match_type=EACLMatchType.STRING_NOT_EQUAL,
header_type=EACLHeaderType.REQUEST, header_type=EACLHeaderType.REQUEST,
) )
OBJ_EQUAL_FILTER = EACLFilter( OBJ_EQUAL_FILTER = EACLFilter(key="check_key", value="check_value", header_type=EACLHeaderType.OBJECT)
key="check_key", value="check_value", header_type=EACLHeaderType.OBJECT
)
NOT_OBJ_EQUAL_FILTER = EACLFilter( NOT_OBJ_EQUAL_FILTER = EACLFilter(
key="check_key", key="check_key",
value="other_value", value="other_value",
@ -128,12 +115,9 @@ class TestEACLFilters(ClusterTestBase):
endpoint=self.cluster.default_rpc_endpoint, endpoint=self.cluster.default_rpc_endpoint,
) )
@allure.title( @pytest.mark.sanity
"Operations with request filter (match_type={match_type}, obj_size={object_size})" @allure.title("Operations with request filter (match_type={match_type}, obj_size={object_size})")
) @pytest.mark.parametrize("match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL])
@pytest.mark.parametrize(
"match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL]
)
def test_extended_acl_filters_request( def test_extended_acl_filters_request(
self, self,
wallets: Wallets, wallets: Wallets,
@ -174,12 +158,8 @@ class TestEACLFilters(ClusterTestBase):
# Filter denies requests where "check_key {match_type} ATTRIBUTE", so when match_type # Filter denies requests where "check_key {match_type} ATTRIBUTE", so when match_type
# is STRING_EQUAL, then requests with "check_key=OTHER_ATTRIBUTE" will be allowed while # is STRING_EQUAL, then requests with "check_key=OTHER_ATTRIBUTE" will be allowed while
# requests with "check_key=ATTRIBUTE" will be denied, and vice versa # requests with "check_key=ATTRIBUTE" will be denied, and vice versa
allow_headers = ( allow_headers = self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE deny_headers = self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
)
deny_headers = (
self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
)
# We test on 3 groups of objects with various headers, # We test on 3 groups of objects with various headers,
# but eACL rule should ignore object headers and # but eACL rule should ignore object headers and
# work only based on request headers # work only based on request headers
@ -198,9 +178,7 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster, cluster=self.cluster,
) )
with allure.step( with allure.step("Check other has full access when sending request with allowed headers"):
"Check other has full access when sending request with allowed headers"
):
check_full_access_to_container( check_full_access_to_container(
other_wallet.wallet_path, other_wallet.wallet_path,
cid, cid,
@ -223,16 +201,12 @@ class TestEACLFilters(ClusterTestBase):
) )
with allure.step( with allure.step(
"Check other has full access when sending request " "Check other has full access when sending request " "with denied headers and using bearer token"
"with denied headers and using bearer token"
): ):
bearer_other = form_bearertoken_file( bearer_other = form_bearertoken_file(
user_wallet.wallet_path, user_wallet.wallet_path,
cid, cid,
[ [EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) for op in EACLOperation],
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS)
for op in EACLOperation
],
shell=self.shell, shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint, endpoint=self.cluster.default_rpc_endpoint,
) )
@ -247,12 +221,8 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other, bearer=bearer_other,
) )
@allure.title( @allure.title("Operations with deny user headers filter (match_type={match_type}, obj_size={object_size})")
"Operations with deny user headers filter (match_type={match_type}, obj_size={object_size})" @pytest.mark.parametrize("match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL])
)
@pytest.mark.parametrize(
"match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL]
)
def test_extended_acl_deny_filters_object( def test_extended_acl_deny_filters_object(
self, self,
wallets: Wallets, wallets: Wallets,
@ -290,16 +260,8 @@ class TestEACLFilters(ClusterTestBase):
) )
wait_for_cache_expired() wait_for_cache_expired()
allow_objects = ( allow_objects = objects_with_other_header if match_type == EACLMatchType.STRING_EQUAL else objects_with_header
objects_with_other_header deny_objects = objects_with_header if match_type == EACLMatchType.STRING_EQUAL else objects_with_other_header
if match_type == EACLMatchType.STRING_EQUAL
else objects_with_header
)
deny_objects = (
objects_with_header
if match_type == EACLMatchType.STRING_EQUAL
else objects_with_other_header
)
# We will attempt requests with various headers, # We will attempt requests with various headers,
# but eACL rule should ignore request headers and validate # but eACL rule should ignore request headers and validate
@ -348,9 +310,7 @@ class TestEACLFilters(ClusterTestBase):
xhdr=xhdr, xhdr=xhdr,
) )
with allure.step( with allure.step("Check other have access to objects with deny attribute and using bearer token"):
"Check other have access to objects with deny attribute and using bearer token"
):
bearer_other = form_bearertoken_file( bearer_other = form_bearertoken_file(
user_wallet.wallet_path, user_wallet.wallet_path,
cid, cid,
@ -376,9 +336,7 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other, bearer=bearer_other,
) )
allow_attribute = ( allow_attribute = self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
)
with allure.step("Check other can PUT objects without denied attribute"): with allure.step("Check other can PUT objects without denied attribute"):
assert can_put_object( assert can_put_object(
other_wallet.wallet_path, other_wallet.wallet_path,
@ -388,13 +346,9 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster, cluster=self.cluster,
attributes=allow_attribute, attributes=allow_attribute,
) )
assert can_put_object( assert can_put_object(other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster)
other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster
)
deny_attribute = ( deny_attribute = self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
)
with allure.step("Check other can not PUT objects with denied attribute"): with allure.step("Check other can not PUT objects with denied attribute"):
with pytest.raises(AssertionError): with pytest.raises(AssertionError):
assert can_put_object( assert can_put_object(
@ -406,9 +360,7 @@ class TestEACLFilters(ClusterTestBase):
attributes=deny_attribute, attributes=deny_attribute,
) )
with allure.step( with allure.step("Check other can PUT objects with denied attribute and using bearer token"):
"Check other can PUT objects with denied attribute and using bearer token"
):
bearer_other_for_put = form_bearertoken_file( bearer_other_for_put = form_bearertoken_file(
user_wallet.wallet_path, user_wallet.wallet_path,
cid, cid,
@ -432,12 +384,8 @@ class TestEACLFilters(ClusterTestBase):
bearer=bearer_other_for_put, bearer=bearer_other_for_put,
) )
@allure.title( @allure.title("Operations with allow eACL user headers filters (match_type={match_type}, obj_size={object_size})")
"Operations with allow eACL user headers filters (match_type={match_type}, obj_size={object_size})" @pytest.mark.parametrize("match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL])
)
@pytest.mark.parametrize(
"match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL]
)
def test_extended_acl_allow_filters_object( def test_extended_acl_allow_filters_object(
self, self,
wallets: Wallets, wallets: Wallets,
@ -454,9 +402,7 @@ class TestEACLFilters(ClusterTestBase):
file_path, file_path,
) = eacl_container_with_objects ) = eacl_container_with_objects
with allure.step( with allure.step("Deny all operations for others except few operations allowed by object filter"):
"Deny all operations for others except few operations allowed by object filter"
):
equal_filter = EACLFilter(**self.OBJ_EQUAL_FILTER.__dict__) equal_filter = EACLFilter(**self.OBJ_EQUAL_FILTER.__dict__)
equal_filter.match_type = match_type equal_filter.match_type = match_type
eacl = [ eacl = [
@ -511,13 +457,9 @@ class TestEACLFilters(ClusterTestBase):
cluster=self.cluster, cluster=self.cluster,
) )
with pytest.raises(AssertionError): with pytest.raises(AssertionError):
assert can_put_object( assert can_put_object(other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster)
other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster
)
with allure.step( with allure.step("Check other can get and put objects without attributes and using bearer token"):
"Check other can get and put objects without attributes and using bearer token"
):
bearer_other = form_bearertoken_file( bearer_other = form_bearertoken_file(
user_wallet.wallet_path, user_wallet.wallet_path,
cid, cid,
@ -613,8 +555,7 @@ class TestEACLFilters(ClusterTestBase):
) )
with allure.step( with allure.step(
"Check other can get objects without attributes matching the filter " "Check other can get objects without attributes matching the filter " "and using bearer token"
"and using bearer token"
): ):
oid = deny_objects.pop() oid = deny_objects.pop()
assert can_get_head_object( assert can_get_head_object(

View file

@ -8,7 +8,9 @@ import allure
import pytest import pytest
import yaml import yaml
from dateutil import parser from dateutil import parser
from frostfs_testlib.healthcheck.interfaces import Healthcheck
from frostfs_testlib.hosting import Hosting from frostfs_testlib.hosting import Hosting
from frostfs_testlib.plugins import load_plugin
from frostfs_testlib.reporter import AllureHandler, get_reporter from frostfs_testlib.reporter import AllureHandler, get_reporter
from frostfs_testlib.resources.common import ( from frostfs_testlib.resources.common import (
ASSETS_DIR, ASSETS_DIR,
@ -17,7 +19,7 @@ from frostfs_testlib.resources.common import (
DEFAULT_WALLET_PASS, DEFAULT_WALLET_PASS,
SIMPLE_OBJECT_SIZE, SIMPLE_OBJECT_SIZE,
) )
from frostfs_testlib.s3.interfaces import S3ClientWrapper, VersioningStatus from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.shell import LocalShell, Shell from frostfs_testlib.shell import LocalShell, Shell
from frostfs_testlib.steps.cli.container import list_containers from frostfs_testlib.steps.cli.container import list_containers
from frostfs_testlib.steps.cli.object import get_netmap_netinfo from frostfs_testlib.steps.cli.object import get_netmap_netinfo
@ -144,7 +146,10 @@ def complex_object_size(max_object_size: int) -> ObjectSize:
# By default we want all tests to be executed with both object sizes # By default we want all tests to be executed with both object sizes
# This can be overriden in choosen tests if needed # This can be overriden in choosen tests if needed
@pytest.fixture(scope="session", params=["simple", "complex"]) @pytest.fixture(
scope="session",
params=[pytest.param("simple", marks=pytest.mark.simple), pytest.param("complex", marks=pytest.mark.complex)],
)
def object_size( def object_size(
simple_object_size: ObjectSize, complex_object_size: ObjectSize, request: pytest.FixtureRequest simple_object_size: ObjectSize, complex_object_size: ObjectSize, request: pytest.FixtureRequest
) -> ObjectSize: ) -> ObjectSize:
@ -182,13 +187,29 @@ def s3_policy(request: pytest.FixtureRequest):
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def cluster_state_controller(client_shell: Shell, cluster: Cluster) -> ClusterStateController: @allure.title("[Session] Create healthcheck object")
controller = ClusterStateController(client_shell, cluster) def healthcheck(cluster: Cluster) -> Healthcheck:
healthcheck_cls = load_plugin(
"frostfs.testlib.healthcheck", cluster.cluster_nodes[0].host.config.healthcheck_plugin_name
)
return healthcheck_cls()
@pytest.fixture(scope="session")
def cluster_state_controller(client_shell: Shell, cluster: Cluster, healthcheck: Healthcheck) -> ClusterStateController:
controller = ClusterStateController(client_shell, cluster, healthcheck)
yield controller yield controller
@allure.step("[Class]: Create S3 client") @allure.step("[Class]: Create S3 client")
@pytest.fixture(scope="class") @pytest.fixture(
scope="class",
params=[
pytest.param(AwsCliClient, marks=pytest.mark.aws),
pytest.param(Boto3ClientWrapper, marks=pytest.mark.boto3),
],
)
def s3_client( def s3_client(
default_wallet: str, default_wallet: str,
client_shell: Shell, client_shell: Shell,
@ -225,22 +246,27 @@ def versioning_status(request: pytest.FixtureRequest) -> VersioningStatus:
@allure.step("Create/delete bucket") @allure.step("Create/delete bucket")
@pytest.fixture @pytest.fixture
def bucket(s3_client: S3ClientWrapper, versioning_status: VersioningStatus): def bucket(s3_client: S3ClientWrapper, versioning_status: VersioningStatus, request: pytest.FixtureRequest):
bucket_name = s3_client.create_bucket() bucket_name = s3_client.create_bucket()
if versioning_status: if versioning_status:
s3_helper.set_bucket_versioning(s3_client, bucket_name, versioning_status) s3_helper.set_bucket_versioning(s3_client, bucket_name, versioning_status)
yield bucket_name yield bucket_name
if "sanity" not in request.config.option.markexpr:
s3_helper.delete_bucket_with_objects(s3_client, bucket_name) s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
@allure.step("Create two buckets") @allure.step("Create two buckets")
@pytest.fixture @pytest.fixture
def two_buckets(s3_client: S3ClientWrapper): def two_buckets(s3_client: S3ClientWrapper, request: pytest.FixtureRequest):
bucket_1 = s3_client.create_bucket() bucket_1 = s3_client.create_bucket()
bucket_2 = s3_client.create_bucket() bucket_2 = s3_client.create_bucket()
yield bucket_1, bucket_2 yield bucket_1, bucket_2
if "sanity" not in request.config.option.markexpr:
for bucket_name in [bucket_1, bucket_2]: for bucket_name in [bucket_1, bucket_2]:
s3_helper.delete_bucket_with_objects(s3_client, bucket_name) s3_helper.delete_bucket_with_objects(s3_client, bucket_name)

View file

@ -92,6 +92,7 @@ class TestPolicy(ClusterTestBase):
endpoint=endpoint, endpoint=endpoint,
) )
@pytest.mark.sanity
@allure.title("Simple policy results with one node") @allure.title("Simple policy results with one node")
def test_simple_policy_results_with_one_node( def test_simple_policy_results_with_one_node(
self, self,
@ -166,6 +167,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"): with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint) delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Policy with SELECT and FILTER results with one node") @allure.title("Policy with SELECT and FILTER results with one node")
def test_policy_with_select_and_filter_results_with_one_node( def test_policy_with_select_and_filter_results_with_one_node(
self, self,
@ -179,7 +181,7 @@ class TestPolicy(ClusterTestBase):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 1 expected_copies = 1
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
expected_params = {"country": "Russia"} placement_params = {"country": "Russia"}
with allure.step(f"Create container with policy {placement_rule}"): with allure.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(
@ -202,9 +204,10 @@ class TestPolicy(ClusterTestBase):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0] node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0]
with allure.step(f"Check the node is selected from {placement_params['country']}"):
assert ( assert (
expected_params["country"] == netmap[node_address]["country"] placement_params["country"] == netmap[node_address]["country"]
), f"The node is selected from the wrong country. Expected {expected_params['country']} and got {netmap[node_address]['country']}" ), f"The node is selected from the wrong country. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -222,8 +225,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with one node. This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with one node.
""" """
placement_rule = "REP 1 IN RUS REP 1 IN RUS CBF 1 SELECT 1 FROM RU AS RUS FILTER Country NE Sweden AS NotSE FILTER @NotSE AND NOT (CountryCode EQ FI) AND Country EQ 'Russia' AS RU" placement_rule = "REP 1 IN RUS REP 1 IN RUS CBF 1 SELECT 1 FROM RU AS RUS FILTER Country NE Sweden AS NotSE FILTER @NotSE AND NOT (CountryCode EQ FI) AND Country EQ 'Russia' AS RU"
expected_params = {"country": ["Russia", "Sweden"]} placement_params = {"country": ["Russia", "Sweden"], "country_code": "FI"}
unexpected_params = {"country": "Sweden", "country_code": "FI"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 1 expected_copies = 1
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -249,12 +251,15 @@ class TestPolicy(ClusterTestBase):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0] node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0]
with allure.step(
f"Check the node is selected from {placement_params['country'][0]}"
):
assert ( assert (
(expected_params["country"][1] == netmap[node_address]["country"]) not (placement_params["country"][1] == netmap[node_address]["country"])
or not (unexpected_params["country"] == netmap[node_address]["country"]) or not (placement_params["country"][1] == netmap[node_address]["country"])
and not (unexpected_params["country_code"] == netmap[node_address]["country_code"]) and not (placement_params["country_code"] == netmap[node_address]["country_code"])
and (expected_params["country"][0] == netmap[node_address]["country"]) and (placement_params["country"][0] == netmap[node_address]["country"])
), f"The node is selected from the wrong country or country code. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected from the wrong country or country code. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -262,6 +267,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"): with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint) delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Policy with Multi SELECTs and FILTERs results with one node") @allure.title("Policy with Multi SELECTs and FILTERs results with one node")
def test_policy_with_multi_selects_and_filters_results_with_one_node( def test_policy_with_multi_selects_and_filters_results_with_one_node(
self, self,
@ -272,8 +278,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with one nodes. This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with one nodes.
""" """
placement_rule = "REP 1 IN RU REP 1 IN EU REP 1 IN RU CBF 1 SELECT 1 FROM RUS AS RU SELECT 1 FROM EUR AS EU FILTER Country EQ Russia AS RUS FILTER NOT (@RUS) AND Country EQ Sweden OR CountryCode EQ FI AS EUR" placement_rule = "REP 1 IN RU REP 1 IN EU REP 1 IN RU CBF 1 SELECT 1 FROM RUS AS RU SELECT 1 FROM EUR AS EU FILTER Country EQ Russia AS RUS FILTER NOT (@RUS) AND Country EQ Sweden OR CountryCode EQ FI AS EUR"
expected_params = {"country": ["Sweden", "Russia"], "country_code": "FI"} placement_params = {"country": ["Sweden", "Russia"], "country_code": "FI"}
unexpected_params = {"country": "Russia"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 2 expected_copies = 2
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -298,13 +303,14 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check two nodes are selected from any country"):
for node in resulting_copies: for node in resulting_copies:
node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0] node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0]
assert (expected_params["country"][1] == netmap[node_address]["country"]) or ( assert (placement_params["country"][1] == netmap[node_address]["country"]) or (
not (unexpected_params["country"][1] == netmap[node_address]["country"]) not (placement_params["country"][1] == netmap[node_address]["country"])
and (expected_params["country"][0] == netmap[node_address]["country"]) and (placement_params["country"][0] == netmap[node_address]["country"])
or (expected_params["country_code"] == netmap[node_address]["country_code"]) or (placement_params["country_code"] == netmap[node_address]["country_code"])
), f"The node is selected from the wrong country or country code. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected from the wrong country or country code. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -312,6 +318,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"): with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint) delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Policy with SELECT and FILTER results with UNIQUE nodes") @allure.title("Policy with SELECT and FILTER results with UNIQUE nodes")
def test_policy_with_select_and_filter_results_with_unique_nodes( def test_policy_with_select_and_filter_results_with_unique_nodes(
self, self,
@ -322,10 +329,10 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with SELECT and FILTER results with UNIQUE nodes. This test checks object's copies based on container's placement policy with SELECT and FILTER results with UNIQUE nodes.
""" """
placement_rule = "UNIQUE REP 1 IN MyRussianNodes REP 1 IN MyRussianNodes CBF 1 SELECT 1 FROM RussianNodes AS MyRussianNodes FILTER Country EQ 'Russia' AS RussianNodes" placement_rule = "UNIQUE REP 1 IN MyRussianNodes REP 1 IN MyRussianNodes CBF 1 SELECT 1 FROM RussianNodes AS MyRussianNodes FILTER Country EQ 'Russia' AS RussianNodes"
placement_params = {"country": "Russia"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 2 expected_copies = 2
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
expected_params = {"country": "Russia"}
with allure.step(f"Create container with policy {placement_rule}"): with allure.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(
@ -345,14 +352,14 @@ class TestPolicy(ClusterTestBase):
), f"Expected {expected_copies} copies, got {len(resulting_copies)}" ), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check two nodes are selected from {placement_params['country']}"):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert ( assert (
expected_params["country"] == netmap[node_address]["country"] placement_params["country"] == netmap[node_address]["country"]
), f"The node is selected from the wrong country. Expected {expected_params['country']} and got {netmap[node_address]['country']}" ), f"The node is selected from the wrong country. Got {netmap[node_address]['country']}"
with allure.step(f"Delete the container"): with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint) delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@ -445,7 +452,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with SELECT and FILTER results with 25% of available nodes. This test checks object's copies based on container's placement policy with SELECT and FILTER results with 25% of available nodes.
""" """
placement_rule = "REP 1 IN Nodes25 SELECT 1 FROM LE10 AS Nodes25 FILTER Price LE 10 AS LE10" placement_rule = "REP 1 IN Nodes25 SELECT 1 FROM LE10 AS Nodes25 FILTER Price LE 10 AS LE10"
expected_params = {"price": 10} placement_params = {"price": 10}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 1 expected_copies = 1
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -470,11 +477,11 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
for node in resulting_copies: node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0]
node_address = node.get_rpc_endpoint().split(":")[0] with allure.step(f"Check the node is selected with price <= {placement_params['price']}"):
assert ( assert (
int(netmap[node_address]["price"]) <= expected_params["price"] int(netmap[node_address]["price"]) <= placement_params["price"]
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected with the wrong price. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -493,7 +500,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 25% of available nodes. This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 25% of available nodes.
""" """
placement_rule = "REP 1 IN Nodes25 SELECT 1 FROM BET0AND10 AS Nodes25 FILTER Price LE 10 AS LE10 FILTER Price GT 0 AS GT0 FILTER @LE10 AND @GT0 AS BET0AND10" placement_rule = "REP 1 IN Nodes25 SELECT 1 FROM BET0AND10 AS Nodes25 FILTER Price LE 10 AS LE10 FILTER Price GT 0 AS GT0 FILTER @LE10 AND @GT0 AS BET0AND10"
expected_params = {"price": [10, 0]} placement_params = {"price": [10, 0]}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 1 expected_copies = 1
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -518,16 +525,13 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check the node is selected with price between 1 and 10"):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert ( assert (
int(netmap[node_address]["price"]) > expected_params["price"][1] int(netmap[node_address]["price"]) > placement_params["price"][1]
or int(netmap[node_address]["price"]) <= expected_params["price"][0] and int(netmap[node_address]["price"]) <= placement_params["price"][0]
or ( ), f"The node is selected with the wrong price. Got {netmap[node_address]}"
int(netmap[node_address]["price"]) > expected_params["price"][1]
and int(netmap[node_address]["price"]) <= expected_params["price"][0]
)
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -546,7 +550,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 25% of available nodes. This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 25% of available nodes.
""" """
placement_rule = "UNIQUE REP 1 IN One REP 1 IN One CBF 1 SELECT 1 FROM MINMAX AS One FILTER Price LT 15 AS LT15 FILTER Price GT 55 AS GT55 FILTER @LT15 OR @GT55 AS MINMAX" placement_rule = "UNIQUE REP 1 IN One REP 1 IN One CBF 1 SELECT 1 FROM MINMAX AS One FILTER Price LT 15 AS LT15 FILTER Price GT 55 AS GT55 FILTER @LT15 OR @GT55 AS MINMAX"
expected_params = {"price": [15, 55]} placement_params = {"price": [15, 55]}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 2 expected_copies = 2
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -571,17 +575,13 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check two nodes are selected with max and min prices"):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert ( assert (
int(netmap[node_address]["price"]) > expected_params["price"][1] int(netmap[node_address]["price"]) > placement_params["price"][1]
or int(netmap[node_address]["price"]) < expected_params["price"][0] or int(netmap[node_address]["price"]) < placement_params["price"][0]
or ( ), f"The node is selected with the wrong price. Got {netmap[node_address]}"
int(netmap[node_address]["price"]) > expected_params["price"][1]
or int(netmap[node_address]["price"]) < expected_params["price"][0]
)
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}"
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -674,7 +674,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with SELECT and FILTER results with 50% of available nodes. This test checks object's copies based on container's placement policy with SELECT and FILTER results with 50% of available nodes.
""" """
placement_rule = "REP 2 IN HALF CBF 1 SELECT 2 FROM GT15 AS HALF FILTER Price GT 15 AS GT15" placement_rule = "REP 2 IN HALF CBF 1 SELECT 2 FROM GT15 AS HALF FILTER Price GT 15 AS GT15"
expected_params = {"price": 15} placement_params = {"price": 15}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 2 expected_copies = 2
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -699,11 +699,12 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check two nodes are selected with price > {placement_params['price']}"):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert ( assert (
int(netmap[node_address]["price"]) > expected_params["price"] int(netmap[node_address]["price"]) > placement_params["price"]
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected with the wrong price. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -722,8 +723,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 50% of available nodes. This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 50% of available nodes.
""" """
placement_rule = "REP 2 IN HALF CBF 2 SELECT 2 FROM GE15 AS HALF FILTER CountryCode NE RU AS NOTRU FILTER @NOTRU AND Price GE 15 AS GE15" placement_rule = "REP 2 IN HALF CBF 2 SELECT 2 FROM GE15 AS HALF FILTER CountryCode NE RU AS NOTRU FILTER @NOTRU AND Price GE 15 AS GE15"
expected_params = {"price": 15} placement_params = {"price": 15, "country_code": "RU"}
unexpected_params = {"country_code": "RU"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 2 expected_copies = 2
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -748,12 +748,13 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check two nodes are selected not with country code '{placement_params['country_code']}'"):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert (not netmap[node_address]["country_code"] == unexpected_params["country_code"]) or ( assert (not netmap[node_address]["country_code"] == placement_params["country_code"]
not netmap[node_address]["country_code"] == unexpected_params["country_code"] or not netmap[node_address]["country_code"] == placement_params["country_code"]
and int(netmap[node_address]["price"]) >= expected_params["price"] and int(netmap[node_address]["price"]) >= placement_params["price"]
), f"The node is selected from the wrong price or country_code. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected with the wrong price or country code. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -772,7 +773,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 50% of available nodes. This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 50% of available nodes.
""" """
placement_rule = "REP 2 IN FH REP 1 IN SH CBF 2 SELECT 2 FROM LE55 AS FH SELECT 2 FROM GE15 AS SH FILTER 'UN-LOCODE' EQ RU_LED OR 'UN-LOCODE' EQ RU_MOW AS RU FILTER NOT(@RU) AS NOTRU FILTER @NOTRU AND Price GE 15 AS GE15 FILTER @RU AND Price LE 55 AS LE55" placement_rule = "REP 2 IN FH REP 1 IN SH CBF 2 SELECT 2 FROM LE55 AS FH SELECT 2 FROM GE15 AS SH FILTER 'UN-LOCODE' EQ RU_LED OR 'UN-LOCODE' EQ RU_MOW AS RU FILTER NOT(@RU) AS NOTRU FILTER @NOTRU AND Price GE 15 AS GE15 FILTER @RU AND Price LE 55 AS LE55"
expected_params = {"un_locode": ["RU_LED", "RU_MOW"], "price": [15, 55]} placement_params = {"un_locode": ["RU_LED", "RU_MOW"], "price": [15, 55]}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 4 expected_copies = 4
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -797,20 +798,21 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check all nodes are selected"):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert ( assert (
(netmap[node_address]["un_locode"] in expected_params["un_locode"]) netmap[node_address]["un_locode"] in placement_params["un_locode"]
or (not netmap[node_address]["un_locode"] in expected_params["un_locode"]) or not netmap[node_address]["un_locode"] == placement_params["un_locode"][1]
or ( or (
not netmap[node_address]["un_locode"] in expected_params["un_locode"] not netmap[node_address]["un_locode"] == placement_params["un_locode"][1]
and int(netmap[node_address]["price"]) >= expected_params["price"][0] and int(netmap[node_address]["price"]) >= placement_params["price"][0]
) )
or ( or (
netmap[node_address]["un_locode"] in expected_params["un_locode"] netmap[node_address]["un_locode"] == placement_params["un_locode"][1]
and int(netmap[node_address]["price"]) <= expected_params["price"][1] and int(netmap[node_address]["price"]) <= placement_params["price"][1]
) )
), f"The node is selected from the wrong price or un_locode. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected with the wrong price or un_locode. Expected {placement_params} and got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -818,6 +820,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"): with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint) delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Simple policy results with 75% of available nodes") @allure.title("Simple policy results with 75% of available nodes")
def test_simple_policy_results_with_75_of_available_nodes( def test_simple_policy_results_with_75_of_available_nodes(
self, self,
@ -903,7 +906,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with SELECT and FILTER results with 75% of available nodes. This test checks object's copies based on container's placement policy with SELECT and FILTER results with 75% of available nodes.
""" """
placement_rule = "REP 2 IN NODES75 SELECT 2 FROM LT65 AS NODES75 FILTER Price LT 65 AS LT65" placement_rule = "REP 2 IN NODES75 SELECT 2 FROM LT65 AS NODES75 FILTER Price LT 65 AS LT65"
expected_params = {"price": 65} placement_params = {"price": 65}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 2 expected_copies = 2
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -928,11 +931,12 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check two nodes are selected with price < {placement_params['price']}"):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert ( assert (
int(netmap[node_address]["price"]) < expected_params["price"] int(netmap[node_address]["price"]) < placement_params["price"]
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected with the wrong price. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -951,8 +955,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 75% of available nodes. This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 75% of available nodes.
""" """
placement_rule = "REP 2 IN NODES75 SELECT 2 FROM LT65 AS NODES75 FILTER Continent NE America AS NOAM FILTER @NOAM AND Price LT 65 AS LT65" placement_rule = "REP 2 IN NODES75 SELECT 2 FROM LT65 AS NODES75 FILTER Continent NE America AS NOAM FILTER @NOAM AND Price LT 65 AS LT65"
expected_params = {"price": 65} placement_params = {"price": 65, "continent": "America"}
unexpected_params = {"continent": "America"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 3 expected_copies = 3
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -977,14 +980,16 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check three nodes are selected not from {placement_params['continent']}"
):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert ( assert (
int(netmap[node_address]["price"]) < expected_params["price"] int(netmap[node_address]["price"]) < placement_params["price"]
and netmap[node_address]["continent"] == unexpected_params["continent"] and not netmap[node_address]["continent"] == placement_params["continent"]
) or ( ) or (
netmap[node_address]["continent"] == unexpected_params["continent"] not netmap[node_address]["continent"] == placement_params["continent"]
), f"The node is selected from the wrong price or continent. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected with the wrong price or continent. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1003,8 +1008,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 75% of available nodes. This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 75% of available nodes.
""" """
placement_rule = "REP 2 IN EXPNSV REP 2 IN CHEAP SELECT 3 FROM GT10 AS EXPNSV SELECT 3 FROM LT65 AS CHEAP FILTER NOT(Continent EQ America) AS NOAM FILTER @NOAM AND Price LT 65 AS LT65 FILTER @NOAM AND Price GT 10 AS GT10" placement_rule = "REP 2 IN EXPNSV REP 2 IN CHEAP SELECT 3 FROM GT10 AS EXPNSV SELECT 3 FROM LT65 AS CHEAP FILTER NOT(Continent EQ America) AS NOAM FILTER @NOAM AND Price LT 65 AS LT65 FILTER @NOAM AND Price GT 10 AS GT10"
expected_params = {"price": [65, 10]} placement_params = {"price": [65, 10], "continent": "America"}
unexpected_params = {"continent": "America"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 4 expected_copies = 4
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -1029,19 +1033,20 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check all nodes are selected"):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert ( assert (
( (
int(netmap[node_address]["price"]) > expected_params["price"][1] int(netmap[node_address]["price"]) > placement_params["price"][1]
and netmap[node_address]["continent"] == unexpected_params["continent"] and not netmap[node_address]["continent"] == placement_params["continent"]
) )
or ( or (
int(netmap[node_address]["price"]) < expected_params["price"][0] int(netmap[node_address]["price"]) < placement_params["price"][0]
and netmap[node_address]["continent"] == unexpected_params["continent"] and not netmap[node_address]["continent"] == placement_params["continent"]
) )
or (netmap[node_address]["continent"] == unexpected_params["continent"]) or not (netmap[node_address]["continent"] == placement_params["continent"])
), f"The node is selected from the wrong price or continent. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected with the wrong price or continent. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1049,6 +1054,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"): with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint) delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Simple policy results with 100% of available nodes") @allure.title("Simple policy results with 100% of available nodes")
def test_simple_policy_results_with_100_of_available_nodes( def test_simple_policy_results_with_100_of_available_nodes(
self, self,
@ -1134,7 +1140,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with SELECT and FILTER results with 100% of available nodes. This test checks object's copies based on container's placement policy with SELECT and FILTER results with 100% of available nodes.
""" """
placement_rule = "REP 1 IN All SELECT 4 FROM AllNodes AS All FILTER Price GE 0 AS AllNodes" placement_rule = "REP 1 IN All SELECT 4 FROM AllNodes AS All FILTER Price GE 0 AS AllNodes"
expected_params = {"price": 0} placement_params = {"price": 0}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 1 expected_copies = 1
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -1157,13 +1163,13 @@ class TestPolicy(ClusterTestBase):
), f"Expected {expected_copies} copies, got {len(resulting_copies)}" ), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0] node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0]
assert netmap[node_address]["price"] >= int( with allure.step(f"Check the node is selected with price >= {placement_params['price']}"):
expected_params["price"] assert (
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}" netmap[node_address]["price"] >= int(placement_params["price"])
), f"The node is selected with the wrong price. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1181,7 +1187,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 100% of available nodes. This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 100% of available nodes.
""" """
placement_rule = "REP 2 IN All SELECT 4 FROM AllNodes AS All FILTER Country EQ Russia OR Country EQ Sweden OR Country EQ Finland AS AllCountries FILTER @AllCountries AND Continent EQ Europe AS AllNodes" placement_rule = "REP 2 IN All SELECT 4 FROM AllNodes AS All FILTER Country EQ Russia OR Country EQ Sweden OR Country EQ Finland AS AllCountries FILTER @AllCountries AND Continent EQ Europe AS AllNodes"
expected_params = {"country": ["Russia", "Sweden", "Finland"], "continent": "Europe"} placement_params = {"country": ["Russia", "Sweden", "Finland"], "continent": "Europe"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 2 expected_copies = 2
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -1206,13 +1212,14 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check two nodes are selected from {' or '.join(placement_params['country'])}"):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert ( assert (
(netmap[node_address]["country"] in expected_params["country"]) (netmap[node_address]["country"] in placement_params["country"])
or (netmap[node_address]["country"] in expected_params["country"]) or (netmap[node_address]["country"] in placement_params["country"])
and (netmap[node_address]["continent"] in expected_params["continent"]) and (netmap[node_address]["continent"] == placement_params["continent"])
), f"The node is selected from the wrong country or continent. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected from the wrong country or continent. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1230,7 +1237,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 100% of available nodes. This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 100% of available nodes.
""" """
placement_rule = "REP 4 IN AllOne REP 4 IN AllTwo CBF 4 SELECT 2 FROM GEZero AS AllOne SELECT 2 FROM AllCountries AS AllTwo FILTER Country EQ Russia OR Country EQ Sweden OR Country EQ Finland AS AllCountries FILTER Price GE 0 AS GEZero" placement_rule = "REP 4 IN AllOne REP 4 IN AllTwo CBF 4 SELECT 2 FROM GEZero AS AllOne SELECT 2 FROM AllCountries AS AllTwo FILTER Country EQ Russia OR Country EQ Sweden OR Country EQ Finland AS AllCountries FILTER Price GE 0 AS GEZero"
expected_params = {"country": ["Russia", "Sweden", "Finland"], "price": "0"} placement_params = {"country": ["Russia", "Sweden", "Finland"], "price": 0}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 4 expected_copies = 4
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -1255,11 +1262,12 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check all node are selected"):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert (netmap[node_address]["country"] in expected_params["country"]) or ( assert (netmap[node_address]["country"] in placement_params["country"]) or (
netmap[node_address]["price"] in expected_params["price"] int(netmap[node_address]["price"]) >= placement_params["price"]
), f"The node is selected from the wrong country or price. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected from the wrong country or with wrong price. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1267,6 +1275,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"): with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint) delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Simple policy results with UNIQUE nodes") @allure.title("Simple policy results with UNIQUE nodes")
def test_simple_policy_results_with_unique_nodes( def test_simple_policy_results_with_unique_nodes(
self, self,
@ -1297,10 +1306,6 @@ class TestPolicy(ClusterTestBase):
assert ( assert (
len(resulting_copies) == expected_copies len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}" ), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
with allure.step(f"Check the object appearance"):
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1339,15 +1344,13 @@ class TestPolicy(ClusterTestBase):
len(resulting_copies) == expected_copies len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}" ), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with allure.step(f"Check the object appearance"):
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
with allure.step(f"Delete the container"): with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint) delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Policy with SELECT and Complex FILTER results with UNIQUE nodes") @allure.title("Policy with SELECT and Complex FILTER results with UNIQUE nodes")
def test_policy_with_select_and_complex_filter_results_with_unique_nodes( def test_policy_with_select_and_complex_filter_results_with_unique_nodes(
self, self,
@ -1358,8 +1361,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with UNIQUE nodes. This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with UNIQUE nodes.
""" """
placement_rule = "UNIQUE REP 1 IN RUS REP 1 IN RUS CBF 1 SELECT 1 FROM RU AS RUS FILTER Country NE Sweden AS NotSE FILTER @NotSE AND NOT (CountryCode EQ FI) AND Country EQ 'Russia' AS RU" placement_rule = "UNIQUE REP 1 IN RUS REP 1 IN RUS CBF 1 SELECT 1 FROM RU AS RUS FILTER Country NE Sweden AS NotSE FILTER @NotSE AND NOT (CountryCode EQ FI) AND Country EQ 'Russia' AS RU"
expected_params = {"country": "Russia"} placement_params = {"country_code": "FI", "country": ["Sweden", "Russia"]}
unexpected_params = {"country_code": "FI", "country": "Sweden"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 2 expected_copies = 2
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -1382,16 +1384,16 @@ class TestPolicy(ClusterTestBase):
), f"Expected {expected_copies} copies, got {len(resulting_copies)}" ), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(f"Check two nodes are selected not from {placement_params['country'][0]}"):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert not (unexpected_params["country"] == netmap[node_address]["country"]) or ( assert not (placement_params["country"][0] == netmap[node_address]["country"]) or (
not (unexpected_params["country"] == netmap[node_address]["country"]) not (placement_params["country"][0] == netmap[node_address]["country"])
and not (expected_params["country_code"] == netmap[node_address]["country_code"]) and not (placement_params["country_code"] == netmap[node_address]["country_code"])
or (expected_params["country"] == netmap[node_address]["country"]) and placement_params["country"][1] == netmap[node_address]["country"]
), f"The node is selected from the wrong country or country code. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected from the wrong country or with wrong country code. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1399,6 +1401,7 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Delete the container"): with allure.step(f"Delete the container"):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint) delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
@pytest.mark.sanity
@allure.title("Policy with Multi SELECTs and FILTERs results with UNIQUE nodes") @allure.title("Policy with Multi SELECTs and FILTERs results with UNIQUE nodes")
def test_policy_with_multi_selects_and_filters_results_with_unique_nodes( def test_policy_with_multi_selects_and_filters_results_with_unique_nodes(
self, self,
@ -1409,8 +1412,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with UNIQUE nodes. This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with UNIQUE nodes.
""" """
placement_rule = "UNIQUE REP 1 IN RU REP 1 IN EU REP 1 IN RU CBF 1 SELECT 1 FROM RUS AS RU SELECT 1 FROM EUR AS EU FILTER Country EQ Russia AS RUS FILTER NOT (@RUS) AND Country EQ Sweden OR CountryCode EQ FI AS EUR" placement_rule = "UNIQUE REP 1 IN RU REP 1 IN EU REP 1 IN RU CBF 1 SELECT 1 FROM RUS AS RU SELECT 1 FROM EUR AS EU FILTER Country EQ Russia AS RUS FILTER NOT (@RUS) AND Country EQ Sweden OR CountryCode EQ FI AS EUR"
expected_params = {"country": ["Russia", "Sweden"], "country_code": "FI"} placement_params = {"country": ["Russia", "Sweden"], "country_code": "FI"}
unexpected_params = {"country": "Russia"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 3 expected_copies = 3
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -1433,16 +1435,18 @@ class TestPolicy(ClusterTestBase):
), f"Expected {expected_copies} copies, got {len(resulting_copies)}" ), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with allure.step(
f"Check three nodes are selected from any country"
):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert (expected_params["country"][0] == netmap[node_address]["country"]) or ( assert (placement_params["country"][0] == netmap[node_address]["country"]) or (
not (unexpected_params["country"][0] == netmap[node_address]["country"]) not (placement_params["country"][0] == netmap[node_address]["country"])
and (expected_params["country"][1] == netmap[node_address]["country"]) and (placement_params["country"][1] == netmap[node_address]["country"])
or (expected_params["country_code"] == netmap[node_address]["country_code"]) or (placement_params["country_code"] == netmap[node_address]["country_code"])
), f"The node is selected from the wrong country or country code. Expected {expected_params} and got {netmap[node_address]}" ), f"The node is selected from the wrong country or with wrong country code. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1460,7 +1464,7 @@ class TestPolicy(ClusterTestBase):
This test checks object's copies based on container's placement policy: REP 1 IN SPB REP 1 IN MSK REP 3. This test checks object's copies based on container's placement policy: REP 1 IN SPB REP 1 IN MSK REP 3.
""" """
placement_rule = "REP 1 IN SPB REP 1 IN MSK REP 3 CBF 1 SELECT 2 FROM LED AS SPB SELECT 2 FROM MOW AS MSK FILTER Location EQ 'Saint Petersburg (ex Leningrad)' AS LED FILTER Location EQ Moskva AS MOW" placement_rule = "REP 1 IN SPB REP 1 IN MSK REP 3 CBF 1 SELECT 2 FROM LED AS SPB SELECT 2 FROM MOW AS MSK FILTER Location EQ 'Saint Petersburg (ex Leningrad)' AS LED FILTER Location EQ Moskva AS MOW"
expected_params = {"location": ["Saint Petersburg (ex Leningrad)", "Moskva"]} placement_params = {"location": ["Saint Petersburg (ex Leningrad)", "Moskva"]}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 3 expected_copies = 3
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -1485,13 +1489,17 @@ class TestPolicy(ClusterTestBase):
with allure.step(f"Check the object appearance"): with allure.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
list_of_location = []
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert expected_params["location"][0] == netmap[node_address]["location"] or ( list_of_location.append(netmap[node_address]["location"])
expected_params["location"][1] == netmap[node_address]["location"]
), f"The node is selected from the wrong location. Expected {expected_params} and got {netmap[node_address]}"
self.check_for_the_uniqueness_of_the_nodes(resulting_copies) with allure.step(f"Check two or three nodes are selected from Russia and from any other country"):
assert (
placement_params["location"][0] in list_of_location
and placement_params["location"][1] in list_of_location
and len(resulting_copies) > 2
), f"The node is selected from the wrong location. Got {netmap[node_address]}"
with allure.step(f"Delete the object from the container"): with allure.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1507,9 +1515,6 @@ class TestPolicy(ClusterTestBase):
"'", "" "'", ""
), f"Expected \n{placement_rule} and got policy \n{got_policy} are the same" ), f"Expected \n{placement_rule} and got policy \n{got_policy} are the same"
def check_for_the_uniqueness_of_the_nodes(self, nodes: list):
assert len(set(nodes)) == len(nodes), f"The nodes found must be unique, but {nodes}"
def get_netmap_param(self, netmap_info: list[NodeNetmapInfo]) -> dict: def get_netmap_param(self, netmap_info: list[NodeNetmapInfo]) -> dict:
dict_external = dict() dict_external = dict()
for node in netmap_info: for node in netmap_info:

View file

@ -6,7 +6,6 @@ from time import sleep
import allure import allure
import pytest import pytest
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.s3 import AwsCliClient
from frostfs_testlib.steps.cli.container import create_container from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import ( from frostfs_testlib.steps.cli.object import (
get_object, get_object,
@ -38,11 +37,6 @@ OBJECT_ATTRIBUTES = [
] ]
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient], indirect=True)
@pytest.mark.failover @pytest.mark.failover
@pytest.mark.failover_network @pytest.mark.failover_network
class TestFailoverNetwork(ClusterTestBase): class TestFailoverNetwork(ClusterTestBase):

View file

@ -99,8 +99,10 @@ class TestFailoverServer(ClusterTestBase):
self, nodes: list[StorageNode], storage_objects: list[StorageObjectInfo] self, nodes: list[StorageNode], storage_objects: list[StorageObjectInfo]
) -> list[StorageObjectInfo]: ) -> list[StorageObjectInfo]:
corrupted_objects = [] corrupted_objects = []
errors_get = []
for node in nodes: for node in nodes:
for storage_object in storage_objects: for storage_object in storage_objects:
try:
got_file_path = get_object( got_file_path = get_object(
storage_object.wallet_file_path, storage_object.wallet_file_path,
storage_object.cid, storage_object.cid,
@ -112,7 +114,10 @@ class TestFailoverServer(ClusterTestBase):
if storage_object.file_hash != get_file_hash(got_file_path): if storage_object.file_hash != get_file_hash(got_file_path):
corrupted_objects.append(storage_object) corrupted_objects.append(storage_object)
os.remove(got_file_path) os.remove(got_file_path)
except RuntimeError:
errors_get.append(storage_object.oid)
assert len(errors_get) == 0, f"Get failed - {errors_get}"
return corrupted_objects return corrupted_objects
def check_objects_replication( def check_objects_replication(

View file

@ -5,10 +5,9 @@ from time import sleep
import allure import allure
import pytest import pytest
from frostfs_testlib.hosting import Host
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.shell import CommandOptions from frostfs_testlib.shell import CommandOptions
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
from frostfs_testlib.steps.cli.object import get_object, put_object_to_random_node from frostfs_testlib.steps.cli.object import get_object, put_object_to_random_node
@ -21,7 +20,7 @@ from frostfs_testlib.steps.node_management import (
wait_for_node_to_be_ready, wait_for_node_to_be_ready,
) )
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, StorageNode from frostfs_testlib.storage.cluster import Cluster, ClusterNode, S3Gate, StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController, ShardsWatcher from frostfs_testlib.storage.controllers import ClusterStateController, ShardsWatcher
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
@ -37,11 +36,6 @@ logger = logging.getLogger("NeoLogger")
stopped_nodes: list[StorageNode] = [] stopped_nodes: list[StorageNode] = []
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
@allure.title("Provide File Keeper") @allure.title("Provide File Keeper")
def file_keeper(): def file_keeper():
@ -57,26 +51,11 @@ def after_run_return_all_stopped_hosts(cluster_state_controller: ClusterStateCon
cluster_state_controller.start_stopped_hosts() cluster_state_controller.start_stopped_hosts()
@allure.step("Return all stopped storage services after test") @allure.step("Return all stopped services after test")
@pytest.fixture(scope="function") @pytest.fixture(scope="function")
def after_run_return_all_stopped_services(cluster_state_controller: ClusterStateController): def after_run_return_all_stopped_services(cluster_state_controller: ClusterStateController):
yield yield
cluster_state_controller.start_stopped_storage_services() cluster_state_controller.start_all_stopped_services()
@allure.step("Return all stopped S3 GateWay services after test")
@pytest.fixture(scope="function")
def after_run_return_all_stopped_s3(cluster_state_controller: ClusterStateController):
yield
cluster_state_controller.start_stopped_s3_gates()
def panic_reboot_host(host: Host) -> None:
shell = host.get_shell()
shell.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"')
options = CommandOptions(close_stdin=True, timeout=1, check=False)
shell.exec('sudo sh -c "echo b > /proc/sysrq-trigger"', options)
@pytest.mark.failover @pytest.mark.failover
@ -209,22 +188,21 @@ class TestFailoverStorage(ClusterTestBase):
s3_client: S3ClientWrapper, s3_client: S3ClientWrapper,
simple_object_size: ObjectSize, simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController, cluster_state_controller: ClusterStateController,
after_run_return_all_stopped_s3,
after_run_return_all_stopped_services, after_run_return_all_stopped_services,
): ):
default_node = self.cluster.cluster_nodes[0] default_node = self.cluster.cluster_nodes[0]
with allure.step("Turn S3 GW off on default node"): with allure.step("Turn S3 GW off on default node"):
cluster_state_controller.stop_s3_gate(default_node) cluster_state_controller.stop_service_of_type(default_node, S3Gate)
with allure.step("Turn off storage on default node"): with allure.step("Turn off storage on default node"):
cluster_state_controller.stop_storage_service(default_node) cluster_state_controller.stop_service_of_type(default_node, StorageNode)
with allure.step("Turn on S3 GW on default node"): with allure.step("Turn on S3 GW on default node"):
cluster_state_controller.start_s3_gate(default_node) cluster_state_controller.start_service_of_type(default_node, S3Gate)
with allure.step("Turn on storage on default node"): with allure.step("Turn on storage on default node"):
cluster_state_controller.start_storage_service(default_node) cluster_state_controller.start_service_of_type(default_node, StorageNode)
with allure.step("Create bucket with REP 1 SELECT 1 policy"): with allure.step("Create bucket with REP 1 SELECT 1 policy"):
bucket = s3_client.create_bucket( bucket = s3_client.create_bucket(
@ -240,13 +218,13 @@ class TestFailoverStorage(ClusterTestBase):
with allure.step("Turn off all storage nodes except default"): with allure.step("Turn off all storage nodes except default"):
for node in self.cluster.cluster_nodes[1:]: for node in self.cluster.cluster_nodes[1:]:
with allure.step(f"Stop storage service on node: {node}"): with allure.step(f"Stop storage service on node: {node}"):
cluster_state_controller.stop_storage_service(node) cluster_state_controller.stop_service_of_type(node, StorageNode)
with allure.step("Check that object is available"): with allure.step("Check that object is available"):
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name]) s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
with allure.step("Start storage nodes"): with allure.step("Start storage nodes"):
cluster_state_controller.start_stopped_storage_services() cluster_state_controller.start_all_stopped_services()
@pytest.mark.failover @pytest.mark.failover
@ -320,7 +298,7 @@ class TestEmptyMap(ClusterTestBase):
def empty_map_stop_service_teardown(self, cluster_state_controller: ClusterStateController): def empty_map_stop_service_teardown(self, cluster_state_controller: ClusterStateController):
yield yield
with allure.step("Return all storage nodes to network map"): with allure.step("Return all storage nodes to network map"):
cluster_state_controller.start_stopped_storage_services() cluster_state_controller.start_all_stopped_services()
for node in stopped_nodes: for node in stopped_nodes:
check_node_in_map(node, shell=self.shell, alive_node=node) check_node_in_map(node, shell=self.shell, alive_node=node)
@ -365,7 +343,7 @@ class TestEmptyMap(ClusterTestBase):
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_objects) s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_objects)
with allure.step("Stop all storage nodes"): with allure.step("Stop all storage nodes"):
cluster_state_controller.stop_all_storage_services() cluster_state_controller.stop_services_of_type(StorageNode)
with allure.step("Remove all nodes from network map"): with allure.step("Remove all nodes from network map"):
remove_nodes_from_map_morph( remove_nodes_from_map_morph(
@ -383,7 +361,7 @@ class TestEmptyMap(ClusterTestBase):
first_node = self.cluster.cluster_nodes[0].service(StorageNode) first_node = self.cluster.cluster_nodes[0].service(StorageNode)
with allure.step("Start first node and check network map"): with allure.step("Start first node and check network map"):
cluster_state_controller.start_storage_service(self.cluster.cluster_nodes[0]) cluster_state_controller.start_service_of_type(self.cluster.cluster_nodes[0], StorageNode)
wait_for_node_to_be_ready(first_node) wait_for_node_to_be_ready(first_node)
for check_node in self.cluster.storage_nodes: for check_node in self.cluster.storage_nodes:
@ -392,7 +370,7 @@ class TestEmptyMap(ClusterTestBase):
for node in self.cluster.cluster_nodes[1:]: for node in self.cluster.cluster_nodes[1:]:
storage_node = node.service(StorageNode) storage_node = node.service(StorageNode)
cluster_state_controller.start_storage_service(node) cluster_state_controller.start_service_of_type(node, StorageNode)
wait_for_node_to_be_ready(storage_node) wait_for_node_to_be_ready(storage_node)
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
@ -420,9 +398,7 @@ class TestEmptyMap(ClusterTestBase):
object_versions.append(put_object) object_versions.append(put_object)
with allure.step("Stop all storage nodes"): with allure.step("Stop all storage nodes"):
for node in self.cluster.cluster_nodes: cluster_state_controller.stop_services_of_type(StorageNode)
with allure.step(f"Stop storage service on node: {node}"):
cluster_state_controller.stop_storage_service(node)
with allure.step("Delete blobovnicza and fstree from all nodes"): with allure.step("Delete blobovnicza and fstree from all nodes"):
for node in self.cluster.storage_nodes: for node in self.cluster.storage_nodes:
@ -430,7 +406,7 @@ class TestEmptyMap(ClusterTestBase):
node.delete_fstree() node.delete_fstree()
with allure.step("Start all storage nodes"): with allure.step("Start all storage nodes"):
cluster_state_controller.start_stopped_storage_services() cluster_state_controller.start_all_stopped_services()
# need to get Delete Marker first # need to get Delete Marker first
with allure.step("Delete the object from the bucket"): with allure.step("Delete the object from the bucket"):
@ -462,9 +438,7 @@ class TestEmptyMap(ClusterTestBase):
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name]) s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
with allure.step("Stop all storage nodes"): with allure.step("Stop all storage nodes"):
for node in self.cluster.cluster_nodes: cluster_state_controller.stop_services_of_type(StorageNode)
with allure.step(f"Stop storage service on node: {node}"):
cluster_state_controller.stop_storage_service(node)
with allure.step("Delete blobovnicza and fstree from all nodes"): with allure.step("Delete blobovnicza and fstree from all nodes"):
for node in self.cluster.storage_nodes: for node in self.cluster.storage_nodes:
@ -472,7 +446,7 @@ class TestEmptyMap(ClusterTestBase):
node.delete_fstree() node.delete_fstree()
with allure.step("Start all storage nodes"): with allure.step("Start all storage nodes"):
cluster_state_controller.start_stopped_storage_services() cluster_state_controller.start_all_stopped_services()
with allure.step("Delete the object from the bucket"): with allure.step("Delete the object from the bucket"):
s3_client.delete_object(bucket, file_name) s3_client.delete_object(bucket, file_name)
@ -507,16 +481,14 @@ class TestEmptyMap(ClusterTestBase):
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name]) s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
with allure.step("Stop all storage nodes"): with allure.step("Stop all storage nodes"):
for node in self.cluster.cluster_nodes: cluster_state_controller.stop_services_of_type(StorageNode)
with allure.step(f"Stop storage service on node: {node}"):
cluster_state_controller.stop_storage_service(node)
with allure.step("Delete pilorama.db from all nodes"): with allure.step("Delete pilorama.db from all nodes"):
for node in self.cluster.storage_nodes: for node in self.cluster.storage_nodes:
node.delete_pilorama() node.delete_pilorama()
with allure.step("Start all storage nodes"): with allure.step("Start all storage nodes"):
cluster_state_controller.start_stopped_storage_services() cluster_state_controller.start_all_stopped_services()
with allure.step("Check list objects first time"): with allure.step("Check list objects first time"):
objects_list = s3_client.list_objects(bucket) objects_list = s3_client.list_objects(bucket)
@ -578,7 +550,7 @@ class TestStorageDataLoss(ClusterTestBase):
) )
with allure.step("Stop storage services on all nodes"): with allure.step("Stop storage services on all nodes"):
cluster_state_controller.stop_all_storage_services() cluster_state_controller.stop_services_of_type(StorageNode)
with allure.step("Delete metabase from all nodes"): with allure.step("Delete metabase from all nodes"):
for node in cluster_state_controller.cluster.storage_nodes: for node in cluster_state_controller.cluster.storage_nodes:
@ -594,7 +566,11 @@ class TestStorageDataLoss(ClusterTestBase):
storage_node.save_config(config, config_file_path) storage_node.save_config(config, config_file_path)
with allure.step("Start storage services on all nodes"): with allure.step("Start storage services on all nodes"):
cluster_state_controller.start_stopped_storage_services() cluster_state_controller.start_all_stopped_services()
with allure.step("Wait for tree rebalance"):
# TODO: Use product metric when we have proper ones for this check
sleep(30)
with allure.step("Delete objects from bucket"): with allure.step("Delete objects from bucket"):
with allure.step("Delete simple object from bucket"): with allure.step("Delete simple object from bucket"):
@ -655,13 +631,13 @@ class TestStorageDataLoss(ClusterTestBase):
shards_watcher.take_shards_snapshot() shards_watcher.take_shards_snapshot()
with allure.step(f"Stop storage service on node {node_under_test}"): with allure.step(f"Stop storage service on node {node_under_test}"):
cluster_state_controller.stop_storage_service(node_under_test) cluster_state_controller.stop_service_of_type(node_under_test, StorageNode)
with allure.step(f"Delete write cache from node {node_under_test}"): with allure.step(f"Delete write cache from node {node_under_test}"):
node_under_test.storage_node.delete_write_cache() node_under_test.storage_node.delete_write_cache()
with allure.step(f"Start storage service on node {node_under_test}"): with allure.step(f"Start storage service on node {node_under_test}"):
cluster_state_controller.start_storage_service(node_under_test) cluster_state_controller.start_all_stopped_services()
with allure.step("Objects should be available"): with allure.step("Objects should be available"):
for storage_object in storage_objects: for storage_object in storage_objects:
@ -710,7 +686,7 @@ class TestStorageDataLoss(ClusterTestBase):
with allure.step("Stop one node and wait for rebalance connection of s3 gate to storage service"): with allure.step("Stop one node and wait for rebalance connection of s3 gate to storage service"):
current_node = self.cluster.cluster_nodes[0] current_node = self.cluster.cluster_nodes[0]
cluster_state_controller.stop_storage_service(current_node) cluster_state_controller.stop_service_of_type(current_node, StorageNode)
# waiting for rebalance connection of s3 gate to storage service # waiting for rebalance connection of s3 gate to storage service
sleep(60) sleep(60)
@ -752,15 +728,13 @@ class TestStorageDataLoss(ClusterTestBase):
piloramas_list_before_removing = self.get_piloramas_list(node_to_check) piloramas_list_before_removing = self.get_piloramas_list(node_to_check)
with allure.step("Stop all storage nodes"): with allure.step("Stop all storage nodes"):
for node in self.cluster.cluster_nodes: cluster_state_controller.stop_services_of_type(StorageNode)
with allure.step(f"Stop storage service on node: {node}"):
cluster_state_controller.stop_storage_service(node)
with allure.step("Delete pilorama.db from one node"): with allure.step("Delete pilorama.db from one node"):
node_to_check.delete_pilorama() node_to_check.delete_pilorama()
with allure.step("Start all storage nodes"): with allure.step("Start all storage nodes"):
cluster_state_controller.start_stopped_storage_services() cluster_state_controller.start_all_stopped_services()
with allure.step("Tick epoch to trigger sync and then wait for 1 minute"): with allure.step("Tick epoch to trigger sync and then wait for 1 minute"):
self.tick_epochs(1) self.tick_epochs(1)

View file

@ -79,9 +79,7 @@ def generate_ranges(
range_length = random.randint(RANGE_MIN_LEN, RANGE_MAX_LEN) range_length = random.randint(RANGE_MIN_LEN, RANGE_MAX_LEN)
range_start = random.randint(offset, offset + length) range_start = random.randint(offset, offset + length)
file_ranges_to_test.append( file_ranges_to_test.append((range_start, min(range_length, storage_object.size - range_start)))
(range_start, min(range_length, storage_object.size - range_start))
)
file_ranges_to_test.extend(STATIC_RANGES.get(storage_object.size, [])) file_ranges_to_test.extend(STATIC_RANGES.get(storage_object.size, []))
@ -250,9 +248,7 @@ class TestObjectApi(ClusterTestBase):
assert sorted(expected_oids) == sorted(result) assert sorted(expected_oids) == sorted(result)
@allure.title("Search objects with removed items (obj_size={object_size})") @allure.title("Search objects with removed items (obj_size={object_size})")
def test_object_search_should_return_tombstone_items( def test_object_search_should_return_tombstone_items(self, default_wallet: str, object_size: ObjectSize):
self, default_wallet: str, object_size: ObjectSize
):
""" """
Validate object search with removed items Validate object search with removed items
""" """
@ -275,9 +271,7 @@ class TestObjectApi(ClusterTestBase):
with allure.step("Search object"): with allure.step("Search object"):
# Root Search object should return root object oid # Root Search object should return root object oid
result = search_object( result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True)
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True
)
assert result == [storage_object.oid] assert result == [storage_object.oid]
with allure.step("Delete file"): with allure.step("Delete file"):
@ -285,22 +279,14 @@ class TestObjectApi(ClusterTestBase):
with allure.step("Search deleted object with --root"): with allure.step("Search deleted object with --root"):
# Root Search object should return nothing # Root Search object should return nothing
result = search_object( result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True)
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True
)
assert len(result) == 0 assert len(result) == 0
with allure.step("Search deleted object with --phy should return only tombstones"): with allure.step("Search deleted object with --phy should return only tombstones"):
# Physical Search object should return only tombstones # Physical Search object should return only tombstones
result = search_object( result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, phy=True)
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, phy=True assert storage_object.tombstone in result, "Search result should contain tombstone of removed object"
) assert storage_object.oid not in result, "Search result should not contain ObjectId of removed object"
assert (
storage_object.tombstone in result
), "Search result should contain tombstone of removed object"
assert (
storage_object.oid not in result
), "Search result should not contain ObjectId of removed object"
for tombstone_oid in result: for tombstone_oid in result:
header = head_object( header = head_object(
wallet, wallet,
@ -315,7 +301,6 @@ class TestObjectApi(ClusterTestBase):
), f"Object wasn't deleted properly. Found object {tombstone_oid} with type {object_type}" ), f"Object wasn't deleted properly. Found object {tombstone_oid} with type {object_type}"
@allure.title("Get range hash by native API (obj_size={object_size})") @allure.title("Get range hash by native API (obj_size={object_size})")
@pytest.mark.sanity
@pytest.mark.grpc_api @pytest.mark.grpc_api
def test_object_get_range_hash(self, storage_objects: list[StorageObjectInfo], max_object_size): def test_object_get_range_hash(self, storage_objects: list[StorageObjectInfo], max_object_size):
""" """
@ -327,9 +312,7 @@ class TestObjectApi(ClusterTestBase):
oids = [storage_object.oid for storage_object in storage_objects[:2]] oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path file_path = storage_objects[0].file_path
file_ranges_to_test = generate_ranges( file_ranges_to_test = generate_ranges(storage_objects[0], max_object_size, self.shell, self.cluster)
storage_objects[0], max_object_size, self.shell, self.cluster
)
logging.info(f"Ranges used in test {file_ranges_to_test}") logging.info(f"Ranges used in test {file_ranges_to_test}")
for range_start, range_len in file_ranges_to_test: for range_start, range_len in file_ranges_to_test:
@ -349,7 +332,6 @@ class TestObjectApi(ClusterTestBase):
), f"Expected range hash to match {range_cut} slice of file payload" ), f"Expected range hash to match {range_cut} slice of file payload"
@allure.title("Get range by native API (obj_size={object_size})") @allure.title("Get range by native API (obj_size={object_size})")
@pytest.mark.sanity
@pytest.mark.grpc_api @pytest.mark.grpc_api
def test_object_get_range(self, storage_objects: list[StorageObjectInfo], max_object_size): def test_object_get_range(self, storage_objects: list[StorageObjectInfo], max_object_size):
""" """
@ -361,9 +343,7 @@ class TestObjectApi(ClusterTestBase):
oids = [storage_object.oid for storage_object in storage_objects[:2]] oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path file_path = storage_objects[0].file_path
file_ranges_to_test = generate_ranges( file_ranges_to_test = generate_ranges(storage_objects[0], max_object_size, self.shell, self.cluster)
storage_objects[0], max_object_size, self.shell, self.cluster
)
logging.info(f"Ranges used in test {file_ranges_to_test}") logging.info(f"Ranges used in test {file_ranges_to_test}")
for range_start, range_len in file_ranges_to_test: for range_start, range_len in file_ranges_to_test:
@ -379,14 +359,11 @@ class TestObjectApi(ClusterTestBase):
range_cut=range_cut, range_cut=range_cut,
) )
assert ( assert (
get_file_content( get_file_content(file_path, content_len=range_len, mode="rb", offset=range_start)
file_path, content_len=range_len, mode="rb", offset=range_start
)
== range_content == range_content
), f"Expected range content to match {range_cut} slice of file payload" ), f"Expected range content to match {range_cut} slice of file payload"
@allure.title("[NEGATIVE] Get invalid range by native API (obj_size={object_size})") @allure.title("[NEGATIVE] Get invalid range by native API (obj_size={object_size})")
@pytest.mark.sanity
@pytest.mark.grpc_api @pytest.mark.grpc_api
def test_object_get_range_negatives( def test_object_get_range_negatives(
self, self,
@ -421,11 +398,7 @@ class TestObjectApi(ClusterTestBase):
for range_start, range_len, expected_error in file_ranges_to_test: for range_start, range_len, expected_error in file_ranges_to_test:
range_cut = f"{range_start}:{range_len}" range_cut = f"{range_start}:{range_len}"
expected_error = ( expected_error = expected_error.format(range=range_cut) if "{range}" in expected_error else expected_error
expected_error.format(range=range_cut)
if "{range}" in expected_error
else expected_error
)
with allure.step(f"Get range ({range_cut})"): with allure.step(f"Get range ({range_cut})"):
for oid in oids: for oid in oids:
with pytest.raises(Exception, match=expected_error): with pytest.raises(Exception, match=expected_error):
@ -472,11 +445,7 @@ class TestObjectApi(ClusterTestBase):
for range_start, range_len, expected_error in file_ranges_to_test: for range_start, range_len, expected_error in file_ranges_to_test:
range_cut = f"{range_start}:{range_len}" range_cut = f"{range_start}:{range_len}"
expected_error = ( expected_error = expected_error.format(range=range_cut) if "{range}" in expected_error else expected_error
expected_error.format(range=range_cut)
if "{range}" in expected_error
else expected_error
)
with allure.step(f"Get range hash ({range_cut})"): with allure.step(f"Get range hash ({range_cut})"):
for oid in oids: for oid in oids:
with pytest.raises(Exception, match=expected_error): with pytest.raises(Exception, match=expected_error):
@ -491,9 +460,7 @@ class TestObjectApi(ClusterTestBase):
def check_header_is_presented(self, head_info: dict, object_header: dict) -> None: def check_header_is_presented(self, head_info: dict, object_header: dict) -> None:
for key_to_check, val_to_check in object_header.items(): for key_to_check, val_to_check in object_header.items():
assert ( assert key_to_check in head_info["header"]["attributes"], f"Key {key_to_check} is found in {head_object}"
key_to_check in head_info["header"]["attributes"]
), f"Key {key_to_check} is found in {head_object}"
assert head_info["header"]["attributes"].get(key_to_check) == str( assert head_info["header"]["attributes"].get(key_to_check) == str(
val_to_check val_to_check
), f"Value {val_to_check} is equal" ), f"Value {val_to_check} is equal"

View file

@ -28,10 +28,7 @@ def bearer_token_file_all_allow(default_wallet: str, client_shell: Shell, cluste
bearer = form_bearertoken_file( bearer = form_bearertoken_file(
default_wallet, default_wallet,
"", "",
[ [EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) for op in EACLOperation],
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS)
for op in EACLOperation
],
shell=client_shell, shell=client_shell,
endpoint=cluster.default_rpc_endpoint, endpoint=cluster.default_rpc_endpoint,
) )
@ -85,9 +82,7 @@ def storage_objects(
@pytest.mark.smoke @pytest.mark.smoke
@pytest.mark.bearer @pytest.mark.bearer
class TestObjectApiWithBearerToken(ClusterTestBase): class TestObjectApiWithBearerToken(ClusterTestBase):
@allure.title( @allure.title("Object can be deleted from any node using s3gate wallet with bearer token (obj_size={object_size})")
"Object can be deleted from any node using s3gate wallet with bearer token (obj_size={object_size})"
)
@pytest.mark.parametrize( @pytest.mark.parametrize(
"user_container", "user_container",
[SINGLE_PLACEMENT_RULE], [SINGLE_PLACEMENT_RULE],
@ -112,9 +107,7 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
wallet_config=s3_gate_wallet.get_wallet_config_path(), wallet_config=s3_gate_wallet.get_wallet_config_path(),
) )
@allure.title( @allure.title("Object can be fetched from any node using s3gate wallet with bearer token (obj_size={object_size})")
"Object can be fetched from any node using s3gate wallet with bearer token (obj_size={object_size})"
)
@pytest.mark.parametrize( @pytest.mark.parametrize(
"user_container", "user_container",
[REP_2_FOR_3_NODES_PLACEMENT_RULE], [REP_2_FOR_3_NODES_PLACEMENT_RULE],

View file

@ -14,11 +14,7 @@ from frostfs_testlib.resources.error_patterns import (
OBJECT_NOT_FOUND, OBJECT_NOT_FOUND,
) )
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import ( from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
StorageContainer,
StorageContainerInfo,
create_container,
)
from frostfs_testlib.steps.cli.object import delete_object, head_object, lock_object from frostfs_testlib.steps.cli.object import delete_object, head_object, lock_object
from frostfs_testlib.steps.complex_object_actions import get_link_object, get_storage_object_chunks from frostfs_testlib.steps.complex_object_actions import get_link_object, get_storage_object_chunks
from frostfs_testlib.steps.epoch import ensure_fresh_epoch, get_epoch, tick_epoch from frostfs_testlib.steps.epoch import ensure_fresh_epoch, get_epoch, tick_epoch
@ -27,10 +23,7 @@ from frostfs_testlib.steps.storage_object import delete_objects
from frostfs_testlib.steps.storage_policy import get_nodes_with_object from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import Cluster from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.storage_object_info import ( from frostfs_testlib.storage.dataclasses.storage_object_info import LockObjectInfo, StorageObjectInfo
LockObjectInfo,
StorageObjectInfo,
)
from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success
@ -57,9 +50,7 @@ def user_wallet(wallet_factory: WalletFactory):
scope="module", scope="module",
) )
def user_container(user_wallet: WalletInfo, client_shell: Shell, cluster: Cluster): def user_container(user_wallet: WalletInfo, client_shell: Shell, cluster: Cluster):
container_id = create_container( container_id = create_container(user_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
user_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint
)
return StorageContainer(StorageContainerInfo(container_id, user_wallet), client_shell, cluster) return StorageContainer(StorageContainerInfo(container_id, user_wallet), client_shell, cluster)
@ -91,9 +82,7 @@ def locked_storage_object(
lifetime=FIXTURE_LOCK_LIFETIME, lifetime=FIXTURE_LOCK_LIFETIME,
) )
storage_object.locks = [ storage_object.locks = [
LockObjectInfo( LockObjectInfo(storage_object.cid, lock_object_id, FIXTURE_LOCK_LIFETIME, expiration_epoch)
storage_object.cid, lock_object_id, FIXTURE_LOCK_LIFETIME, expiration_epoch
)
] ]
yield storage_object yield storage_object
@ -117,17 +106,13 @@ def locked_storage_object(
except Exception as ex: except Exception as ex:
ex_message = str(ex) ex_message = str(ex)
# It's okay if object already removed # It's okay if object already removed
if not re.search(OBJECT_NOT_FOUND, ex_message) and not re.search( if not re.search(OBJECT_NOT_FOUND, ex_message) and not re.search(OBJECT_ALREADY_REMOVED, ex_message):
OBJECT_ALREADY_REMOVED, ex_message
):
raise ex raise ex
logger.debug(ex_message) logger.debug(ex_message)
@wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME)) @wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME))
def check_object_not_found( def check_object_not_found(wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str
):
with pytest.raises(Exception, match=OBJECT_NOT_FOUND): with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object( head_object(
wallet_file_path, wallet_file_path,
@ -138,9 +123,7 @@ def check_object_not_found(
) )
def verify_object_available( def verify_object_available(wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str
):
with expect_not_raises(): with expect_not_raises():
head_object( head_object(
wallet_file_path, wallet_file_path,
@ -151,13 +134,10 @@ def verify_object_available(
) )
@pytest.mark.sanity
@pytest.mark.grpc_object_lock @pytest.mark.grpc_object_lock
class TestObjectLockWithGrpc(ClusterTestBase): class TestObjectLockWithGrpc(ClusterTestBase):
@pytest.fixture() @pytest.fixture()
def new_locked_storage_object( def new_locked_storage_object(self, user_container: StorageContainer, object_size: ObjectSize) -> StorageObjectInfo:
self, user_container: StorageContainer, object_size: ObjectSize
) -> StorageObjectInfo:
""" """
Intention of this fixture is to provide new storage object for tests which may delete or corrupt the object or it's complementary objects Intention of this fixture is to provide new storage object for tests which may delete or corrupt the object or it's complementary objects
So we need a new one each time we ask for it So we need a new one each time we ask for it
@ -284,6 +264,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
expire_at=wrong_expire_at, expire_at=wrong_expire_at,
) )
@pytest.mark.sanity
@allure.title("Expired object is deleted when locks are expired (obj_size={object_size})") @allure.title("Expired object is deleted when locks are expired (obj_size={object_size})")
def test_expired_object_should_be_deleted_after_locks_are_expired( def test_expired_object_should_be_deleted_after_locks_are_expired(
self, self,
@ -295,9 +276,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
""" """
current_epoch = self.ensure_fresh_epoch() current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object( storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 1)
object_size.value, expire_at=current_epoch + 1
)
with allure.step("Lock object for couple epochs"): with allure.step("Lock object for couple epochs"):
lock_object( lock_object(
@ -355,9 +334,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with allure.step("Generate three objects"): with allure.step("Generate three objects"):
for _ in range(3): for _ in range(3):
storage_objects.append( storage_objects.append(user_container.generate_object(object_size.value, expire_at=current_epoch + 5))
user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
)
lock_object( lock_object(
storage_objects[0].wallet_file_path, storage_objects[0].wallet_file_path,
@ -398,16 +375,12 @@ class TestObjectLockWithGrpc(ClusterTestBase):
current_epoch = self.ensure_fresh_epoch() current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object( storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 1)
object_size.value, expire_at=current_epoch + 1
)
expiration_epoch = current_epoch - 1 expiration_epoch = current_epoch - 1
with pytest.raises( with pytest.raises(
Exception, Exception,
match=LOCK_OBJECT_EXPIRATION.format( match=LOCK_OBJECT_EXPIRATION.format(expiration_epoch=expiration_epoch, current_epoch=current_epoch),
expiration_epoch=expiration_epoch, current_epoch=current_epoch
),
): ):
lock_object( lock_object(
storage_object.wallet_file_path, storage_object.wallet_file_path,
@ -418,6 +391,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
expire_at=expiration_epoch, expire_at=expiration_epoch,
) )
@pytest.mark.sanity
@allure.title("Delete object when lock is expired by lifetime (obj_size={object_size})") @allure.title("Delete object when lock is expired by lifetime (obj_size={object_size})")
@expect_not_raises() @expect_not_raises()
def test_after_lock_expiration_with_lifetime_user_should_be_able_to_delete_object( def test_after_lock_expiration_with_lifetime_user_should_be_able_to_delete_object(
@ -430,9 +404,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
""" """
current_epoch = self.ensure_fresh_epoch() current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object( storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
object_size.value, expire_at=current_epoch + 5
)
lock_object( lock_object(
storage_object.wallet_file_path, storage_object.wallet_file_path,
@ -466,9 +438,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
current_epoch = self.ensure_fresh_epoch() current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object( storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
object_size.value, expire_at=current_epoch + 5
)
lock_object( lock_object(
storage_object.wallet_file_path, storage_object.wallet_file_path,
@ -505,9 +475,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
Complex object chunks should also be protected from deletion Complex object chunks should also be protected from deletion
""" """
chunk_object_ids = get_storage_object_chunks( chunk_object_ids = get_storage_object_chunks(locked_storage_object, self.shell, self.cluster)
locked_storage_object, self.shell, self.cluster
)
for chunk_object_id in chunk_object_ids: for chunk_object_id in chunk_object_ids:
with allure.step(f"Try to delete chunk object {chunk_object_id}"): with allure.step(f"Try to delete chunk object {chunk_object_id}"):
with pytest.raises(Exception, match=OBJECT_IS_LOCKED): with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
@ -527,9 +495,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
["complex"], ["complex"],
indirect=True, indirect=True,
) )
def test_link_object_of_locked_complex_object_can_be_dropped( def test_link_object_of_locked_complex_object_can_be_dropped(self, new_locked_storage_object: StorageObjectInfo):
self, new_locked_storage_object: StorageObjectInfo
):
link_object_id = get_link_object( link_object_id = get_link_object(
new_locked_storage_object.wallet_file_path, new_locked_storage_object.wallet_file_path,
new_locked_storage_object.cid, new_locked_storage_object.cid,
@ -557,12 +523,8 @@ class TestObjectLockWithGrpc(ClusterTestBase):
["complex"], ["complex"],
indirect=True, indirect=True,
) )
def test_chunks_of_locked_complex_object_can_be_dropped( def test_chunks_of_locked_complex_object_can_be_dropped(self, new_locked_storage_object: StorageObjectInfo):
self, new_locked_storage_object: StorageObjectInfo chunk_objects = get_storage_object_chunks(new_locked_storage_object, self.shell, self.cluster)
):
chunk_objects = get_storage_object_chunks(
new_locked_storage_object, self.shell, self.cluster
)
for chunk_object_id in chunk_objects: for chunk_object_id in chunk_objects:
with allure.step(f"Drop chunk object with id {chunk_object_id} from nodes"): with allure.step(f"Drop chunk object with id {chunk_object_id} from nodes"):
@ -630,9 +592,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
object_size: ObjectSize, object_size: ObjectSize,
): ):
current_epoch = self.ensure_fresh_epoch() current_epoch = self.ensure_fresh_epoch()
storage_object = user_container.generate_object( storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 1)
object_size.value, expire_at=current_epoch + 1
)
with allure.step("Apply first lock to object for 3 epochs"): with allure.step("Apply first lock to object for 3 epochs"):
lock_object_id_0 = lock_object( lock_object_id_0 = lock_object(
@ -705,9 +665,8 @@ class TestObjectLockWithGrpc(ClusterTestBase):
self.cluster.default_rpc_endpoint, self.cluster.default_rpc_endpoint,
) )
@allure.title( @pytest.mark.sanity
"Two expired objects with one lock are deleted after lock expiration (obj_size={object_size})" @allure.title("Two expired objects with one lock are deleted after lock expiration (obj_size={object_size})")
)
def test_two_objects_expiration_with_one_lock( def test_two_objects_expiration_with_one_lock(
self, self,
user_container: StorageContainer, user_container: StorageContainer,
@ -720,9 +679,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with allure.step("Generate two objects"): with allure.step("Generate two objects"):
for epoch_i in range(2): for epoch_i in range(2):
storage_objects.append( storage_objects.append(
user_container.generate_object( user_container.generate_object(object_size.value, expire_at=current_epoch + epoch_i + 3)
object_size.value, expire_at=current_epoch + epoch_i + 3
)
) )
self.tick_epoch() self.tick_epoch()

View file

@ -20,6 +20,7 @@ OBJECT_ATTRIBUTES = {"common_key": "common_value"}
WAIT_FOR_REPLICATION = 60 WAIT_FOR_REPLICATION = 60
# Adding failover mark because it may make cluster unhealthy # Adding failover mark because it may make cluster unhealthy
@pytest.mark.sanity
@pytest.mark.failover @pytest.mark.failover
@pytest.mark.replication @pytest.mark.replication
class TestReplication(ClusterTestBase): class TestReplication(ClusterTestBase):

View file

@ -21,7 +21,6 @@ from frostfs_testlib.utils.file_utils import generate_file
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@pytest.mark.sanity
@pytest.mark.http_gate @pytest.mark.http_gate
@pytest.mark.skip("Skipped due to deprecated PUT via http") @pytest.mark.skip("Skipped due to deprecated PUT via http")
@pytest.mark.http_put @pytest.mark.http_put
@ -46,9 +45,7 @@ class Test_http_bearer(ClusterTestBase):
@pytest.fixture(scope="class") @pytest.fixture(scope="class")
def eacl_deny_for_others(self, user_container: str) -> None: def eacl_deny_for_others(self, user_container: str) -> None:
with allure.step(f"Set deny all operations for {EACLRole.OTHERS} via eACL"): with allure.step(f"Set deny all operations for {EACLRole.OTHERS} via eACL"):
eacl = EACLRule( eacl = EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=EACLOperation.PUT)
access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=EACLOperation.PUT
)
set_eacl( set_eacl(
self.wallet, self.wallet,
user_container, user_container,
@ -64,10 +61,7 @@ class Test_http_bearer(ClusterTestBase):
bearer = form_bearertoken_file( bearer = form_bearertoken_file(
self.wallet, self.wallet,
user_container, user_container,
[ [EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) for op in EACLOperation],
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS)
for op in EACLOperation
],
shell=self.shell, shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint, endpoint=self.cluster.default_rpc_endpoint,
sign=False, sign=False,
@ -105,9 +99,7 @@ class Test_http_bearer(ClusterTestBase):
eacl_deny_for_others eacl_deny_for_others
bearer = bearer_token_no_limit_for_others bearer = bearer_token_no_limit_for_others
file_path = generate_file(object_size.value) file_path = generate_file(object_size.value)
with allure.step( with allure.step(f"Put object with bearer token for {EACLRole.OTHERS}, then get and verify hashes"):
f"Put object with bearer token for {EACLRole.OTHERS}, then get and verify hashes"
):
headers = [f" -H 'Authorization: Bearer {bearer}'"] headers = [f" -H 'Authorization: Bearer {bearer}'"]
oid = upload_via_http_gate_curl( oid = upload_via_http_gate_curl(
cid=user_container, cid=user_container,

View file

@ -44,9 +44,7 @@ class TestHttpGate(ClusterTestBase):
TestHttpGate.wallet = default_wallet TestHttpGate.wallet = default_wallet
@allure.title("Put over gRPC, Get over HTTP") @allure.title("Put over gRPC, Get over HTTP")
def test_put_grpc_get_http( def test_put_grpc_get_http(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
self, complex_object_size: ObjectSize, simple_object_size: ObjectSize
):
""" """
Test that object can be put using gRPC interface and get using HTTP. Test that object can be put using gRPC interface and get using HTTP.
@ -106,7 +104,6 @@ class TestHttpGate(ClusterTestBase):
) )
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading") @allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading") @allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@pytest.mark.sanity
@pytest.mark.http_gate @pytest.mark.http_gate
@pytest.mark.http_put @pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http") @pytest.mark.skip("Skipped due to deprecated PUT via http")
@ -115,9 +112,7 @@ class TestHttpPut(ClusterTestBase):
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading") @allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@allure.title("Put over HTTP, Get over HTTP") @allure.title("Put over HTTP, Get over HTTP")
@pytest.mark.smoke @pytest.mark.smoke
def test_put_http_get_http( def test_put_http_get_http(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
self, complex_object_size: ObjectSize, simple_object_size: ObjectSize
):
""" """
Test that object can be put and get using HTTP interface. Test that object can be put and get using HTTP interface.
@ -165,19 +160,17 @@ class TestHttpPut(ClusterTestBase):
name="download by attributes", name="download by attributes",
) )
@pytest.mark.skip("Skipped due to deprecated PUT via http") @pytest.mark.skip("Skipped due to deprecated PUT via http")
@allure.title("Put over HTTP, Get over HTTP with header") @allure.title("Put over HTTP, Get over HTTP with {id} header")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"attributes", "attributes,id",
[ [
{"fileName": "simple_obj_filename"}, ({"fileName": "simple_obj_filename"}, "simple"),
{"file-Name": "simple obj filename"}, ({"file-Name": "simple obj filename"}, "hyphen"),
{"cat%jpeg": "cat%jpeg"}, ({"cat%jpeg": "cat%jpeg"}, "percent"),
], ],
ids=["simple", "hyphen", "percent"], ids=["simple", "hyphen", "percent"],
) )
def test_put_http_get_http_with_headers( def test_put_http_get_http_with_headers(self, attributes: dict, simple_object_size: ObjectSize, id: str):
self, attributes: dict, simple_object_size: ObjectSize, request: pytest.FixtureRequest
):
""" """
Test that object can be downloaded using different attributes in HTTP header. Test that object can be downloaded using different attributes in HTTP header.
@ -190,7 +183,6 @@ class TestHttpPut(ClusterTestBase):
Expected result: Expected result:
Hashes must be the same. Hashes must be the same.
""" """
allure.dynamic.title(f"Put over HTTP, Get over HTTP with {request.node.callspec.id} header")
cid = create_container( cid = create_container(
self.wallet, self.wallet,
shell=self.shell, shell=self.shell,
@ -341,9 +333,7 @@ class TestHttpPut(ClusterTestBase):
file_path = generate_file(complex_object_size.value) file_path = generate_file(complex_object_size.value)
with allure.step("Put objects using HTTP"): with allure.step("Put objects using HTTP"):
oid_gate = upload_via_http_gate( oid_gate = upload_via_http_gate(cid=cid, path=file_path, endpoint=self.cluster.default_http_gate_endpoint)
cid=cid, path=file_path, endpoint=self.cluster.default_http_gate_endpoint
)
oid_curl = upload_via_http_gate_curl( oid_curl = upload_via_http_gate_curl(
cid=cid, cid=cid,
filepath=file_path, filepath=file_path,
@ -374,9 +364,7 @@ class TestHttpPut(ClusterTestBase):
@pytest.mark.skip("Skipped due to deprecated PUT via http") @pytest.mark.skip("Skipped due to deprecated PUT via http")
@allure.title("Put/Get over HTTP using Curl utility") @allure.title("Put/Get over HTTP using Curl utility")
def test_put_http_get_http_curl( def test_put_http_get_http_curl(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
self, complex_object_size: ObjectSize, simple_object_size: ObjectSize
):
""" """
Test checks upload and download over HTTP using curl utility. Test checks upload and download over HTTP using curl utility.
""" """

View file

@ -27,7 +27,6 @@ OBJECT_ALREADY_REMOVED_ERROR = "object already removed"
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@pytest.mark.sanity
@pytest.mark.http_gate @pytest.mark.http_gate
@pytest.mark.http_put @pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http") @pytest.mark.skip("Skipped due to deprecated PUT via http")
@ -79,9 +78,7 @@ class Test_http_headers(ClusterTestBase):
yield storage_objects yield storage_objects
@allure.title("Get object1 by attribute") @allure.title("Get object1 by attribute")
def test_object1_can_be_get_by_attr( def test_object1_can_be_get_by_attr(self, storage_objects_with_attributes: list[StorageObjectInfo]):
self, storage_objects_with_attributes: list[StorageObjectInfo]
):
""" """
Test to get object#1 by attribute and comapre hashes Test to get object#1 by attribute and comapre hashes
@ -104,9 +101,7 @@ class Test_http_headers(ClusterTestBase):
) )
@allure.title("Get object2 with different attributes, then delete object2 and get object1") @allure.title("Get object2 with different attributes, then delete object2 and get object1")
def test_object2_can_be_get_by_attr( def test_object2_can_be_get_by_attr(self, storage_objects_with_attributes: list[StorageObjectInfo]):
self, storage_objects_with_attributes: list[StorageObjectInfo]
):
""" """
Test to get object2 with different attributes, then delete object2 and get object1 using 1st attribute. Note: obj1 and obj2 have the same attribute#1, Test to get object2 with different attributes, then delete object2 and get object1 using 1st attribute. Note: obj1 and obj2 have the same attribute#1,
and when obj2 is deleted you can get obj1 by 1st attribute and when obj2 is deleted you can get obj1 by 1st attribute
@ -167,9 +162,7 @@ class Test_http_headers(ClusterTestBase):
) )
@allure.title("[NEGATIVE] Put object and get right after container is deleted") @allure.title("[NEGATIVE] Put object and get right after container is deleted")
def test_negative_put_and_get_object3( def test_negative_put_and_get_object3(self, storage_objects_with_attributes: list[StorageObjectInfo]):
self, storage_objects_with_attributes: list[StorageObjectInfo]
):
""" """
Test to attempt to put object and try to download it right after the container has been deleted Test to attempt to put object and try to download it right after the container has been deleted
@ -203,6 +196,7 @@ class Test_http_headers(ClusterTestBase):
cid=storage_object_1.cid, cid=storage_object_1.cid,
shell=self.shell, shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint, endpoint=self.cluster.default_rpc_endpoint,
await_mode=True,
) )
self.tick_epoch() self.tick_epoch()
wait_for_container_deletion( wait_for_container_deletion(
@ -214,9 +208,7 @@ class Test_http_headers(ClusterTestBase):
assert storage_object_1.cid not in list_containers( assert storage_object_1.cid not in list_containers(
self.wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint self.wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
) )
with allure.step( with allure.step("[Negative] Try to download (wget) object via wget with attributes [peace=peace]"):
"[Negative] Try to download (wget) object via wget with attributes [peace=peace]"
):
request = f"/get/{storage_object_1.cid}/peace/peace" request = f"/get/{storage_object_1.cid}/peace/peace"
error_pattern = "404 Not Found" error_pattern = "404 Not Found"
try_to_get_object_via_passed_request_and_expect_error( try_to_get_object_via_passed_request_and_expect_error(

View file

@ -12,7 +12,6 @@ from frostfs_testlib.utils.file_utils import generate_file
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@pytest.mark.sanity
@pytest.mark.http_gate @pytest.mark.http_gate
@pytest.mark.http_put @pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http") @pytest.mark.skip("Skipped due to deprecated PUT via http")
@ -50,9 +49,7 @@ class Test_http_streaming(ClusterTestBase):
# Generate file # Generate file
file_path = generate_file(complex_object_size.value) file_path = generate_file(complex_object_size.value)
with allure.step( with allure.step("Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]"):
"Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]"
):
oid = upload_via_http_gate_curl( oid = upload_via_http_gate_curl(
cid=cid, filepath=file_path, endpoint=self.cluster.default_http_gate_endpoint cid=cid, filepath=file_path, endpoint=self.cluster.default_http_gate_endpoint
) )

View file

@ -8,11 +8,7 @@ import pytest
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import ( from frostfs_testlib.steps.cli.object import get_netmap_netinfo, get_object_from_random_node, head_object
get_netmap_netinfo,
get_object_from_random_node,
head_object,
)
from frostfs_testlib.steps.epoch import get_epoch, wait_for_epochs_align from frostfs_testlib.steps.epoch import get_epoch, wait_for_epochs_align
from frostfs_testlib.steps.http.http_gate import ( from frostfs_testlib.steps.http.http_gate import (
attr_into_str_header_curl, attr_into_str_header_curl,
@ -35,7 +31,6 @@ SYSTEM_EXPIRATION_TIMESTAMP = "System-Expiration-Timestamp"
SYSTEM_EXPIRATION_RFC3339 = "System-Expiration-RFC3339" SYSTEM_EXPIRATION_RFC3339 = "System-Expiration-RFC3339"
@pytest.mark.sanity
@pytest.mark.http_gate @pytest.mark.http_gate
@pytest.mark.http_put @pytest.mark.http_put
@pytest.mark.skip("Skipped due to deprecated PUT via http") @pytest.mark.skip("Skipped due to deprecated PUT via http")
@ -76,9 +71,7 @@ class Test_http_system_header(ClusterTestBase):
return f"{mins}m" return f"{mins}m"
@allure.title("Return future timestamp after N epochs are passed") @allure.title("Return future timestamp after N epochs are passed")
def epoch_count_into_timestamp( def epoch_count_into_timestamp(self, epoch_duration: int, epoch: int, rfc3339: Optional[bool] = False) -> str:
self, epoch_duration: int, epoch: int, rfc3339: Optional[bool] = False
) -> str:
current_datetime = datetime.datetime.utcnow() current_datetime = datetime.datetime.utcnow()
epoch_count_in_seconds = epoch_duration * epoch epoch_count_in_seconds = epoch_duration * epoch
future_datetime = current_datetime + datetime.timedelta(seconds=epoch_count_in_seconds) future_datetime = current_datetime + datetime.timedelta(seconds=epoch_count_in_seconds)
@ -96,9 +89,7 @@ class Test_http_system_header(ClusterTestBase):
return False return False
return True return True
@allure.title( @allure.title(f"Validate that only {EXPIRATION_EPOCH_HEADER} exists in header and other headers are abesent")
f"Validate that only {EXPIRATION_EPOCH_HEADER} exists in header and other headers are abesent"
)
def validation_for_http_header_attr(self, head_info: dict, expected_epoch: int) -> None: def validation_for_http_header_attr(self, head_info: dict, expected_epoch: int) -> None:
# check that __SYSTEM__EXPIRATION_EPOCH attribute has corresponding epoch # check that __SYSTEM__EXPIRATION_EPOCH attribute has corresponding epoch
assert self.check_key_value_presented_header( assert self.check_key_value_presented_header(
@ -146,13 +137,9 @@ class Test_http_system_header(ClusterTestBase):
@allure.title("[NEGATIVE] Put object with expired epoch") @allure.title("[NEGATIVE] Put object with expired epoch")
def test_unable_put_expired_epoch(self, user_container: str, simple_object_size: ObjectSize): def test_unable_put_expired_epoch(self, user_container: str, simple_object_size: ObjectSize):
headers = attr_into_str_header_curl( headers = attr_into_str_header_curl({"System-Expiration-Epoch": str(get_epoch(self.shell, self.cluster) - 1)})
{"System-Expiration-Epoch": str(get_epoch(self.shell, self.cluster) - 1)}
)
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
with allure.step( with allure.step("Put object using HTTP with attribute Expiration-Epoch where epoch is expired"):
"Put object using HTTP with attribute Expiration-Epoch where epoch is expired"
):
upload_via_http_gate_curl( upload_via_http_gate_curl(
cid=user_container, cid=user_container,
filepath=file_path, filepath=file_path,
@ -162,14 +149,10 @@ class Test_http_system_header(ClusterTestBase):
) )
@allure.title("[NEGATIVE] Put object with negative System-Expiration-Duration") @allure.title("[NEGATIVE] Put object with negative System-Expiration-Duration")
def test_unable_put_negative_duration( def test_unable_put_negative_duration(self, user_container: str, simple_object_size: ObjectSize):
self, user_container: str, simple_object_size: ObjectSize
):
headers = attr_into_str_header_curl({"System-Expiration-Duration": "-1h"}) headers = attr_into_str_header_curl({"System-Expiration-Duration": "-1h"})
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
with allure.step( with allure.step("Put object using HTTP with attribute System-Expiration-Duration where duration is negative"):
"Put object using HTTP with attribute System-Expiration-Duration where duration is negative"
):
upload_via_http_gate_curl( upload_via_http_gate_curl(
cid=user_container, cid=user_container,
filepath=file_path, filepath=file_path,
@ -179,9 +162,7 @@ class Test_http_system_header(ClusterTestBase):
) )
@allure.title("[NEGATIVE] Put object with System-Expiration-Timestamp value in the past") @allure.title("[NEGATIVE] Put object with System-Expiration-Timestamp value in the past")
def test_unable_put_expired_timestamp( def test_unable_put_expired_timestamp(self, user_container: str, simple_object_size: ObjectSize):
self, user_container: str, simple_object_size: ObjectSize
):
headers = attr_into_str_header_curl({"System-Expiration-Timestamp": "1635075727"}) headers = attr_into_str_header_curl({"System-Expiration-Timestamp": "1635075727"})
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
with allure.step( with allure.step(
@ -211,9 +192,7 @@ class Test_http_system_header(ClusterTestBase):
@allure.title("Priority of attributes epoch>duration (obj_size={object_size})") @allure.title("Priority of attributes epoch>duration (obj_size={object_size})")
@pytest.mark.skip("Temp disable for v0.37") @pytest.mark.skip("Temp disable for v0.37")
def test_http_attr_priority_epoch_duration( def test_http_attr_priority_epoch_duration(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
self, user_container: str, object_size: ObjectSize, epoch_duration: int
):
self.tick_epoch() self.tick_epoch()
epoch_count = 1 epoch_count = 1
expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count
@ -247,15 +226,11 @@ class Test_http_system_header(ClusterTestBase):
) )
# check that object is not available via grpc # check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND): with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node( get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
self.wallet, user_container, oid, self.shell, self.cluster
)
@allure.title("Priority of attributes duration>timestamp (obj_size={object_size})") @allure.title("Priority of attributes duration>timestamp (obj_size={object_size})")
@pytest.mark.skip("Temp disable for v0.37") @pytest.mark.skip("Temp disable for v0.37")
def test_http_attr_priority_dur_timestamp( def test_http_attr_priority_dur_timestamp(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
self, user_container: str, object_size: ObjectSize, epoch_duration: int
):
self.tick_epoch() self.tick_epoch()
epoch_count = 2 epoch_count = 2
expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count
@ -263,12 +238,8 @@ class Test_http_system_header(ClusterTestBase):
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}" f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
) )
attributes = { attributes = {
SYSTEM_EXPIRATION_DURATION: self.epoch_count_into_mins( SYSTEM_EXPIRATION_DURATION: self.epoch_count_into_mins(epoch_duration=epoch_duration, epoch=2),
epoch_duration=epoch_duration, epoch=2 SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(epoch_duration=epoch_duration, epoch=1),
),
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(
epoch_duration=epoch_duration, epoch=1
),
} }
file_path = generate_file(object_size.value) file_path = generate_file(object_size.value)
with allure.step( with allure.step(
@ -296,15 +267,11 @@ class Test_http_system_header(ClusterTestBase):
) )
# check that object is not available via grpc # check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND): with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node( get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
self.wallet, user_container, oid, self.shell, self.cluster
)
@allure.title("Priority of attributes timestamp>Expiration-RFC (obj_size={object_size})") @allure.title("Priority of attributes timestamp>Expiration-RFC (obj_size={object_size})")
@pytest.mark.skip("Temp disable for v0.37") @pytest.mark.skip("Temp disable for v0.37")
def test_http_attr_priority_timestamp_rfc( def test_http_attr_priority_timestamp_rfc(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
self, user_container: str, object_size: ObjectSize, epoch_duration: int
):
self.tick_epoch() self.tick_epoch()
epoch_count = 2 epoch_count = 2
expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count
@ -312,9 +279,7 @@ class Test_http_system_header(ClusterTestBase):
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}" f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
) )
attributes = { attributes = {
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp( SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(epoch_duration=epoch_duration, epoch=2),
epoch_duration=epoch_duration, epoch=2
),
SYSTEM_EXPIRATION_RFC3339: self.epoch_count_into_timestamp( SYSTEM_EXPIRATION_RFC3339: self.epoch_count_into_timestamp(
epoch_duration=epoch_duration, epoch=1, rfc3339=True epoch_duration=epoch_duration, epoch=1, rfc3339=True
), ),
@ -345,9 +310,7 @@ class Test_http_system_header(ClusterTestBase):
) )
# check that object is not available via grpc # check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND): with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node( get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
self.wallet, user_container, oid, self.shell, self.cluster
)
@allure.title("Object should be deleted when expiration passed (obj_size={object_size})") @allure.title("Object should be deleted when expiration passed (obj_size={object_size})")
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -399,6 +362,4 @@ class Test_http_system_header(ClusterTestBase):
) )
# check that object is not available via grpc # check that object is not available via grpc
with pytest.raises(Exception, match=OBJECT_NOT_FOUND): with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node( get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
self.wallet, user_container, oid, self.shell, self.cluster
)

View file

@ -6,15 +6,12 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.sanity
@pytest.mark.acl @pytest.mark.acl
@pytest.mark.s3_gate @pytest.mark.s3_gate
class TestS3GateACL: class TestS3GateACL:
@allure.title("Object ACL (s3_client={s3_client})") @allure.title("Object ACL (s3_client={s3_client})")
@pytest.mark.parametrize("s3_client", [AwsCliClient], indirect=True) @pytest.mark.parametrize("s3_client", [AwsCliClient], indirect=True)
def test_s3_object_ACL( def test_s3_object_ACL(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
@ -33,9 +30,7 @@ class TestS3GateACL:
obj_acl = s3_client.get_object_acl(bucket, file_name) obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser") s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
with allure.step( with allure.step("Put object with grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"):
"Put object with grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
):
s3_client.put_object_acl( s3_client.put_object_acl(
bucket, bucket,
file_name, file_name,
@ -48,9 +43,7 @@ class TestS3GateACL:
@pytest.mark.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True) @pytest.mark.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
def test_s3_bucket_ACL(self, s3_client: S3ClientWrapper): def test_s3_bucket_ACL(self, s3_client: S3ClientWrapper):
with allure.step("Create bucket with ACL = public-read-write"): with allure.step("Create bucket with ACL = public-read-write"):
bucket = s3_client.create_bucket( bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read-write")
object_lock_enabled_for_bucket=True, acl="public-read-write"
)
bucket_acl = s3_client.get_bucket_acl(bucket) bucket_acl = s3_client.get_bucket_acl(bucket)
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers") s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers")
@ -59,9 +52,7 @@ class TestS3GateACL:
bucket_acl = s3_client.get_bucket_acl(bucket) bucket_acl = s3_client.get_bucket_acl(bucket)
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser") s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser")
with allure.step( with allure.step("Change bucket acl to --grant-write uri=http://acs.amazonaws.com/groups/global/AllUsers"):
"Change bucket acl to --grant-write uri=http://acs.amazonaws.com/groups/global/AllUsers"
):
s3_client.put_bucket_acl( s3_client.put_bucket_acl(
bucket, bucket,
grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers", grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers",

View file

@ -2,18 +2,12 @@ from datetime import datetime, timedelta
import allure import allure
import pytest import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_bucket @pytest.mark.s3_gate_bucket
class TestS3GateBucket: class TestS3GateBucket:
@ -26,23 +20,17 @@ class TestS3GateBucket:
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser") s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser")
with allure.step("Create bucket with ACL = public-read"): with allure.step("Create bucket with ACL = public-read"):
bucket_1 = s3_client.create_bucket( bucket_1 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read")
object_lock_enabled_for_bucket=True, acl="public-read"
)
bucket_acl_1 = s3_client.get_bucket_acl(bucket_1) bucket_acl_1 = s3_client.get_bucket_acl(bucket_1)
s3_helper.assert_s3_acl(acl_grants=bucket_acl_1, permitted_users="AllUsers") s3_helper.assert_s3_acl(acl_grants=bucket_acl_1, permitted_users="AllUsers")
with allure.step("Create bucket with ACL public-read-write"): with allure.step("Create bucket with ACL public-read-write"):
bucket_2 = s3_client.create_bucket( bucket_2 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read-write")
object_lock_enabled_for_bucket=True, acl="public-read-write"
)
bucket_acl_2 = s3_client.get_bucket_acl(bucket_2) bucket_acl_2 = s3_client.get_bucket_acl(bucket_2)
s3_helper.assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers") s3_helper.assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers")
with allure.step("Create bucket with ACL = authenticated-read"): with allure.step("Create bucket with ACL = authenticated-read"):
bucket_3 = s3_client.create_bucket( bucket_3 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="authenticated-read")
object_lock_enabled_for_bucket=True, acl="authenticated-read"
)
bucket_acl_3 = s3_client.get_bucket_acl(bucket_3) bucket_acl_3 = s3_client.get_bucket_acl(bucket_3)
s3_helper.assert_s3_acl(acl_grants=bucket_acl_3, permitted_users="AllUsers") s3_helper.assert_s3_acl(acl_grants=bucket_acl_3, permitted_users="AllUsers")
@ -74,18 +62,14 @@ class TestS3GateBucket:
s3_helper.assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers") s3_helper.assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers")
@allure.title("Create bucket with object lock (s3_client={s3_client})") @allure.title("Create bucket with object lock (s3_client={s3_client})")
def test_s3_bucket_object_lock( def test_s3_bucket_object_lock(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize
):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
with allure.step("Create bucket with --no-object-lock-enabled-for-bucket"): with allure.step("Create bucket with --no-object-lock-enabled-for-bucket"):
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=False) bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=False)
date_obj = datetime.utcnow() + timedelta(days=1) date_obj = datetime.utcnow() + timedelta(days=1)
with pytest.raises( with pytest.raises(Exception, match=r".*Object Lock configuration does not exist for this bucket.*"):
Exception, match=r".*Object Lock configuration does not exist for this bucket.*"
):
# An error occurred (ObjectLockConfigurationNotFoundError) when calling the PutObject operation (reached max retries: 0): # An error occurred (ObjectLockConfigurationNotFoundError) when calling the PutObject operation (reached max retries: 0):
# Object Lock configuration does not exist for this bucket # Object Lock configuration does not exist for this bucket
s3_client.put_object( s3_client.put_object(
@ -104,9 +88,7 @@ class TestS3GateBucket:
object_lock_retain_until_date=date_obj_1.strftime("%Y-%m-%dT%H:%M:%S"), object_lock_retain_until_date=date_obj_1.strftime("%Y-%m-%dT%H:%M:%S"),
object_lock_legal_hold_status="ON", object_lock_legal_hold_status="ON",
) )
s3_helper.assert_object_lock_mode( s3_helper.assert_object_lock_mode(s3_client, bucket_1, file_name, "COMPLIANCE", date_obj_1, "ON")
s3_client, bucket_1, file_name, "COMPLIANCE", date_obj_1, "ON"
)
@allure.title("Delete bucket (s3_client={s3_client})") @allure.title("Delete bucket (s3_client={s3_client})")
def test_s3_delete_bucket(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize): def test_s3_delete_bucket(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):

View file

@ -22,14 +22,7 @@ from frostfs_testlib.utils.file_utils import (
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
def pytest_generate_tests(metafunc: pytest.Metafunc): @allure.link("https://github.com/TrueCloudLab/frostfs-s3-gw#frostfs-s3-gw", name="frostfs-s3-gateway")
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@allure.link(
"https://github.com/TrueCloudLab/frostfs-s3-gw#frostfs-s3-gw", name="frostfs-s3-gateway"
)
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_base @pytest.mark.s3_gate_base
@ -73,9 +66,7 @@ class TestS3Gate:
s3_client.head_object(bucket_1, file_name) s3_client.head_object(bucket_1, file_name)
bucket_objects = s3_client.list_objects(bucket_1) bucket_objects = s3_client.list_objects(bucket_1)
assert ( assert file_name in bucket_objects, f"Expected file {file_name} in objects list {bucket_objects}"
file_name in bucket_objects
), f"Expected file {file_name} in objects list {bucket_objects}"
with allure.step("Try to delete not empty bucket and get error"): with allure.step("Try to delete not empty bucket and get error"):
with pytest.raises(Exception, match=r".*The bucket you tried to delete is not empty.*"): with pytest.raises(Exception, match=r".*The bucket you tried to delete is not empty.*"):
@ -136,18 +127,14 @@ class TestS3Gate:
s3_client.head_object(bucket, file_name) s3_client.head_object(bucket, file_name)
bucket_objects = s3_client.list_objects(bucket) bucket_objects = s3_client.list_objects(bucket)
assert ( assert file_name in bucket_objects, f"Expected file {file_name} in objects list {bucket_objects}"
file_name in bucket_objects
), f"Expected file {file_name} in objects list {bucket_objects}"
with allure.step("Check object's attributes"): with allure.step("Check object's attributes"):
for attrs in (["ETag"], ["ObjectSize", "StorageClass"]): for attrs in (["ETag"], ["ObjectSize", "StorageClass"]):
s3_client.get_object_attributes(bucket, file_name, attrs) s3_client.get_object_attributes(bucket, file_name, attrs)
@allure.title("Sync directory (s3_client={s3_client})") @allure.title("Sync directory (s3_client={s3_client})")
def test_s3_sync_dir( def test_s3_sync_dir(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
""" """
Test checks sync directory with AWS CLI utility. Test checks sync directory with AWS CLI utility.
""" """
@ -167,9 +154,7 @@ class TestS3Gate:
objects = s3_client.list_objects(bucket) objects = s3_client.list_objects(bucket)
with allure.step("Check these are the same objects"): with allure.step("Check these are the same objects"):
assert set(key_to_path.keys()) == set( assert set(key_to_path.keys()) == set(objects), f"Expected all objects saved. Got {objects}"
objects
), f"Expected all objects saved. Got {objects}"
for obj_key in objects: for obj_key in objects:
got_object = s3_client.get_object(bucket, obj_key) got_object = s3_client.get_object(bucket, obj_key)
assert get_file_hash(got_object) == get_file_hash( assert get_file_hash(got_object) == get_file_hash(
@ -177,32 +162,24 @@ class TestS3Gate:
), "Expected hashes are the same" ), "Expected hashes are the same"
@allure.title("Object versioning (s3_client={s3_client})") @allure.title("Object versioning (s3_client={s3_client})")
def test_s3_api_versioning( def test_s3_api_versioning(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
""" """
Test checks basic versioning functionality for S3 bucket. Test checks basic versioning functionality for S3 bucket.
""" """
version_1_content = "Version 1" version_1_content = "Version 1"
version_2_content = "Version 2" version_2_content = "Version 2"
file_name_simple = generate_file_with_content( file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
simple_object_size.value, content=version_1_content
)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with allure.step("Put several versions of object into bucket"): with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_name_simple) version_id_1 = s3_client.put_object(bucket, file_name_simple)
generate_file_with_content( generate_file_with_content(simple_object_size.value, file_path=file_name_simple, content=version_2_content)
simple_object_size.value, file_path=file_name_simple, content=version_2_content
)
version_id_2 = s3_client.put_object(bucket, file_name_simple) version_id_2 = s3_client.put_object(bucket, file_name_simple)
with allure.step("Check bucket shows all versions"): with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket) versions = s3_client.list_objects_versions(bucket)
obj_versions = { obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
assert obj_versions == { assert obj_versions == {
version_id_1, version_id_1,
version_id_2, version_id_2,
@ -213,20 +190,14 @@ class TestS3Gate:
response = s3_client.head_object(bucket, obj_key, version_id=version_id) response = s3_client.head_object(bucket, obj_key, version_id=version_id)
assert "LastModified" in response, "Expected LastModified field" assert "LastModified" in response, "Expected LastModified field"
assert "ETag" in response, "Expected ETag field" assert "ETag" in response, "Expected ETag field"
assert ( assert response.get("VersionId") == version_id, f"Expected VersionId is {version_id}"
response.get("VersionId") == version_id
), f"Expected VersionId is {version_id}"
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero" assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
with allure.step("Check object's attributes"): with allure.step("Check object's attributes"):
for version_id in (version_id_1, version_id_2): for version_id in (version_id_1, version_id_2):
got_attrs = s3_client.get_object_attributes( got_attrs = s3_client.get_object_attributes(bucket, obj_key, ["ETag"], version_id=version_id)
bucket, obj_key, ["ETag"], version_id=version_id
)
if got_attrs: if got_attrs:
assert ( assert got_attrs.get("VersionId") == version_id, f"Expected VersionId is {version_id}"
got_attrs.get("VersionId") == version_id
), f"Expected VersionId is {version_id}"
with allure.step("Delete object and check it was deleted"): with allure.step("Delete object and check it was deleted"):
response = s3_client.delete_object(bucket, obj_key) response = s3_client.delete_object(bucket, obj_key)
@ -242,9 +213,7 @@ class TestS3Gate:
): ):
file_name = s3_client.get_object(bucket, obj_key, version_id=version) file_name = s3_client.get_object(bucket, obj_key, version_id=version)
got_content = get_file_content(file_name) got_content = get_file_content(file_name)
assert ( assert got_content == content, f"Expected object content is\n{content}\nGot\n{got_content}"
got_content == content
), f"Expected object content is\n{content}\nGot\n{got_content}"
with allure.step("Restore previous object version"): with allure.step("Restore previous object version"):
s3_client.delete_object(bucket, obj_key, version_id=version_id_delete) s3_client.delete_object(bucket, obj_key, version_id=version_id_delete)
@ -257,17 +226,13 @@ class TestS3Gate:
@pytest.mark.s3_gate_multipart @pytest.mark.s3_gate_multipart
@allure.title("Object Multipart API (s3_client={s3_client})") @allure.title("Object Multipart API (s3_client={s3_client})")
def test_s3_api_multipart( def test_s3_api_multipart(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
""" """
Test checks S3 Multipart API (Create multipart upload/Abort multipart upload/List multipart upload/ Test checks S3 Multipart API (Create multipart upload/Abort multipart upload/List multipart upload/
Upload part/List parts/Complete multipart upload). Upload part/List parts/Complete multipart upload).
""" """
parts_count = 3 parts_count = 3
file_name_large = generate_file( file_name_large = generate_file(simple_object_size.value * 1024 * 6 * parts_count) # 5Mb - min part
simple_object_size.value * 1024 * 6 * parts_count
) # 5Mb - min part
object_key = s3_helper.object_key_from_file_path(file_name_large) object_key = s3_helper.object_key_from_file_path(file_name_large)
part_files = split_file(file_name_large, parts_count) part_files = split_file(file_name_large, parts_count)
parts = [] parts = []
@ -279,12 +244,8 @@ class TestS3Gate:
upload_id = s3_client.create_multipart_upload(bucket, object_key) upload_id = s3_client.create_multipart_upload(bucket, object_key)
uploads = s3_client.list_multipart_uploads(bucket) uploads = s3_client.list_multipart_uploads(bucket)
assert uploads, f"Expected there one upload in bucket {bucket}" assert uploads, f"Expected there one upload in bucket {bucket}"
assert ( assert uploads[0].get("Key") == object_key, f"Expected correct key {object_key} in upload {uploads}"
uploads[0].get("Key") == object_key assert uploads[0].get("UploadId") == upload_id, f"Expected correct UploadId {upload_id} in upload {uploads}"
), f"Expected correct key {object_key} in upload {uploads}"
assert (
uploads[0].get("UploadId") == upload_id
), f"Expected correct UploadId {upload_id} in upload {uploads}"
s3_client.abort_multipart_upload(bucket, object_key, upload_id) s3_client.abort_multipart_upload(bucket, object_key, upload_id)
uploads = s3_client.list_multipart_uploads(bucket) uploads = s3_client.list_multipart_uploads(bucket)
@ -298,9 +259,7 @@ class TestS3Gate:
with allure.step("Check all parts are visible in bucket"): with allure.step("Check all parts are visible in bucket"):
got_parts = s3_client.list_parts(bucket, object_key, upload_id) got_parts = s3_client.list_parts(bucket, object_key, upload_id)
assert len(got_parts) == len( assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
part_files
), f"Expected {parts_count} parts, got\n{got_parts}"
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts) s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
@ -327,9 +286,7 @@ class TestS3Gate:
s3_helper.check_tags_by_bucket(s3_client, bucket, []) s3_helper.check_tags_by_bucket(s3_client, bucket, [])
@allure.title("Object tagging API (s3_client={s3_client})") @allure.title("Object tagging API (s3_client={s3_client})")
def test_s3_api_object_tagging( def test_s3_api_object_tagging(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
""" """
Test checks S3 Object tagging API (Put tag/Get tag/Update tag). Test checks S3 Object tagging API (Put tag/Get tag/Update tag).
""" """
@ -458,9 +415,7 @@ class TestS3Gate:
with allure.step("Check copied object has the same content"): with allure.step("Check copied object has the same content"):
got_copied_file = s3_client.get_object(bucket, copy_obj_path) got_copied_file = s3_client.get_object(bucket, copy_obj_path)
assert get_file_hash(file_path_simple) == get_file_hash( assert get_file_hash(file_path_simple) == get_file_hash(got_copied_file), "Hashes must be the same"
got_copied_file
), "Hashes must be the same"
with allure.step("Delete one object from bucket"): with allure.step("Delete one object from bucket"):
s3_client.delete_object(bucket, file_name_simple) s3_client.delete_object(bucket, file_name_simple)
@ -509,9 +464,7 @@ class TestS3Gate:
with allure.step("Check copied object has the same content"): with allure.step("Check copied object has the same content"):
got_copied_file_b2 = s3_client.get_object(bucket_2, copy_obj_path_b2) got_copied_file_b2 = s3_client.get_object(bucket_2, copy_obj_path_b2)
assert get_file_hash(file_path_large) == get_file_hash( assert get_file_hash(file_path_large) == get_file_hash(got_copied_file_b2), "Hashes must be the same"
got_copied_file_b2
), "Hashes must be the same"
with allure.step("Delete one object from first bucket"): with allure.step("Delete one object from first bucket"):
s3_client.delete_object(bucket_1, file_name_simple) s3_client.delete_object(bucket_1, file_name_simple)
@ -524,23 +477,15 @@ class TestS3Gate:
s3_client.delete_object(bucket_2, copy_obj_path_b2) s3_client.delete_object(bucket_2, copy_obj_path_b2)
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[]) s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[])
def check_object_attributes( def check_object_attributes(self, s3_client: S3ClientWrapper, bucket: str, object_key: str, parts_count: int):
self, s3_client: S3ClientWrapper, bucket: str, object_key: str, parts_count: int
):
if not isinstance(s3_client, AwsCliClient): if not isinstance(s3_client, AwsCliClient):
logger.warning("Attributes check is not supported for boto3 implementation") logger.warning("Attributes check is not supported for boto3 implementation")
return return
with allure.step("Check object's attributes"): with allure.step("Check object's attributes"):
obj_parts = s3_client.get_object_attributes( obj_parts = s3_client.get_object_attributes(bucket, object_key, ["ObjectParts"], full_output=False)
bucket, object_key, ["ObjectParts"], full_output=False assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
) assert len(obj_parts.get("Parts")) == parts_count, f"Expected Parts cunt is {parts_count}"
assert (
obj_parts.get("TotalPartsCount") == parts_count
), f"Expected TotalPartsCount is {parts_count}"
assert (
len(obj_parts.get("Parts")) == parts_count
), f"Expected Parts cunt is {parts_count}"
with allure.step("Check object's attribute max-parts"): with allure.step("Check object's attribute max-parts"):
max_parts = 2 max_parts = 2
@ -551,13 +496,9 @@ class TestS3Gate:
max_parts=max_parts, max_parts=max_parts,
full_output=False, full_output=False,
) )
assert ( assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
obj_parts.get("TotalPartsCount") == parts_count
), f"Expected TotalPartsCount is {parts_count}"
assert obj_parts.get("MaxParts") == max_parts, f"Expected MaxParts is {parts_count}" assert obj_parts.get("MaxParts") == max_parts, f"Expected MaxParts is {parts_count}"
assert ( assert len(obj_parts.get("Parts")) == max_parts, f"Expected Parts count is {parts_count}"
len(obj_parts.get("Parts")) == max_parts
), f"Expected Parts count is {parts_count}"
with allure.step("Check object's attribute part-number-marker"): with allure.step("Check object's attribute part-number-marker"):
part_number_marker = 3 part_number_marker = 3
@ -568,9 +509,7 @@ class TestS3Gate:
part_number=part_number_marker, part_number=part_number_marker,
full_output=False, full_output=False,
) )
assert ( assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
obj_parts.get("TotalPartsCount") == parts_count
), f"Expected TotalPartsCount is {parts_count}"
assert ( assert (
obj_parts.get("PartNumberMarker") == part_number_marker obj_parts.get("PartNumberMarker") == part_number_marker
), f"Expected PartNumberMarker is {part_number_marker}" ), f"Expected PartNumberMarker is {part_number_marker}"

View file

@ -3,28 +3,18 @@ from datetime import datetime, timedelta
import allure import allure
import pytest import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_locking @pytest.mark.s3_gate_locking
@pytest.mark.parametrize("version_id", [None, "second"]) @pytest.mark.parametrize("version_id", [None, "second"])
class TestS3GateLocking: class TestS3GateLocking:
@allure.title( @allure.title("Retention period and legal lock on object (version_id={version_id}, s3_client={s3_client})")
"Retention period and legal lock on object (version_id={version_id}, s3_client={s3_client})" def test_s3_object_locking(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
)
def test_s3_object_locking(
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 2 retention_period = 2
@ -46,15 +36,11 @@ class TestS3GateLocking:
"RetainUntilDate": date_obj, "RetainUntilDate": date_obj,
} }
s3_client.put_object_retention(bucket, file_name, retention, version_id) s3_client.put_object_retention(bucket, file_name, retention, version_id)
s3_helper.assert_object_lock_mode( s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF"
)
with allure.step(f"Put legal hold to object {file_name}"): with allure.step(f"Put legal hold to object {file_name}"):
s3_client.put_object_legal_hold(bucket, file_name, "ON", version_id) s3_client.put_object_legal_hold(bucket, file_name, "ON", version_id)
s3_helper.assert_object_lock_mode( s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON"
)
with allure.step("Fail with deleting object with legal hold and retention period"): with allure.step("Fail with deleting object with legal hold and retention period"):
if version_id: if version_id:
@ -64,9 +50,7 @@ class TestS3GateLocking:
with allure.step("Check retention period is no longer set on the uploaded object"): with allure.step("Check retention period is no longer set on the uploaded object"):
time.sleep((retention_period + 1) * 60) time.sleep((retention_period + 1) * 60)
s3_helper.assert_object_lock_mode( s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON"
)
with allure.step("Fail with deleting object with legal hold and retention period"): with allure.step("Fail with deleting object with legal hold and retention period"):
if version_id: if version_id:
@ -76,12 +60,8 @@ class TestS3GateLocking:
else: else:
s3_client.delete_object(bucket, file_name, version_id) s3_client.delete_object(bucket, file_name, version_id)
@allure.title( @allure.title("Impossible to change retention mode COMPLIANCE (version_id={version_id}, s3_client={s3_client})")
"Impossible to change retention mode COMPLIANCE (version_id={version_id}, s3_client={s3_client})" def test_s3_mode_compliance(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
)
def test_s3_mode_compliance(
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 2 retention_period = 2
@ -102,13 +82,9 @@ class TestS3GateLocking:
"RetainUntilDate": date_obj, "RetainUntilDate": date_obj,
} }
s3_client.put_object_retention(bucket, file_name, retention, version_id) s3_client.put_object_retention(bucket, file_name, retention, version_id)
s3_helper.assert_object_lock_mode( s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF"
)
with allure.step( with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
f"Try to change retention period {retention_period_1}min to object {file_name}"
):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1) date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
retention = { retention = {
"Mode": "COMPLIANCE", "Mode": "COMPLIANCE",
@ -117,12 +93,8 @@ class TestS3GateLocking:
with pytest.raises(Exception): with pytest.raises(Exception):
s3_client.put_object_retention(bucket, file_name, retention, version_id) s3_client.put_object_retention(bucket, file_name, retention, version_id)
@allure.title( @allure.title("Change retention mode GOVERNANCE (version_id={version_id}, s3_client={s3_client})")
"Change retention mode GOVERNANCE (version_id={version_id}, s3_client={s3_client})" def test_s3_mode_governance(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
)
def test_s3_mode_governance(
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 3 retention_period = 3
@ -144,13 +116,9 @@ class TestS3GateLocking:
"RetainUntilDate": date_obj, "RetainUntilDate": date_obj,
} }
s3_client.put_object_retention(bucket, file_name, retention, version_id) s3_client.put_object_retention(bucket, file_name, retention, version_id)
s3_helper.assert_object_lock_mode( s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF"
)
with allure.step( with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
f"Try to change retention period {retention_period_1}min to object {file_name}"
):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1) date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
retention = { retention = {
"Mode": "GOVERNANCE", "Mode": "GOVERNANCE",
@ -159,9 +127,7 @@ class TestS3GateLocking:
with pytest.raises(Exception): with pytest.raises(Exception):
s3_client.put_object_retention(bucket, file_name, retention, version_id) s3_client.put_object_retention(bucket, file_name, retention, version_id)
with allure.step( with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
f"Try to change retention period {retention_period_1}min to object {file_name}"
):
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1) date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
retention = { retention = {
"Mode": "GOVERNANCE", "Mode": "GOVERNANCE",
@ -177,16 +143,12 @@ class TestS3GateLocking:
"RetainUntilDate": date_obj, "RetainUntilDate": date_obj,
} }
s3_client.put_object_retention(bucket, file_name, retention, version_id, True) s3_client.put_object_retention(bucket, file_name, retention, version_id, True)
s3_helper.assert_object_lock_mode( s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF"
)
@allure.title( @allure.title(
"[NEGATIVE] Lock object in bucket with disabled locking (version_id={version_id}, s3_client={s3_client})" "[NEGATIVE] Lock object in bucket with disabled locking (version_id={version_id}, s3_client={s3_client})"
) )
def test_s3_legal_hold( def test_s3_legal_hold(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
@ -227,6 +189,4 @@ class TestS3GateLockingBucket:
with allure.step("Put object into bucket"): with allure.step("Put object into bucket"):
s3_client.put_object(bucket, file_path) s3_client.put_object(bucket, file_path)
s3_helper.assert_object_lock_mode( s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", None, "OFF", 1)
s3_client, bucket, file_name, "COMPLIANCE", None, "OFF", 1
)

View file

@ -1,6 +1,6 @@
import allure import allure
import pytest import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.cli.container import list_objects, search_container_by_name from frostfs_testlib.steps.cli.container import list_objects, search_container_by_name
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
@ -10,28 +10,27 @@ from frostfs_testlib.utils.file_utils import generate_file, get_file_hash, split
PART_SIZE = 5 * 1024 * 1024 PART_SIZE = 5 * 1024 * 1024
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_multipart @pytest.mark.s3_gate_multipart
class TestS3GateMultipart(ClusterTestBase): class TestS3GateMultipart(ClusterTestBase):
NO_SUCH_UPLOAD = ( NO_SUCH_UPLOAD = "The upload ID may be invalid, or the upload may have been aborted or completed."
"The upload ID may be invalid, or the upload may have been aborted or completed."
)
@allure.title("Object Multipart API (s3_client={s3_client})") @allure.title("Object Multipart API (s3_client={s3_client}, bucket versioning = {versioning_status})")
@pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED], indirect=True) @pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED, VersioningStatus.UNDEFINED], indirect=True)
def test_s3_object_multipart(self, s3_client: S3ClientWrapper, bucket: str): def test_s3_object_multipart(
self, s3_client: S3ClientWrapper, bucket: str, default_wallet: str, versioning_status: str
):
parts_count = 5 parts_count = 5
file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part
object_key = s3_helper.object_key_from_file_path(file_name_large) object_key = s3_helper.object_key_from_file_path(file_name_large)
part_files = split_file(file_name_large, parts_count) part_files = split_file(file_name_large, parts_count)
parts = [] parts = []
with allure.step(f"Get related container_id for bucket"):
for cluster_node in self.cluster.cluster_nodes:
container_id = search_container_by_name(bucket, cluster_node)
if container_id:
break
with allure.step("Upload first part"): with allure.step("Upload first part"):
upload_id = s3_client.create_multipart_upload(bucket, object_key) upload_id = s3_client.create_multipart_upload(bucket, object_key)
uploads = s3_client.list_multipart_uploads(bucket) uploads = s3_client.list_multipart_uploads(bucket)
@ -45,10 +44,12 @@ class TestS3GateMultipart(ClusterTestBase):
etag = s3_client.upload_part(bucket, object_key, upload_id, part_id, file_path) etag = s3_client.upload_part(bucket, object_key, upload_id, part_id, file_path)
parts.append((part_id, etag)) parts.append((part_id, etag))
got_parts = s3_client.list_parts(bucket, object_key, upload_id) got_parts = s3_client.list_parts(bucket, object_key, upload_id)
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts) response = s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
assert len(got_parts) == len(
part_files version_id = None
), f"Expected {parts_count} parts, got\n{got_parts}" if versioning_status == VersioningStatus.ENABLED:
version_id = response["VersionId"]
assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
with allure.step("Check upload list is empty"): with allure.step("Check upload list is empty"):
uploads = s3_client.list_multipart_uploads(bucket) uploads = s3_client.list_multipart_uploads(bucket)
@ -58,6 +59,21 @@ class TestS3GateMultipart(ClusterTestBase):
got_object = s3_client.get_object(bucket, object_key) got_object = s3_client.get_object(bucket, object_key)
assert get_file_hash(got_object) == get_file_hash(file_name_large) assert get_file_hash(got_object) == get_file_hash(file_name_large)
if version_id:
with allure.step("Delete the object version"):
s3_client.delete_object(bucket, object_key, version_id)
else:
with allure.step("Delete the object"):
s3_client.delete_object(bucket, object_key)
with allure.step("List objects in the bucket, expect to be empty"):
objects_list = s3_client.list_objects(bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"
with allure.step("List objects in the container via rpc, expect to be empty"):
objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
assert len(objects) == 0, f"Expected no objects in container, got\n{objects}"
@allure.title("Abort Multipart Upload (s3_client={s3_client})") @allure.title("Abort Multipart Upload (s3_client={s3_client})")
@pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED], indirect=True) @pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED], indirect=True)
def test_s3_abort_multipart( def test_s3_abort_multipart(
@ -91,12 +107,8 @@ class TestS3GateMultipart(ClusterTestBase):
assert len(parts) == files_count, f"Expected {files_count} parts, got\n{parts}" assert len(parts) == files_count, f"Expected {files_count} parts, got\n{parts}"
with allure.step(f"Check that we have {files_count} files in container '{container_id}'"): with allure.step(f"Check that we have {files_count} files in container '{container_id}'"):
objects = list_objects( objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint assert len(objects) == files_count, f"Expected {files_count} objects in container, got\n{objects}"
)
assert (
len(objects) == files_count
), f"Expected {files_count} objects in container, got\n{objects}"
with allure.step("Abort multipart upload"): with allure.step("Abort multipart upload"):
s3_client.abort_multipart_upload(bucket, upload_key, upload_id) s3_client.abort_multipart_upload(bucket, upload_key, upload_id)
@ -108,9 +120,7 @@ class TestS3GateMultipart(ClusterTestBase):
s3_client.list_parts(bucket, upload_key, upload_id) s3_client.list_parts(bucket, upload_key, upload_id)
with allure.step("Check that we have no files in container since upload was aborted"): with allure.step("Check that we have no files in container since upload was aborted"):
objects = list_objects( objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint
)
assert len(objects) == 0, f"Expected no objects in container, got\n{objects}" assert len(objects) == 0, f"Expected no objects in container, got\n{objects}"
@allure.title("Upload Part Copy (s3_client={s3_client})") @allure.title("Upload Part Copy (s3_client={s3_client})")
@ -136,17 +146,13 @@ class TestS3GateMultipart(ClusterTestBase):
with allure.step("Upload parts to multipart upload"): with allure.step("Upload parts to multipart upload"):
for part_id, obj_key in enumerate(objs, start=1): for part_id, obj_key in enumerate(objs, start=1):
etag = s3_client.upload_part_copy( etag = s3_client.upload_part_copy(bucket, object_key, upload_id, part_id, f"{bucket}/{obj_key}")
bucket, object_key, upload_id, part_id, f"{bucket}/{obj_key}"
)
parts.append((part_id, etag)) parts.append((part_id, etag))
got_parts = s3_client.list_parts(bucket, object_key, upload_id) got_parts = s3_client.list_parts(bucket, object_key, upload_id)
with allure.step("Complete multipart upload"): with allure.step("Complete multipart upload"):
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts) s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
assert len(got_parts) == len( assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
part_files
), f"Expected {parts_count} parts, got\n{got_parts}"
with allure.step("Check we can get whole object from bucket"): with allure.step("Check we can get whole object from bucket"):
got_object = s3_client.get_object(bucket, object_key) got_object = s3_client.get_object(bucket, object_key)

View file

@ -9,25 +9,14 @@ import allure
import pytest import pytest
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_PASS from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_PASS
from frostfs_testlib.resources.error_patterns import S3_MALFORMED_XML_REQUEST from frostfs_testlib.resources.error_patterns import S3_MALFORMED_XML_REQUEST
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus from frostfs_testlib.s3 import AwsCliClient, S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils import wallet_utils from frostfs_testlib.utils import wallet_utils
from frostfs_testlib.utils.file_utils import ( from frostfs_testlib.utils.file_utils import concat_files, generate_file, generate_file_with_content, get_file_hash
concat_files,
generate_file,
generate_file_with_content,
get_file_hash,
)
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_object @pytest.mark.s3_gate_object
class TestS3GateObject: class TestS3GateObject:
@ -67,28 +56,18 @@ class TestS3GateObject:
with allure.step("Copy object from first bucket into second"): with allure.step("Copy object from first bucket into second"):
copy_obj_path_b2 = s3_client.copy_object(bucket_1, file_name, bucket=bucket_2) copy_obj_path_b2 = s3_client.copy_object(bucket_1, file_name, bucket=bucket_2)
s3_helper.check_objects_in_bucket( s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_client, bucket_1, expected_objects=bucket_1_objects s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
)
s3_helper.check_objects_in_bucket(
s3_client, bucket_2, expected_objects=[copy_obj_path_b2]
)
with allure.step("Check copied object has the same content"): with allure.step("Check copied object has the same content"):
got_copied_file_b2 = s3_client.get_object(bucket_2, copy_obj_path_b2) got_copied_file_b2 = s3_client.get_object(bucket_2, copy_obj_path_b2)
assert get_file_hash(file_path) == get_file_hash( assert get_file_hash(file_path) == get_file_hash(got_copied_file_b2), "Hashes must be the same"
got_copied_file_b2
), "Hashes must be the same"
with allure.step("Delete one object from first bucket"): with allure.step("Delete one object from first bucket"):
s3_client.delete_object(bucket_1, file_name) s3_client.delete_object(bucket_1, file_name)
bucket_1_objects.remove(file_name) bucket_1_objects.remove(file_name)
s3_helper.check_objects_in_bucket( s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_client, bucket_1, expected_objects=bucket_1_objects s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
)
s3_helper.check_objects_in_bucket(
s3_client, bucket_2, expected_objects=[copy_obj_path_b2]
)
with allure.step("Copy one object into the same bucket"): with allure.step("Copy one object into the same bucket"):
with pytest.raises(Exception): with pytest.raises(Exception):
@ -102,9 +81,7 @@ class TestS3GateObject:
simple_object_size: ObjectSize, simple_object_size: ObjectSize,
): ):
version_1_content = "Version 1" version_1_content = "Version 1"
file_name_simple = generate_file_with_content( file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
simple_object_size.value, content=version_1_content
)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
bucket_1, bucket_2 = two_buckets bucket_1, bucket_2 = two_buckets
@ -123,32 +100,22 @@ class TestS3GateObject:
s3_helper.set_bucket_versioning(s3_client, bucket_2, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket_2, VersioningStatus.ENABLED)
with allure.step("Copy object from first bucket into second"): with allure.step("Copy object from first bucket into second"):
copy_obj_path_b2 = s3_client.copy_object(bucket_1, obj_key, bucket=bucket_2) copy_obj_path_b2 = s3_client.copy_object(bucket_1, obj_key, bucket=bucket_2)
s3_helper.check_objects_in_bucket( s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_client, bucket_1, expected_objects=bucket_1_objects s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
)
s3_helper.check_objects_in_bucket(
s3_client, bucket_2, expected_objects=[copy_obj_path_b2]
)
with allure.step("Delete one object from first bucket and check object in bucket"): with allure.step("Delete one object from first bucket and check object in bucket"):
s3_client.delete_object(bucket_1, obj_key) s3_client.delete_object(bucket_1, obj_key)
bucket_1_objects.remove(obj_key) bucket_1_objects.remove(obj_key)
s3_helper.check_objects_in_bucket( s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
s3_client, bucket_1, expected_objects=bucket_1_objects
)
with allure.step("Copy one object into the same bucket"): with allure.step("Copy one object into the same bucket"):
with pytest.raises(Exception): with pytest.raises(Exception):
s3_client.copy_object(bucket_1, obj_key) s3_client.copy_object(bucket_1, obj_key)
@allure.title("Copy with acl (s3_client={s3_client})") @allure.title("Copy with acl (s3_client={s3_client})")
def test_s3_copy_acl( def test_s3_copy_acl(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
version_1_content = "Version 1" version_1_content = "Version 1"
file_name_simple = generate_file_with_content( file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
simple_object_size.value, content=version_1_content
)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
@ -163,9 +130,7 @@ class TestS3GateObject:
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser") s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
@allure.title("Copy object with metadata (s3_client={s3_client})") @allure.title("Copy object with metadata (s3_client={s3_client})")
def test_s3_copy_metadate( def test_s3_copy_metadate(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
@ -183,17 +148,13 @@ class TestS3GateObject:
bucket_1_objects.append(copy_obj_path) bucket_1_objects.append(copy_obj_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_1_objects) s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_1_objects)
obj_head = s3_client.head_object(bucket, copy_obj_path) obj_head = s3_client.head_object(bucket, copy_obj_path)
assert ( assert obj_head.get("Metadata") == object_metadata, f"Metadata must be {object_metadata}"
obj_head.get("Metadata") == object_metadata
), f"Metadata must be {object_metadata}"
with allure.step("Copy one object with metadata"): with allure.step("Copy one object with metadata"):
copy_obj_path = s3_client.copy_object(bucket, file_name, metadata_directive="COPY") copy_obj_path = s3_client.copy_object(bucket, file_name, metadata_directive="COPY")
bucket_1_objects.append(copy_obj_path) bucket_1_objects.append(copy_obj_path)
obj_head = s3_client.head_object(bucket, copy_obj_path) obj_head = s3_client.head_object(bucket, copy_obj_path)
assert ( assert obj_head.get("Metadata") == object_metadata, f"Metadata must be {object_metadata}"
obj_head.get("Metadata") == object_metadata
), f"Metadata must be {object_metadata}"
with allure.step("Copy one object with new metadata"): with allure.step("Copy one object with new metadata"):
object_metadata_1 = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} object_metadata_1 = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
@ -205,14 +166,10 @@ class TestS3GateObject:
) )
bucket_1_objects.append(copy_obj_path) bucket_1_objects.append(copy_obj_path)
obj_head = s3_client.head_object(bucket, copy_obj_path) obj_head = s3_client.head_object(bucket, copy_obj_path)
assert ( assert obj_head.get("Metadata") == object_metadata_1, f"Metadata must be {object_metadata_1}"
obj_head.get("Metadata") == object_metadata_1
), f"Metadata must be {object_metadata_1}"
@allure.title("Copy object with tagging (s3_client={s3_client})") @allure.title("Copy object with tagging (s3_client={s3_client})")
def test_s3_copy_tagging( def test_s3_copy_tagging(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
object_tagging = [(f"{uuid.uuid4()}", f"{uuid.uuid4()}")] object_tagging = [(f"{uuid.uuid4()}", f"{uuid.uuid4()}")]
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name_simple = s3_helper.object_key_from_file_path(file_path) file_name_simple = s3_helper.object_key_from_file_path(file_path)
@ -235,9 +192,7 @@ class TestS3GateObject:
assert tag in got_tags, f"Expected tag {tag} in {got_tags}" assert tag in got_tags, f"Expected tag {tag} in {got_tags}"
with allure.step("Copy one object with tag"): with allure.step("Copy one object with tag"):
copy_obj_path_1 = s3_client.copy_object( copy_obj_path_1 = s3_client.copy_object(bucket, file_name_simple, tagging_directive="COPY")
bucket, file_name_simple, tagging_directive="COPY"
)
got_tags = s3_client.get_object_tagging(bucket, copy_obj_path_1) got_tags = s3_client.get_object_tagging(bucket, copy_obj_path_1)
assert got_tags, f"Expected tags, got {got_tags}" assert got_tags, f"Expected tags, got {got_tags}"
expected_tags = [{"Key": key, "Value": value} for key, value in object_tagging] expected_tags = [{"Key": key, "Value": value} for key, value in object_tagging]
@ -270,9 +225,7 @@ class TestS3GateObject:
): ):
version_1_content = "Version 1" version_1_content = "Version 1"
version_2_content = "Version 2" version_2_content = "Version 2"
file_name_simple = generate_file_with_content( file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
simple_object_size.value, content=version_1_content
)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
@ -286,9 +239,7 @@ class TestS3GateObject:
with allure.step("Check bucket shows all versions"): with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket) versions = s3_client.list_objects_versions(bucket)
obj_versions = { obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
assert obj_versions == { assert obj_versions == {
version_id_1, version_id_1,
version_id_2, version_id_2,
@ -297,18 +248,14 @@ class TestS3GateObject:
with allure.step("Delete 1 version of object"): with allure.step("Delete 1 version of object"):
delete_obj = s3_client.delete_object(bucket, obj_key, version_id=version_id_1) delete_obj = s3_client.delete_object(bucket, obj_key, version_id=version_id_1)
versions = s3_client.list_objects_versions(bucket) versions = s3_client.list_objects_versions(bucket)
obj_versions = { obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
assert obj_versions == {version_id_2}, f"Object should have versions: {version_id_2}" assert obj_versions == {version_id_2}, f"Object should have versions: {version_id_2}"
assert "DeleteMarker" not in delete_obj.keys(), "Delete markers should not be created" assert "DeleteMarker" not in delete_obj.keys(), "Delete markers should not be created"
with allure.step("Delete second version of object"): with allure.step("Delete second version of object"):
delete_obj = s3_client.delete_object(bucket, obj_key, version_id=version_id_2) delete_obj = s3_client.delete_object(bucket, obj_key, version_id=version_id_2)
versions = s3_client.list_objects_versions(bucket) versions = s3_client.list_objects_versions(bucket)
obj_versions = { obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
assert not obj_versions, "Expected object not found" assert not obj_versions, "Expected object not found"
assert "DeleteMarker" not in delete_obj.keys(), "Delete markers should not be created" assert "DeleteMarker" not in delete_obj.keys(), "Delete markers should not be created"
@ -324,16 +271,12 @@ class TestS3GateObject:
assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker" assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
@allure.title("Bulk delete version of object (s3_client={s3_client})") @allure.title("Bulk delete version of object (s3_client={s3_client})")
def test_s3_bulk_delete_versioning( def test_s3_bulk_delete_versioning(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
version_1_content = "Version 1" version_1_content = "Version 1"
version_2_content = "Version 2" version_2_content = "Version 2"
version_3_content = "Version 3" version_3_content = "Version 3"
version_4_content = "Version 4" version_4_content = "Version 4"
file_name_1 = generate_file_with_content( file_name_1 = generate_file_with_content(simple_object_size.value, content=version_1_content)
simple_object_size.value, content=version_1_content
)
obj_key = os.path.basename(file_name_1) obj_key = os.path.basename(file_name_1)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
@ -356,37 +299,25 @@ class TestS3GateObject:
with allure.step("Check bucket shows all versions"): with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket) versions = s3_client.list_objects_versions(bucket)
obj_versions = { obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
version.get("VersionId") for version in versions if version.get("Key") == obj_key
}
assert obj_versions == version_ids, f"Object should have versions: {version_ids}" assert obj_versions == version_ids, f"Object should have versions: {version_ids}"
with allure.step("Delete two objects from bucket one by one"): with allure.step("Delete two objects from bucket one by one"):
version_to_delete_b1 = sample( version_to_delete_b1 = sample([version_id_1, version_id_2, version_id_3, version_id_4], k=2)
[version_id_1, version_id_2, version_id_3, version_id_4], k=2
)
version_to_save = list(set(version_ids) - set(version_to_delete_b1)) version_to_save = list(set(version_ids) - set(version_to_delete_b1))
for ver in version_to_delete_b1: for ver in version_to_delete_b1:
s3_client.delete_object(bucket, obj_key, ver) s3_client.delete_object(bucket, obj_key, ver)
with allure.step("Check bucket shows all versions"): with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket) versions = s3_client.list_objects_versions(bucket)
obj_versions = [ obj_versions = [version.get("VersionId") for version in versions if version.get("Key") == obj_key]
version.get("VersionId") for version in versions if version.get("Key") == obj_key assert obj_versions.sort() == version_to_save.sort(), f"Object should have versions: {version_to_save}"
]
assert (
obj_versions.sort() == version_to_save.sort()
), f"Object should have versions: {version_to_save}"
@allure.title("Get versions of object (s3_client={s3_client})") @allure.title("Get versions of object (s3_client={s3_client})")
def test_s3_get_versioning( def test_s3_get_versioning(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
version_1_content = "Version 1" version_1_content = "Version 1"
version_2_content = "Version 2" version_2_content = "Version 2"
file_name_simple = generate_file_with_content( file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
simple_object_size.value, content=version_1_content
)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
@ -399,21 +330,15 @@ class TestS3GateObject:
with allure.step("Get first version of object"): with allure.step("Get first version of object"):
object_1 = s3_client.get_object(bucket, obj_key, version_id_1, full_output=True) object_1 = s3_client.get_object(bucket, obj_key, version_id_1, full_output=True)
assert ( assert object_1.get("VersionId") == version_id_1, f"Get object with version {version_id_1}"
object_1.get("VersionId") == version_id_1
), f"Get object with version {version_id_1}"
with allure.step("Get second version of object"): with allure.step("Get second version of object"):
object_2 = s3_client.get_object(bucket, obj_key, version_id_2, full_output=True) object_2 = s3_client.get_object(bucket, obj_key, version_id_2, full_output=True)
assert ( assert object_2.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
object_2.get("VersionId") == version_id_2
), f"Get object with version {version_id_2}"
with allure.step("Get object"): with allure.step("Get object"):
object_3 = s3_client.get_object(bucket, obj_key, full_output=True) object_3 = s3_client.get_object(bucket, obj_key, full_output=True)
assert ( assert object_3.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
object_3.get("VersionId") == version_id_2
), f"Get object with version {version_id_2}"
@allure.title("Get range (s3_client={s3_client})") @allure.title("Get range (s3_client={s3_client})")
def test_s3_get_range( def test_s3_get_range(
@ -483,9 +408,7 @@ class TestS3GateObject:
object_range=[2 * int(simple_object_size.value / 3) + 1, simple_object_size.value], object_range=[2 * int(simple_object_size.value / 3) + 1, simple_object_size.value],
) )
con_file_1 = concat_files([object_2_part_1, object_2_part_2, object_2_part_3]) con_file_1 = concat_files([object_2_part_1, object_2_part_2, object_2_part_3])
assert get_file_hash(con_file_1) == get_file_hash( assert get_file_hash(con_file_1) == get_file_hash(file_name_1), "Hashes must be the same"
file_name_1
), "Hashes must be the same"
with allure.step("Get object"): with allure.step("Get object"):
object_3_part_1 = s3_client.get_object( object_3_part_1 = s3_client.get_object(
@ -568,26 +491,18 @@ class TestS3GateObject:
assert "LastModified" in response, "Expected LastModified field" assert "LastModified" in response, "Expected LastModified field"
assert "ETag" in response, "Expected ETag field" assert "ETag" in response, "Expected ETag field"
assert response.get("Metadata") == {}, "Expected Metadata empty" assert response.get("Metadata") == {}, "Expected Metadata empty"
assert ( assert response.get("VersionId") == version_id_2, f"Expected VersionId is {version_id_2}"
response.get("VersionId") == version_id_2
), f"Expected VersionId is {version_id_2}"
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero" assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
with allure.step("Get head ob first version of object"): with allure.step("Get head ob first version of object"):
response = s3_client.head_object(bucket, file_name, version_id=version_id_1) response = s3_client.head_object(bucket, file_name, version_id=version_id_1)
assert "LastModified" in response, "Expected LastModified field" assert "LastModified" in response, "Expected LastModified field"
assert "ETag" in response, "Expected ETag field" assert "ETag" in response, "Expected ETag field"
assert ( assert response.get("Metadata") == object_metadata, f"Expected Metadata is {object_metadata}"
response.get("Metadata") == object_metadata assert response.get("VersionId") == version_id_1, f"Expected VersionId is {version_id_1}"
), f"Expected Metadata is {object_metadata}"
assert (
response.get("VersionId") == version_id_1
), f"Expected VersionId is {version_id_1}"
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero" assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
@allure.title( @allure.title("List of objects with version (method_version={list_type}, s3_client={s3_client})")
"List of objects with version (method_version={list_type}, s3_client={s3_client})"
)
@pytest.mark.parametrize("list_type", ["v1", "v2"]) @pytest.mark.parametrize("list_type", ["v1", "v2"])
def test_s3_list_object( def test_s3_list_object(
self, self,
@ -624,9 +539,7 @@ class TestS3GateObject:
list_obj_1 = s3_client.list_objects_v2(bucket, full_output=True) list_obj_1 = s3_client.list_objects_v2(bucket, full_output=True)
contents = list_obj_1.get("Contents", []) contents = list_obj_1.get("Contents", [])
assert len(contents) == 1, "bucket should have only 1 object" assert len(contents) == 1, "bucket should have only 1 object"
assert ( assert contents[0].get("Key") == file_name_2, f"bucket should have object key {file_name_2}"
contents[0].get("Key") == file_name_2
), f"bucket should have object key {file_name_2}"
assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker" assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
@allure.title("Put object (s3_client={s3_client})") @allure.title("Put object (s3_client={s3_client})")
@ -655,22 +568,16 @@ class TestS3GateObject:
assert obj_head.get("Metadata") == object_1_metadata, "Metadata must be the same" assert obj_head.get("Metadata") == object_1_metadata, "Metadata must be the same"
got_tags = s3_client.get_object_tagging(bucket, file_name) got_tags = s3_client.get_object_tagging(bucket, file_name)
assert got_tags, f"Expected tags, got {got_tags}" assert got_tags, f"Expected tags, got {got_tags}"
assert got_tags == [ assert got_tags == [{"Key": tag_key_1, "Value": str(tag_value_1)}], "Tags must be the same"
{"Key": tag_key_1, "Value": str(tag_value_1)}
], "Tags must be the same"
with allure.step("Rewrite file into bucket"): with allure.step("Rewrite file into bucket"):
file_path_2 = generate_file_with_content( file_path_2 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
simple_object_size.value, file_path=file_path_1
)
s3_client.put_object(bucket, file_path_2, metadata=object_2_metadata, tagging=tag_2) s3_client.put_object(bucket, file_path_2, metadata=object_2_metadata, tagging=tag_2)
obj_head = s3_client.head_object(bucket, file_name) obj_head = s3_client.head_object(bucket, file_name)
assert obj_head.get("Metadata") == object_2_metadata, "Metadata must be the same" assert obj_head.get("Metadata") == object_2_metadata, "Metadata must be the same"
got_tags_1 = s3_client.get_object_tagging(bucket, file_name) got_tags_1 = s3_client.get_object_tagging(bucket, file_name)
assert got_tags_1, f"Expected tags, got {got_tags_1}" assert got_tags_1, f"Expected tags, got {got_tags_1}"
assert got_tags_1 == [ assert got_tags_1 == [{"Key": tag_key_2, "Value": str(tag_value_2)}], "Tags must be the same"
{"Key": tag_key_2, "Value": str(tag_value_2)}
], "Tags must be the same"
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
@ -683,28 +590,18 @@ class TestS3GateObject:
tag_3 = f"{tag_key_3}={tag_value_3}" tag_3 = f"{tag_key_3}={tag_value_3}"
with allure.step("Put third object into bucket"): with allure.step("Put third object into bucket"):
version_id_1 = s3_client.put_object( version_id_1 = s3_client.put_object(bucket, file_path_3, metadata=object_3_metadata, tagging=tag_3)
bucket, file_path_3, metadata=object_3_metadata, tagging=tag_3
)
obj_head_3 = s3_client.head_object(bucket, file_name_3) obj_head_3 = s3_client.head_object(bucket, file_name_3)
assert obj_head_3.get("Metadata") == object_3_metadata, "Matadata must be the same" assert obj_head_3.get("Metadata") == object_3_metadata, "Matadata must be the same"
got_tags_3 = s3_client.get_object_tagging(bucket, file_name_3) got_tags_3 = s3_client.get_object_tagging(bucket, file_name_3)
assert got_tags_3, f"Expected tags, got {got_tags_3}" assert got_tags_3, f"Expected tags, got {got_tags_3}"
assert got_tags_3 == [ assert got_tags_3 == [{"Key": tag_key_3, "Value": str(tag_value_3)}], "Tags must be the same"
{"Key": tag_key_3, "Value": str(tag_value_3)}
], "Tags must be the same"
with allure.step("Put new version of file into bucket"): with allure.step("Put new version of file into bucket"):
file_path_4 = generate_file_with_content( file_path_4 = generate_file_with_content(simple_object_size.value, file_path=file_path_3)
simple_object_size.value, file_path=file_path_3
)
version_id_2 = s3_client.put_object(bucket, file_path_4) version_id_2 = s3_client.put_object(bucket, file_path_4)
versions = s3_client.list_objects_versions(bucket) versions = s3_client.list_objects_versions(bucket)
obj_versions = { obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == file_name_3}
version.get("VersionId")
for version in versions
if version.get("Key") == file_name_3
}
assert obj_versions == { assert obj_versions == {
version_id_1, version_id_1,
version_id_2, version_id_2,
@ -714,26 +611,20 @@ class TestS3GateObject:
with allure.step("Get object"): with allure.step("Get object"):
object_3 = s3_client.get_object(bucket, file_name_3, full_output=True) object_3 = s3_client.get_object(bucket, file_name_3, full_output=True)
assert ( assert object_3.get("VersionId") == version_id_2, f"get object with version {version_id_2}"
object_3.get("VersionId") == version_id_2
), f"get object with version {version_id_2}"
object_3 = s3_client.get_object(bucket, file_name_3) object_3 = s3_client.get_object(bucket, file_name_3)
assert get_file_hash(file_path_4) == get_file_hash(object_3), "Hashes must be the same" assert get_file_hash(file_path_4) == get_file_hash(object_3), "Hashes must be the same"
with allure.step("Get first version of object"): with allure.step("Get first version of object"):
object_4 = s3_client.get_object(bucket, file_name_3, version_id_1, full_output=True) object_4 = s3_client.get_object(bucket, file_name_3, version_id_1, full_output=True)
assert ( assert object_4.get("VersionId") == version_id_1, f"get object with version {version_id_1}"
object_4.get("VersionId") == version_id_1
), f"get object with version {version_id_1}"
object_4 = s3_client.get_object(bucket, file_name_3, version_id_1) object_4 = s3_client.get_object(bucket, file_name_3, version_id_1)
assert file_hash == get_file_hash(object_4), "Hashes must be the same" assert file_hash == get_file_hash(object_4), "Hashes must be the same"
obj_head_3 = s3_client.head_object(bucket, file_name_3, version_id_1) obj_head_3 = s3_client.head_object(bucket, file_name_3, version_id_1)
assert obj_head_3.get("Metadata") == object_3_metadata, "Metadata must be the same" assert obj_head_3.get("Metadata") == object_3_metadata, "Metadata must be the same"
got_tags_3 = s3_client.get_object_tagging(bucket, file_name_3, version_id_1) got_tags_3 = s3_client.get_object_tagging(bucket, file_name_3, version_id_1)
assert got_tags_3, f"Expected tags, got {got_tags_3}" assert got_tags_3, f"Expected tags, got {got_tags_3}"
assert got_tags_3 == [ assert got_tags_3 == [{"Key": tag_key_3, "Value": str(tag_value_3)}], "Tags must be the same"
{"Key": tag_key_3, "Value": str(tag_value_3)}
], "Tags must be the same"
@allure.title("Put object with ACL (versioning={bucket_versioning}, s3_client={s3_client})") @allure.title("Put object with ACL (versioning={bucket_versioning}, s3_client={s3_client})")
@pytest.mark.parametrize("bucket_versioning", ["ENABLED", "SUSPENDED"]) @pytest.mark.parametrize("bucket_versioning", ["ENABLED", "SUSPENDED"])
@ -762,9 +653,7 @@ class TestS3GateObject:
assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same" assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same"
with allure.step("Put object with acl public-read"): with allure.step("Put object with acl public-read"):
file_path_2 = generate_file_with_content( file_path_2 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
simple_object_size.value, file_path=file_path_1
)
s3_client.put_object(bucket, file_path_2, acl="public-read") s3_client.put_object(bucket, file_path_2, acl="public-read")
obj_acl = s3_client.get_object_acl(bucket, file_name) obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers") s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
@ -772,9 +661,7 @@ class TestS3GateObject:
assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same" assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same"
with allure.step("Put object with acl public-read-write"): with allure.step("Put object with acl public-read-write"):
file_path_3 = generate_file_with_content( file_path_3 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
simple_object_size.value, file_path=file_path_1
)
s3_client.put_object(bucket, file_path_3, acl="public-read-write") s3_client.put_object(bucket, file_path_3, acl="public-read-write")
obj_acl = s3_client.get_object_acl(bucket, file_name) obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers") s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
@ -782,9 +669,7 @@ class TestS3GateObject:
assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same" assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same"
with allure.step("Put object with acl authenticated-read"): with allure.step("Put object with acl authenticated-read"):
file_path_4 = generate_file_with_content( file_path_4 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
simple_object_size.value, file_path=file_path_1
)
s3_client.put_object(bucket, file_path_4, acl="authenticated-read") s3_client.put_object(bucket, file_path_4, acl="authenticated-read")
obj_acl = s3_client.get_object_acl(bucket, file_name) obj_acl = s3_client.get_object_acl(bucket, file_name)
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers") s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
@ -806,9 +691,7 @@ class TestS3GateObject:
object_5 = s3_client.get_object(bucket, file_name_5) object_5 = s3_client.get_object(bucket, file_name_5)
assert get_file_hash(file_path_5) == get_file_hash(object_5), "Hashes must be the same" assert get_file_hash(file_path_5) == get_file_hash(object_5), "Hashes must be the same"
with allure.step( with allure.step("Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"):
"Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
):
generate_file_with_content(simple_object_size.value, file_path=file_path_5) generate_file_with_content(simple_object_size.value, file_path=file_path_5)
s3_client.put_object( s3_client.put_object(
bucket, bucket,
@ -833,9 +716,7 @@ class TestS3GateObject:
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True) bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True)
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with allure.step( with allure.step("Put object with lock-mode GOVERNANCE lock-retain-until-date +1day, lock-legal-hold-status"):
"Put object with lock-mode GOVERNANCE lock-retain-until-date +1day, lock-legal-hold-status"
):
date_obj = datetime.utcnow() + timedelta(days=1) date_obj = datetime.utcnow() + timedelta(days=1)
s3_client.put_object( s3_client.put_object(
bucket, bucket,
@ -844,9 +725,7 @@ class TestS3GateObject:
object_lock_retain_until_date=date_obj.strftime("%Y-%m-%dT%H:%M:%S"), object_lock_retain_until_date=date_obj.strftime("%Y-%m-%dT%H:%M:%S"),
object_lock_legal_hold_status="OFF", object_lock_legal_hold_status="OFF",
) )
s3_helper.assert_object_lock_mode( s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF"
)
with allure.step( with allure.step(
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]" "Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"
@ -859,9 +738,7 @@ class TestS3GateObject:
object_lock_mode="COMPLIANCE", object_lock_mode="COMPLIANCE",
object_lock_retain_until_date=date_obj, object_lock_retain_until_date=date_obj,
) )
s3_helper.assert_object_lock_mode( s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF"
)
with allure.step( with allure.step(
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]" "Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"
@ -875,9 +752,7 @@ class TestS3GateObject:
object_lock_retain_until_date=date_obj, object_lock_retain_until_date=date_obj,
object_lock_legal_hold_status="ON", object_lock_legal_hold_status="ON",
) )
s3_helper.assert_object_lock_mode( s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON"
)
with allure.step("Put object with lock-mode"): with allure.step("Put object with lock-mode"):
with pytest.raises( with pytest.raises(
@ -939,9 +814,7 @@ class TestS3GateObject:
with allure.step("Check objects are synced"): with allure.step("Check objects are synced"):
objects = s3_client.list_objects(bucket) objects = s3_client.list_objects(bucket)
assert set(key_to_path.keys()) == set( assert set(key_to_path.keys()) == set(objects), f"Expected all abjects saved. Got {objects}"
objects
), f"Expected all abjects saved. Got {objects}"
with allure.step("Check these are the same objects"): with allure.step("Check these are the same objects"):
for obj_key in objects: for obj_key in objects:
@ -950,9 +823,7 @@ class TestS3GateObject:
key_to_path.get(obj_key) key_to_path.get(obj_key)
), "Expected hashes are the same" ), "Expected hashes are the same"
obj_head = s3_client.head_object(bucket, obj_key) obj_head = s3_client.head_object(bucket, obj_key)
assert ( assert obj_head.get("Metadata") == object_metadata, f"Metadata of object is {object_metadata}"
obj_head.get("Metadata") == object_metadata
), f"Metadata of object is {object_metadata}"
# Uncomment after https://github.com/nspcc-dev/neofs-s3-gw/issues/685 is solved # Uncomment after https://github.com/nspcc-dev/neofs-s3-gw/issues/685 is solved
# obj_acl = s3_client.get_object_acl(bucket, obj_key) # obj_acl = s3_client.get_object_acl(bucket, obj_key)
# s3_helper.assert_s3_acl(acl_grants = obj_acl, permitted_users = "AllUsers") # s3_helper.assert_s3_acl(acl_grants = obj_acl, permitted_users = "AllUsers")
@ -994,9 +865,7 @@ class TestS3GateObject:
assert not objects_list, f"Expected empty bucket, got {objects_list}" assert not objects_list, f"Expected empty bucket, got {objects_list}"
@allure.title("Delete the same object twice (s3_client={s3_client})") @allure.title("Delete the same object twice (s3_client={s3_client})")
def test_s3_delete_twice( def test_s3_delete_twice(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
objects_list = s3_client.list_objects(bucket) objects_list = s3_client.list_objects(bucket)
with allure.step("Check that bucket is empty"): with allure.step("Check that bucket is empty"):
@ -1012,9 +881,7 @@ class TestS3GateObject:
delete_object = s3_client.delete_object(bucket, file_name) delete_object = s3_client.delete_object(bucket, file_name)
versions = s3_client.list_objects_versions(bucket) versions = s3_client.list_objects_versions(bucket)
obj_versions = { obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == file_name}
version.get("VersionId") for version in versions if version.get("Key") == file_name
}
assert obj_versions, f"Object versions were not found {objects_list}" assert obj_versions, f"Object versions were not found {objects_list}"
assert "DeleteMarker" in delete_object.keys(), "Delete markers not found" assert "DeleteMarker" in delete_object.keys(), "Delete markers not found"
@ -1022,9 +889,7 @@ class TestS3GateObject:
delete_object_2nd_attempt = s3_client.delete_object(bucket, file_name) delete_object_2nd_attempt = s3_client.delete_object(bucket, file_name)
versions_2nd_attempt = s3_client.list_objects_versions(bucket) versions_2nd_attempt = s3_client.list_objects_versions(bucket)
assert ( assert delete_object.keys() == delete_object_2nd_attempt.keys(), "Delete markers are not the same"
delete_object.keys() == delete_object_2nd_attempt.keys()
), "Delete markers are not the same"
# check that nothing was changed # check that nothing was changed
# checking here not VersionId only, but all data (for example LastModified) # checking here not VersionId only, but all data (for example LastModified)
assert versions == versions_2nd_attempt, "Versions are not the same" assert versions == versions_2nd_attempt, "Versions are not the same"

View file

@ -2,7 +2,7 @@ import os
import allure import allure
import pytest import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.cli.container import search_container_by_name from frostfs_testlib.steps.cli.container import search_container_by_name
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.steps.storage_policy import get_simple_object_copies from frostfs_testlib.steps.storage_policy import get_simple_object_copies
@ -12,23 +12,11 @@ from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
def pytest_generate_tests(metafunc: pytest.Metafunc):
policy = f"{os.getcwd()}/pytest_tests/resources/files/policy.json"
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize(
"s3_client, s3_policy",
[(AwsCliClient, policy), (Boto3ClientWrapper, policy)],
indirect=True,
ids=["aws cli", "boto3"],
)
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True)
class TestS3GatePolicy(ClusterTestBase): class TestS3GatePolicy(ClusterTestBase):
@allure.title("Bucket creation with retention policy applied (s3_client={s3_client})") @allure.title("Bucket creation with retention policy applied (s3_client={s3_client})")
def test_s3_bucket_location( def test_s3_bucket_location(self, default_wallet: str, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
self, default_wallet: str, s3_client: S3ClientWrapper, simple_object_size: ObjectSize
):
file_path_1 = generate_file(simple_object_size.value) file_path_1 = generate_file(simple_object_size.value)
file_name_1 = s3_helper.object_key_from_file_path(file_path_1) file_name_1 = s3_helper.object_key_from_file_path(file_path_1)
file_path_2 = generate_file(simple_object_size.value) file_path_2 = generate_file(simple_object_size.value)
@ -156,9 +144,7 @@ class TestS3GatePolicy(ClusterTestBase):
} }
s3_client.put_bucket_cors(bucket, cors) s3_client.put_bucket_cors(bucket, cors)
bucket_cors = s3_client.get_bucket_cors(bucket) bucket_cors = s3_client.get_bucket_cors(bucket)
assert bucket_cors == cors.get( assert bucket_cors == cors.get("CORSRules"), f"Expected CORSRules must be {cors.get('CORSRules')}"
"CORSRules"
), f"Expected CORSRules must be {cors.get('CORSRules')}"
with allure.step("delete bucket cors"): with allure.step("delete bucket cors"):
s3_client.delete_bucket_cors(bucket) s3_client.delete_bucket_cors(bucket)

View file

@ -4,18 +4,12 @@ from typing import Tuple
import allure import allure
import pytest import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_tagging @pytest.mark.s3_gate_tagging
class TestS3GateTagging: class TestS3GateTagging:
@ -29,9 +23,7 @@ class TestS3GateTagging:
return tags return tags
@allure.title("Object tagging (s3_client={s3_client})") @allure.title("Object tagging (s3_client={s3_client})")
def test_s3_object_tagging( def test_s3_object_tagging(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
@ -45,9 +37,7 @@ class TestS3GateTagging:
with allure.step("Put 10 new tags for object"): with allure.step("Put 10 new tags for object"):
tags_2 = self.create_tags(10) tags_2 = self.create_tags(10)
s3_client.put_object_tagging(bucket, file_name, tags=tags_2) s3_client.put_object_tagging(bucket, file_name, tags=tags_2)
s3_helper.check_tags_by_object( s3_helper.check_tags_by_object(s3_client, bucket, file_name, tags_2, [("Tag1", "Value1")])
s3_client, bucket, file_name, tags_2, [("Tag1", "Value1")]
)
with allure.step("Put 10 extra new tags for object"): with allure.step("Put 10 extra new tags for object"):
tags_3 = self.create_tags(10) tags_3 = self.create_tags(10)

View file

@ -1,19 +1,11 @@
import os
import allure import allure
import pytest import pytest
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
@pytest.mark.sanity
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_versioning @pytest.mark.s3_gate_versioning
class TestS3GateVersioning: class TestS3GateVersioning:
@ -34,18 +26,10 @@ class TestS3GateVersioning:
with allure.step("Put object into bucket"): with allure.step("Put object into bucket"):
s3_client.put_object(bucket, file_path) s3_client.put_object(bucket, file_path)
objects_list = s3_client.list_objects(bucket) objects_list = s3_client.list_objects(bucket)
assert ( assert objects_list == bucket_objects, f"Expected list with single objects in bucket, got {objects_list}"
objects_list == bucket_objects
), f"Expected list with single objects in bucket, got {objects_list}"
object_version = s3_client.list_objects_versions(bucket) object_version = s3_client.list_objects_versions(bucket)
actual_version = [ actual_version = [version.get("VersionId") for version in object_version if version.get("Key") == file_name]
version.get("VersionId") assert actual_version == ["null"], f"Expected version is null in list-object-versions, got {object_version}"
for version in object_version
if version.get("Key") == file_name
]
assert actual_version == [
"null"
], f"Expected version is null in list-object-versions, got {object_version}"
object_0 = s3_client.head_object(bucket, file_name) object_0 = s3_client.head_object(bucket, file_name)
assert ( assert (
object_0.get("VersionId") == "null" object_0.get("VersionId") == "null"
@ -60,27 +44,19 @@ class TestS3GateVersioning:
with allure.step("Check bucket shows all versions"): with allure.step("Check bucket shows all versions"):
versions = s3_client.list_objects_versions(bucket) versions = s3_client.list_objects_versions(bucket)
obj_versions = [ obj_versions = [version.get("VersionId") for version in versions if version.get("Key") == file_name]
version.get("VersionId") for version in versions if version.get("Key") == file_name
]
assert ( assert (
obj_versions.sort() == [version_id_1, version_id_2, "null"].sort() obj_versions.sort() == [version_id_1, version_id_2, "null"].sort()
), f"Expected object has versions: {version_id_1, version_id_2, 'null'}" ), f"Expected object has versions: {version_id_1, version_id_2, 'null'}"
with allure.step("Get object"): with allure.step("Get object"):
object_1 = s3_client.get_object(bucket, file_name, full_output=True) object_1 = s3_client.get_object(bucket, file_name, full_output=True)
assert ( assert object_1.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
object_1.get("VersionId") == version_id_2
), f"Get object with version {version_id_2}"
with allure.step("Get first version of object"): with allure.step("Get first version of object"):
object_2 = s3_client.get_object(bucket, file_name, version_id_1, full_output=True) object_2 = s3_client.get_object(bucket, file_name, version_id_1, full_output=True)
assert ( assert object_2.get("VersionId") == version_id_1, f"Get object with version {version_id_1}"
object_2.get("VersionId") == version_id_1
), f"Get object with version {version_id_1}"
with allure.step("Get second version of object"): with allure.step("Get second version of object"):
object_3 = s3_client.get_object(bucket, file_name, version_id_2, full_output=True) object_3 = s3_client.get_object(bucket, file_name, version_id_2, full_output=True)
assert ( assert object_3.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
object_3.get("VersionId") == version_id_2
), f"Get object with version {version_id_2}"

View file

@ -1,7 +1,7 @@
import logging import logging
import os import os
from http import HTTPStatus from http import HTTPStatus
from re import match, fullmatch from re import fullmatch, match
import allure import allure
import pytest import pytest
@ -14,8 +14,8 @@ from pytest import FixtureRequest
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@allure.title("Check binaries versions") @allure.title("Check binaries versions")
@pytest.mark.sanity
@pytest.mark.check_binaries @pytest.mark.check_binaries
def test_binaries_versions(request: FixtureRequest, hosting: Hosting): def test_binaries_versions(request: FixtureRequest, hosting: Hosting):
""" """
@ -56,9 +56,7 @@ def download_versions_info(url: str) -> dict:
response = requests.get(url) response = requests.get(url)
assert ( assert response.status_code == HTTPStatus.OK, f"Got {response.status_code} code. Content {response.json()}"
response.status_code == HTTPStatus.OK
), f"Got {response.status_code} code. Content {response.json()}"
content = response.text content = response.text
assert content, f"Expected file with content, got {response}" assert content, f"Expected file with content, got {response}"

View file

@ -48,15 +48,9 @@ RANGE_OFFSET_FOR_COMPLEX_OBJECT = 200
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def storage_containers( def storage_containers(owner_wallet: WalletInfo, client_shell: Shell, cluster: Cluster) -> list[str]:
owner_wallet: WalletInfo, client_shell: Shell, cluster: Cluster cid = create_container(owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
) -> list[str]: other_cid = create_container(owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
cid = create_container(
owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint
)
other_cid = create_container(
owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint
)
yield [cid, other_cid] yield [cid, other_cid]
@ -99,9 +93,7 @@ def storage_objects(
@allure.step("Get ranges for test") @allure.step("Get ranges for test")
def get_ranges( def get_ranges(storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, endpoint: str) -> list[str]:
storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, endpoint: str
) -> list[str]:
""" """
Returns ranges to test range/hash methods via static session Returns ranges to test range/hash methods via static session
""" """
@ -112,8 +104,7 @@ def get_ranges(
return [ return [
"0:10", "0:10",
f"{object_size-10}:10", f"{object_size-10}:10",
f"{max_object_size - RANGE_OFFSET_FOR_COMPLEX_OBJECT}:" f"{max_object_size - RANGE_OFFSET_FOR_COMPLEX_OBJECT}:" f"{RANGE_OFFSET_FOR_COMPLEX_OBJECT * 2}",
f"{RANGE_OFFSET_FOR_COMPLEX_OBJECT * 2}",
] ]
else: else:
return ["0:10", f"{object_size-10}:10"] return ["0:10", f"{object_size-10}:10"]
@ -148,9 +139,7 @@ def static_sessions(
@pytest.mark.static_session @pytest.mark.static_session
class TestObjectStaticSession(ClusterTestBase): class TestObjectStaticSession(ClusterTestBase):
@allure.title( @allure.title("Read operations with static session (method={method_under_test.__name__}, obj_size={object_size})")
"Read operations with static session (method={method_under_test.__name__}, obj_size={object_size})"
)
@pytest.mark.parametrize( @pytest.mark.parametrize(
"method_under_test,verb", "method_under_test,verb",
[ [
@ -181,13 +170,12 @@ class TestObjectStaticSession(ClusterTestBase):
session=static_sessions[verb], session=static_sessions[verb],
) )
@allure.title( @allure.title("Range operations with static session (method={method_under_test.__name__}, obj_size={object_size})")
"Range operations with static session (method={method_under_test.__name__}, obj_size={object_size})"
)
@pytest.mark.parametrize( @pytest.mark.parametrize(
"method_under_test,verb", "method_under_test,verb",
[(get_range, ObjectVerb.RANGE), (get_range_hash, ObjectVerb.RANGEHASH)], [(get_range, ObjectVerb.RANGE), (get_range_hash, ObjectVerb.RANGEHASH)],
) )
@pytest.mark.sanity
def test_static_session_range( def test_static_session_range(
self, self,
user_wallet: WalletInfo, user_wallet: WalletInfo,
@ -201,9 +189,7 @@ class TestObjectStaticSession(ClusterTestBase):
Validate static session with range operations Validate static session with range operations
""" """
storage_object = storage_objects[0] storage_object = storage_objects[0]
ranges_to_test = get_ranges( ranges_to_test = get_ranges(storage_object, max_object_size, self.shell, self.cluster.default_rpc_endpoint)
storage_object, max_object_size, self.shell, self.cluster.default_rpc_endpoint
)
for range_to_test in ranges_to_test: for range_to_test in ranges_to_test:
with allure.step(f"Check range {range_to_test}"): with allure.step(f"Check range {range_to_test}"):
@ -241,9 +227,7 @@ class TestObjectStaticSession(ClusterTestBase):
) )
assert sorted(expected_object_ids) == sorted(actual_object_ids) assert sorted(expected_object_ids) == sorted(actual_object_ids)
@allure.title( @allure.title("[NEGATIVE] Static session with object id not in session (obj_size={object_size})")
"[NEGATIVE] Static session with object id not in session (obj_size={object_size})"
)
def test_static_session_unrelated_object( def test_static_session_unrelated_object(
self, self,
user_wallet: WalletInfo, user_wallet: WalletInfo,
@ -307,9 +291,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=static_sessions[ObjectVerb.HEAD], session=static_sessions[ObjectVerb.HEAD],
) )
@allure.title( @allure.title("[NEGATIVE] Static session with container id not in session (obj_size={object_size})")
"[NEGATIVE] Static session with container id not in session (obj_size={object_size})"
)
def test_static_session_unrelated_container( def test_static_session_unrelated_container(
self, self,
user_wallet: WalletInfo, user_wallet: WalletInfo,
@ -473,9 +455,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_expire_at_next_epoch, session=token_expire_at_next_epoch,
) )
with allure.step( with allure.step("Object should be available at last epoch before session token expiration"):
"Object should be available at last epoch before session token expiration"
):
self.tick_epoch() self.tick_epoch()
with expect_not_raises(): with expect_not_raises():
head_object( head_object(
@ -499,6 +479,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_expire_at_next_epoch, session=token_expire_at_next_epoch,
) )
@pytest.mark.sanity
@allure.title("Static session which is valid since next epoch (obj_size={object_size})") @allure.title("Static session which is valid since next epoch (obj_size={object_size})")
def test_static_session_start_at_next( def test_static_session_start_at_next(
self, self,
@ -540,9 +521,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_start_at_next_epoch, session=token_start_at_next_epoch,
) )
with allure.step( with allure.step("Object should be available with session token starting from token nbf epoch"):
"Object should be available with session token starting from token nbf epoch"
):
self.tick_epoch() self.tick_epoch()
with expect_not_raises(): with expect_not_raises():
head_object( head_object(
@ -554,9 +533,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=token_start_at_next_epoch, session=token_start_at_next_epoch,
) )
with allure.step( with allure.step("Object should be available at last epoch before session token expiration"):
"Object should be available at last epoch before session token expiration"
):
self.tick_epoch() self.tick_epoch()
with expect_not_raises(): with expect_not_raises():
head_object( head_object(
@ -640,6 +617,7 @@ class TestObjectStaticSession(ClusterTestBase):
session=static_sessions[ObjectVerb.DELETE], session=static_sessions[ObjectVerb.DELETE],
) )
@pytest.mark.sanity
@allure.title("Put verb is restricted for static session (obj_size={object_size})") @allure.title("Put verb is restricted for static session (obj_size={object_size})")
def test_static_session_put_verb( def test_static_session_put_verb(
self, self,

View file

@ -3,12 +3,7 @@ import pytest
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.acl import create_eacl, set_eacl, wait_for_cache_expired from frostfs_testlib.steps.acl import create_eacl, set_eacl, wait_for_cache_expired
from frostfs_testlib.steps.cli.container import ( from frostfs_testlib.steps.cli.container import create_container, delete_container, get_container, list_containers
create_container,
delete_container,
get_container,
list_containers,
)
from frostfs_testlib.steps.session_token import ContainerVerb, get_container_signed_token from frostfs_testlib.steps.session_token import ContainerVerb, get_container_signed_token
from frostfs_testlib.storage.dataclasses.acl import EACLAccess, EACLOperation, EACLRole, EACLRule from frostfs_testlib.storage.dataclasses.acl import EACLAccess, EACLOperation, EACLRole, EACLRule
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
@ -33,9 +28,7 @@ class TestSessionTokenContainer(ClusterTestBase):
Returns dict with static session token file paths for all verbs with default lifetime Returns dict with static session token file paths for all verbs with default lifetime
""" """
return { return {
verb: get_container_signed_token( verb: get_container_signed_token(owner_wallet, user_wallet, verb, client_shell, temp_directory)
owner_wallet, user_wallet, verb, client_shell, temp_directory
)
for verb in ContainerVerb for verb in ContainerVerb
} }
@ -65,9 +58,7 @@ class TestSessionTokenContainer(ClusterTestBase):
assert cid not in list_containers( assert cid not in list_containers(
user_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint user_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
) )
assert cid in list_containers( assert cid in list_containers(owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
def test_static_session_token_container_create_with_other_verb( def test_static_session_token_container_create_with_other_verb(
self, self,
@ -136,6 +127,7 @@ class TestSessionTokenContainer(ClusterTestBase):
owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
) )
@pytest.mark.sanity
def test_static_session_token_container_set_eacl( def test_static_session_token_container_set_eacl(
self, self,
owner_wallet: WalletInfo, owner_wallet: WalletInfo,
@ -158,10 +150,7 @@ class TestSessionTokenContainer(ClusterTestBase):
assert can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster) assert can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster)
with allure.step("Deny all operations for other via eACL"): with allure.step("Deny all operations for other via eACL"):
eacl_deny = [ eacl_deny = [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op)
for op in EACLOperation
]
set_eacl( set_eacl(
user_wallet.path, user_wallet.path,
cid, cid,

View file

@ -66,8 +66,7 @@ class Shard:
blobstor_count = Shard._get_blobstor_count_from_section(config_object, shard_id) blobstor_count = Shard._get_blobstor_count_from_section(config_object, shard_id)
blobstors = [ blobstors = [
Blobstor.from_config_object(config_object, shard_id, blobstor_id) Blobstor.from_config_object(config_object, shard_id, blobstor_id) for blobstor_id in range(blobstor_count)
for blobstor_id in range(blobstor_count)
] ]
write_cache_enabled = config_object.as_bool(f"{var_prefix}_WRITECACHE_ENABLED") write_cache_enabled = config_object.as_bool(f"{var_prefix}_WRITECACHE_ENABLED")
@ -81,15 +80,10 @@ class Shard:
@staticmethod @staticmethod
def from_object(shard): def from_object(shard):
metabase = shard["metabase"]["path"] if "path" in shard["metabase"] else shard["metabase"] metabase = shard["metabase"]["path"] if "path" in shard["metabase"] else shard["metabase"]
writecache = ( writecache = shard["writecache"]["path"] if "path" in shard["writecache"] else shard["writecache"]
shard["writecache"]["path"] if "path" in shard["writecache"] else shard["writecache"]
)
return Shard( return Shard(
blobstor=[ blobstor=[Blobstor(path=blobstor["path"], path_type=blobstor["type"]) for blobstor in shard["blobstor"]],
Blobstor(path=blobstor["path"], path_type=blobstor["type"])
for blobstor in shard["blobstor"]
],
metabase=metabase, metabase=metabase,
writecache=writecache, writecache=writecache,
) )
@ -111,7 +105,6 @@ def shards_from_env(contents: str) -> list[Shard]:
return [Shard.from_config_object(configObj, shard_id) for shard_id in range(num_shards)] return [Shard.from_config_object(configObj, shard_id) for shard_id in range(num_shards)]
@pytest.mark.sanity
@pytest.mark.shard @pytest.mark.shard
class TestControlShard: class TestControlShard:
@staticmethod @staticmethod

View file

@ -1,5 +1,6 @@
import os import os
import shutil import shutil
import time
from datetime import datetime from datetime import datetime
import allure import allure
@ -35,9 +36,12 @@ class TestLogs:
os.makedirs(logs_dir) os.makedirs(logs_dir)
# Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID # Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas" issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas"
exclude_filter = r"too many requests"
time.sleep(2)
futures = parallel( futures = parallel(
self._collect_logs_on_host, cluster.hosts, logs_dir, issues_regex, session_start_time, end_time self._collect_logs_on_host, cluster.hosts, logs_dir, issues_regex, session_start_time, end_time, exclude_filter
) )
hosts_with_problems = [ hosts_with_problems = [
@ -50,9 +54,9 @@ class TestLogs:
not hosts_with_problems not hosts_with_problems
), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}" ), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}"
def _collect_logs_on_host(self, host: Host, logs_dir: str, regex: str, since: datetime, until: datetime): def _collect_logs_on_host(self, host: Host, logs_dir: str, regex: str, since: datetime, until: datetime, exclude_filter: str):
with allure.step(f"Get logs from {host.config.address}"): with allure.step(f"Get logs from {host.config.address}"):
logs = host.get_filtered_logs(regex, since, until) logs = host.get_filtered_logs(filter_regex=regex, since=since, until=until, exclude_filter=exclude_filter)
if not logs: if not logs:
return None return None