Compare commits
17 commits
master
...
support/v0
| Author | SHA1 | Date | |
|---|---|---|---|
| c51e4bd44f | |||
| 048f713f3f | |||
| 370c1059f1 | |||
| 6980cd15bd | |||
| 8115d28bcd | |||
| 8d2b3aee0e | |||
| fdd5fd55d4 | |||
| 7f9517075b | |||
| 849e052ad2 | |||
| 354c054a8a | |||
| 6d32372d93 | |||
| b8dbf086be | |||
| e2e4a0c667 | |||
| 8f339ecbcd | |||
| 9f8485f5eb | |||
| 1629caddec | |||
| cd06a073a2 |
30 changed files with 585 additions and 1161 deletions
|
|
@ -9,10 +9,7 @@ from frostfs_testlib.storage.dataclasses.acl import EACLAccess, EACLOperation, E
|
|||
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
||||
from frostfs_testlib.utils.failover_utils import wait_object_replication
|
||||
|
||||
from pytest_tests.helpers.container_access import (
|
||||
check_full_access_to_container,
|
||||
check_no_access_to_container,
|
||||
)
|
||||
from pytest_tests.helpers.container_access import check_full_access_to_container, check_no_access_to_container
|
||||
from pytest_tests.helpers.object_access import (
|
||||
can_delete_object,
|
||||
can_get_head_object,
|
||||
|
|
@ -25,7 +22,6 @@ from pytest_tests.helpers.object_access import (
|
|||
from pytest_tests.testsuites.acl.conftest import Wallets
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.acl
|
||||
@pytest.mark.acl_extended
|
||||
class TestEACLContainer(ClusterTestBase):
|
||||
|
|
@ -58,6 +54,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
|
||||
yield cid, oid, file_path
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Deny operations (role={deny_role.value}, obj_size={object_size})")
|
||||
@pytest.mark.parametrize("deny_role", [EACLRole.USER, EACLRole.OTHERS])
|
||||
def test_extended_acl_deny_all_operations(
|
||||
|
|
@ -75,10 +72,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
cid, object_oids, file_path = eacl_container_with_objects
|
||||
|
||||
with allure.step(f"Deny all operations for {deny_role_str} via eACL"):
|
||||
eacl_deny = [
|
||||
EACLRule(access=EACLAccess.DENY, role=deny_role, operation=op)
|
||||
for op in EACLOperation
|
||||
]
|
||||
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=deny_role, operation=op) for op in EACLOperation]
|
||||
set_eacl(
|
||||
user_wallet.wallet_path,
|
||||
cid,
|
||||
|
|
@ -89,9 +83,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
wait_for_cache_expired()
|
||||
|
||||
with allure.step(f"Check only {not_deny_role_str} has full access to container"):
|
||||
with allure.step(
|
||||
f"Check {deny_role_str} has not access to any operations with container"
|
||||
):
|
||||
with allure.step(f"Check {deny_role_str} has not access to any operations with container"):
|
||||
check_no_access_to_container(
|
||||
deny_role_wallet.wallet_path,
|
||||
cid,
|
||||
|
|
@ -101,9 +93,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
cluster=self.cluster,
|
||||
)
|
||||
|
||||
with allure.step(
|
||||
f"Check {not_deny_role_wallet} has full access to eACL public container"
|
||||
):
|
||||
with allure.step(f"Check {not_deny_role_wallet} has full access to eACL public container"):
|
||||
check_full_access_to_container(
|
||||
not_deny_role_wallet.wallet_path,
|
||||
cid,
|
||||
|
|
@ -114,10 +104,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
)
|
||||
|
||||
with allure.step(f"Allow all operations for {deny_role_str} via eACL"):
|
||||
eacl_deny = [
|
||||
EACLRule(access=EACLAccess.ALLOW, role=deny_role, operation=op)
|
||||
for op in EACLOperation
|
||||
]
|
||||
eacl_deny = [EACLRule(access=EACLAccess.ALLOW, role=deny_role, operation=op) for op in EACLOperation]
|
||||
set_eacl(
|
||||
user_wallet.wallet_path,
|
||||
cid,
|
||||
|
|
@ -162,10 +149,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
)
|
||||
for op in EACLOperation
|
||||
]
|
||||
eacl += [
|
||||
EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op)
|
||||
for op in EACLOperation
|
||||
]
|
||||
eacl += [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
|
||||
set_eacl(
|
||||
user_wallet.wallet_path,
|
||||
cid,
|
||||
|
|
@ -218,14 +202,8 @@ class TestEACLContainer(ClusterTestBase):
|
|||
storage_node = self.cluster.storage_nodes[0]
|
||||
|
||||
with allure.step("Deny all operations for user via eACL"):
|
||||
eacl_deny = [
|
||||
EACLRule(access=EACLAccess.DENY, role=EACLRole.USER, operation=op)
|
||||
for op in EACLOperation
|
||||
]
|
||||
eacl_deny += [
|
||||
EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op)
|
||||
for op in EACLOperation
|
||||
]
|
||||
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=EACLRole.USER, operation=op) for op in EACLOperation]
|
||||
eacl_deny += [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
|
||||
set_eacl(
|
||||
user_wallet.wallet_path,
|
||||
cid,
|
||||
|
|
@ -249,9 +227,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Operations with extended ACL for SYSTEM (obj_size={object_size})")
|
||||
def test_extended_actions_system(
|
||||
self, wallets: Wallets, eacl_container_with_objects: tuple[str, list[str], str]
|
||||
):
|
||||
def test_extended_actions_system(self, wallets: Wallets, eacl_container_with_objects: tuple[str, list[str], str]):
|
||||
user_wallet = wallets.get_wallet()
|
||||
ir_wallet, storage_wallet = wallets.get_wallets_list(role=EACLRole.SYSTEM)[:2]
|
||||
|
||||
|
|
@ -392,8 +368,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
create_eacl(
|
||||
cid=cid,
|
||||
rules_list=[
|
||||
EACLRule(access=EACLAccess.DENY, role=EACLRole.SYSTEM, operation=op)
|
||||
for op in EACLOperation
|
||||
EACLRule(access=EACLAccess.DENY, role=EACLRole.SYSTEM, operation=op) for op in EACLOperation
|
||||
],
|
||||
shell=self.shell,
|
||||
),
|
||||
|
|
@ -543,8 +518,7 @@ class TestEACLContainer(ClusterTestBase):
|
|||
create_eacl(
|
||||
cid=cid,
|
||||
rules_list=[
|
||||
EACLRule(access=EACLAccess.ALLOW, role=EACLRole.SYSTEM, operation=op)
|
||||
for op in EACLOperation
|
||||
EACLRule(access=EACLAccess.ALLOW, role=EACLRole.SYSTEM, operation=op) for op in EACLOperation
|
||||
],
|
||||
shell=self.shell,
|
||||
),
|
||||
|
|
|
|||
|
|
@ -1,12 +1,7 @@
|
|||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
|
||||
from frostfs_testlib.steps.acl import (
|
||||
create_eacl,
|
||||
form_bearertoken_file,
|
||||
set_eacl,
|
||||
wait_for_cache_expired,
|
||||
)
|
||||
from frostfs_testlib.steps.acl import create_eacl, form_bearertoken_file, set_eacl, wait_for_cache_expired
|
||||
from frostfs_testlib.steps.cli.container import create_container, delete_container
|
||||
from frostfs_testlib.steps.cli.object import put_object_to_random_node
|
||||
from frostfs_testlib.storage.dataclasses.acl import (
|
||||
|
|
@ -21,15 +16,11 @@ from frostfs_testlib.storage.dataclasses.acl import (
|
|||
)
|
||||
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
||||
|
||||
from pytest_tests.helpers.container_access import (
|
||||
check_full_access_to_container,
|
||||
check_no_access_to_container,
|
||||
)
|
||||
from pytest_tests.helpers.container_access import check_full_access_to_container, check_no_access_to_container
|
||||
from pytest_tests.helpers.object_access import can_get_head_object, can_get_object, can_put_object
|
||||
from pytest_tests.testsuites.acl.conftest import Wallets
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.acl
|
||||
@pytest.mark.acl_filters
|
||||
class TestEACLFilters(ClusterTestBase):
|
||||
|
|
@ -46,18 +37,14 @@ class TestEACLFilters(ClusterTestBase):
|
|||
"x_key": "other_value",
|
||||
"check_key": "other_value",
|
||||
}
|
||||
REQ_EQUAL_FILTER = EACLFilter(
|
||||
key="check_key", value="check_value", header_type=EACLHeaderType.REQUEST
|
||||
)
|
||||
REQ_EQUAL_FILTER = EACLFilter(key="check_key", value="check_value", header_type=EACLHeaderType.REQUEST)
|
||||
NOT_REQ_EQUAL_FILTER = EACLFilter(
|
||||
key="check_key",
|
||||
value="other_value",
|
||||
match_type=EACLMatchType.STRING_NOT_EQUAL,
|
||||
header_type=EACLHeaderType.REQUEST,
|
||||
)
|
||||
OBJ_EQUAL_FILTER = EACLFilter(
|
||||
key="check_key", value="check_value", header_type=EACLHeaderType.OBJECT
|
||||
)
|
||||
OBJ_EQUAL_FILTER = EACLFilter(key="check_key", value="check_value", header_type=EACLHeaderType.OBJECT)
|
||||
NOT_OBJ_EQUAL_FILTER = EACLFilter(
|
||||
key="check_key",
|
||||
value="other_value",
|
||||
|
|
@ -128,12 +115,9 @@ class TestEACLFilters(ClusterTestBase):
|
|||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
|
||||
@allure.title(
|
||||
"Operations with request filter (match_type={match_type}, obj_size={object_size})"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL]
|
||||
)
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Operations with request filter (match_type={match_type}, obj_size={object_size})")
|
||||
@pytest.mark.parametrize("match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL])
|
||||
def test_extended_acl_filters_request(
|
||||
self,
|
||||
wallets: Wallets,
|
||||
|
|
@ -174,12 +158,8 @@ class TestEACLFilters(ClusterTestBase):
|
|||
# Filter denies requests where "check_key {match_type} ATTRIBUTE", so when match_type
|
||||
# is STRING_EQUAL, then requests with "check_key=OTHER_ATTRIBUTE" will be allowed while
|
||||
# requests with "check_key=ATTRIBUTE" will be denied, and vice versa
|
||||
allow_headers = (
|
||||
self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
|
||||
)
|
||||
deny_headers = (
|
||||
self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
|
||||
)
|
||||
allow_headers = self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
|
||||
deny_headers = self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
|
||||
# We test on 3 groups of objects with various headers,
|
||||
# but eACL rule should ignore object headers and
|
||||
# work only based on request headers
|
||||
|
|
@ -198,9 +178,7 @@ class TestEACLFilters(ClusterTestBase):
|
|||
cluster=self.cluster,
|
||||
)
|
||||
|
||||
with allure.step(
|
||||
"Check other has full access when sending request with allowed headers"
|
||||
):
|
||||
with allure.step("Check other has full access when sending request with allowed headers"):
|
||||
check_full_access_to_container(
|
||||
other_wallet.wallet_path,
|
||||
cid,
|
||||
|
|
@ -223,16 +201,12 @@ class TestEACLFilters(ClusterTestBase):
|
|||
)
|
||||
|
||||
with allure.step(
|
||||
"Check other has full access when sending request "
|
||||
"with denied headers and using bearer token"
|
||||
"Check other has full access when sending request " "with denied headers and using bearer token"
|
||||
):
|
||||
bearer_other = form_bearertoken_file(
|
||||
user_wallet.wallet_path,
|
||||
cid,
|
||||
[
|
||||
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS)
|
||||
for op in EACLOperation
|
||||
],
|
||||
[EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) for op in EACLOperation],
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
|
|
@ -247,12 +221,8 @@ class TestEACLFilters(ClusterTestBase):
|
|||
bearer=bearer_other,
|
||||
)
|
||||
|
||||
@allure.title(
|
||||
"Operations with deny user headers filter (match_type={match_type}, obj_size={object_size})"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL]
|
||||
)
|
||||
@allure.title("Operations with deny user headers filter (match_type={match_type}, obj_size={object_size})")
|
||||
@pytest.mark.parametrize("match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL])
|
||||
def test_extended_acl_deny_filters_object(
|
||||
self,
|
||||
wallets: Wallets,
|
||||
|
|
@ -290,16 +260,8 @@ class TestEACLFilters(ClusterTestBase):
|
|||
)
|
||||
wait_for_cache_expired()
|
||||
|
||||
allow_objects = (
|
||||
objects_with_other_header
|
||||
if match_type == EACLMatchType.STRING_EQUAL
|
||||
else objects_with_header
|
||||
)
|
||||
deny_objects = (
|
||||
objects_with_header
|
||||
if match_type == EACLMatchType.STRING_EQUAL
|
||||
else objects_with_other_header
|
||||
)
|
||||
allow_objects = objects_with_other_header if match_type == EACLMatchType.STRING_EQUAL else objects_with_header
|
||||
deny_objects = objects_with_header if match_type == EACLMatchType.STRING_EQUAL else objects_with_other_header
|
||||
|
||||
# We will attempt requests with various headers,
|
||||
# but eACL rule should ignore request headers and validate
|
||||
|
|
@ -348,9 +310,7 @@ class TestEACLFilters(ClusterTestBase):
|
|||
xhdr=xhdr,
|
||||
)
|
||||
|
||||
with allure.step(
|
||||
"Check other have access to objects with deny attribute and using bearer token"
|
||||
):
|
||||
with allure.step("Check other have access to objects with deny attribute and using bearer token"):
|
||||
bearer_other = form_bearertoken_file(
|
||||
user_wallet.wallet_path,
|
||||
cid,
|
||||
|
|
@ -376,9 +336,7 @@ class TestEACLFilters(ClusterTestBase):
|
|||
bearer=bearer_other,
|
||||
)
|
||||
|
||||
allow_attribute = (
|
||||
self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
|
||||
)
|
||||
allow_attribute = self.OTHER_ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.ATTRIBUTE
|
||||
with allure.step("Check other can PUT objects without denied attribute"):
|
||||
assert can_put_object(
|
||||
other_wallet.wallet_path,
|
||||
|
|
@ -388,13 +346,9 @@ class TestEACLFilters(ClusterTestBase):
|
|||
cluster=self.cluster,
|
||||
attributes=allow_attribute,
|
||||
)
|
||||
assert can_put_object(
|
||||
other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster
|
||||
)
|
||||
assert can_put_object(other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster)
|
||||
|
||||
deny_attribute = (
|
||||
self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
|
||||
)
|
||||
deny_attribute = self.ATTRIBUTE if match_type == EACLMatchType.STRING_EQUAL else self.OTHER_ATTRIBUTE
|
||||
with allure.step("Check other can not PUT objects with denied attribute"):
|
||||
with pytest.raises(AssertionError):
|
||||
assert can_put_object(
|
||||
|
|
@ -406,9 +360,7 @@ class TestEACLFilters(ClusterTestBase):
|
|||
attributes=deny_attribute,
|
||||
)
|
||||
|
||||
with allure.step(
|
||||
"Check other can PUT objects with denied attribute and using bearer token"
|
||||
):
|
||||
with allure.step("Check other can PUT objects with denied attribute and using bearer token"):
|
||||
bearer_other_for_put = form_bearertoken_file(
|
||||
user_wallet.wallet_path,
|
||||
cid,
|
||||
|
|
@ -432,12 +384,8 @@ class TestEACLFilters(ClusterTestBase):
|
|||
bearer=bearer_other_for_put,
|
||||
)
|
||||
|
||||
@allure.title(
|
||||
"Operations with allow eACL user headers filters (match_type={match_type}, obj_size={object_size})"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL]
|
||||
)
|
||||
@allure.title("Operations with allow eACL user headers filters (match_type={match_type}, obj_size={object_size})")
|
||||
@pytest.mark.parametrize("match_type", [EACLMatchType.STRING_EQUAL, EACLMatchType.STRING_NOT_EQUAL])
|
||||
def test_extended_acl_allow_filters_object(
|
||||
self,
|
||||
wallets: Wallets,
|
||||
|
|
@ -454,9 +402,7 @@ class TestEACLFilters(ClusterTestBase):
|
|||
file_path,
|
||||
) = eacl_container_with_objects
|
||||
|
||||
with allure.step(
|
||||
"Deny all operations for others except few operations allowed by object filter"
|
||||
):
|
||||
with allure.step("Deny all operations for others except few operations allowed by object filter"):
|
||||
equal_filter = EACLFilter(**self.OBJ_EQUAL_FILTER.__dict__)
|
||||
equal_filter.match_type = match_type
|
||||
eacl = [
|
||||
|
|
@ -511,13 +457,9 @@ class TestEACLFilters(ClusterTestBase):
|
|||
cluster=self.cluster,
|
||||
)
|
||||
with pytest.raises(AssertionError):
|
||||
assert can_put_object(
|
||||
other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster
|
||||
)
|
||||
assert can_put_object(other_wallet.wallet_path, cid, file_path, shell=self.shell, cluster=self.cluster)
|
||||
|
||||
with allure.step(
|
||||
"Check other can get and put objects without attributes and using bearer token"
|
||||
):
|
||||
with allure.step("Check other can get and put objects without attributes and using bearer token"):
|
||||
bearer_other = form_bearertoken_file(
|
||||
user_wallet.wallet_path,
|
||||
cid,
|
||||
|
|
@ -613,8 +555,7 @@ class TestEACLFilters(ClusterTestBase):
|
|||
)
|
||||
|
||||
with allure.step(
|
||||
"Check other can get objects without attributes matching the filter "
|
||||
"and using bearer token"
|
||||
"Check other can get objects without attributes matching the filter " "and using bearer token"
|
||||
):
|
||||
oid = deny_objects.pop()
|
||||
assert can_get_head_object(
|
||||
|
|
|
|||
|
|
@ -8,7 +8,9 @@ import allure
|
|||
import pytest
|
||||
import yaml
|
||||
from dateutil import parser
|
||||
from frostfs_testlib.healthcheck.interfaces import Healthcheck
|
||||
from frostfs_testlib.hosting import Hosting
|
||||
from frostfs_testlib.plugins import load_plugin
|
||||
from frostfs_testlib.reporter import AllureHandler, get_reporter
|
||||
from frostfs_testlib.resources.common import (
|
||||
ASSETS_DIR,
|
||||
|
|
@ -17,7 +19,7 @@ from frostfs_testlib.resources.common import (
|
|||
DEFAULT_WALLET_PASS,
|
||||
SIMPLE_OBJECT_SIZE,
|
||||
)
|
||||
from frostfs_testlib.s3.interfaces import S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.shell import LocalShell, Shell
|
||||
from frostfs_testlib.steps.cli.container import list_containers
|
||||
from frostfs_testlib.steps.cli.object import get_netmap_netinfo
|
||||
|
|
@ -144,7 +146,10 @@ def complex_object_size(max_object_size: int) -> ObjectSize:
|
|||
|
||||
# By default we want all tests to be executed with both object sizes
|
||||
# This can be overriden in choosen tests if needed
|
||||
@pytest.fixture(scope="session", params=["simple", "complex"])
|
||||
@pytest.fixture(
|
||||
scope="session",
|
||||
params=[pytest.param("simple", marks=pytest.mark.simple), pytest.param("complex", marks=pytest.mark.complex)],
|
||||
)
|
||||
def object_size(
|
||||
simple_object_size: ObjectSize, complex_object_size: ObjectSize, request: pytest.FixtureRequest
|
||||
) -> ObjectSize:
|
||||
|
|
@ -182,13 +187,29 @@ def s3_policy(request: pytest.FixtureRequest):
|
|||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def cluster_state_controller(client_shell: Shell, cluster: Cluster) -> ClusterStateController:
|
||||
controller = ClusterStateController(client_shell, cluster)
|
||||
@allure.title("[Session] Create healthcheck object")
|
||||
def healthcheck(cluster: Cluster) -> Healthcheck:
|
||||
healthcheck_cls = load_plugin(
|
||||
"frostfs.testlib.healthcheck", cluster.cluster_nodes[0].host.config.healthcheck_plugin_name
|
||||
)
|
||||
|
||||
return healthcheck_cls()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def cluster_state_controller(client_shell: Shell, cluster: Cluster, healthcheck: Healthcheck) -> ClusterStateController:
|
||||
controller = ClusterStateController(client_shell, cluster, healthcheck)
|
||||
yield controller
|
||||
|
||||
|
||||
@allure.step("[Class]: Create S3 client")
|
||||
@pytest.fixture(scope="class")
|
||||
@pytest.fixture(
|
||||
scope="class",
|
||||
params=[
|
||||
pytest.param(AwsCliClient, marks=pytest.mark.aws),
|
||||
pytest.param(Boto3ClientWrapper, marks=pytest.mark.boto3),
|
||||
],
|
||||
)
|
||||
def s3_client(
|
||||
default_wallet: str,
|
||||
client_shell: Shell,
|
||||
|
|
@ -225,24 +246,29 @@ def versioning_status(request: pytest.FixtureRequest) -> VersioningStatus:
|
|||
|
||||
@allure.step("Create/delete bucket")
|
||||
@pytest.fixture
|
||||
def bucket(s3_client: S3ClientWrapper, versioning_status: VersioningStatus):
|
||||
def bucket(s3_client: S3ClientWrapper, versioning_status: VersioningStatus, request: pytest.FixtureRequest):
|
||||
|
||||
bucket_name = s3_client.create_bucket()
|
||||
|
||||
if versioning_status:
|
||||
s3_helper.set_bucket_versioning(s3_client, bucket_name, versioning_status)
|
||||
|
||||
yield bucket_name
|
||||
s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
|
||||
|
||||
if "sanity" not in request.config.option.markexpr:
|
||||
s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
|
||||
|
||||
|
||||
@allure.step("Create two buckets")
|
||||
@pytest.fixture
|
||||
def two_buckets(s3_client: S3ClientWrapper):
|
||||
def two_buckets(s3_client: S3ClientWrapper, request: pytest.FixtureRequest):
|
||||
bucket_1 = s3_client.create_bucket()
|
||||
bucket_2 = s3_client.create_bucket()
|
||||
yield bucket_1, bucket_2
|
||||
for bucket_name in [bucket_1, bucket_2]:
|
||||
s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
|
||||
|
||||
if "sanity" not in request.config.option.markexpr:
|
||||
for bucket_name in [bucket_1, bucket_2]:
|
||||
s3_helper.delete_bucket_with_objects(s3_client, bucket_name)
|
||||
|
||||
|
||||
@allure.step("[Autouse/Session] Check binary versions")
|
||||
|
|
|
|||
|
|
@ -92,6 +92,7 @@ class TestPolicy(ClusterTestBase):
|
|||
endpoint=endpoint,
|
||||
)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Simple policy results with one node")
|
||||
def test_simple_policy_results_with_one_node(
|
||||
self,
|
||||
|
|
@ -166,6 +167,7 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Delete the container"):
|
||||
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Policy with SELECT and FILTER results with one node")
|
||||
def test_policy_with_select_and_filter_results_with_one_node(
|
||||
self,
|
||||
|
|
@ -179,7 +181,7 @@ class TestPolicy(ClusterTestBase):
|
|||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 1
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
expected_params = {"country": "Russia"}
|
||||
placement_params = {"country": "Russia"}
|
||||
|
||||
with allure.step(f"Create container with policy {placement_rule}"):
|
||||
cid = create_container(
|
||||
|
|
@ -202,9 +204,10 @@ class TestPolicy(ClusterTestBase):
|
|||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
expected_params["country"] == netmap[node_address]["country"]
|
||||
), f"The node is selected from the wrong country. Expected {expected_params['country']} and got {netmap[node_address]['country']}"
|
||||
with allure.step(f"Check the node is selected from {placement_params['country']}"):
|
||||
assert (
|
||||
placement_params["country"] == netmap[node_address]["country"]
|
||||
), f"The node is selected from the wrong country. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -222,8 +225,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with one node.
|
||||
"""
|
||||
placement_rule = "REP 1 IN RUS REP 1 IN RUS CBF 1 SELECT 1 FROM RU AS RUS FILTER Country NE Sweden AS NotSE FILTER @NotSE AND NOT (CountryCode EQ FI) AND Country EQ 'Russia' AS RU"
|
||||
expected_params = {"country": ["Russia", "Sweden"]}
|
||||
unexpected_params = {"country": "Sweden", "country_code": "FI"}
|
||||
placement_params = {"country": ["Russia", "Sweden"], "country_code": "FI"}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 1
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -249,12 +251,15 @@ class TestPolicy(ClusterTestBase):
|
|||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
(expected_params["country"][1] == netmap[node_address]["country"])
|
||||
or not (unexpected_params["country"] == netmap[node_address]["country"])
|
||||
and not (unexpected_params["country_code"] == netmap[node_address]["country_code"])
|
||||
and (expected_params["country"][0] == netmap[node_address]["country"])
|
||||
), f"The node is selected from the wrong country or country code. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(
|
||||
f"Check the node is selected from {placement_params['country'][0]}"
|
||||
):
|
||||
assert (
|
||||
not (placement_params["country"][1] == netmap[node_address]["country"])
|
||||
or not (placement_params["country"][1] == netmap[node_address]["country"])
|
||||
and not (placement_params["country_code"] == netmap[node_address]["country_code"])
|
||||
and (placement_params["country"][0] == netmap[node_address]["country"])
|
||||
), f"The node is selected from the wrong country or country code. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -262,6 +267,7 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Delete the container"):
|
||||
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Policy with Multi SELECTs and FILTERs results with one node")
|
||||
def test_policy_with_multi_selects_and_filters_results_with_one_node(
|
||||
self,
|
||||
|
|
@ -272,8 +278,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with one nodes.
|
||||
"""
|
||||
placement_rule = "REP 1 IN RU REP 1 IN EU REP 1 IN RU CBF 1 SELECT 1 FROM RUS AS RU SELECT 1 FROM EUR AS EU FILTER Country EQ Russia AS RUS FILTER NOT (@RUS) AND Country EQ Sweden OR CountryCode EQ FI AS EUR"
|
||||
expected_params = {"country": ["Sweden", "Russia"], "country_code": "FI"}
|
||||
unexpected_params = {"country": "Russia"}
|
||||
placement_params = {"country": ["Sweden", "Russia"], "country_code": "FI"}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 2
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -298,13 +303,14 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0]
|
||||
assert (expected_params["country"][1] == netmap[node_address]["country"]) or (
|
||||
not (unexpected_params["country"][1] == netmap[node_address]["country"])
|
||||
and (expected_params["country"][0] == netmap[node_address]["country"])
|
||||
or (expected_params["country_code"] == netmap[node_address]["country_code"])
|
||||
), f"The node is selected from the wrong country or country code. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check two nodes are selected from any country"):
|
||||
for node in resulting_copies:
|
||||
node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0]
|
||||
assert (placement_params["country"][1] == netmap[node_address]["country"]) or (
|
||||
not (placement_params["country"][1] == netmap[node_address]["country"])
|
||||
and (placement_params["country"][0] == netmap[node_address]["country"])
|
||||
or (placement_params["country_code"] == netmap[node_address]["country_code"])
|
||||
), f"The node is selected from the wrong country or country code. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -312,6 +318,7 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Delete the container"):
|
||||
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Policy with SELECT and FILTER results with UNIQUE nodes")
|
||||
def test_policy_with_select_and_filter_results_with_unique_nodes(
|
||||
self,
|
||||
|
|
@ -322,11 +329,11 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with SELECT and FILTER results with UNIQUE nodes.
|
||||
"""
|
||||
placement_rule = "UNIQUE REP 1 IN MyRussianNodes REP 1 IN MyRussianNodes CBF 1 SELECT 1 FROM RussianNodes AS MyRussianNodes FILTER Country EQ 'Russia' AS RussianNodes"
|
||||
placement_params = {"country": "Russia"}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 2
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
expected_params = {"country": "Russia"}
|
||||
|
||||
|
||||
with allure.step(f"Create container with policy {placement_rule}"):
|
||||
cid = create_container(
|
||||
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
|
||||
|
|
@ -345,14 +352,14 @@ class TestPolicy(ClusterTestBase):
|
|||
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
|
||||
|
||||
with allure.step(f"Check the object appearance"):
|
||||
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
expected_params["country"] == netmap[node_address]["country"]
|
||||
), f"The node is selected from the wrong country. Expected {expected_params['country']} and got {netmap[node_address]['country']}"
|
||||
with allure.step(f"Check two nodes are selected from {placement_params['country']}"):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
placement_params["country"] == netmap[node_address]["country"]
|
||||
), f"The node is selected from the wrong country. Got {netmap[node_address]['country']}"
|
||||
|
||||
with allure.step(f"Delete the container"):
|
||||
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -445,7 +452,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with SELECT and FILTER results with 25% of available nodes.
|
||||
"""
|
||||
placement_rule = "REP 1 IN Nodes25 SELECT 1 FROM LE10 AS Nodes25 FILTER Price LE 10 AS LE10"
|
||||
expected_params = {"price": 10}
|
||||
placement_params = {"price": 10}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 1
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -470,11 +477,11 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0]
|
||||
with allure.step(f"Check the node is selected with price <= {placement_params['price']}"):
|
||||
assert (
|
||||
int(netmap[node_address]["price"]) <= expected_params["price"]
|
||||
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}"
|
||||
int(netmap[node_address]["price"]) <= placement_params["price"]
|
||||
), f"The node is selected with the wrong price. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -493,7 +500,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 25% of available nodes.
|
||||
"""
|
||||
placement_rule = "REP 1 IN Nodes25 SELECT 1 FROM BET0AND10 AS Nodes25 FILTER Price LE 10 AS LE10 FILTER Price GT 0 AS GT0 FILTER @LE10 AND @GT0 AS BET0AND10"
|
||||
expected_params = {"price": [10, 0]}
|
||||
placement_params = {"price": [10, 0]}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 1
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -518,16 +525,13 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
int(netmap[node_address]["price"]) > expected_params["price"][1]
|
||||
or int(netmap[node_address]["price"]) <= expected_params["price"][0]
|
||||
or (
|
||||
int(netmap[node_address]["price"]) > expected_params["price"][1]
|
||||
and int(netmap[node_address]["price"]) <= expected_params["price"][0]
|
||||
)
|
||||
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check the node is selected with price between 1 and 10"):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
int(netmap[node_address]["price"]) > placement_params["price"][1]
|
||||
and int(netmap[node_address]["price"]) <= placement_params["price"][0]
|
||||
), f"The node is selected with the wrong price. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -546,7 +550,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 25% of available nodes.
|
||||
"""
|
||||
placement_rule = "UNIQUE REP 1 IN One REP 1 IN One CBF 1 SELECT 1 FROM MINMAX AS One FILTER Price LT 15 AS LT15 FILTER Price GT 55 AS GT55 FILTER @LT15 OR @GT55 AS MINMAX"
|
||||
expected_params = {"price": [15, 55]}
|
||||
placement_params = {"price": [15, 55]}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 2
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -571,17 +575,13 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
int(netmap[node_address]["price"]) > expected_params["price"][1]
|
||||
or int(netmap[node_address]["price"]) < expected_params["price"][0]
|
||||
or (
|
||||
int(netmap[node_address]["price"]) > expected_params["price"][1]
|
||||
or int(netmap[node_address]["price"]) < expected_params["price"][0]
|
||||
)
|
||||
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}"
|
||||
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
|
||||
with allure.step(f"Check two nodes are selected with max and min prices"):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
int(netmap[node_address]["price"]) > placement_params["price"][1]
|
||||
or int(netmap[node_address]["price"]) < placement_params["price"][0]
|
||||
), f"The node is selected with the wrong price. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -674,7 +674,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with SELECT and FILTER results with 50% of available nodes.
|
||||
"""
|
||||
placement_rule = "REP 2 IN HALF CBF 1 SELECT 2 FROM GT15 AS HALF FILTER Price GT 15 AS GT15"
|
||||
expected_params = {"price": 15}
|
||||
placement_params = {"price": 15}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 2
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -699,11 +699,12 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
int(netmap[node_address]["price"]) > expected_params["price"]
|
||||
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check two nodes are selected with price > {placement_params['price']}"):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
int(netmap[node_address]["price"]) > placement_params["price"]
|
||||
), f"The node is selected with the wrong price. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -722,8 +723,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 50% of available nodes.
|
||||
"""
|
||||
placement_rule = "REP 2 IN HALF CBF 2 SELECT 2 FROM GE15 AS HALF FILTER CountryCode NE RU AS NOTRU FILTER @NOTRU AND Price GE 15 AS GE15"
|
||||
expected_params = {"price": 15}
|
||||
unexpected_params = {"country_code": "RU"}
|
||||
placement_params = {"price": 15, "country_code": "RU"}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 2
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -748,12 +748,13 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (not netmap[node_address]["country_code"] == unexpected_params["country_code"]) or (
|
||||
not netmap[node_address]["country_code"] == unexpected_params["country_code"]
|
||||
and int(netmap[node_address]["price"]) >= expected_params["price"]
|
||||
), f"The node is selected from the wrong price or country_code. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check two nodes are selected not with country code '{placement_params['country_code']}'"):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (not netmap[node_address]["country_code"] == placement_params["country_code"]
|
||||
or not netmap[node_address]["country_code"] == placement_params["country_code"]
|
||||
and int(netmap[node_address]["price"]) >= placement_params["price"]
|
||||
), f"The node is selected with the wrong price or country code. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -772,7 +773,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 50% of available nodes.
|
||||
"""
|
||||
placement_rule = "REP 2 IN FH REP 1 IN SH CBF 2 SELECT 2 FROM LE55 AS FH SELECT 2 FROM GE15 AS SH FILTER 'UN-LOCODE' EQ RU_LED OR 'UN-LOCODE' EQ RU_MOW AS RU FILTER NOT(@RU) AS NOTRU FILTER @NOTRU AND Price GE 15 AS GE15 FILTER @RU AND Price LE 55 AS LE55"
|
||||
expected_params = {"un_locode": ["RU_LED", "RU_MOW"], "price": [15, 55]}
|
||||
placement_params = {"un_locode": ["RU_LED", "RU_MOW"], "price": [15, 55]}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 4
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -797,20 +798,21 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
(netmap[node_address]["un_locode"] in expected_params["un_locode"])
|
||||
or (not netmap[node_address]["un_locode"] in expected_params["un_locode"])
|
||||
or (
|
||||
not netmap[node_address]["un_locode"] in expected_params["un_locode"]
|
||||
and int(netmap[node_address]["price"]) >= expected_params["price"][0]
|
||||
)
|
||||
or (
|
||||
netmap[node_address]["un_locode"] in expected_params["un_locode"]
|
||||
and int(netmap[node_address]["price"]) <= expected_params["price"][1]
|
||||
)
|
||||
), f"The node is selected from the wrong price or un_locode. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check all nodes are selected"):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
netmap[node_address]["un_locode"] in placement_params["un_locode"]
|
||||
or not netmap[node_address]["un_locode"] == placement_params["un_locode"][1]
|
||||
or (
|
||||
not netmap[node_address]["un_locode"] == placement_params["un_locode"][1]
|
||||
and int(netmap[node_address]["price"]) >= placement_params["price"][0]
|
||||
)
|
||||
or (
|
||||
netmap[node_address]["un_locode"] == placement_params["un_locode"][1]
|
||||
and int(netmap[node_address]["price"]) <= placement_params["price"][1]
|
||||
)
|
||||
), f"The node is selected with the wrong price or un_locode. Expected {placement_params} and got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -818,6 +820,7 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Delete the container"):
|
||||
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Simple policy results with 75% of available nodes")
|
||||
def test_simple_policy_results_with_75_of_available_nodes(
|
||||
self,
|
||||
|
|
@ -903,7 +906,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with SELECT and FILTER results with 75% of available nodes.
|
||||
"""
|
||||
placement_rule = "REP 2 IN NODES75 SELECT 2 FROM LT65 AS NODES75 FILTER Price LT 65 AS LT65"
|
||||
expected_params = {"price": 65}
|
||||
placement_params = {"price": 65}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 2
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -928,11 +931,12 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
int(netmap[node_address]["price"]) < expected_params["price"]
|
||||
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check two nodes are selected with price < {placement_params['price']}"):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
int(netmap[node_address]["price"]) < placement_params["price"]
|
||||
), f"The node is selected with the wrong price. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -951,8 +955,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 75% of available nodes.
|
||||
"""
|
||||
placement_rule = "REP 2 IN NODES75 SELECT 2 FROM LT65 AS NODES75 FILTER Continent NE America AS NOAM FILTER @NOAM AND Price LT 65 AS LT65"
|
||||
expected_params = {"price": 65}
|
||||
unexpected_params = {"continent": "America"}
|
||||
placement_params = {"price": 65, "continent": "America"}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 3
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -977,14 +980,16 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
int(netmap[node_address]["price"]) < expected_params["price"]
|
||||
and netmap[node_address]["continent"] == unexpected_params["continent"]
|
||||
) or (
|
||||
netmap[node_address]["continent"] == unexpected_params["continent"]
|
||||
), f"The node is selected from the wrong price or continent. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check three nodes are selected not from {placement_params['continent']}"
|
||||
):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
int(netmap[node_address]["price"]) < placement_params["price"]
|
||||
and not netmap[node_address]["continent"] == placement_params["continent"]
|
||||
) or (
|
||||
not netmap[node_address]["continent"] == placement_params["continent"]
|
||||
), f"The node is selected with the wrong price or continent. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -1003,8 +1008,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 75% of available nodes.
|
||||
"""
|
||||
placement_rule = "REP 2 IN EXPNSV REP 2 IN CHEAP SELECT 3 FROM GT10 AS EXPNSV SELECT 3 FROM LT65 AS CHEAP FILTER NOT(Continent EQ America) AS NOAM FILTER @NOAM AND Price LT 65 AS LT65 FILTER @NOAM AND Price GT 10 AS GT10"
|
||||
expected_params = {"price": [65, 10]}
|
||||
unexpected_params = {"continent": "America"}
|
||||
placement_params = {"price": [65, 10], "continent": "America"}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 4
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -1029,19 +1033,20 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
(
|
||||
int(netmap[node_address]["price"]) > expected_params["price"][1]
|
||||
and netmap[node_address]["continent"] == unexpected_params["continent"]
|
||||
)
|
||||
or (
|
||||
int(netmap[node_address]["price"]) < expected_params["price"][0]
|
||||
and netmap[node_address]["continent"] == unexpected_params["continent"]
|
||||
)
|
||||
or (netmap[node_address]["continent"] == unexpected_params["continent"])
|
||||
), f"The node is selected from the wrong price or continent. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check all nodes are selected"):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
(
|
||||
int(netmap[node_address]["price"]) > placement_params["price"][1]
|
||||
and not netmap[node_address]["continent"] == placement_params["continent"]
|
||||
)
|
||||
or (
|
||||
int(netmap[node_address]["price"]) < placement_params["price"][0]
|
||||
and not netmap[node_address]["continent"] == placement_params["continent"]
|
||||
)
|
||||
or not (netmap[node_address]["continent"] == placement_params["continent"])
|
||||
), f"The node is selected with the wrong price or continent. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -1049,6 +1054,7 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Delete the container"):
|
||||
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Simple policy results with 100% of available nodes")
|
||||
def test_simple_policy_results_with_100_of_available_nodes(
|
||||
self,
|
||||
|
|
@ -1134,7 +1140,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with SELECT and FILTER results with 100% of available nodes.
|
||||
"""
|
||||
placement_rule = "REP 1 IN All SELECT 4 FROM AllNodes AS All FILTER Price GE 0 AS AllNodes"
|
||||
expected_params = {"price": 0}
|
||||
placement_params = {"price": 0}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 1
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -1157,13 +1163,13 @@ class TestPolicy(ClusterTestBase):
|
|||
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
|
||||
|
||||
with allure.step(f"Check the object appearance"):
|
||||
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
node_address = resulting_copies[0].get_rpc_endpoint().split(":")[0]
|
||||
assert netmap[node_address]["price"] >= int(
|
||||
expected_params["price"]
|
||||
), f"The node is selected from the wrong price. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check the node is selected with price >= {placement_params['price']}"):
|
||||
assert (
|
||||
netmap[node_address]["price"] >= int(placement_params["price"])
|
||||
), f"The node is selected with the wrong price. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -1181,7 +1187,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 100% of available nodes.
|
||||
"""
|
||||
placement_rule = "REP 2 IN All SELECT 4 FROM AllNodes AS All FILTER Country EQ Russia OR Country EQ Sweden OR Country EQ Finland AS AllCountries FILTER @AllCountries AND Continent EQ Europe AS AllNodes"
|
||||
expected_params = {"country": ["Russia", "Sweden", "Finland"], "continent": "Europe"}
|
||||
placement_params = {"country": ["Russia", "Sweden", "Finland"], "continent": "Europe"}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 2
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -1206,13 +1212,14 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
(netmap[node_address]["country"] in expected_params["country"])
|
||||
or (netmap[node_address]["country"] in expected_params["country"])
|
||||
and (netmap[node_address]["continent"] in expected_params["continent"])
|
||||
), f"The node is selected from the wrong country or continent. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check two nodes are selected from {' or '.join(placement_params['country'])}"):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (
|
||||
(netmap[node_address]["country"] in placement_params["country"])
|
||||
or (netmap[node_address]["country"] in placement_params["country"])
|
||||
and (netmap[node_address]["continent"] == placement_params["continent"])
|
||||
), f"The node is selected from the wrong country or continent. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -1230,7 +1237,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with 100% of available nodes.
|
||||
"""
|
||||
placement_rule = "REP 4 IN AllOne REP 4 IN AllTwo CBF 4 SELECT 2 FROM GEZero AS AllOne SELECT 2 FROM AllCountries AS AllTwo FILTER Country EQ Russia OR Country EQ Sweden OR Country EQ Finland AS AllCountries FILTER Price GE 0 AS GEZero"
|
||||
expected_params = {"country": ["Russia", "Sweden", "Finland"], "price": "0"}
|
||||
placement_params = {"country": ["Russia", "Sweden", "Finland"], "price": 0}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 4
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -1255,11 +1262,12 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (netmap[node_address]["country"] in expected_params["country"]) or (
|
||||
netmap[node_address]["price"] in expected_params["price"]
|
||||
), f"The node is selected from the wrong country or price. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check all node are selected"):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (netmap[node_address]["country"] in placement_params["country"]) or (
|
||||
int(netmap[node_address]["price"]) >= placement_params["price"]
|
||||
), f"The node is selected from the wrong country or with wrong price. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -1267,6 +1275,7 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Delete the container"):
|
||||
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Simple policy results with UNIQUE nodes")
|
||||
def test_simple_policy_results_with_unique_nodes(
|
||||
self,
|
||||
|
|
@ -1297,10 +1306,6 @@ class TestPolicy(ClusterTestBase):
|
|||
assert (
|
||||
len(resulting_copies) == expected_copies
|
||||
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
|
||||
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
|
||||
|
||||
with allure.step(f"Check the object appearance"):
|
||||
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -1339,15 +1344,13 @@ class TestPolicy(ClusterTestBase):
|
|||
len(resulting_copies) == expected_copies
|
||||
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
|
||||
|
||||
with allure.step(f"Check the object appearance"):
|
||||
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
||||
with allure.step(f"Delete the container"):
|
||||
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Policy with SELECT and Complex FILTER results with UNIQUE nodes")
|
||||
def test_policy_with_select_and_complex_filter_results_with_unique_nodes(
|
||||
self,
|
||||
|
|
@ -1358,8 +1361,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with UNIQUE nodes.
|
||||
"""
|
||||
placement_rule = "UNIQUE REP 1 IN RUS REP 1 IN RUS CBF 1 SELECT 1 FROM RU AS RUS FILTER Country NE Sweden AS NotSE FILTER @NotSE AND NOT (CountryCode EQ FI) AND Country EQ 'Russia' AS RU"
|
||||
expected_params = {"country": "Russia"}
|
||||
unexpected_params = {"country_code": "FI", "country": "Sweden"}
|
||||
placement_params = {"country_code": "FI", "country": ["Sweden", "Russia"]}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 2
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -1382,16 +1384,16 @@ class TestPolicy(ClusterTestBase):
|
|||
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
|
||||
|
||||
with allure.step(f"Check the object appearance"):
|
||||
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert not (unexpected_params["country"] == netmap[node_address]["country"]) or (
|
||||
not (unexpected_params["country"] == netmap[node_address]["country"])
|
||||
and not (expected_params["country_code"] == netmap[node_address]["country_code"])
|
||||
or (expected_params["country"] == netmap[node_address]["country"])
|
||||
), f"The node is selected from the wrong country or country code. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(f"Check two nodes are selected not from {placement_params['country'][0]}"):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert not (placement_params["country"][0] == netmap[node_address]["country"]) or (
|
||||
not (placement_params["country"][0] == netmap[node_address]["country"])
|
||||
and not (placement_params["country_code"] == netmap[node_address]["country_code"])
|
||||
and placement_params["country"][1] == netmap[node_address]["country"]
|
||||
), f"The node is selected from the wrong country or with wrong country code. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -1399,6 +1401,7 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Delete the container"):
|
||||
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Policy with Multi SELECTs and FILTERs results with UNIQUE nodes")
|
||||
def test_policy_with_multi_selects_and_filters_results_with_unique_nodes(
|
||||
self,
|
||||
|
|
@ -1409,8 +1412,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy with Multi SELECTs and FILTERs results with UNIQUE nodes.
|
||||
"""
|
||||
placement_rule = "UNIQUE REP 1 IN RU REP 1 IN EU REP 1 IN RU CBF 1 SELECT 1 FROM RUS AS RU SELECT 1 FROM EUR AS EU FILTER Country EQ Russia AS RUS FILTER NOT (@RUS) AND Country EQ Sweden OR CountryCode EQ FI AS EUR"
|
||||
expected_params = {"country": ["Russia", "Sweden"], "country_code": "FI"}
|
||||
unexpected_params = {"country": "Russia"}
|
||||
placement_params = {"country": ["Russia", "Sweden"], "country_code": "FI"}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 3
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -1433,16 +1435,18 @@ class TestPolicy(ClusterTestBase):
|
|||
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
|
||||
|
||||
with allure.step(f"Check the object appearance"):
|
||||
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (expected_params["country"][0] == netmap[node_address]["country"]) or (
|
||||
not (unexpected_params["country"][0] == netmap[node_address]["country"])
|
||||
and (expected_params["country"][1] == netmap[node_address]["country"])
|
||||
or (expected_params["country_code"] == netmap[node_address]["country_code"])
|
||||
), f"The node is selected from the wrong country or country code. Expected {expected_params} and got {netmap[node_address]}"
|
||||
with allure.step(
|
||||
f"Check three nodes are selected from any country"
|
||||
):
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert (placement_params["country"][0] == netmap[node_address]["country"]) or (
|
||||
not (placement_params["country"][0] == netmap[node_address]["country"])
|
||||
and (placement_params["country"][1] == netmap[node_address]["country"])
|
||||
or (placement_params["country_code"] == netmap[node_address]["country_code"])
|
||||
), f"The node is selected from the wrong country or with wrong country code. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -1460,7 +1464,7 @@ class TestPolicy(ClusterTestBase):
|
|||
This test checks object's copies based on container's placement policy: REP 1 IN SPB REP 1 IN MSK REP 3.
|
||||
"""
|
||||
placement_rule = "REP 1 IN SPB REP 1 IN MSK REP 3 CBF 1 SELECT 2 FROM LED AS SPB SELECT 2 FROM MOW AS MSK FILTER Location EQ 'Saint Petersburg (ex Leningrad)' AS LED FILTER Location EQ Moskva AS MOW"
|
||||
expected_params = {"location": ["Saint Petersburg (ex Leningrad)", "Moskva"]}
|
||||
placement_params = {"location": ["Saint Petersburg (ex Leningrad)", "Moskva"]}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
expected_copies = 3
|
||||
endpoint = self.cluster.default_rpc_endpoint
|
||||
|
|
@ -1485,13 +1489,17 @@ class TestPolicy(ClusterTestBase):
|
|||
with allure.step(f"Check the object appearance"):
|
||||
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
|
||||
netmap = self.get_netmap_param(netmap)
|
||||
list_of_location = []
|
||||
for node in resulting_copies:
|
||||
node_address = node.get_rpc_endpoint().split(":")[0]
|
||||
assert expected_params["location"][0] == netmap[node_address]["location"] or (
|
||||
expected_params["location"][1] == netmap[node_address]["location"]
|
||||
), f"The node is selected from the wrong location. Expected {expected_params} and got {netmap[node_address]}"
|
||||
list_of_location.append(netmap[node_address]["location"])
|
||||
|
||||
self.check_for_the_uniqueness_of_the_nodes(resulting_copies)
|
||||
with allure.step(f"Check two or three nodes are selected from Russia and from any other country"):
|
||||
assert (
|
||||
placement_params["location"][0] in list_of_location
|
||||
and placement_params["location"][1] in list_of_location
|
||||
and len(resulting_copies) > 2
|
||||
), f"The node is selected from the wrong location. Got {netmap[node_address]}"
|
||||
|
||||
with allure.step(f"Delete the object from the container"):
|
||||
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
|
||||
|
|
@ -1507,9 +1515,6 @@ class TestPolicy(ClusterTestBase):
|
|||
"'", ""
|
||||
), f"Expected \n{placement_rule} and got policy \n{got_policy} are the same"
|
||||
|
||||
def check_for_the_uniqueness_of_the_nodes(self, nodes: list):
|
||||
assert len(set(nodes)) == len(nodes), f"The nodes found must be unique, but {nodes}"
|
||||
|
||||
def get_netmap_param(self, netmap_info: list[NodeNetmapInfo]) -> dict:
|
||||
dict_external = dict()
|
||||
for node in netmap_info:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ from time import sleep
|
|||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
|
||||
from frostfs_testlib.s3 import AwsCliClient
|
||||
from frostfs_testlib.steps.cli.container import create_container
|
||||
from frostfs_testlib.steps.cli.object import (
|
||||
get_object,
|
||||
|
|
@ -38,11 +37,6 @@ OBJECT_ATTRIBUTES = [
|
|||
]
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc):
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize("s3_client", [AwsCliClient], indirect=True)
|
||||
|
||||
|
||||
@pytest.mark.failover
|
||||
@pytest.mark.failover_network
|
||||
class TestFailoverNetwork(ClusterTestBase):
|
||||
|
|
|
|||
|
|
@ -99,20 +99,25 @@ class TestFailoverServer(ClusterTestBase):
|
|||
self, nodes: list[StorageNode], storage_objects: list[StorageObjectInfo]
|
||||
) -> list[StorageObjectInfo]:
|
||||
corrupted_objects = []
|
||||
errors_get = []
|
||||
for node in nodes:
|
||||
for storage_object in storage_objects:
|
||||
got_file_path = get_object(
|
||||
storage_object.wallet_file_path,
|
||||
storage_object.cid,
|
||||
storage_object.oid,
|
||||
endpoint=node.get_rpc_endpoint(),
|
||||
shell=self.shell,
|
||||
timeout="60s",
|
||||
)
|
||||
if storage_object.file_hash != get_file_hash(got_file_path):
|
||||
corrupted_objects.append(storage_object)
|
||||
os.remove(got_file_path)
|
||||
try:
|
||||
got_file_path = get_object(
|
||||
storage_object.wallet_file_path,
|
||||
storage_object.cid,
|
||||
storage_object.oid,
|
||||
endpoint=node.get_rpc_endpoint(),
|
||||
shell=self.shell,
|
||||
timeout="60s",
|
||||
)
|
||||
if storage_object.file_hash != get_file_hash(got_file_path):
|
||||
corrupted_objects.append(storage_object)
|
||||
os.remove(got_file_path)
|
||||
except RuntimeError:
|
||||
errors_get.append(storage_object.oid)
|
||||
|
||||
assert len(errors_get) == 0, f"Get failed - {errors_get}"
|
||||
return corrupted_objects
|
||||
|
||||
def check_objects_replication(
|
||||
|
|
|
|||
|
|
@ -5,10 +5,9 @@ from time import sleep
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.hosting import Host
|
||||
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
|
||||
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
|
||||
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.shell import CommandOptions
|
||||
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
|
||||
from frostfs_testlib.steps.cli.object import get_object, put_object_to_random_node
|
||||
|
|
@ -21,7 +20,7 @@ from frostfs_testlib.steps.node_management import (
|
|||
wait_for_node_to_be_ready,
|
||||
)
|
||||
from frostfs_testlib.steps.s3 import s3_helper
|
||||
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, StorageNode
|
||||
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, S3Gate, StorageNode
|
||||
from frostfs_testlib.storage.controllers import ClusterStateController, ShardsWatcher
|
||||
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
|
||||
|
|
@ -37,11 +36,6 @@ logger = logging.getLogger("NeoLogger")
|
|||
stopped_nodes: list[StorageNode] = []
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc):
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
@allure.title("Provide File Keeper")
|
||||
def file_keeper():
|
||||
|
|
@ -57,26 +51,11 @@ def after_run_return_all_stopped_hosts(cluster_state_controller: ClusterStateCon
|
|||
cluster_state_controller.start_stopped_hosts()
|
||||
|
||||
|
||||
@allure.step("Return all stopped storage services after test")
|
||||
@allure.step("Return all stopped services after test")
|
||||
@pytest.fixture(scope="function")
|
||||
def after_run_return_all_stopped_services(cluster_state_controller: ClusterStateController):
|
||||
yield
|
||||
cluster_state_controller.start_stopped_storage_services()
|
||||
|
||||
|
||||
@allure.step("Return all stopped S3 GateWay services after test")
|
||||
@pytest.fixture(scope="function")
|
||||
def after_run_return_all_stopped_s3(cluster_state_controller: ClusterStateController):
|
||||
yield
|
||||
cluster_state_controller.start_stopped_s3_gates()
|
||||
|
||||
|
||||
def panic_reboot_host(host: Host) -> None:
|
||||
shell = host.get_shell()
|
||||
shell.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"')
|
||||
|
||||
options = CommandOptions(close_stdin=True, timeout=1, check=False)
|
||||
shell.exec('sudo sh -c "echo b > /proc/sysrq-trigger"', options)
|
||||
cluster_state_controller.start_all_stopped_services()
|
||||
|
||||
|
||||
@pytest.mark.failover
|
||||
|
|
@ -209,22 +188,21 @@ class TestFailoverStorage(ClusterTestBase):
|
|||
s3_client: S3ClientWrapper,
|
||||
simple_object_size: ObjectSize,
|
||||
cluster_state_controller: ClusterStateController,
|
||||
after_run_return_all_stopped_s3,
|
||||
after_run_return_all_stopped_services,
|
||||
):
|
||||
default_node = self.cluster.cluster_nodes[0]
|
||||
|
||||
with allure.step("Turn S3 GW off on default node"):
|
||||
cluster_state_controller.stop_s3_gate(default_node)
|
||||
cluster_state_controller.stop_service_of_type(default_node, S3Gate)
|
||||
|
||||
with allure.step("Turn off storage on default node"):
|
||||
cluster_state_controller.stop_storage_service(default_node)
|
||||
cluster_state_controller.stop_service_of_type(default_node, StorageNode)
|
||||
|
||||
with allure.step("Turn on S3 GW on default node"):
|
||||
cluster_state_controller.start_s3_gate(default_node)
|
||||
cluster_state_controller.start_service_of_type(default_node, S3Gate)
|
||||
|
||||
with allure.step("Turn on storage on default node"):
|
||||
cluster_state_controller.start_storage_service(default_node)
|
||||
cluster_state_controller.start_service_of_type(default_node, StorageNode)
|
||||
|
||||
with allure.step("Create bucket with REP 1 SELECT 1 policy"):
|
||||
bucket = s3_client.create_bucket(
|
||||
|
|
@ -240,13 +218,13 @@ class TestFailoverStorage(ClusterTestBase):
|
|||
with allure.step("Turn off all storage nodes except default"):
|
||||
for node in self.cluster.cluster_nodes[1:]:
|
||||
with allure.step(f"Stop storage service on node: {node}"):
|
||||
cluster_state_controller.stop_storage_service(node)
|
||||
cluster_state_controller.stop_service_of_type(node, StorageNode)
|
||||
|
||||
with allure.step("Check that object is available"):
|
||||
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
|
||||
|
||||
with allure.step("Start storage nodes"):
|
||||
cluster_state_controller.start_stopped_storage_services()
|
||||
cluster_state_controller.start_all_stopped_services()
|
||||
|
||||
|
||||
@pytest.mark.failover
|
||||
|
|
@ -320,7 +298,7 @@ class TestEmptyMap(ClusterTestBase):
|
|||
def empty_map_stop_service_teardown(self, cluster_state_controller: ClusterStateController):
|
||||
yield
|
||||
with allure.step("Return all storage nodes to network map"):
|
||||
cluster_state_controller.start_stopped_storage_services()
|
||||
cluster_state_controller.start_all_stopped_services()
|
||||
for node in stopped_nodes:
|
||||
check_node_in_map(node, shell=self.shell, alive_node=node)
|
||||
|
||||
|
|
@ -365,7 +343,7 @@ class TestEmptyMap(ClusterTestBase):
|
|||
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_objects)
|
||||
|
||||
with allure.step("Stop all storage nodes"):
|
||||
cluster_state_controller.stop_all_storage_services()
|
||||
cluster_state_controller.stop_services_of_type(StorageNode)
|
||||
|
||||
with allure.step("Remove all nodes from network map"):
|
||||
remove_nodes_from_map_morph(
|
||||
|
|
@ -383,7 +361,7 @@ class TestEmptyMap(ClusterTestBase):
|
|||
first_node = self.cluster.cluster_nodes[0].service(StorageNode)
|
||||
|
||||
with allure.step("Start first node and check network map"):
|
||||
cluster_state_controller.start_storage_service(self.cluster.cluster_nodes[0])
|
||||
cluster_state_controller.start_service_of_type(self.cluster.cluster_nodes[0], StorageNode)
|
||||
wait_for_node_to_be_ready(first_node)
|
||||
|
||||
for check_node in self.cluster.storage_nodes:
|
||||
|
|
@ -392,7 +370,7 @@ class TestEmptyMap(ClusterTestBase):
|
|||
for node in self.cluster.cluster_nodes[1:]:
|
||||
storage_node = node.service(StorageNode)
|
||||
|
||||
cluster_state_controller.start_storage_service(node)
|
||||
cluster_state_controller.start_service_of_type(node, StorageNode)
|
||||
wait_for_node_to_be_ready(storage_node)
|
||||
|
||||
sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
|
||||
|
|
@ -420,9 +398,7 @@ class TestEmptyMap(ClusterTestBase):
|
|||
object_versions.append(put_object)
|
||||
|
||||
with allure.step("Stop all storage nodes"):
|
||||
for node in self.cluster.cluster_nodes:
|
||||
with allure.step(f"Stop storage service on node: {node}"):
|
||||
cluster_state_controller.stop_storage_service(node)
|
||||
cluster_state_controller.stop_services_of_type(StorageNode)
|
||||
|
||||
with allure.step("Delete blobovnicza and fstree from all nodes"):
|
||||
for node in self.cluster.storage_nodes:
|
||||
|
|
@ -430,7 +406,7 @@ class TestEmptyMap(ClusterTestBase):
|
|||
node.delete_fstree()
|
||||
|
||||
with allure.step("Start all storage nodes"):
|
||||
cluster_state_controller.start_stopped_storage_services()
|
||||
cluster_state_controller.start_all_stopped_services()
|
||||
|
||||
# need to get Delete Marker first
|
||||
with allure.step("Delete the object from the bucket"):
|
||||
|
|
@ -462,9 +438,7 @@ class TestEmptyMap(ClusterTestBase):
|
|||
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
|
||||
|
||||
with allure.step("Stop all storage nodes"):
|
||||
for node in self.cluster.cluster_nodes:
|
||||
with allure.step(f"Stop storage service on node: {node}"):
|
||||
cluster_state_controller.stop_storage_service(node)
|
||||
cluster_state_controller.stop_services_of_type(StorageNode)
|
||||
|
||||
with allure.step("Delete blobovnicza and fstree from all nodes"):
|
||||
for node in self.cluster.storage_nodes:
|
||||
|
|
@ -472,7 +446,7 @@ class TestEmptyMap(ClusterTestBase):
|
|||
node.delete_fstree()
|
||||
|
||||
with allure.step("Start all storage nodes"):
|
||||
cluster_state_controller.start_stopped_storage_services()
|
||||
cluster_state_controller.start_all_stopped_services()
|
||||
|
||||
with allure.step("Delete the object from the bucket"):
|
||||
s3_client.delete_object(bucket, file_name)
|
||||
|
|
@ -507,16 +481,14 @@ class TestEmptyMap(ClusterTestBase):
|
|||
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
|
||||
|
||||
with allure.step("Stop all storage nodes"):
|
||||
for node in self.cluster.cluster_nodes:
|
||||
with allure.step(f"Stop storage service on node: {node}"):
|
||||
cluster_state_controller.stop_storage_service(node)
|
||||
cluster_state_controller.stop_services_of_type(StorageNode)
|
||||
|
||||
with allure.step("Delete pilorama.db from all nodes"):
|
||||
for node in self.cluster.storage_nodes:
|
||||
node.delete_pilorama()
|
||||
|
||||
with allure.step("Start all storage nodes"):
|
||||
cluster_state_controller.start_stopped_storage_services()
|
||||
cluster_state_controller.start_all_stopped_services()
|
||||
|
||||
with allure.step("Check list objects first time"):
|
||||
objects_list = s3_client.list_objects(bucket)
|
||||
|
|
@ -578,7 +550,7 @@ class TestStorageDataLoss(ClusterTestBase):
|
|||
)
|
||||
|
||||
with allure.step("Stop storage services on all nodes"):
|
||||
cluster_state_controller.stop_all_storage_services()
|
||||
cluster_state_controller.stop_services_of_type(StorageNode)
|
||||
|
||||
with allure.step("Delete metabase from all nodes"):
|
||||
for node in cluster_state_controller.cluster.storage_nodes:
|
||||
|
|
@ -594,7 +566,11 @@ class TestStorageDataLoss(ClusterTestBase):
|
|||
storage_node.save_config(config, config_file_path)
|
||||
|
||||
with allure.step("Start storage services on all nodes"):
|
||||
cluster_state_controller.start_stopped_storage_services()
|
||||
cluster_state_controller.start_all_stopped_services()
|
||||
|
||||
with allure.step("Wait for tree rebalance"):
|
||||
# TODO: Use product metric when we have proper ones for this check
|
||||
sleep(30)
|
||||
|
||||
with allure.step("Delete objects from bucket"):
|
||||
with allure.step("Delete simple object from bucket"):
|
||||
|
|
@ -655,13 +631,13 @@ class TestStorageDataLoss(ClusterTestBase):
|
|||
shards_watcher.take_shards_snapshot()
|
||||
|
||||
with allure.step(f"Stop storage service on node {node_under_test}"):
|
||||
cluster_state_controller.stop_storage_service(node_under_test)
|
||||
cluster_state_controller.stop_service_of_type(node_under_test, StorageNode)
|
||||
|
||||
with allure.step(f"Delete write cache from node {node_under_test}"):
|
||||
node_under_test.storage_node.delete_write_cache()
|
||||
|
||||
with allure.step(f"Start storage service on node {node_under_test}"):
|
||||
cluster_state_controller.start_storage_service(node_under_test)
|
||||
cluster_state_controller.start_all_stopped_services()
|
||||
|
||||
with allure.step("Objects should be available"):
|
||||
for storage_object in storage_objects:
|
||||
|
|
@ -710,7 +686,7 @@ class TestStorageDataLoss(ClusterTestBase):
|
|||
|
||||
with allure.step("Stop one node and wait for rebalance connection of s3 gate to storage service"):
|
||||
current_node = self.cluster.cluster_nodes[0]
|
||||
cluster_state_controller.stop_storage_service(current_node)
|
||||
cluster_state_controller.stop_service_of_type(current_node, StorageNode)
|
||||
# waiting for rebalance connection of s3 gate to storage service
|
||||
sleep(60)
|
||||
|
||||
|
|
@ -752,15 +728,13 @@ class TestStorageDataLoss(ClusterTestBase):
|
|||
piloramas_list_before_removing = self.get_piloramas_list(node_to_check)
|
||||
|
||||
with allure.step("Stop all storage nodes"):
|
||||
for node in self.cluster.cluster_nodes:
|
||||
with allure.step(f"Stop storage service on node: {node}"):
|
||||
cluster_state_controller.stop_storage_service(node)
|
||||
cluster_state_controller.stop_services_of_type(StorageNode)
|
||||
|
||||
with allure.step("Delete pilorama.db from one node"):
|
||||
node_to_check.delete_pilorama()
|
||||
|
||||
with allure.step("Start all storage nodes"):
|
||||
cluster_state_controller.start_stopped_storage_services()
|
||||
cluster_state_controller.start_all_stopped_services()
|
||||
|
||||
with allure.step("Tick epoch to trigger sync and then wait for 1 minute"):
|
||||
self.tick_epochs(1)
|
||||
|
|
|
|||
|
|
@ -79,9 +79,7 @@ def generate_ranges(
|
|||
range_length = random.randint(RANGE_MIN_LEN, RANGE_MAX_LEN)
|
||||
range_start = random.randint(offset, offset + length)
|
||||
|
||||
file_ranges_to_test.append(
|
||||
(range_start, min(range_length, storage_object.size - range_start))
|
||||
)
|
||||
file_ranges_to_test.append((range_start, min(range_length, storage_object.size - range_start)))
|
||||
|
||||
file_ranges_to_test.extend(STATIC_RANGES.get(storage_object.size, []))
|
||||
|
||||
|
|
@ -250,9 +248,7 @@ class TestObjectApi(ClusterTestBase):
|
|||
assert sorted(expected_oids) == sorted(result)
|
||||
|
||||
@allure.title("Search objects with removed items (obj_size={object_size})")
|
||||
def test_object_search_should_return_tombstone_items(
|
||||
self, default_wallet: str, object_size: ObjectSize
|
||||
):
|
||||
def test_object_search_should_return_tombstone_items(self, default_wallet: str, object_size: ObjectSize):
|
||||
"""
|
||||
Validate object search with removed items
|
||||
"""
|
||||
|
|
@ -275,9 +271,7 @@ class TestObjectApi(ClusterTestBase):
|
|||
|
||||
with allure.step("Search object"):
|
||||
# Root Search object should return root object oid
|
||||
result = search_object(
|
||||
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True
|
||||
)
|
||||
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True)
|
||||
assert result == [storage_object.oid]
|
||||
|
||||
with allure.step("Delete file"):
|
||||
|
|
@ -285,22 +279,14 @@ class TestObjectApi(ClusterTestBase):
|
|||
|
||||
with allure.step("Search deleted object with --root"):
|
||||
# Root Search object should return nothing
|
||||
result = search_object(
|
||||
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True
|
||||
)
|
||||
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True)
|
||||
assert len(result) == 0
|
||||
|
||||
with allure.step("Search deleted object with --phy should return only tombstones"):
|
||||
# Physical Search object should return only tombstones
|
||||
result = search_object(
|
||||
wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, phy=True
|
||||
)
|
||||
assert (
|
||||
storage_object.tombstone in result
|
||||
), "Search result should contain tombstone of removed object"
|
||||
assert (
|
||||
storage_object.oid not in result
|
||||
), "Search result should not contain ObjectId of removed object"
|
||||
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, phy=True)
|
||||
assert storage_object.tombstone in result, "Search result should contain tombstone of removed object"
|
||||
assert storage_object.oid not in result, "Search result should not contain ObjectId of removed object"
|
||||
for tombstone_oid in result:
|
||||
header = head_object(
|
||||
wallet,
|
||||
|
|
@ -315,7 +301,6 @@ class TestObjectApi(ClusterTestBase):
|
|||
), f"Object wasn't deleted properly. Found object {tombstone_oid} with type {object_type}"
|
||||
|
||||
@allure.title("Get range hash by native API (obj_size={object_size})")
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.grpc_api
|
||||
def test_object_get_range_hash(self, storage_objects: list[StorageObjectInfo], max_object_size):
|
||||
"""
|
||||
|
|
@ -327,9 +312,7 @@ class TestObjectApi(ClusterTestBase):
|
|||
oids = [storage_object.oid for storage_object in storage_objects[:2]]
|
||||
file_path = storage_objects[0].file_path
|
||||
|
||||
file_ranges_to_test = generate_ranges(
|
||||
storage_objects[0], max_object_size, self.shell, self.cluster
|
||||
)
|
||||
file_ranges_to_test = generate_ranges(storage_objects[0], max_object_size, self.shell, self.cluster)
|
||||
logging.info(f"Ranges used in test {file_ranges_to_test}")
|
||||
|
||||
for range_start, range_len in file_ranges_to_test:
|
||||
|
|
@ -349,7 +332,6 @@ class TestObjectApi(ClusterTestBase):
|
|||
), f"Expected range hash to match {range_cut} slice of file payload"
|
||||
|
||||
@allure.title("Get range by native API (obj_size={object_size})")
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.grpc_api
|
||||
def test_object_get_range(self, storage_objects: list[StorageObjectInfo], max_object_size):
|
||||
"""
|
||||
|
|
@ -361,9 +343,7 @@ class TestObjectApi(ClusterTestBase):
|
|||
oids = [storage_object.oid for storage_object in storage_objects[:2]]
|
||||
file_path = storage_objects[0].file_path
|
||||
|
||||
file_ranges_to_test = generate_ranges(
|
||||
storage_objects[0], max_object_size, self.shell, self.cluster
|
||||
)
|
||||
file_ranges_to_test = generate_ranges(storage_objects[0], max_object_size, self.shell, self.cluster)
|
||||
logging.info(f"Ranges used in test {file_ranges_to_test}")
|
||||
|
||||
for range_start, range_len in file_ranges_to_test:
|
||||
|
|
@ -379,14 +359,11 @@ class TestObjectApi(ClusterTestBase):
|
|||
range_cut=range_cut,
|
||||
)
|
||||
assert (
|
||||
get_file_content(
|
||||
file_path, content_len=range_len, mode="rb", offset=range_start
|
||||
)
|
||||
get_file_content(file_path, content_len=range_len, mode="rb", offset=range_start)
|
||||
== range_content
|
||||
), f"Expected range content to match {range_cut} slice of file payload"
|
||||
|
||||
@allure.title("[NEGATIVE] Get invalid range by native API (obj_size={object_size})")
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.grpc_api
|
||||
def test_object_get_range_negatives(
|
||||
self,
|
||||
|
|
@ -421,11 +398,7 @@ class TestObjectApi(ClusterTestBase):
|
|||
|
||||
for range_start, range_len, expected_error in file_ranges_to_test:
|
||||
range_cut = f"{range_start}:{range_len}"
|
||||
expected_error = (
|
||||
expected_error.format(range=range_cut)
|
||||
if "{range}" in expected_error
|
||||
else expected_error
|
||||
)
|
||||
expected_error = expected_error.format(range=range_cut) if "{range}" in expected_error else expected_error
|
||||
with allure.step(f"Get range ({range_cut})"):
|
||||
for oid in oids:
|
||||
with pytest.raises(Exception, match=expected_error):
|
||||
|
|
@ -472,11 +445,7 @@ class TestObjectApi(ClusterTestBase):
|
|||
|
||||
for range_start, range_len, expected_error in file_ranges_to_test:
|
||||
range_cut = f"{range_start}:{range_len}"
|
||||
expected_error = (
|
||||
expected_error.format(range=range_cut)
|
||||
if "{range}" in expected_error
|
||||
else expected_error
|
||||
)
|
||||
expected_error = expected_error.format(range=range_cut) if "{range}" in expected_error else expected_error
|
||||
with allure.step(f"Get range hash ({range_cut})"):
|
||||
for oid in oids:
|
||||
with pytest.raises(Exception, match=expected_error):
|
||||
|
|
@ -491,9 +460,7 @@ class TestObjectApi(ClusterTestBase):
|
|||
|
||||
def check_header_is_presented(self, head_info: dict, object_header: dict) -> None:
|
||||
for key_to_check, val_to_check in object_header.items():
|
||||
assert (
|
||||
key_to_check in head_info["header"]["attributes"]
|
||||
), f"Key {key_to_check} is found in {head_object}"
|
||||
assert key_to_check in head_info["header"]["attributes"], f"Key {key_to_check} is found in {head_object}"
|
||||
assert head_info["header"]["attributes"].get(key_to_check) == str(
|
||||
val_to_check
|
||||
), f"Value {val_to_check} is equal"
|
||||
|
|
|
|||
|
|
@ -28,10 +28,7 @@ def bearer_token_file_all_allow(default_wallet: str, client_shell: Shell, cluste
|
|||
bearer = form_bearertoken_file(
|
||||
default_wallet,
|
||||
"",
|
||||
[
|
||||
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS)
|
||||
for op in EACLOperation
|
||||
],
|
||||
[EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) for op in EACLOperation],
|
||||
shell=client_shell,
|
||||
endpoint=cluster.default_rpc_endpoint,
|
||||
)
|
||||
|
|
@ -85,9 +82,7 @@ def storage_objects(
|
|||
@pytest.mark.smoke
|
||||
@pytest.mark.bearer
|
||||
class TestObjectApiWithBearerToken(ClusterTestBase):
|
||||
@allure.title(
|
||||
"Object can be deleted from any node using s3gate wallet with bearer token (obj_size={object_size})"
|
||||
)
|
||||
@allure.title("Object can be deleted from any node using s3gate wallet with bearer token (obj_size={object_size})")
|
||||
@pytest.mark.parametrize(
|
||||
"user_container",
|
||||
[SINGLE_PLACEMENT_RULE],
|
||||
|
|
@ -112,9 +107,7 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
|
|||
wallet_config=s3_gate_wallet.get_wallet_config_path(),
|
||||
)
|
||||
|
||||
@allure.title(
|
||||
"Object can be fetched from any node using s3gate wallet with bearer token (obj_size={object_size})"
|
||||
)
|
||||
@allure.title("Object can be fetched from any node using s3gate wallet with bearer token (obj_size={object_size})")
|
||||
@pytest.mark.parametrize(
|
||||
"user_container",
|
||||
[REP_2_FOR_3_NODES_PLACEMENT_RULE],
|
||||
|
|
|
|||
|
|
@ -14,11 +14,7 @@ from frostfs_testlib.resources.error_patterns import (
|
|||
OBJECT_NOT_FOUND,
|
||||
)
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.steps.cli.container import (
|
||||
StorageContainer,
|
||||
StorageContainerInfo,
|
||||
create_container,
|
||||
)
|
||||
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
|
||||
from frostfs_testlib.steps.cli.object import delete_object, head_object, lock_object
|
||||
from frostfs_testlib.steps.complex_object_actions import get_link_object, get_storage_object_chunks
|
||||
from frostfs_testlib.steps.epoch import ensure_fresh_epoch, get_epoch, tick_epoch
|
||||
|
|
@ -27,10 +23,7 @@ from frostfs_testlib.steps.storage_object import delete_objects
|
|||
from frostfs_testlib.steps.storage_policy import get_nodes_with_object
|
||||
from frostfs_testlib.storage.cluster import Cluster
|
||||
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||
from frostfs_testlib.storage.dataclasses.storage_object_info import (
|
||||
LockObjectInfo,
|
||||
StorageObjectInfo,
|
||||
)
|
||||
from frostfs_testlib.storage.dataclasses.storage_object_info import LockObjectInfo, StorageObjectInfo
|
||||
from frostfs_testlib.storage.dataclasses.wallet import WalletFactory, WalletInfo
|
||||
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
||||
from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success
|
||||
|
|
@ -57,9 +50,7 @@ def user_wallet(wallet_factory: WalletFactory):
|
|||
scope="module",
|
||||
)
|
||||
def user_container(user_wallet: WalletInfo, client_shell: Shell, cluster: Cluster):
|
||||
container_id = create_container(
|
||||
user_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint
|
||||
)
|
||||
container_id = create_container(user_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
|
||||
return StorageContainer(StorageContainerInfo(container_id, user_wallet), client_shell, cluster)
|
||||
|
||||
|
||||
|
|
@ -91,9 +82,7 @@ def locked_storage_object(
|
|||
lifetime=FIXTURE_LOCK_LIFETIME,
|
||||
)
|
||||
storage_object.locks = [
|
||||
LockObjectInfo(
|
||||
storage_object.cid, lock_object_id, FIXTURE_LOCK_LIFETIME, expiration_epoch
|
||||
)
|
||||
LockObjectInfo(storage_object.cid, lock_object_id, FIXTURE_LOCK_LIFETIME, expiration_epoch)
|
||||
]
|
||||
|
||||
yield storage_object
|
||||
|
|
@ -117,17 +106,13 @@ def locked_storage_object(
|
|||
except Exception as ex:
|
||||
ex_message = str(ex)
|
||||
# It's okay if object already removed
|
||||
if not re.search(OBJECT_NOT_FOUND, ex_message) and not re.search(
|
||||
OBJECT_ALREADY_REMOVED, ex_message
|
||||
):
|
||||
if not re.search(OBJECT_NOT_FOUND, ex_message) and not re.search(OBJECT_ALREADY_REMOVED, ex_message):
|
||||
raise ex
|
||||
logger.debug(ex_message)
|
||||
|
||||
|
||||
@wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME))
|
||||
def check_object_not_found(
|
||||
wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str
|
||||
):
|
||||
def check_object_not_found(wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
|
||||
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
|
||||
head_object(
|
||||
wallet_file_path,
|
||||
|
|
@ -138,9 +123,7 @@ def check_object_not_found(
|
|||
)
|
||||
|
||||
|
||||
def verify_object_available(
|
||||
wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str
|
||||
):
|
||||
def verify_object_available(wallet_file_path: str, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
|
||||
with expect_not_raises():
|
||||
head_object(
|
||||
wallet_file_path,
|
||||
|
|
@ -151,13 +134,10 @@ def verify_object_available(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.grpc_object_lock
|
||||
class TestObjectLockWithGrpc(ClusterTestBase):
|
||||
@pytest.fixture()
|
||||
def new_locked_storage_object(
|
||||
self, user_container: StorageContainer, object_size: ObjectSize
|
||||
) -> StorageObjectInfo:
|
||||
def new_locked_storage_object(self, user_container: StorageContainer, object_size: ObjectSize) -> StorageObjectInfo:
|
||||
"""
|
||||
Intention of this fixture is to provide new storage object for tests which may delete or corrupt the object or it's complementary objects
|
||||
So we need a new one each time we ask for it
|
||||
|
|
@ -284,6 +264,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
expire_at=wrong_expire_at,
|
||||
)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Expired object is deleted when locks are expired (obj_size={object_size})")
|
||||
def test_expired_object_should_be_deleted_after_locks_are_expired(
|
||||
self,
|
||||
|
|
@ -295,9 +276,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
"""
|
||||
|
||||
current_epoch = self.ensure_fresh_epoch()
|
||||
storage_object = user_container.generate_object(
|
||||
object_size.value, expire_at=current_epoch + 1
|
||||
)
|
||||
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 1)
|
||||
|
||||
with allure.step("Lock object for couple epochs"):
|
||||
lock_object(
|
||||
|
|
@ -355,9 +334,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
|
||||
with allure.step("Generate three objects"):
|
||||
for _ in range(3):
|
||||
storage_objects.append(
|
||||
user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
|
||||
)
|
||||
storage_objects.append(user_container.generate_object(object_size.value, expire_at=current_epoch + 5))
|
||||
|
||||
lock_object(
|
||||
storage_objects[0].wallet_file_path,
|
||||
|
|
@ -398,16 +375,12 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
|
||||
current_epoch = self.ensure_fresh_epoch()
|
||||
|
||||
storage_object = user_container.generate_object(
|
||||
object_size.value, expire_at=current_epoch + 1
|
||||
)
|
||||
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 1)
|
||||
|
||||
expiration_epoch = current_epoch - 1
|
||||
with pytest.raises(
|
||||
Exception,
|
||||
match=LOCK_OBJECT_EXPIRATION.format(
|
||||
expiration_epoch=expiration_epoch, current_epoch=current_epoch
|
||||
),
|
||||
match=LOCK_OBJECT_EXPIRATION.format(expiration_epoch=expiration_epoch, current_epoch=current_epoch),
|
||||
):
|
||||
lock_object(
|
||||
storage_object.wallet_file_path,
|
||||
|
|
@ -418,6 +391,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
expire_at=expiration_epoch,
|
||||
)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Delete object when lock is expired by lifetime (obj_size={object_size})")
|
||||
@expect_not_raises()
|
||||
def test_after_lock_expiration_with_lifetime_user_should_be_able_to_delete_object(
|
||||
|
|
@ -430,9 +404,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
"""
|
||||
|
||||
current_epoch = self.ensure_fresh_epoch()
|
||||
storage_object = user_container.generate_object(
|
||||
object_size.value, expire_at=current_epoch + 5
|
||||
)
|
||||
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
|
||||
|
||||
lock_object(
|
||||
storage_object.wallet_file_path,
|
||||
|
|
@ -466,9 +438,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
|
||||
current_epoch = self.ensure_fresh_epoch()
|
||||
|
||||
storage_object = user_container.generate_object(
|
||||
object_size.value, expire_at=current_epoch + 5
|
||||
)
|
||||
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 5)
|
||||
|
||||
lock_object(
|
||||
storage_object.wallet_file_path,
|
||||
|
|
@ -505,9 +475,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
Complex object chunks should also be protected from deletion
|
||||
"""
|
||||
|
||||
chunk_object_ids = get_storage_object_chunks(
|
||||
locked_storage_object, self.shell, self.cluster
|
||||
)
|
||||
chunk_object_ids = get_storage_object_chunks(locked_storage_object, self.shell, self.cluster)
|
||||
for chunk_object_id in chunk_object_ids:
|
||||
with allure.step(f"Try to delete chunk object {chunk_object_id}"):
|
||||
with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
|
||||
|
|
@ -527,9 +495,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
["complex"],
|
||||
indirect=True,
|
||||
)
|
||||
def test_link_object_of_locked_complex_object_can_be_dropped(
|
||||
self, new_locked_storage_object: StorageObjectInfo
|
||||
):
|
||||
def test_link_object_of_locked_complex_object_can_be_dropped(self, new_locked_storage_object: StorageObjectInfo):
|
||||
link_object_id = get_link_object(
|
||||
new_locked_storage_object.wallet_file_path,
|
||||
new_locked_storage_object.cid,
|
||||
|
|
@ -557,12 +523,8 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
["complex"],
|
||||
indirect=True,
|
||||
)
|
||||
def test_chunks_of_locked_complex_object_can_be_dropped(
|
||||
self, new_locked_storage_object: StorageObjectInfo
|
||||
):
|
||||
chunk_objects = get_storage_object_chunks(
|
||||
new_locked_storage_object, self.shell, self.cluster
|
||||
)
|
||||
def test_chunks_of_locked_complex_object_can_be_dropped(self, new_locked_storage_object: StorageObjectInfo):
|
||||
chunk_objects = get_storage_object_chunks(new_locked_storage_object, self.shell, self.cluster)
|
||||
|
||||
for chunk_object_id in chunk_objects:
|
||||
with allure.step(f"Drop chunk object with id {chunk_object_id} from nodes"):
|
||||
|
|
@ -630,9 +592,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
object_size: ObjectSize,
|
||||
):
|
||||
current_epoch = self.ensure_fresh_epoch()
|
||||
storage_object = user_container.generate_object(
|
||||
object_size.value, expire_at=current_epoch + 1
|
||||
)
|
||||
storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + 1)
|
||||
|
||||
with allure.step("Apply first lock to object for 3 epochs"):
|
||||
lock_object_id_0 = lock_object(
|
||||
|
|
@ -705,9 +665,8 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
self.cluster.default_rpc_endpoint,
|
||||
)
|
||||
|
||||
@allure.title(
|
||||
"Two expired objects with one lock are deleted after lock expiration (obj_size={object_size})"
|
||||
)
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Two expired objects with one lock are deleted after lock expiration (obj_size={object_size})")
|
||||
def test_two_objects_expiration_with_one_lock(
|
||||
self,
|
||||
user_container: StorageContainer,
|
||||
|
|
@ -720,9 +679,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
|
|||
with allure.step("Generate two objects"):
|
||||
for epoch_i in range(2):
|
||||
storage_objects.append(
|
||||
user_container.generate_object(
|
||||
object_size.value, expire_at=current_epoch + epoch_i + 3
|
||||
)
|
||||
user_container.generate_object(object_size.value, expire_at=current_epoch + epoch_i + 3)
|
||||
)
|
||||
|
||||
self.tick_epoch()
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ OBJECT_ATTRIBUTES = {"common_key": "common_value"}
|
|||
WAIT_FOR_REPLICATION = 60
|
||||
|
||||
# Adding failover mark because it may make cluster unhealthy
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.failover
|
||||
@pytest.mark.replication
|
||||
class TestReplication(ClusterTestBase):
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ from frostfs_testlib.utils.file_utils import generate_file
|
|||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.http_gate
|
||||
@pytest.mark.skip("Skipped due to deprecated PUT via http")
|
||||
@pytest.mark.http_put
|
||||
|
|
@ -46,9 +45,7 @@ class Test_http_bearer(ClusterTestBase):
|
|||
@pytest.fixture(scope="class")
|
||||
def eacl_deny_for_others(self, user_container: str) -> None:
|
||||
with allure.step(f"Set deny all operations for {EACLRole.OTHERS} via eACL"):
|
||||
eacl = EACLRule(
|
||||
access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=EACLOperation.PUT
|
||||
)
|
||||
eacl = EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=EACLOperation.PUT)
|
||||
set_eacl(
|
||||
self.wallet,
|
||||
user_container,
|
||||
|
|
@ -64,10 +61,7 @@ class Test_http_bearer(ClusterTestBase):
|
|||
bearer = form_bearertoken_file(
|
||||
self.wallet,
|
||||
user_container,
|
||||
[
|
||||
EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS)
|
||||
for op in EACLOperation
|
||||
],
|
||||
[EACLRule(operation=op, access=EACLAccess.ALLOW, role=EACLRole.OTHERS) for op in EACLOperation],
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
sign=False,
|
||||
|
|
@ -105,9 +99,7 @@ class Test_http_bearer(ClusterTestBase):
|
|||
eacl_deny_for_others
|
||||
bearer = bearer_token_no_limit_for_others
|
||||
file_path = generate_file(object_size.value)
|
||||
with allure.step(
|
||||
f"Put object with bearer token for {EACLRole.OTHERS}, then get and verify hashes"
|
||||
):
|
||||
with allure.step(f"Put object with bearer token for {EACLRole.OTHERS}, then get and verify hashes"):
|
||||
headers = [f" -H 'Authorization: Bearer {bearer}'"]
|
||||
oid = upload_via_http_gate_curl(
|
||||
cid=user_container,
|
||||
|
|
|
|||
|
|
@ -44,9 +44,7 @@ class TestHttpGate(ClusterTestBase):
|
|||
TestHttpGate.wallet = default_wallet
|
||||
|
||||
@allure.title("Put over gRPC, Get over HTTP")
|
||||
def test_put_grpc_get_http(
|
||||
self, complex_object_size: ObjectSize, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_put_grpc_get_http(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
|
||||
"""
|
||||
Test that object can be put using gRPC interface and get using HTTP.
|
||||
|
||||
|
|
@ -106,7 +104,6 @@ class TestHttpGate(ClusterTestBase):
|
|||
)
|
||||
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
|
||||
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.http_gate
|
||||
@pytest.mark.http_put
|
||||
@pytest.mark.skip("Skipped due to deprecated PUT via http")
|
||||
|
|
@ -115,9 +112,7 @@ class TestHttpPut(ClusterTestBase):
|
|||
@allure.link("https://github.com/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
|
||||
@allure.title("Put over HTTP, Get over HTTP")
|
||||
@pytest.mark.smoke
|
||||
def test_put_http_get_http(
|
||||
self, complex_object_size: ObjectSize, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_put_http_get_http(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
|
||||
"""
|
||||
Test that object can be put and get using HTTP interface.
|
||||
|
||||
|
|
@ -165,19 +160,17 @@ class TestHttpPut(ClusterTestBase):
|
|||
name="download by attributes",
|
||||
)
|
||||
@pytest.mark.skip("Skipped due to deprecated PUT via http")
|
||||
@allure.title("Put over HTTP, Get over HTTP with header")
|
||||
@allure.title("Put over HTTP, Get over HTTP with {id} header")
|
||||
@pytest.mark.parametrize(
|
||||
"attributes",
|
||||
"attributes,id",
|
||||
[
|
||||
{"fileName": "simple_obj_filename"},
|
||||
{"file-Name": "simple obj filename"},
|
||||
{"cat%jpeg": "cat%jpeg"},
|
||||
({"fileName": "simple_obj_filename"}, "simple"),
|
||||
({"file-Name": "simple obj filename"}, "hyphen"),
|
||||
({"cat%jpeg": "cat%jpeg"}, "percent"),
|
||||
],
|
||||
ids=["simple", "hyphen", "percent"],
|
||||
)
|
||||
def test_put_http_get_http_with_headers(
|
||||
self, attributes: dict, simple_object_size: ObjectSize, request: pytest.FixtureRequest
|
||||
):
|
||||
def test_put_http_get_http_with_headers(self, attributes: dict, simple_object_size: ObjectSize, id: str):
|
||||
"""
|
||||
Test that object can be downloaded using different attributes in HTTP header.
|
||||
|
||||
|
|
@ -190,7 +183,6 @@ class TestHttpPut(ClusterTestBase):
|
|||
Expected result:
|
||||
Hashes must be the same.
|
||||
"""
|
||||
allure.dynamic.title(f"Put over HTTP, Get over HTTP with {request.node.callspec.id} header")
|
||||
cid = create_container(
|
||||
self.wallet,
|
||||
shell=self.shell,
|
||||
|
|
@ -341,9 +333,7 @@ class TestHttpPut(ClusterTestBase):
|
|||
file_path = generate_file(complex_object_size.value)
|
||||
|
||||
with allure.step("Put objects using HTTP"):
|
||||
oid_gate = upload_via_http_gate(
|
||||
cid=cid, path=file_path, endpoint=self.cluster.default_http_gate_endpoint
|
||||
)
|
||||
oid_gate = upload_via_http_gate(cid=cid, path=file_path, endpoint=self.cluster.default_http_gate_endpoint)
|
||||
oid_curl = upload_via_http_gate_curl(
|
||||
cid=cid,
|
||||
filepath=file_path,
|
||||
|
|
@ -374,9 +364,7 @@ class TestHttpPut(ClusterTestBase):
|
|||
|
||||
@pytest.mark.skip("Skipped due to deprecated PUT via http")
|
||||
@allure.title("Put/Get over HTTP using Curl utility")
|
||||
def test_put_http_get_http_curl(
|
||||
self, complex_object_size: ObjectSize, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_put_http_get_http_curl(self, complex_object_size: ObjectSize, simple_object_size: ObjectSize):
|
||||
"""
|
||||
Test checks upload and download over HTTP using curl utility.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ OBJECT_ALREADY_REMOVED_ERROR = "object already removed"
|
|||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.http_gate
|
||||
@pytest.mark.http_put
|
||||
@pytest.mark.skip("Skipped due to deprecated PUT via http")
|
||||
|
|
@ -79,9 +78,7 @@ class Test_http_headers(ClusterTestBase):
|
|||
yield storage_objects
|
||||
|
||||
@allure.title("Get object1 by attribute")
|
||||
def test_object1_can_be_get_by_attr(
|
||||
self, storage_objects_with_attributes: list[StorageObjectInfo]
|
||||
):
|
||||
def test_object1_can_be_get_by_attr(self, storage_objects_with_attributes: list[StorageObjectInfo]):
|
||||
"""
|
||||
Test to get object#1 by attribute and comapre hashes
|
||||
|
||||
|
|
@ -104,9 +101,7 @@ class Test_http_headers(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("Get object2 with different attributes, then delete object2 and get object1")
|
||||
def test_object2_can_be_get_by_attr(
|
||||
self, storage_objects_with_attributes: list[StorageObjectInfo]
|
||||
):
|
||||
def test_object2_can_be_get_by_attr(self, storage_objects_with_attributes: list[StorageObjectInfo]):
|
||||
"""
|
||||
Test to get object2 with different attributes, then delete object2 and get object1 using 1st attribute. Note: obj1 and obj2 have the same attribute#1,
|
||||
and when obj2 is deleted you can get obj1 by 1st attribute
|
||||
|
|
@ -167,9 +162,7 @@ class Test_http_headers(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("[NEGATIVE] Put object and get right after container is deleted")
|
||||
def test_negative_put_and_get_object3(
|
||||
self, storage_objects_with_attributes: list[StorageObjectInfo]
|
||||
):
|
||||
def test_negative_put_and_get_object3(self, storage_objects_with_attributes: list[StorageObjectInfo]):
|
||||
"""
|
||||
Test to attempt to put object and try to download it right after the container has been deleted
|
||||
|
||||
|
|
@ -203,6 +196,7 @@ class Test_http_headers(ClusterTestBase):
|
|||
cid=storage_object_1.cid,
|
||||
shell=self.shell,
|
||||
endpoint=self.cluster.default_rpc_endpoint,
|
||||
await_mode=True,
|
||||
)
|
||||
self.tick_epoch()
|
||||
wait_for_container_deletion(
|
||||
|
|
@ -214,9 +208,7 @@ class Test_http_headers(ClusterTestBase):
|
|||
assert storage_object_1.cid not in list_containers(
|
||||
self.wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
|
||||
)
|
||||
with allure.step(
|
||||
"[Negative] Try to download (wget) object via wget with attributes [peace=peace]"
|
||||
):
|
||||
with allure.step("[Negative] Try to download (wget) object via wget with attributes [peace=peace]"):
|
||||
request = f"/get/{storage_object_1.cid}/peace/peace"
|
||||
error_pattern = "404 Not Found"
|
||||
try_to_get_object_via_passed_request_and_expect_error(
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ from frostfs_testlib.utils.file_utils import generate_file
|
|||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.http_gate
|
||||
@pytest.mark.http_put
|
||||
@pytest.mark.skip("Skipped due to deprecated PUT via http")
|
||||
|
|
@ -50,9 +49,7 @@ class Test_http_streaming(ClusterTestBase):
|
|||
# Generate file
|
||||
file_path = generate_file(complex_object_size.value)
|
||||
|
||||
with allure.step(
|
||||
"Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]"
|
||||
):
|
||||
with allure.step("Put objects using curl utility and Get object and verify hashes [ get/$CID/$OID ]"):
|
||||
oid = upload_via_http_gate_curl(
|
||||
cid=cid, filepath=file_path, endpoint=self.cluster.default_http_gate_endpoint
|
||||
)
|
||||
|
|
|
|||
|
|
@ -8,11 +8,7 @@ import pytest
|
|||
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
|
||||
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
|
||||
from frostfs_testlib.steps.cli.container import create_container
|
||||
from frostfs_testlib.steps.cli.object import (
|
||||
get_netmap_netinfo,
|
||||
get_object_from_random_node,
|
||||
head_object,
|
||||
)
|
||||
from frostfs_testlib.steps.cli.object import get_netmap_netinfo, get_object_from_random_node, head_object
|
||||
from frostfs_testlib.steps.epoch import get_epoch, wait_for_epochs_align
|
||||
from frostfs_testlib.steps.http.http_gate import (
|
||||
attr_into_str_header_curl,
|
||||
|
|
@ -35,7 +31,6 @@ SYSTEM_EXPIRATION_TIMESTAMP = "System-Expiration-Timestamp"
|
|||
SYSTEM_EXPIRATION_RFC3339 = "System-Expiration-RFC3339"
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.http_gate
|
||||
@pytest.mark.http_put
|
||||
@pytest.mark.skip("Skipped due to deprecated PUT via http")
|
||||
|
|
@ -76,9 +71,7 @@ class Test_http_system_header(ClusterTestBase):
|
|||
return f"{mins}m"
|
||||
|
||||
@allure.title("Return future timestamp after N epochs are passed")
|
||||
def epoch_count_into_timestamp(
|
||||
self, epoch_duration: int, epoch: int, rfc3339: Optional[bool] = False
|
||||
) -> str:
|
||||
def epoch_count_into_timestamp(self, epoch_duration: int, epoch: int, rfc3339: Optional[bool] = False) -> str:
|
||||
current_datetime = datetime.datetime.utcnow()
|
||||
epoch_count_in_seconds = epoch_duration * epoch
|
||||
future_datetime = current_datetime + datetime.timedelta(seconds=epoch_count_in_seconds)
|
||||
|
|
@ -96,9 +89,7 @@ class Test_http_system_header(ClusterTestBase):
|
|||
return False
|
||||
return True
|
||||
|
||||
@allure.title(
|
||||
f"Validate that only {EXPIRATION_EPOCH_HEADER} exists in header and other headers are abesent"
|
||||
)
|
||||
@allure.title(f"Validate that only {EXPIRATION_EPOCH_HEADER} exists in header and other headers are abesent")
|
||||
def validation_for_http_header_attr(self, head_info: dict, expected_epoch: int) -> None:
|
||||
# check that __SYSTEM__EXPIRATION_EPOCH attribute has corresponding epoch
|
||||
assert self.check_key_value_presented_header(
|
||||
|
|
@ -146,13 +137,9 @@ class Test_http_system_header(ClusterTestBase):
|
|||
|
||||
@allure.title("[NEGATIVE] Put object with expired epoch")
|
||||
def test_unable_put_expired_epoch(self, user_container: str, simple_object_size: ObjectSize):
|
||||
headers = attr_into_str_header_curl(
|
||||
{"System-Expiration-Epoch": str(get_epoch(self.shell, self.cluster) - 1)}
|
||||
)
|
||||
headers = attr_into_str_header_curl({"System-Expiration-Epoch": str(get_epoch(self.shell, self.cluster) - 1)})
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
with allure.step(
|
||||
"Put object using HTTP with attribute Expiration-Epoch where epoch is expired"
|
||||
):
|
||||
with allure.step("Put object using HTTP with attribute Expiration-Epoch where epoch is expired"):
|
||||
upload_via_http_gate_curl(
|
||||
cid=user_container,
|
||||
filepath=file_path,
|
||||
|
|
@ -162,14 +149,10 @@ class Test_http_system_header(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("[NEGATIVE] Put object with negative System-Expiration-Duration")
|
||||
def test_unable_put_negative_duration(
|
||||
self, user_container: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_unable_put_negative_duration(self, user_container: str, simple_object_size: ObjectSize):
|
||||
headers = attr_into_str_header_curl({"System-Expiration-Duration": "-1h"})
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
with allure.step(
|
||||
"Put object using HTTP with attribute System-Expiration-Duration where duration is negative"
|
||||
):
|
||||
with allure.step("Put object using HTTP with attribute System-Expiration-Duration where duration is negative"):
|
||||
upload_via_http_gate_curl(
|
||||
cid=user_container,
|
||||
filepath=file_path,
|
||||
|
|
@ -179,9 +162,7 @@ class Test_http_system_header(ClusterTestBase):
|
|||
)
|
||||
|
||||
@allure.title("[NEGATIVE] Put object with System-Expiration-Timestamp value in the past")
|
||||
def test_unable_put_expired_timestamp(
|
||||
self, user_container: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_unable_put_expired_timestamp(self, user_container: str, simple_object_size: ObjectSize):
|
||||
headers = attr_into_str_header_curl({"System-Expiration-Timestamp": "1635075727"})
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
with allure.step(
|
||||
|
|
@ -211,9 +192,7 @@ class Test_http_system_header(ClusterTestBase):
|
|||
|
||||
@allure.title("Priority of attributes epoch>duration (obj_size={object_size})")
|
||||
@pytest.mark.skip("Temp disable for v0.37")
|
||||
def test_http_attr_priority_epoch_duration(
|
||||
self, user_container: str, object_size: ObjectSize, epoch_duration: int
|
||||
):
|
||||
def test_http_attr_priority_epoch_duration(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
|
||||
self.tick_epoch()
|
||||
epoch_count = 1
|
||||
expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count
|
||||
|
|
@ -247,15 +226,11 @@ class Test_http_system_header(ClusterTestBase):
|
|||
)
|
||||
# check that object is not available via grpc
|
||||
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
|
||||
get_object_from_random_node(
|
||||
self.wallet, user_container, oid, self.shell, self.cluster
|
||||
)
|
||||
get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
|
||||
|
||||
@allure.title("Priority of attributes duration>timestamp (obj_size={object_size})")
|
||||
@pytest.mark.skip("Temp disable for v0.37")
|
||||
def test_http_attr_priority_dur_timestamp(
|
||||
self, user_container: str, object_size: ObjectSize, epoch_duration: int
|
||||
):
|
||||
def test_http_attr_priority_dur_timestamp(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
|
||||
self.tick_epoch()
|
||||
epoch_count = 2
|
||||
expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count
|
||||
|
|
@ -263,12 +238,8 @@ class Test_http_system_header(ClusterTestBase):
|
|||
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
|
||||
)
|
||||
attributes = {
|
||||
SYSTEM_EXPIRATION_DURATION: self.epoch_count_into_mins(
|
||||
epoch_duration=epoch_duration, epoch=2
|
||||
),
|
||||
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(
|
||||
epoch_duration=epoch_duration, epoch=1
|
||||
),
|
||||
SYSTEM_EXPIRATION_DURATION: self.epoch_count_into_mins(epoch_duration=epoch_duration, epoch=2),
|
||||
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(epoch_duration=epoch_duration, epoch=1),
|
||||
}
|
||||
file_path = generate_file(object_size.value)
|
||||
with allure.step(
|
||||
|
|
@ -296,15 +267,11 @@ class Test_http_system_header(ClusterTestBase):
|
|||
)
|
||||
# check that object is not available via grpc
|
||||
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
|
||||
get_object_from_random_node(
|
||||
self.wallet, user_container, oid, self.shell, self.cluster
|
||||
)
|
||||
get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
|
||||
|
||||
@allure.title("Priority of attributes timestamp>Expiration-RFC (obj_size={object_size})")
|
||||
@pytest.mark.skip("Temp disable for v0.37")
|
||||
def test_http_attr_priority_timestamp_rfc(
|
||||
self, user_container: str, object_size: ObjectSize, epoch_duration: int
|
||||
):
|
||||
def test_http_attr_priority_timestamp_rfc(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
|
||||
self.tick_epoch()
|
||||
epoch_count = 2
|
||||
expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count
|
||||
|
|
@ -312,9 +279,7 @@ class Test_http_system_header(ClusterTestBase):
|
|||
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
|
||||
)
|
||||
attributes = {
|
||||
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(
|
||||
epoch_duration=epoch_duration, epoch=2
|
||||
),
|
||||
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(epoch_duration=epoch_duration, epoch=2),
|
||||
SYSTEM_EXPIRATION_RFC3339: self.epoch_count_into_timestamp(
|
||||
epoch_duration=epoch_duration, epoch=1, rfc3339=True
|
||||
),
|
||||
|
|
@ -345,9 +310,7 @@ class Test_http_system_header(ClusterTestBase):
|
|||
)
|
||||
# check that object is not available via grpc
|
||||
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
|
||||
get_object_from_random_node(
|
||||
self.wallet, user_container, oid, self.shell, self.cluster
|
||||
)
|
||||
get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
|
||||
|
||||
@allure.title("Object should be deleted when expiration passed (obj_size={object_size})")
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -399,6 +362,4 @@ class Test_http_system_header(ClusterTestBase):
|
|||
)
|
||||
# check that object is not available via grpc
|
||||
with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
|
||||
get_object_from_random_node(
|
||||
self.wallet, user_container, oid, self.shell, self.cluster
|
||||
)
|
||||
get_object_from_random_node(self.wallet, user_container, oid, self.shell, self.cluster)
|
||||
|
|
|
|||
|
|
@ -6,15 +6,12 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
|||
from frostfs_testlib.utils.file_utils import generate_file
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.acl
|
||||
@pytest.mark.s3_gate
|
||||
class TestS3GateACL:
|
||||
@allure.title("Object ACL (s3_client={s3_client})")
|
||||
@pytest.mark.parametrize("s3_client", [AwsCliClient], indirect=True)
|
||||
def test_s3_object_ACL(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_object_ACL(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
file_name = s3_helper.object_key_from_file_path(file_path)
|
||||
|
||||
|
|
@ -33,9 +30,7 @@ class TestS3GateACL:
|
|||
obj_acl = s3_client.get_object_acl(bucket, file_name)
|
||||
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
|
||||
|
||||
with allure.step(
|
||||
"Put object with grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
|
||||
):
|
||||
with allure.step("Put object with grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"):
|
||||
s3_client.put_object_acl(
|
||||
bucket,
|
||||
file_name,
|
||||
|
|
@ -48,9 +43,7 @@ class TestS3GateACL:
|
|||
@pytest.mark.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
|
||||
def test_s3_bucket_ACL(self, s3_client: S3ClientWrapper):
|
||||
with allure.step("Create bucket with ACL = public-read-write"):
|
||||
bucket = s3_client.create_bucket(
|
||||
object_lock_enabled_for_bucket=True, acl="public-read-write"
|
||||
)
|
||||
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read-write")
|
||||
bucket_acl = s3_client.get_bucket_acl(bucket)
|
||||
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="AllUsers")
|
||||
|
||||
|
|
@ -59,9 +52,7 @@ class TestS3GateACL:
|
|||
bucket_acl = s3_client.get_bucket_acl(bucket)
|
||||
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser")
|
||||
|
||||
with allure.step(
|
||||
"Change bucket acl to --grant-write uri=http://acs.amazonaws.com/groups/global/AllUsers"
|
||||
):
|
||||
with allure.step("Change bucket acl to --grant-write uri=http://acs.amazonaws.com/groups/global/AllUsers"):
|
||||
s3_client.put_bucket_acl(
|
||||
bucket,
|
||||
grant_write="uri=http://acs.amazonaws.com/groups/global/AllUsers",
|
||||
|
|
|
|||
|
|
@ -2,18 +2,12 @@ from datetime import datetime, timedelta
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper
|
||||
from frostfs_testlib.s3 import S3ClientWrapper
|
||||
from frostfs_testlib.steps.s3 import s3_helper
|
||||
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||
from frostfs_testlib.utils.file_utils import generate_file
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc):
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.s3_gate
|
||||
@pytest.mark.s3_gate_bucket
|
||||
class TestS3GateBucket:
|
||||
|
|
@ -26,23 +20,17 @@ class TestS3GateBucket:
|
|||
s3_helper.assert_s3_acl(acl_grants=bucket_acl, permitted_users="CanonicalUser")
|
||||
|
||||
with allure.step("Create bucket with ACL = public-read"):
|
||||
bucket_1 = s3_client.create_bucket(
|
||||
object_lock_enabled_for_bucket=True, acl="public-read"
|
||||
)
|
||||
bucket_1 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read")
|
||||
bucket_acl_1 = s3_client.get_bucket_acl(bucket_1)
|
||||
s3_helper.assert_s3_acl(acl_grants=bucket_acl_1, permitted_users="AllUsers")
|
||||
|
||||
with allure.step("Create bucket with ACL public-read-write"):
|
||||
bucket_2 = s3_client.create_bucket(
|
||||
object_lock_enabled_for_bucket=True, acl="public-read-write"
|
||||
)
|
||||
bucket_2 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read-write")
|
||||
bucket_acl_2 = s3_client.get_bucket_acl(bucket_2)
|
||||
s3_helper.assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers")
|
||||
|
||||
with allure.step("Create bucket with ACL = authenticated-read"):
|
||||
bucket_3 = s3_client.create_bucket(
|
||||
object_lock_enabled_for_bucket=True, acl="authenticated-read"
|
||||
)
|
||||
bucket_3 = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="authenticated-read")
|
||||
bucket_acl_3 = s3_client.get_bucket_acl(bucket_3)
|
||||
s3_helper.assert_s3_acl(acl_grants=bucket_acl_3, permitted_users="AllUsers")
|
||||
|
||||
|
|
@ -74,18 +62,14 @@ class TestS3GateBucket:
|
|||
s3_helper.assert_s3_acl(acl_grants=bucket_acl_2, permitted_users="AllUsers")
|
||||
|
||||
@allure.title("Create bucket with object lock (s3_client={s3_client})")
|
||||
def test_s3_bucket_object_lock(
|
||||
self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_bucket_object_lock(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
file_name = s3_helper.object_key_from_file_path(file_path)
|
||||
|
||||
with allure.step("Create bucket with --no-object-lock-enabled-for-bucket"):
|
||||
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=False)
|
||||
date_obj = datetime.utcnow() + timedelta(days=1)
|
||||
with pytest.raises(
|
||||
Exception, match=r".*Object Lock configuration does not exist for this bucket.*"
|
||||
):
|
||||
with pytest.raises(Exception, match=r".*Object Lock configuration does not exist for this bucket.*"):
|
||||
# An error occurred (ObjectLockConfigurationNotFoundError) when calling the PutObject operation (reached max retries: 0):
|
||||
# Object Lock configuration does not exist for this bucket
|
||||
s3_client.put_object(
|
||||
|
|
@ -104,9 +88,7 @@ class TestS3GateBucket:
|
|||
object_lock_retain_until_date=date_obj_1.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
object_lock_legal_hold_status="ON",
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(
|
||||
s3_client, bucket_1, file_name, "COMPLIANCE", date_obj_1, "ON"
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(s3_client, bucket_1, file_name, "COMPLIANCE", date_obj_1, "ON")
|
||||
|
||||
@allure.title("Delete bucket (s3_client={s3_client})")
|
||||
def test_s3_delete_bucket(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
|
||||
|
|
|
|||
|
|
@ -22,14 +22,7 @@ from frostfs_testlib.utils.file_utils import (
|
|||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc):
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
|
||||
|
||||
|
||||
@allure.link(
|
||||
"https://github.com/TrueCloudLab/frostfs-s3-gw#frostfs-s3-gw", name="frostfs-s3-gateway"
|
||||
)
|
||||
@allure.link("https://github.com/TrueCloudLab/frostfs-s3-gw#frostfs-s3-gw", name="frostfs-s3-gateway")
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.s3_gate
|
||||
@pytest.mark.s3_gate_base
|
||||
|
|
@ -73,9 +66,7 @@ class TestS3Gate:
|
|||
s3_client.head_object(bucket_1, file_name)
|
||||
|
||||
bucket_objects = s3_client.list_objects(bucket_1)
|
||||
assert (
|
||||
file_name in bucket_objects
|
||||
), f"Expected file {file_name} in objects list {bucket_objects}"
|
||||
assert file_name in bucket_objects, f"Expected file {file_name} in objects list {bucket_objects}"
|
||||
|
||||
with allure.step("Try to delete not empty bucket and get error"):
|
||||
with pytest.raises(Exception, match=r".*The bucket you tried to delete is not empty.*"):
|
||||
|
|
@ -136,18 +127,14 @@ class TestS3Gate:
|
|||
s3_client.head_object(bucket, file_name)
|
||||
|
||||
bucket_objects = s3_client.list_objects(bucket)
|
||||
assert (
|
||||
file_name in bucket_objects
|
||||
), f"Expected file {file_name} in objects list {bucket_objects}"
|
||||
assert file_name in bucket_objects, f"Expected file {file_name} in objects list {bucket_objects}"
|
||||
|
||||
with allure.step("Check object's attributes"):
|
||||
for attrs in (["ETag"], ["ObjectSize", "StorageClass"]):
|
||||
s3_client.get_object_attributes(bucket, file_name, attrs)
|
||||
|
||||
@allure.title("Sync directory (s3_client={s3_client})")
|
||||
def test_s3_sync_dir(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_sync_dir(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
"""
|
||||
Test checks sync directory with AWS CLI utility.
|
||||
"""
|
||||
|
|
@ -167,9 +154,7 @@ class TestS3Gate:
|
|||
objects = s3_client.list_objects(bucket)
|
||||
|
||||
with allure.step("Check these are the same objects"):
|
||||
assert set(key_to_path.keys()) == set(
|
||||
objects
|
||||
), f"Expected all objects saved. Got {objects}"
|
||||
assert set(key_to_path.keys()) == set(objects), f"Expected all objects saved. Got {objects}"
|
||||
for obj_key in objects:
|
||||
got_object = s3_client.get_object(bucket, obj_key)
|
||||
assert get_file_hash(got_object) == get_file_hash(
|
||||
|
|
@ -177,32 +162,24 @@ class TestS3Gate:
|
|||
), "Expected hashes are the same"
|
||||
|
||||
@allure.title("Object versioning (s3_client={s3_client})")
|
||||
def test_s3_api_versioning(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_api_versioning(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
"""
|
||||
Test checks basic versioning functionality for S3 bucket.
|
||||
"""
|
||||
version_1_content = "Version 1"
|
||||
version_2_content = "Version 2"
|
||||
file_name_simple = generate_file_with_content(
|
||||
simple_object_size.value, content=version_1_content
|
||||
)
|
||||
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
|
||||
obj_key = os.path.basename(file_name_simple)
|
||||
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
|
||||
|
||||
with allure.step("Put several versions of object into bucket"):
|
||||
version_id_1 = s3_client.put_object(bucket, file_name_simple)
|
||||
generate_file_with_content(
|
||||
simple_object_size.value, file_path=file_name_simple, content=version_2_content
|
||||
)
|
||||
generate_file_with_content(simple_object_size.value, file_path=file_name_simple, content=version_2_content)
|
||||
version_id_2 = s3_client.put_object(bucket, file_name_simple)
|
||||
|
||||
with allure.step("Check bucket shows all versions"):
|
||||
versions = s3_client.list_objects_versions(bucket)
|
||||
obj_versions = {
|
||||
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
||||
}
|
||||
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
|
||||
assert obj_versions == {
|
||||
version_id_1,
|
||||
version_id_2,
|
||||
|
|
@ -213,20 +190,14 @@ class TestS3Gate:
|
|||
response = s3_client.head_object(bucket, obj_key, version_id=version_id)
|
||||
assert "LastModified" in response, "Expected LastModified field"
|
||||
assert "ETag" in response, "Expected ETag field"
|
||||
assert (
|
||||
response.get("VersionId") == version_id
|
||||
), f"Expected VersionId is {version_id}"
|
||||
assert response.get("VersionId") == version_id, f"Expected VersionId is {version_id}"
|
||||
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
|
||||
|
||||
with allure.step("Check object's attributes"):
|
||||
for version_id in (version_id_1, version_id_2):
|
||||
got_attrs = s3_client.get_object_attributes(
|
||||
bucket, obj_key, ["ETag"], version_id=version_id
|
||||
)
|
||||
got_attrs = s3_client.get_object_attributes(bucket, obj_key, ["ETag"], version_id=version_id)
|
||||
if got_attrs:
|
||||
assert (
|
||||
got_attrs.get("VersionId") == version_id
|
||||
), f"Expected VersionId is {version_id}"
|
||||
assert got_attrs.get("VersionId") == version_id, f"Expected VersionId is {version_id}"
|
||||
|
||||
with allure.step("Delete object and check it was deleted"):
|
||||
response = s3_client.delete_object(bucket, obj_key)
|
||||
|
|
@ -242,9 +213,7 @@ class TestS3Gate:
|
|||
):
|
||||
file_name = s3_client.get_object(bucket, obj_key, version_id=version)
|
||||
got_content = get_file_content(file_name)
|
||||
assert (
|
||||
got_content == content
|
||||
), f"Expected object content is\n{content}\nGot\n{got_content}"
|
||||
assert got_content == content, f"Expected object content is\n{content}\nGot\n{got_content}"
|
||||
|
||||
with allure.step("Restore previous object version"):
|
||||
s3_client.delete_object(bucket, obj_key, version_id=version_id_delete)
|
||||
|
|
@ -257,17 +226,13 @@ class TestS3Gate:
|
|||
|
||||
@pytest.mark.s3_gate_multipart
|
||||
@allure.title("Object Multipart API (s3_client={s3_client})")
|
||||
def test_s3_api_multipart(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_api_multipart(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
"""
|
||||
Test checks S3 Multipart API (Create multipart upload/Abort multipart upload/List multipart upload/
|
||||
Upload part/List parts/Complete multipart upload).
|
||||
"""
|
||||
parts_count = 3
|
||||
file_name_large = generate_file(
|
||||
simple_object_size.value * 1024 * 6 * parts_count
|
||||
) # 5Mb - min part
|
||||
file_name_large = generate_file(simple_object_size.value * 1024 * 6 * parts_count) # 5Mb - min part
|
||||
object_key = s3_helper.object_key_from_file_path(file_name_large)
|
||||
part_files = split_file(file_name_large, parts_count)
|
||||
parts = []
|
||||
|
|
@ -279,12 +244,8 @@ class TestS3Gate:
|
|||
upload_id = s3_client.create_multipart_upload(bucket, object_key)
|
||||
uploads = s3_client.list_multipart_uploads(bucket)
|
||||
assert uploads, f"Expected there one upload in bucket {bucket}"
|
||||
assert (
|
||||
uploads[0].get("Key") == object_key
|
||||
), f"Expected correct key {object_key} in upload {uploads}"
|
||||
assert (
|
||||
uploads[0].get("UploadId") == upload_id
|
||||
), f"Expected correct UploadId {upload_id} in upload {uploads}"
|
||||
assert uploads[0].get("Key") == object_key, f"Expected correct key {object_key} in upload {uploads}"
|
||||
assert uploads[0].get("UploadId") == upload_id, f"Expected correct UploadId {upload_id} in upload {uploads}"
|
||||
|
||||
s3_client.abort_multipart_upload(bucket, object_key, upload_id)
|
||||
uploads = s3_client.list_multipart_uploads(bucket)
|
||||
|
|
@ -298,9 +259,7 @@ class TestS3Gate:
|
|||
|
||||
with allure.step("Check all parts are visible in bucket"):
|
||||
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
|
||||
assert len(got_parts) == len(
|
||||
part_files
|
||||
), f"Expected {parts_count} parts, got\n{got_parts}"
|
||||
assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
|
||||
|
||||
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
|
||||
|
||||
|
|
@ -327,9 +286,7 @@ class TestS3Gate:
|
|||
s3_helper.check_tags_by_bucket(s3_client, bucket, [])
|
||||
|
||||
@allure.title("Object tagging API (s3_client={s3_client})")
|
||||
def test_s3_api_object_tagging(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_api_object_tagging(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
"""
|
||||
Test checks S3 Object tagging API (Put tag/Get tag/Update tag).
|
||||
"""
|
||||
|
|
@ -458,9 +415,7 @@ class TestS3Gate:
|
|||
|
||||
with allure.step("Check copied object has the same content"):
|
||||
got_copied_file = s3_client.get_object(bucket, copy_obj_path)
|
||||
assert get_file_hash(file_path_simple) == get_file_hash(
|
||||
got_copied_file
|
||||
), "Hashes must be the same"
|
||||
assert get_file_hash(file_path_simple) == get_file_hash(got_copied_file), "Hashes must be the same"
|
||||
|
||||
with allure.step("Delete one object from bucket"):
|
||||
s3_client.delete_object(bucket, file_name_simple)
|
||||
|
|
@ -509,9 +464,7 @@ class TestS3Gate:
|
|||
|
||||
with allure.step("Check copied object has the same content"):
|
||||
got_copied_file_b2 = s3_client.get_object(bucket_2, copy_obj_path_b2)
|
||||
assert get_file_hash(file_path_large) == get_file_hash(
|
||||
got_copied_file_b2
|
||||
), "Hashes must be the same"
|
||||
assert get_file_hash(file_path_large) == get_file_hash(got_copied_file_b2), "Hashes must be the same"
|
||||
|
||||
with allure.step("Delete one object from first bucket"):
|
||||
s3_client.delete_object(bucket_1, file_name_simple)
|
||||
|
|
@ -524,23 +477,15 @@ class TestS3Gate:
|
|||
s3_client.delete_object(bucket_2, copy_obj_path_b2)
|
||||
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[])
|
||||
|
||||
def check_object_attributes(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, object_key: str, parts_count: int
|
||||
):
|
||||
def check_object_attributes(self, s3_client: S3ClientWrapper, bucket: str, object_key: str, parts_count: int):
|
||||
if not isinstance(s3_client, AwsCliClient):
|
||||
logger.warning("Attributes check is not supported for boto3 implementation")
|
||||
return
|
||||
|
||||
with allure.step("Check object's attributes"):
|
||||
obj_parts = s3_client.get_object_attributes(
|
||||
bucket, object_key, ["ObjectParts"], full_output=False
|
||||
)
|
||||
assert (
|
||||
obj_parts.get("TotalPartsCount") == parts_count
|
||||
), f"Expected TotalPartsCount is {parts_count}"
|
||||
assert (
|
||||
len(obj_parts.get("Parts")) == parts_count
|
||||
), f"Expected Parts cunt is {parts_count}"
|
||||
obj_parts = s3_client.get_object_attributes(bucket, object_key, ["ObjectParts"], full_output=False)
|
||||
assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
|
||||
assert len(obj_parts.get("Parts")) == parts_count, f"Expected Parts cunt is {parts_count}"
|
||||
|
||||
with allure.step("Check object's attribute max-parts"):
|
||||
max_parts = 2
|
||||
|
|
@ -551,13 +496,9 @@ class TestS3Gate:
|
|||
max_parts=max_parts,
|
||||
full_output=False,
|
||||
)
|
||||
assert (
|
||||
obj_parts.get("TotalPartsCount") == parts_count
|
||||
), f"Expected TotalPartsCount is {parts_count}"
|
||||
assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
|
||||
assert obj_parts.get("MaxParts") == max_parts, f"Expected MaxParts is {parts_count}"
|
||||
assert (
|
||||
len(obj_parts.get("Parts")) == max_parts
|
||||
), f"Expected Parts count is {parts_count}"
|
||||
assert len(obj_parts.get("Parts")) == max_parts, f"Expected Parts count is {parts_count}"
|
||||
|
||||
with allure.step("Check object's attribute part-number-marker"):
|
||||
part_number_marker = 3
|
||||
|
|
@ -568,9 +509,7 @@ class TestS3Gate:
|
|||
part_number=part_number_marker,
|
||||
full_output=False,
|
||||
)
|
||||
assert (
|
||||
obj_parts.get("TotalPartsCount") == parts_count
|
||||
), f"Expected TotalPartsCount is {parts_count}"
|
||||
assert obj_parts.get("TotalPartsCount") == parts_count, f"Expected TotalPartsCount is {parts_count}"
|
||||
assert (
|
||||
obj_parts.get("PartNumberMarker") == part_number_marker
|
||||
), f"Expected PartNumberMarker is {part_number_marker}"
|
||||
|
|
|
|||
|
|
@ -3,28 +3,18 @@ from datetime import datetime, timedelta
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper
|
||||
from frostfs_testlib.s3 import S3ClientWrapper
|
||||
from frostfs_testlib.steps.s3 import s3_helper
|
||||
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc):
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.s3_gate
|
||||
@pytest.mark.s3_gate_locking
|
||||
@pytest.mark.parametrize("version_id", [None, "second"])
|
||||
class TestS3GateLocking:
|
||||
@allure.title(
|
||||
"Retention period and legal lock on object (version_id={version_id}, s3_client={s3_client})"
|
||||
)
|
||||
def test_s3_object_locking(
|
||||
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
|
||||
):
|
||||
@allure.title("Retention period and legal lock on object (version_id={version_id}, s3_client={s3_client})")
|
||||
def test_s3_object_locking(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
file_name = s3_helper.object_key_from_file_path(file_path)
|
||||
retention_period = 2
|
||||
|
|
@ -46,15 +36,11 @@ class TestS3GateLocking:
|
|||
"RetainUntilDate": date_obj,
|
||||
}
|
||||
s3_client.put_object_retention(bucket, file_name, retention, version_id)
|
||||
s3_helper.assert_object_lock_mode(
|
||||
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF"
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
|
||||
|
||||
with allure.step(f"Put legal hold to object {file_name}"):
|
||||
s3_client.put_object_legal_hold(bucket, file_name, "ON", version_id)
|
||||
s3_helper.assert_object_lock_mode(
|
||||
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON"
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
|
||||
|
||||
with allure.step("Fail with deleting object with legal hold and retention period"):
|
||||
if version_id:
|
||||
|
|
@ -64,9 +50,7 @@ class TestS3GateLocking:
|
|||
|
||||
with allure.step("Check retention period is no longer set on the uploaded object"):
|
||||
time.sleep((retention_period + 1) * 60)
|
||||
s3_helper.assert_object_lock_mode(
|
||||
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON"
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
|
||||
|
||||
with allure.step("Fail with deleting object with legal hold and retention period"):
|
||||
if version_id:
|
||||
|
|
@ -76,12 +60,8 @@ class TestS3GateLocking:
|
|||
else:
|
||||
s3_client.delete_object(bucket, file_name, version_id)
|
||||
|
||||
@allure.title(
|
||||
"Impossible to change retention mode COMPLIANCE (version_id={version_id}, s3_client={s3_client})"
|
||||
)
|
||||
def test_s3_mode_compliance(
|
||||
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
|
||||
):
|
||||
@allure.title("Impossible to change retention mode COMPLIANCE (version_id={version_id}, s3_client={s3_client})")
|
||||
def test_s3_mode_compliance(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
file_name = s3_helper.object_key_from_file_path(file_path)
|
||||
retention_period = 2
|
||||
|
|
@ -102,13 +82,9 @@ class TestS3GateLocking:
|
|||
"RetainUntilDate": date_obj,
|
||||
}
|
||||
s3_client.put_object_retention(bucket, file_name, retention, version_id)
|
||||
s3_helper.assert_object_lock_mode(
|
||||
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF"
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
|
||||
|
||||
with allure.step(
|
||||
f"Try to change retention period {retention_period_1}min to object {file_name}"
|
||||
):
|
||||
with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
|
||||
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
|
||||
retention = {
|
||||
"Mode": "COMPLIANCE",
|
||||
|
|
@ -117,12 +93,8 @@ class TestS3GateLocking:
|
|||
with pytest.raises(Exception):
|
||||
s3_client.put_object_retention(bucket, file_name, retention, version_id)
|
||||
|
||||
@allure.title(
|
||||
"Change retention mode GOVERNANCE (version_id={version_id}, s3_client={s3_client})"
|
||||
)
|
||||
def test_s3_mode_governance(
|
||||
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
|
||||
):
|
||||
@allure.title("Change retention mode GOVERNANCE (version_id={version_id}, s3_client={s3_client})")
|
||||
def test_s3_mode_governance(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
file_name = s3_helper.object_key_from_file_path(file_path)
|
||||
retention_period = 3
|
||||
|
|
@ -144,13 +116,9 @@ class TestS3GateLocking:
|
|||
"RetainUntilDate": date_obj,
|
||||
}
|
||||
s3_client.put_object_retention(bucket, file_name, retention, version_id)
|
||||
s3_helper.assert_object_lock_mode(
|
||||
s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF"
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
|
||||
|
||||
with allure.step(
|
||||
f"Try to change retention period {retention_period_1}min to object {file_name}"
|
||||
):
|
||||
with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
|
||||
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
|
||||
retention = {
|
||||
"Mode": "GOVERNANCE",
|
||||
|
|
@ -159,9 +127,7 @@ class TestS3GateLocking:
|
|||
with pytest.raises(Exception):
|
||||
s3_client.put_object_retention(bucket, file_name, retention, version_id)
|
||||
|
||||
with allure.step(
|
||||
f"Try to change retention period {retention_period_1}min to object {file_name}"
|
||||
):
|
||||
with allure.step(f"Try to change retention period {retention_period_1}min to object {file_name}"):
|
||||
date_obj = datetime.utcnow() + timedelta(minutes=retention_period_1)
|
||||
retention = {
|
||||
"Mode": "GOVERNANCE",
|
||||
|
|
@ -177,16 +143,12 @@ class TestS3GateLocking:
|
|||
"RetainUntilDate": date_obj,
|
||||
}
|
||||
s3_client.put_object_retention(bucket, file_name, retention, version_id, True)
|
||||
s3_helper.assert_object_lock_mode(
|
||||
s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF"
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
|
||||
|
||||
@allure.title(
|
||||
"[NEGATIVE] Lock object in bucket with disabled locking (version_id={version_id}, s3_client={s3_client})"
|
||||
)
|
||||
def test_s3_legal_hold(
|
||||
self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_legal_hold(self, s3_client: S3ClientWrapper, version_id: str, simple_object_size: ObjectSize):
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
file_name = s3_helper.object_key_from_file_path(file_path)
|
||||
|
||||
|
|
@ -227,6 +189,4 @@ class TestS3GateLockingBucket:
|
|||
|
||||
with allure.step("Put object into bucket"):
|
||||
s3_client.put_object(bucket, file_path)
|
||||
s3_helper.assert_object_lock_mode(
|
||||
s3_client, bucket, file_name, "COMPLIANCE", None, "OFF", 1
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", None, "OFF", 1)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.steps.cli.container import list_objects, search_container_by_name
|
||||
from frostfs_testlib.steps.s3 import s3_helper
|
||||
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||
|
|
@ -10,28 +10,27 @@ from frostfs_testlib.utils.file_utils import generate_file, get_file_hash, split
|
|||
PART_SIZE = 5 * 1024 * 1024
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc):
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.s3_gate
|
||||
@pytest.mark.s3_gate_multipart
|
||||
class TestS3GateMultipart(ClusterTestBase):
|
||||
NO_SUCH_UPLOAD = (
|
||||
"The upload ID may be invalid, or the upload may have been aborted or completed."
|
||||
)
|
||||
NO_SUCH_UPLOAD = "The upload ID may be invalid, or the upload may have been aborted or completed."
|
||||
|
||||
@allure.title("Object Multipart API (s3_client={s3_client})")
|
||||
@pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED], indirect=True)
|
||||
def test_s3_object_multipart(self, s3_client: S3ClientWrapper, bucket: str):
|
||||
@allure.title("Object Multipart API (s3_client={s3_client}, bucket versioning = {versioning_status})")
|
||||
@pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED, VersioningStatus.UNDEFINED], indirect=True)
|
||||
def test_s3_object_multipart(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, default_wallet: str, versioning_status: str
|
||||
):
|
||||
parts_count = 5
|
||||
file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part
|
||||
object_key = s3_helper.object_key_from_file_path(file_name_large)
|
||||
part_files = split_file(file_name_large, parts_count)
|
||||
parts = []
|
||||
|
||||
with allure.step(f"Get related container_id for bucket"):
|
||||
for cluster_node in self.cluster.cluster_nodes:
|
||||
container_id = search_container_by_name(bucket, cluster_node)
|
||||
if container_id:
|
||||
break
|
||||
with allure.step("Upload first part"):
|
||||
upload_id = s3_client.create_multipart_upload(bucket, object_key)
|
||||
uploads = s3_client.list_multipart_uploads(bucket)
|
||||
|
|
@ -45,10 +44,12 @@ class TestS3GateMultipart(ClusterTestBase):
|
|||
etag = s3_client.upload_part(bucket, object_key, upload_id, part_id, file_path)
|
||||
parts.append((part_id, etag))
|
||||
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
|
||||
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
|
||||
assert len(got_parts) == len(
|
||||
part_files
|
||||
), f"Expected {parts_count} parts, got\n{got_parts}"
|
||||
response = s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
|
||||
|
||||
version_id = None
|
||||
if versioning_status == VersioningStatus.ENABLED:
|
||||
version_id = response["VersionId"]
|
||||
assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
|
||||
|
||||
with allure.step("Check upload list is empty"):
|
||||
uploads = s3_client.list_multipart_uploads(bucket)
|
||||
|
|
@ -58,6 +59,21 @@ class TestS3GateMultipart(ClusterTestBase):
|
|||
got_object = s3_client.get_object(bucket, object_key)
|
||||
assert get_file_hash(got_object) == get_file_hash(file_name_large)
|
||||
|
||||
if version_id:
|
||||
with allure.step("Delete the object version"):
|
||||
s3_client.delete_object(bucket, object_key, version_id)
|
||||
else:
|
||||
with allure.step("Delete the object"):
|
||||
s3_client.delete_object(bucket, object_key)
|
||||
|
||||
with allure.step("List objects in the bucket, expect to be empty"):
|
||||
objects_list = s3_client.list_objects(bucket)
|
||||
assert not objects_list, f"Expected empty bucket, got {objects_list}"
|
||||
|
||||
with allure.step("List objects in the container via rpc, expect to be empty"):
|
||||
objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
|
||||
assert len(objects) == 0, f"Expected no objects in container, got\n{objects}"
|
||||
|
||||
@allure.title("Abort Multipart Upload (s3_client={s3_client})")
|
||||
@pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED], indirect=True)
|
||||
def test_s3_abort_multipart(
|
||||
|
|
@ -91,12 +107,8 @@ class TestS3GateMultipart(ClusterTestBase):
|
|||
assert len(parts) == files_count, f"Expected {files_count} parts, got\n{parts}"
|
||||
|
||||
with allure.step(f"Check that we have {files_count} files in container '{container_id}'"):
|
||||
objects = list_objects(
|
||||
default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint
|
||||
)
|
||||
assert (
|
||||
len(objects) == files_count
|
||||
), f"Expected {files_count} objects in container, got\n{objects}"
|
||||
objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
|
||||
assert len(objects) == files_count, f"Expected {files_count} objects in container, got\n{objects}"
|
||||
|
||||
with allure.step("Abort multipart upload"):
|
||||
s3_client.abort_multipart_upload(bucket, upload_key, upload_id)
|
||||
|
|
@ -108,9 +120,7 @@ class TestS3GateMultipart(ClusterTestBase):
|
|||
s3_client.list_parts(bucket, upload_key, upload_id)
|
||||
|
||||
with allure.step("Check that we have no files in container since upload was aborted"):
|
||||
objects = list_objects(
|
||||
default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint
|
||||
)
|
||||
objects = list_objects(default_wallet, self.shell, container_id, self.cluster.default_rpc_endpoint)
|
||||
assert len(objects) == 0, f"Expected no objects in container, got\n{objects}"
|
||||
|
||||
@allure.title("Upload Part Copy (s3_client={s3_client})")
|
||||
|
|
@ -136,17 +146,13 @@ class TestS3GateMultipart(ClusterTestBase):
|
|||
|
||||
with allure.step("Upload parts to multipart upload"):
|
||||
for part_id, obj_key in enumerate(objs, start=1):
|
||||
etag = s3_client.upload_part_copy(
|
||||
bucket, object_key, upload_id, part_id, f"{bucket}/{obj_key}"
|
||||
)
|
||||
etag = s3_client.upload_part_copy(bucket, object_key, upload_id, part_id, f"{bucket}/{obj_key}")
|
||||
parts.append((part_id, etag))
|
||||
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
|
||||
|
||||
with allure.step("Complete multipart upload"):
|
||||
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
|
||||
assert len(got_parts) == len(
|
||||
part_files
|
||||
), f"Expected {parts_count} parts, got\n{got_parts}"
|
||||
assert len(got_parts) == len(part_files), f"Expected {parts_count} parts, got\n{got_parts}"
|
||||
|
||||
with allure.step("Check we can get whole object from bucket"):
|
||||
got_object = s3_client.get_object(bucket, object_key)
|
||||
|
|
|
|||
|
|
@ -9,25 +9,14 @@ import allure
|
|||
import pytest
|
||||
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_PASS
|
||||
from frostfs_testlib.resources.error_patterns import S3_MALFORMED_XML_REQUEST
|
||||
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.s3 import AwsCliClient, S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.steps.s3 import s3_helper
|
||||
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||
from frostfs_testlib.testing.test_control import expect_not_raises
|
||||
from frostfs_testlib.utils import wallet_utils
|
||||
from frostfs_testlib.utils.file_utils import (
|
||||
concat_files,
|
||||
generate_file,
|
||||
generate_file_with_content,
|
||||
get_file_hash,
|
||||
)
|
||||
from frostfs_testlib.utils.file_utils import concat_files, generate_file, generate_file_with_content, get_file_hash
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc):
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.s3_gate
|
||||
@pytest.mark.s3_gate_object
|
||||
class TestS3GateObject:
|
||||
|
|
@ -67,28 +56,18 @@ class TestS3GateObject:
|
|||
|
||||
with allure.step("Copy object from first bucket into second"):
|
||||
copy_obj_path_b2 = s3_client.copy_object(bucket_1, file_name, bucket=bucket_2)
|
||||
s3_helper.check_objects_in_bucket(
|
||||
s3_client, bucket_1, expected_objects=bucket_1_objects
|
||||
)
|
||||
s3_helper.check_objects_in_bucket(
|
||||
s3_client, bucket_2, expected_objects=[copy_obj_path_b2]
|
||||
)
|
||||
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
|
||||
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
|
||||
|
||||
with allure.step("Check copied object has the same content"):
|
||||
got_copied_file_b2 = s3_client.get_object(bucket_2, copy_obj_path_b2)
|
||||
assert get_file_hash(file_path) == get_file_hash(
|
||||
got_copied_file_b2
|
||||
), "Hashes must be the same"
|
||||
assert get_file_hash(file_path) == get_file_hash(got_copied_file_b2), "Hashes must be the same"
|
||||
|
||||
with allure.step("Delete one object from first bucket"):
|
||||
s3_client.delete_object(bucket_1, file_name)
|
||||
bucket_1_objects.remove(file_name)
|
||||
s3_helper.check_objects_in_bucket(
|
||||
s3_client, bucket_1, expected_objects=bucket_1_objects
|
||||
)
|
||||
s3_helper.check_objects_in_bucket(
|
||||
s3_client, bucket_2, expected_objects=[copy_obj_path_b2]
|
||||
)
|
||||
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
|
||||
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
|
||||
|
||||
with allure.step("Copy one object into the same bucket"):
|
||||
with pytest.raises(Exception):
|
||||
|
|
@ -102,9 +81,7 @@ class TestS3GateObject:
|
|||
simple_object_size: ObjectSize,
|
||||
):
|
||||
version_1_content = "Version 1"
|
||||
file_name_simple = generate_file_with_content(
|
||||
simple_object_size.value, content=version_1_content
|
||||
)
|
||||
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
|
||||
obj_key = os.path.basename(file_name_simple)
|
||||
|
||||
bucket_1, bucket_2 = two_buckets
|
||||
|
|
@ -123,32 +100,22 @@ class TestS3GateObject:
|
|||
s3_helper.set_bucket_versioning(s3_client, bucket_2, VersioningStatus.ENABLED)
|
||||
with allure.step("Copy object from first bucket into second"):
|
||||
copy_obj_path_b2 = s3_client.copy_object(bucket_1, obj_key, bucket=bucket_2)
|
||||
s3_helper.check_objects_in_bucket(
|
||||
s3_client, bucket_1, expected_objects=bucket_1_objects
|
||||
)
|
||||
s3_helper.check_objects_in_bucket(
|
||||
s3_client, bucket_2, expected_objects=[copy_obj_path_b2]
|
||||
)
|
||||
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
|
||||
s3_helper.check_objects_in_bucket(s3_client, bucket_2, expected_objects=[copy_obj_path_b2])
|
||||
|
||||
with allure.step("Delete one object from first bucket and check object in bucket"):
|
||||
s3_client.delete_object(bucket_1, obj_key)
|
||||
bucket_1_objects.remove(obj_key)
|
||||
s3_helper.check_objects_in_bucket(
|
||||
s3_client, bucket_1, expected_objects=bucket_1_objects
|
||||
)
|
||||
s3_helper.check_objects_in_bucket(s3_client, bucket_1, expected_objects=bucket_1_objects)
|
||||
|
||||
with allure.step("Copy one object into the same bucket"):
|
||||
with pytest.raises(Exception):
|
||||
s3_client.copy_object(bucket_1, obj_key)
|
||||
|
||||
@allure.title("Copy with acl (s3_client={s3_client})")
|
||||
def test_s3_copy_acl(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_copy_acl(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
version_1_content = "Version 1"
|
||||
file_name_simple = generate_file_with_content(
|
||||
simple_object_size.value, content=version_1_content
|
||||
)
|
||||
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
|
||||
obj_key = os.path.basename(file_name_simple)
|
||||
|
||||
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
|
||||
|
|
@ -163,9 +130,7 @@ class TestS3GateObject:
|
|||
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="CanonicalUser")
|
||||
|
||||
@allure.title("Copy object with metadata (s3_client={s3_client})")
|
||||
def test_s3_copy_metadate(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_copy_metadate(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
file_name = s3_helper.object_key_from_file_path(file_path)
|
||||
|
|
@ -183,17 +148,13 @@ class TestS3GateObject:
|
|||
bucket_1_objects.append(copy_obj_path)
|
||||
s3_helper.check_objects_in_bucket(s3_client, bucket, bucket_1_objects)
|
||||
obj_head = s3_client.head_object(bucket, copy_obj_path)
|
||||
assert (
|
||||
obj_head.get("Metadata") == object_metadata
|
||||
), f"Metadata must be {object_metadata}"
|
||||
assert obj_head.get("Metadata") == object_metadata, f"Metadata must be {object_metadata}"
|
||||
|
||||
with allure.step("Copy one object with metadata"):
|
||||
copy_obj_path = s3_client.copy_object(bucket, file_name, metadata_directive="COPY")
|
||||
bucket_1_objects.append(copy_obj_path)
|
||||
obj_head = s3_client.head_object(bucket, copy_obj_path)
|
||||
assert (
|
||||
obj_head.get("Metadata") == object_metadata
|
||||
), f"Metadata must be {object_metadata}"
|
||||
assert obj_head.get("Metadata") == object_metadata, f"Metadata must be {object_metadata}"
|
||||
|
||||
with allure.step("Copy one object with new metadata"):
|
||||
object_metadata_1 = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
|
||||
|
|
@ -205,14 +166,10 @@ class TestS3GateObject:
|
|||
)
|
||||
bucket_1_objects.append(copy_obj_path)
|
||||
obj_head = s3_client.head_object(bucket, copy_obj_path)
|
||||
assert (
|
||||
obj_head.get("Metadata") == object_metadata_1
|
||||
), f"Metadata must be {object_metadata_1}"
|
||||
assert obj_head.get("Metadata") == object_metadata_1, f"Metadata must be {object_metadata_1}"
|
||||
|
||||
@allure.title("Copy object with tagging (s3_client={s3_client})")
|
||||
def test_s3_copy_tagging(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_copy_tagging(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
object_tagging = [(f"{uuid.uuid4()}", f"{uuid.uuid4()}")]
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
file_name_simple = s3_helper.object_key_from_file_path(file_path)
|
||||
|
|
@ -235,9 +192,7 @@ class TestS3GateObject:
|
|||
assert tag in got_tags, f"Expected tag {tag} in {got_tags}"
|
||||
|
||||
with allure.step("Copy one object with tag"):
|
||||
copy_obj_path_1 = s3_client.copy_object(
|
||||
bucket, file_name_simple, tagging_directive="COPY"
|
||||
)
|
||||
copy_obj_path_1 = s3_client.copy_object(bucket, file_name_simple, tagging_directive="COPY")
|
||||
got_tags = s3_client.get_object_tagging(bucket, copy_obj_path_1)
|
||||
assert got_tags, f"Expected tags, got {got_tags}"
|
||||
expected_tags = [{"Key": key, "Value": value} for key, value in object_tagging]
|
||||
|
|
@ -270,9 +225,7 @@ class TestS3GateObject:
|
|||
):
|
||||
version_1_content = "Version 1"
|
||||
version_2_content = "Version 2"
|
||||
file_name_simple = generate_file_with_content(
|
||||
simple_object_size.value, content=version_1_content
|
||||
)
|
||||
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
|
||||
|
||||
obj_key = os.path.basename(file_name_simple)
|
||||
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
|
||||
|
|
@ -286,9 +239,7 @@ class TestS3GateObject:
|
|||
|
||||
with allure.step("Check bucket shows all versions"):
|
||||
versions = s3_client.list_objects_versions(bucket)
|
||||
obj_versions = {
|
||||
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
||||
}
|
||||
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
|
||||
assert obj_versions == {
|
||||
version_id_1,
|
||||
version_id_2,
|
||||
|
|
@ -297,18 +248,14 @@ class TestS3GateObject:
|
|||
with allure.step("Delete 1 version of object"):
|
||||
delete_obj = s3_client.delete_object(bucket, obj_key, version_id=version_id_1)
|
||||
versions = s3_client.list_objects_versions(bucket)
|
||||
obj_versions = {
|
||||
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
||||
}
|
||||
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
|
||||
assert obj_versions == {version_id_2}, f"Object should have versions: {version_id_2}"
|
||||
assert "DeleteMarker" not in delete_obj.keys(), "Delete markers should not be created"
|
||||
|
||||
with allure.step("Delete second version of object"):
|
||||
delete_obj = s3_client.delete_object(bucket, obj_key, version_id=version_id_2)
|
||||
versions = s3_client.list_objects_versions(bucket)
|
||||
obj_versions = {
|
||||
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
||||
}
|
||||
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
|
||||
assert not obj_versions, "Expected object not found"
|
||||
assert "DeleteMarker" not in delete_obj.keys(), "Delete markers should not be created"
|
||||
|
||||
|
|
@ -324,16 +271,12 @@ class TestS3GateObject:
|
|||
assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
|
||||
|
||||
@allure.title("Bulk delete version of object (s3_client={s3_client})")
|
||||
def test_s3_bulk_delete_versioning(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_bulk_delete_versioning(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
version_1_content = "Version 1"
|
||||
version_2_content = "Version 2"
|
||||
version_3_content = "Version 3"
|
||||
version_4_content = "Version 4"
|
||||
file_name_1 = generate_file_with_content(
|
||||
simple_object_size.value, content=version_1_content
|
||||
)
|
||||
file_name_1 = generate_file_with_content(simple_object_size.value, content=version_1_content)
|
||||
|
||||
obj_key = os.path.basename(file_name_1)
|
||||
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
|
||||
|
|
@ -356,37 +299,25 @@ class TestS3GateObject:
|
|||
|
||||
with allure.step("Check bucket shows all versions"):
|
||||
versions = s3_client.list_objects_versions(bucket)
|
||||
obj_versions = {
|
||||
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
||||
}
|
||||
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == obj_key}
|
||||
assert obj_versions == version_ids, f"Object should have versions: {version_ids}"
|
||||
|
||||
with allure.step("Delete two objects from bucket one by one"):
|
||||
version_to_delete_b1 = sample(
|
||||
[version_id_1, version_id_2, version_id_3, version_id_4], k=2
|
||||
)
|
||||
version_to_delete_b1 = sample([version_id_1, version_id_2, version_id_3, version_id_4], k=2)
|
||||
version_to_save = list(set(version_ids) - set(version_to_delete_b1))
|
||||
for ver in version_to_delete_b1:
|
||||
s3_client.delete_object(bucket, obj_key, ver)
|
||||
|
||||
with allure.step("Check bucket shows all versions"):
|
||||
versions = s3_client.list_objects_versions(bucket)
|
||||
obj_versions = [
|
||||
version.get("VersionId") for version in versions if version.get("Key") == obj_key
|
||||
]
|
||||
assert (
|
||||
obj_versions.sort() == version_to_save.sort()
|
||||
), f"Object should have versions: {version_to_save}"
|
||||
obj_versions = [version.get("VersionId") for version in versions if version.get("Key") == obj_key]
|
||||
assert obj_versions.sort() == version_to_save.sort(), f"Object should have versions: {version_to_save}"
|
||||
|
||||
@allure.title("Get versions of object (s3_client={s3_client})")
|
||||
def test_s3_get_versioning(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_get_versioning(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
version_1_content = "Version 1"
|
||||
version_2_content = "Version 2"
|
||||
file_name_simple = generate_file_with_content(
|
||||
simple_object_size.value, content=version_1_content
|
||||
)
|
||||
file_name_simple = generate_file_with_content(simple_object_size.value, content=version_1_content)
|
||||
|
||||
obj_key = os.path.basename(file_name_simple)
|
||||
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
|
||||
|
|
@ -399,21 +330,15 @@ class TestS3GateObject:
|
|||
|
||||
with allure.step("Get first version of object"):
|
||||
object_1 = s3_client.get_object(bucket, obj_key, version_id_1, full_output=True)
|
||||
assert (
|
||||
object_1.get("VersionId") == version_id_1
|
||||
), f"Get object with version {version_id_1}"
|
||||
assert object_1.get("VersionId") == version_id_1, f"Get object with version {version_id_1}"
|
||||
|
||||
with allure.step("Get second version of object"):
|
||||
object_2 = s3_client.get_object(bucket, obj_key, version_id_2, full_output=True)
|
||||
assert (
|
||||
object_2.get("VersionId") == version_id_2
|
||||
), f"Get object with version {version_id_2}"
|
||||
assert object_2.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
|
||||
|
||||
with allure.step("Get object"):
|
||||
object_3 = s3_client.get_object(bucket, obj_key, full_output=True)
|
||||
assert (
|
||||
object_3.get("VersionId") == version_id_2
|
||||
), f"Get object with version {version_id_2}"
|
||||
assert object_3.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
|
||||
|
||||
@allure.title("Get range (s3_client={s3_client})")
|
||||
def test_s3_get_range(
|
||||
|
|
@ -483,9 +408,7 @@ class TestS3GateObject:
|
|||
object_range=[2 * int(simple_object_size.value / 3) + 1, simple_object_size.value],
|
||||
)
|
||||
con_file_1 = concat_files([object_2_part_1, object_2_part_2, object_2_part_3])
|
||||
assert get_file_hash(con_file_1) == get_file_hash(
|
||||
file_name_1
|
||||
), "Hashes must be the same"
|
||||
assert get_file_hash(con_file_1) == get_file_hash(file_name_1), "Hashes must be the same"
|
||||
|
||||
with allure.step("Get object"):
|
||||
object_3_part_1 = s3_client.get_object(
|
||||
|
|
@ -568,26 +491,18 @@ class TestS3GateObject:
|
|||
assert "LastModified" in response, "Expected LastModified field"
|
||||
assert "ETag" in response, "Expected ETag field"
|
||||
assert response.get("Metadata") == {}, "Expected Metadata empty"
|
||||
assert (
|
||||
response.get("VersionId") == version_id_2
|
||||
), f"Expected VersionId is {version_id_2}"
|
||||
assert response.get("VersionId") == version_id_2, f"Expected VersionId is {version_id_2}"
|
||||
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
|
||||
|
||||
with allure.step("Get head ob first version of object"):
|
||||
response = s3_client.head_object(bucket, file_name, version_id=version_id_1)
|
||||
assert "LastModified" in response, "Expected LastModified field"
|
||||
assert "ETag" in response, "Expected ETag field"
|
||||
assert (
|
||||
response.get("Metadata") == object_metadata
|
||||
), f"Expected Metadata is {object_metadata}"
|
||||
assert (
|
||||
response.get("VersionId") == version_id_1
|
||||
), f"Expected VersionId is {version_id_1}"
|
||||
assert response.get("Metadata") == object_metadata, f"Expected Metadata is {object_metadata}"
|
||||
assert response.get("VersionId") == version_id_1, f"Expected VersionId is {version_id_1}"
|
||||
assert response.get("ContentLength") != 0, "Expected ContentLength is not zero"
|
||||
|
||||
@allure.title(
|
||||
"List of objects with version (method_version={list_type}, s3_client={s3_client})"
|
||||
)
|
||||
@allure.title("List of objects with version (method_version={list_type}, s3_client={s3_client})")
|
||||
@pytest.mark.parametrize("list_type", ["v1", "v2"])
|
||||
def test_s3_list_object(
|
||||
self,
|
||||
|
|
@ -624,9 +539,7 @@ class TestS3GateObject:
|
|||
list_obj_1 = s3_client.list_objects_v2(bucket, full_output=True)
|
||||
contents = list_obj_1.get("Contents", [])
|
||||
assert len(contents) == 1, "bucket should have only 1 object"
|
||||
assert (
|
||||
contents[0].get("Key") == file_name_2
|
||||
), f"bucket should have object key {file_name_2}"
|
||||
assert contents[0].get("Key") == file_name_2, f"bucket should have object key {file_name_2}"
|
||||
assert "DeleteMarker" in delete_obj.keys(), "Expected delete Marker"
|
||||
|
||||
@allure.title("Put object (s3_client={s3_client})")
|
||||
|
|
@ -655,22 +568,16 @@ class TestS3GateObject:
|
|||
assert obj_head.get("Metadata") == object_1_metadata, "Metadata must be the same"
|
||||
got_tags = s3_client.get_object_tagging(bucket, file_name)
|
||||
assert got_tags, f"Expected tags, got {got_tags}"
|
||||
assert got_tags == [
|
||||
{"Key": tag_key_1, "Value": str(tag_value_1)}
|
||||
], "Tags must be the same"
|
||||
assert got_tags == [{"Key": tag_key_1, "Value": str(tag_value_1)}], "Tags must be the same"
|
||||
|
||||
with allure.step("Rewrite file into bucket"):
|
||||
file_path_2 = generate_file_with_content(
|
||||
simple_object_size.value, file_path=file_path_1
|
||||
)
|
||||
file_path_2 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
|
||||
s3_client.put_object(bucket, file_path_2, metadata=object_2_metadata, tagging=tag_2)
|
||||
obj_head = s3_client.head_object(bucket, file_name)
|
||||
assert obj_head.get("Metadata") == object_2_metadata, "Metadata must be the same"
|
||||
got_tags_1 = s3_client.get_object_tagging(bucket, file_name)
|
||||
assert got_tags_1, f"Expected tags, got {got_tags_1}"
|
||||
assert got_tags_1 == [
|
||||
{"Key": tag_key_2, "Value": str(tag_value_2)}
|
||||
], "Tags must be the same"
|
||||
assert got_tags_1 == [{"Key": tag_key_2, "Value": str(tag_value_2)}], "Tags must be the same"
|
||||
|
||||
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
|
||||
|
||||
|
|
@ -683,28 +590,18 @@ class TestS3GateObject:
|
|||
tag_3 = f"{tag_key_3}={tag_value_3}"
|
||||
|
||||
with allure.step("Put third object into bucket"):
|
||||
version_id_1 = s3_client.put_object(
|
||||
bucket, file_path_3, metadata=object_3_metadata, tagging=tag_3
|
||||
)
|
||||
version_id_1 = s3_client.put_object(bucket, file_path_3, metadata=object_3_metadata, tagging=tag_3)
|
||||
obj_head_3 = s3_client.head_object(bucket, file_name_3)
|
||||
assert obj_head_3.get("Metadata") == object_3_metadata, "Matadata must be the same"
|
||||
got_tags_3 = s3_client.get_object_tagging(bucket, file_name_3)
|
||||
assert got_tags_3, f"Expected tags, got {got_tags_3}"
|
||||
assert got_tags_3 == [
|
||||
{"Key": tag_key_3, "Value": str(tag_value_3)}
|
||||
], "Tags must be the same"
|
||||
assert got_tags_3 == [{"Key": tag_key_3, "Value": str(tag_value_3)}], "Tags must be the same"
|
||||
|
||||
with allure.step("Put new version of file into bucket"):
|
||||
file_path_4 = generate_file_with_content(
|
||||
simple_object_size.value, file_path=file_path_3
|
||||
)
|
||||
file_path_4 = generate_file_with_content(simple_object_size.value, file_path=file_path_3)
|
||||
version_id_2 = s3_client.put_object(bucket, file_path_4)
|
||||
versions = s3_client.list_objects_versions(bucket)
|
||||
obj_versions = {
|
||||
version.get("VersionId")
|
||||
for version in versions
|
||||
if version.get("Key") == file_name_3
|
||||
}
|
||||
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == file_name_3}
|
||||
assert obj_versions == {
|
||||
version_id_1,
|
||||
version_id_2,
|
||||
|
|
@ -714,26 +611,20 @@ class TestS3GateObject:
|
|||
|
||||
with allure.step("Get object"):
|
||||
object_3 = s3_client.get_object(bucket, file_name_3, full_output=True)
|
||||
assert (
|
||||
object_3.get("VersionId") == version_id_2
|
||||
), f"get object with version {version_id_2}"
|
||||
assert object_3.get("VersionId") == version_id_2, f"get object with version {version_id_2}"
|
||||
object_3 = s3_client.get_object(bucket, file_name_3)
|
||||
assert get_file_hash(file_path_4) == get_file_hash(object_3), "Hashes must be the same"
|
||||
|
||||
with allure.step("Get first version of object"):
|
||||
object_4 = s3_client.get_object(bucket, file_name_3, version_id_1, full_output=True)
|
||||
assert (
|
||||
object_4.get("VersionId") == version_id_1
|
||||
), f"get object with version {version_id_1}"
|
||||
assert object_4.get("VersionId") == version_id_1, f"get object with version {version_id_1}"
|
||||
object_4 = s3_client.get_object(bucket, file_name_3, version_id_1)
|
||||
assert file_hash == get_file_hash(object_4), "Hashes must be the same"
|
||||
obj_head_3 = s3_client.head_object(bucket, file_name_3, version_id_1)
|
||||
assert obj_head_3.get("Metadata") == object_3_metadata, "Metadata must be the same"
|
||||
got_tags_3 = s3_client.get_object_tagging(bucket, file_name_3, version_id_1)
|
||||
assert got_tags_3, f"Expected tags, got {got_tags_3}"
|
||||
assert got_tags_3 == [
|
||||
{"Key": tag_key_3, "Value": str(tag_value_3)}
|
||||
], "Tags must be the same"
|
||||
assert got_tags_3 == [{"Key": tag_key_3, "Value": str(tag_value_3)}], "Tags must be the same"
|
||||
|
||||
@allure.title("Put object with ACL (versioning={bucket_versioning}, s3_client={s3_client})")
|
||||
@pytest.mark.parametrize("bucket_versioning", ["ENABLED", "SUSPENDED"])
|
||||
|
|
@ -762,9 +653,7 @@ class TestS3GateObject:
|
|||
assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same"
|
||||
|
||||
with allure.step("Put object with acl public-read"):
|
||||
file_path_2 = generate_file_with_content(
|
||||
simple_object_size.value, file_path=file_path_1
|
||||
)
|
||||
file_path_2 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
|
||||
s3_client.put_object(bucket, file_path_2, acl="public-read")
|
||||
obj_acl = s3_client.get_object_acl(bucket, file_name)
|
||||
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
|
||||
|
|
@ -772,9 +661,7 @@ class TestS3GateObject:
|
|||
assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same"
|
||||
|
||||
with allure.step("Put object with acl public-read-write"):
|
||||
file_path_3 = generate_file_with_content(
|
||||
simple_object_size.value, file_path=file_path_1
|
||||
)
|
||||
file_path_3 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
|
||||
s3_client.put_object(bucket, file_path_3, acl="public-read-write")
|
||||
obj_acl = s3_client.get_object_acl(bucket, file_name)
|
||||
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
|
||||
|
|
@ -782,9 +669,7 @@ class TestS3GateObject:
|
|||
assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same"
|
||||
|
||||
with allure.step("Put object with acl authenticated-read"):
|
||||
file_path_4 = generate_file_with_content(
|
||||
simple_object_size.value, file_path=file_path_1
|
||||
)
|
||||
file_path_4 = generate_file_with_content(simple_object_size.value, file_path=file_path_1)
|
||||
s3_client.put_object(bucket, file_path_4, acl="authenticated-read")
|
||||
obj_acl = s3_client.get_object_acl(bucket, file_name)
|
||||
s3_helper.assert_s3_acl(acl_grants=obj_acl, permitted_users="AllUsers")
|
||||
|
|
@ -806,9 +691,7 @@ class TestS3GateObject:
|
|||
object_5 = s3_client.get_object(bucket, file_name_5)
|
||||
assert get_file_hash(file_path_5) == get_file_hash(object_5), "Hashes must be the same"
|
||||
|
||||
with allure.step(
|
||||
"Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
|
||||
):
|
||||
with allure.step("Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"):
|
||||
generate_file_with_content(simple_object_size.value, file_path=file_path_5)
|
||||
s3_client.put_object(
|
||||
bucket,
|
||||
|
|
@ -833,9 +716,7 @@ class TestS3GateObject:
|
|||
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True)
|
||||
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
|
||||
|
||||
with allure.step(
|
||||
"Put object with lock-mode GOVERNANCE lock-retain-until-date +1day, lock-legal-hold-status"
|
||||
):
|
||||
with allure.step("Put object with lock-mode GOVERNANCE lock-retain-until-date +1day, lock-legal-hold-status"):
|
||||
date_obj = datetime.utcnow() + timedelta(days=1)
|
||||
s3_client.put_object(
|
||||
bucket,
|
||||
|
|
@ -844,9 +725,7 @@ class TestS3GateObject:
|
|||
object_lock_retain_until_date=date_obj.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
object_lock_legal_hold_status="OFF",
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(
|
||||
s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF"
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
|
||||
|
||||
with allure.step(
|
||||
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"
|
||||
|
|
@ -859,9 +738,7 @@ class TestS3GateObject:
|
|||
object_lock_mode="COMPLIANCE",
|
||||
object_lock_retain_until_date=date_obj,
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(
|
||||
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF"
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
|
||||
|
||||
with allure.step(
|
||||
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"
|
||||
|
|
@ -875,9 +752,7 @@ class TestS3GateObject:
|
|||
object_lock_retain_until_date=date_obj,
|
||||
object_lock_legal_hold_status="ON",
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(
|
||||
s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON"
|
||||
)
|
||||
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "ON")
|
||||
|
||||
with allure.step("Put object with lock-mode"):
|
||||
with pytest.raises(
|
||||
|
|
@ -939,9 +814,7 @@ class TestS3GateObject:
|
|||
|
||||
with allure.step("Check objects are synced"):
|
||||
objects = s3_client.list_objects(bucket)
|
||||
assert set(key_to_path.keys()) == set(
|
||||
objects
|
||||
), f"Expected all abjects saved. Got {objects}"
|
||||
assert set(key_to_path.keys()) == set(objects), f"Expected all abjects saved. Got {objects}"
|
||||
|
||||
with allure.step("Check these are the same objects"):
|
||||
for obj_key in objects:
|
||||
|
|
@ -950,9 +823,7 @@ class TestS3GateObject:
|
|||
key_to_path.get(obj_key)
|
||||
), "Expected hashes are the same"
|
||||
obj_head = s3_client.head_object(bucket, obj_key)
|
||||
assert (
|
||||
obj_head.get("Metadata") == object_metadata
|
||||
), f"Metadata of object is {object_metadata}"
|
||||
assert obj_head.get("Metadata") == object_metadata, f"Metadata of object is {object_metadata}"
|
||||
# Uncomment after https://github.com/nspcc-dev/neofs-s3-gw/issues/685 is solved
|
||||
# obj_acl = s3_client.get_object_acl(bucket, obj_key)
|
||||
# s3_helper.assert_s3_acl(acl_grants = obj_acl, permitted_users = "AllUsers")
|
||||
|
|
@ -994,9 +865,7 @@ class TestS3GateObject:
|
|||
assert not objects_list, f"Expected empty bucket, got {objects_list}"
|
||||
|
||||
@allure.title("Delete the same object twice (s3_client={s3_client})")
|
||||
def test_s3_delete_twice(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_delete_twice(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
|
||||
objects_list = s3_client.list_objects(bucket)
|
||||
with allure.step("Check that bucket is empty"):
|
||||
|
|
@ -1012,9 +881,7 @@ class TestS3GateObject:
|
|||
delete_object = s3_client.delete_object(bucket, file_name)
|
||||
versions = s3_client.list_objects_versions(bucket)
|
||||
|
||||
obj_versions = {
|
||||
version.get("VersionId") for version in versions if version.get("Key") == file_name
|
||||
}
|
||||
obj_versions = {version.get("VersionId") for version in versions if version.get("Key") == file_name}
|
||||
assert obj_versions, f"Object versions were not found {objects_list}"
|
||||
assert "DeleteMarker" in delete_object.keys(), "Delete markers not found"
|
||||
|
||||
|
|
@ -1022,9 +889,7 @@ class TestS3GateObject:
|
|||
delete_object_2nd_attempt = s3_client.delete_object(bucket, file_name)
|
||||
versions_2nd_attempt = s3_client.list_objects_versions(bucket)
|
||||
|
||||
assert (
|
||||
delete_object.keys() == delete_object_2nd_attempt.keys()
|
||||
), "Delete markers are not the same"
|
||||
assert delete_object.keys() == delete_object_2nd_attempt.keys(), "Delete markers are not the same"
|
||||
# check that nothing was changed
|
||||
# checking here not VersionId only, but all data (for example LastModified)
|
||||
assert versions == versions_2nd_attempt, "Versions are not the same"
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import os
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.steps.cli.container import search_container_by_name
|
||||
from frostfs_testlib.steps.s3 import s3_helper
|
||||
from frostfs_testlib.steps.storage_policy import get_simple_object_copies
|
||||
|
|
@ -12,23 +12,11 @@ from frostfs_testlib.testing.test_control import expect_not_raises
|
|||
from frostfs_testlib.utils.file_utils import generate_file
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc):
|
||||
policy = f"{os.getcwd()}/pytest_tests/resources/files/policy.json"
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize(
|
||||
"s3_client, s3_policy",
|
||||
[(AwsCliClient, policy), (Boto3ClientWrapper, policy)],
|
||||
indirect=True,
|
||||
ids=["aws cli", "boto3"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.s3_gate
|
||||
@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True)
|
||||
class TestS3GatePolicy(ClusterTestBase):
|
||||
@allure.title("Bucket creation with retention policy applied (s3_client={s3_client})")
|
||||
def test_s3_bucket_location(
|
||||
self, default_wallet: str, s3_client: S3ClientWrapper, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_bucket_location(self, default_wallet: str, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
|
||||
file_path_1 = generate_file(simple_object_size.value)
|
||||
file_name_1 = s3_helper.object_key_from_file_path(file_path_1)
|
||||
file_path_2 = generate_file(simple_object_size.value)
|
||||
|
|
@ -156,9 +144,7 @@ class TestS3GatePolicy(ClusterTestBase):
|
|||
}
|
||||
s3_client.put_bucket_cors(bucket, cors)
|
||||
bucket_cors = s3_client.get_bucket_cors(bucket)
|
||||
assert bucket_cors == cors.get(
|
||||
"CORSRules"
|
||||
), f"Expected CORSRules must be {cors.get('CORSRules')}"
|
||||
assert bucket_cors == cors.get("CORSRules"), f"Expected CORSRules must be {cors.get('CORSRules')}"
|
||||
|
||||
with allure.step("delete bucket cors"):
|
||||
s3_client.delete_bucket_cors(bucket)
|
||||
|
|
|
|||
|
|
@ -4,18 +4,12 @@ from typing import Tuple
|
|||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper
|
||||
from frostfs_testlib.s3 import S3ClientWrapper
|
||||
from frostfs_testlib.steps.s3 import s3_helper
|
||||
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||
from frostfs_testlib.utils.file_utils import generate_file
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc):
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.s3_gate
|
||||
@pytest.mark.s3_gate_tagging
|
||||
class TestS3GateTagging:
|
||||
|
|
@ -29,9 +23,7 @@ class TestS3GateTagging:
|
|||
return tags
|
||||
|
||||
@allure.title("Object tagging (s3_client={s3_client})")
|
||||
def test_s3_object_tagging(
|
||||
self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize
|
||||
):
|
||||
def test_s3_object_tagging(self, s3_client: S3ClientWrapper, bucket: str, simple_object_size: ObjectSize):
|
||||
file_path = generate_file(simple_object_size.value)
|
||||
file_name = s3_helper.object_key_from_file_path(file_path)
|
||||
|
||||
|
|
@ -45,9 +37,7 @@ class TestS3GateTagging:
|
|||
with allure.step("Put 10 new tags for object"):
|
||||
tags_2 = self.create_tags(10)
|
||||
s3_client.put_object_tagging(bucket, file_name, tags=tags_2)
|
||||
s3_helper.check_tags_by_object(
|
||||
s3_client, bucket, file_name, tags_2, [("Tag1", "Value1")]
|
||||
)
|
||||
s3_helper.check_tags_by_object(s3_client, bucket, file_name, tags_2, [("Tag1", "Value1")])
|
||||
|
||||
with allure.step("Put 10 extra new tags for object"):
|
||||
tags_3 = self.create_tags(10)
|
||||
|
|
|
|||
|
|
@ -1,19 +1,11 @@
|
|||
import os
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
|
||||
from frostfs_testlib.steps.s3 import s3_helper
|
||||
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc):
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize("s3_client", [AwsCliClient, Boto3ClientWrapper], indirect=True)
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.s3_gate
|
||||
@pytest.mark.s3_gate_versioning
|
||||
class TestS3GateVersioning:
|
||||
|
|
@ -34,18 +26,10 @@ class TestS3GateVersioning:
|
|||
with allure.step("Put object into bucket"):
|
||||
s3_client.put_object(bucket, file_path)
|
||||
objects_list = s3_client.list_objects(bucket)
|
||||
assert (
|
||||
objects_list == bucket_objects
|
||||
), f"Expected list with single objects in bucket, got {objects_list}"
|
||||
assert objects_list == bucket_objects, f"Expected list with single objects in bucket, got {objects_list}"
|
||||
object_version = s3_client.list_objects_versions(bucket)
|
||||
actual_version = [
|
||||
version.get("VersionId")
|
||||
for version in object_version
|
||||
if version.get("Key") == file_name
|
||||
]
|
||||
assert actual_version == [
|
||||
"null"
|
||||
], f"Expected version is null in list-object-versions, got {object_version}"
|
||||
actual_version = [version.get("VersionId") for version in object_version if version.get("Key") == file_name]
|
||||
assert actual_version == ["null"], f"Expected version is null in list-object-versions, got {object_version}"
|
||||
object_0 = s3_client.head_object(bucket, file_name)
|
||||
assert (
|
||||
object_0.get("VersionId") == "null"
|
||||
|
|
@ -60,27 +44,19 @@ class TestS3GateVersioning:
|
|||
|
||||
with allure.step("Check bucket shows all versions"):
|
||||
versions = s3_client.list_objects_versions(bucket)
|
||||
obj_versions = [
|
||||
version.get("VersionId") for version in versions if version.get("Key") == file_name
|
||||
]
|
||||
obj_versions = [version.get("VersionId") for version in versions if version.get("Key") == file_name]
|
||||
assert (
|
||||
obj_versions.sort() == [version_id_1, version_id_2, "null"].sort()
|
||||
), f"Expected object has versions: {version_id_1, version_id_2, 'null'}"
|
||||
|
||||
with allure.step("Get object"):
|
||||
object_1 = s3_client.get_object(bucket, file_name, full_output=True)
|
||||
assert (
|
||||
object_1.get("VersionId") == version_id_2
|
||||
), f"Get object with version {version_id_2}"
|
||||
assert object_1.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
|
||||
|
||||
with allure.step("Get first version of object"):
|
||||
object_2 = s3_client.get_object(bucket, file_name, version_id_1, full_output=True)
|
||||
assert (
|
||||
object_2.get("VersionId") == version_id_1
|
||||
), f"Get object with version {version_id_1}"
|
||||
assert object_2.get("VersionId") == version_id_1, f"Get object with version {version_id_1}"
|
||||
|
||||
with allure.step("Get second version of object"):
|
||||
object_3 = s3_client.get_object(bucket, file_name, version_id_2, full_output=True)
|
||||
assert (
|
||||
object_3.get("VersionId") == version_id_2
|
||||
), f"Get object with version {version_id_2}"
|
||||
assert object_3.get("VersionId") == version_id_2, f"Get object with version {version_id_2}"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import logging
|
||||
import os
|
||||
from http import HTTPStatus
|
||||
from re import match, fullmatch
|
||||
from re import fullmatch, match
|
||||
|
||||
import allure
|
||||
import pytest
|
||||
|
|
@ -14,8 +14,8 @@ from pytest import FixtureRequest
|
|||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
||||
@allure.title("Check binaries versions")
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.check_binaries
|
||||
def test_binaries_versions(request: FixtureRequest, hosting: Hosting):
|
||||
"""
|
||||
|
|
@ -31,8 +31,8 @@ def test_binaries_versions(request: FixtureRequest, hosting: Hosting):
|
|||
# compare versions from servers and file
|
||||
exeptions = []
|
||||
additional_env_properties = {}
|
||||
|
||||
for binary, version in got_versions.items():
|
||||
|
||||
for binary, version in got_versions.items():
|
||||
if not fullmatch(r"^\d+\.\d+\.\d+(-.*)?(?<!dirty)", version):
|
||||
exeptions.append(f"{binary}: Actual version doesn't conform to format '0.0.0-000-aaaaaaa': {version}")
|
||||
|
||||
|
|
@ -56,9 +56,7 @@ def download_versions_info(url: str) -> dict:
|
|||
|
||||
response = requests.get(url)
|
||||
|
||||
assert (
|
||||
response.status_code == HTTPStatus.OK
|
||||
), f"Got {response.status_code} code. Content {response.json()}"
|
||||
assert response.status_code == HTTPStatus.OK, f"Got {response.status_code} code. Content {response.json()}"
|
||||
|
||||
content = response.text
|
||||
assert content, f"Expected file with content, got {response}"
|
||||
|
|
|
|||
|
|
@ -48,15 +48,9 @@ RANGE_OFFSET_FOR_COMPLEX_OBJECT = 200
|
|||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def storage_containers(
|
||||
owner_wallet: WalletInfo, client_shell: Shell, cluster: Cluster
|
||||
) -> list[str]:
|
||||
cid = create_container(
|
||||
owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint
|
||||
)
|
||||
other_cid = create_container(
|
||||
owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint
|
||||
)
|
||||
def storage_containers(owner_wallet: WalletInfo, client_shell: Shell, cluster: Cluster) -> list[str]:
|
||||
cid = create_container(owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
|
||||
other_cid = create_container(owner_wallet.path, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
|
||||
yield [cid, other_cid]
|
||||
|
||||
|
||||
|
|
@ -99,9 +93,7 @@ def storage_objects(
|
|||
|
||||
|
||||
@allure.step("Get ranges for test")
|
||||
def get_ranges(
|
||||
storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, endpoint: str
|
||||
) -> list[str]:
|
||||
def get_ranges(storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, endpoint: str) -> list[str]:
|
||||
"""
|
||||
Returns ranges to test range/hash methods via static session
|
||||
"""
|
||||
|
|
@ -112,8 +104,7 @@ def get_ranges(
|
|||
return [
|
||||
"0:10",
|
||||
f"{object_size-10}:10",
|
||||
f"{max_object_size - RANGE_OFFSET_FOR_COMPLEX_OBJECT}:"
|
||||
f"{RANGE_OFFSET_FOR_COMPLEX_OBJECT * 2}",
|
||||
f"{max_object_size - RANGE_OFFSET_FOR_COMPLEX_OBJECT}:" f"{RANGE_OFFSET_FOR_COMPLEX_OBJECT * 2}",
|
||||
]
|
||||
else:
|
||||
return ["0:10", f"{object_size-10}:10"]
|
||||
|
|
@ -148,9 +139,7 @@ def static_sessions(
|
|||
|
||||
@pytest.mark.static_session
|
||||
class TestObjectStaticSession(ClusterTestBase):
|
||||
@allure.title(
|
||||
"Read operations with static session (method={method_under_test.__name__}, obj_size={object_size})"
|
||||
)
|
||||
@allure.title("Read operations with static session (method={method_under_test.__name__}, obj_size={object_size})")
|
||||
@pytest.mark.parametrize(
|
||||
"method_under_test,verb",
|
||||
[
|
||||
|
|
@ -181,13 +170,12 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
session=static_sessions[verb],
|
||||
)
|
||||
|
||||
@allure.title(
|
||||
"Range operations with static session (method={method_under_test.__name__}, obj_size={object_size})"
|
||||
)
|
||||
@allure.title("Range operations with static session (method={method_under_test.__name__}, obj_size={object_size})")
|
||||
@pytest.mark.parametrize(
|
||||
"method_under_test,verb",
|
||||
[(get_range, ObjectVerb.RANGE), (get_range_hash, ObjectVerb.RANGEHASH)],
|
||||
)
|
||||
@pytest.mark.sanity
|
||||
def test_static_session_range(
|
||||
self,
|
||||
user_wallet: WalletInfo,
|
||||
|
|
@ -201,9 +189,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
Validate static session with range operations
|
||||
"""
|
||||
storage_object = storage_objects[0]
|
||||
ranges_to_test = get_ranges(
|
||||
storage_object, max_object_size, self.shell, self.cluster.default_rpc_endpoint
|
||||
)
|
||||
ranges_to_test = get_ranges(storage_object, max_object_size, self.shell, self.cluster.default_rpc_endpoint)
|
||||
|
||||
for range_to_test in ranges_to_test:
|
||||
with allure.step(f"Check range {range_to_test}"):
|
||||
|
|
@ -241,9 +227,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
)
|
||||
assert sorted(expected_object_ids) == sorted(actual_object_ids)
|
||||
|
||||
@allure.title(
|
||||
"[NEGATIVE] Static session with object id not in session (obj_size={object_size})"
|
||||
)
|
||||
@allure.title("[NEGATIVE] Static session with object id not in session (obj_size={object_size})")
|
||||
def test_static_session_unrelated_object(
|
||||
self,
|
||||
user_wallet: WalletInfo,
|
||||
|
|
@ -307,9 +291,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
session=static_sessions[ObjectVerb.HEAD],
|
||||
)
|
||||
|
||||
@allure.title(
|
||||
"[NEGATIVE] Static session with container id not in session (obj_size={object_size})"
|
||||
)
|
||||
@allure.title("[NEGATIVE] Static session with container id not in session (obj_size={object_size})")
|
||||
def test_static_session_unrelated_container(
|
||||
self,
|
||||
user_wallet: WalletInfo,
|
||||
|
|
@ -473,9 +455,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
session=token_expire_at_next_epoch,
|
||||
)
|
||||
|
||||
with allure.step(
|
||||
"Object should be available at last epoch before session token expiration"
|
||||
):
|
||||
with allure.step("Object should be available at last epoch before session token expiration"):
|
||||
self.tick_epoch()
|
||||
with expect_not_raises():
|
||||
head_object(
|
||||
|
|
@ -499,6 +479,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
session=token_expire_at_next_epoch,
|
||||
)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Static session which is valid since next epoch (obj_size={object_size})")
|
||||
def test_static_session_start_at_next(
|
||||
self,
|
||||
|
|
@ -540,9 +521,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
session=token_start_at_next_epoch,
|
||||
)
|
||||
|
||||
with allure.step(
|
||||
"Object should be available with session token starting from token nbf epoch"
|
||||
):
|
||||
with allure.step("Object should be available with session token starting from token nbf epoch"):
|
||||
self.tick_epoch()
|
||||
with expect_not_raises():
|
||||
head_object(
|
||||
|
|
@ -554,9 +533,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
session=token_start_at_next_epoch,
|
||||
)
|
||||
|
||||
with allure.step(
|
||||
"Object should be available at last epoch before session token expiration"
|
||||
):
|
||||
with allure.step("Object should be available at last epoch before session token expiration"):
|
||||
self.tick_epoch()
|
||||
with expect_not_raises():
|
||||
head_object(
|
||||
|
|
@ -640,6 +617,7 @@ class TestObjectStaticSession(ClusterTestBase):
|
|||
session=static_sessions[ObjectVerb.DELETE],
|
||||
)
|
||||
|
||||
@pytest.mark.sanity
|
||||
@allure.title("Put verb is restricted for static session (obj_size={object_size})")
|
||||
def test_static_session_put_verb(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -3,12 +3,7 @@ import pytest
|
|||
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.steps.acl import create_eacl, set_eacl, wait_for_cache_expired
|
||||
from frostfs_testlib.steps.cli.container import (
|
||||
create_container,
|
||||
delete_container,
|
||||
get_container,
|
||||
list_containers,
|
||||
)
|
||||
from frostfs_testlib.steps.cli.container import create_container, delete_container, get_container, list_containers
|
||||
from frostfs_testlib.steps.session_token import ContainerVerb, get_container_signed_token
|
||||
from frostfs_testlib.storage.dataclasses.acl import EACLAccess, EACLOperation, EACLRole, EACLRule
|
||||
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
||||
|
|
@ -33,9 +28,7 @@ class TestSessionTokenContainer(ClusterTestBase):
|
|||
Returns dict with static session token file paths for all verbs with default lifetime
|
||||
"""
|
||||
return {
|
||||
verb: get_container_signed_token(
|
||||
owner_wallet, user_wallet, verb, client_shell, temp_directory
|
||||
)
|
||||
verb: get_container_signed_token(owner_wallet, user_wallet, verb, client_shell, temp_directory)
|
||||
for verb in ContainerVerb
|
||||
}
|
||||
|
||||
|
|
@ -65,9 +58,7 @@ class TestSessionTokenContainer(ClusterTestBase):
|
|||
assert cid not in list_containers(
|
||||
user_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
|
||||
)
|
||||
assert cid in list_containers(
|
||||
owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
|
||||
)
|
||||
assert cid in list_containers(owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint)
|
||||
|
||||
def test_static_session_token_container_create_with_other_verb(
|
||||
self,
|
||||
|
|
@ -136,6 +127,7 @@ class TestSessionTokenContainer(ClusterTestBase):
|
|||
owner_wallet.path, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
|
||||
)
|
||||
|
||||
@pytest.mark.sanity
|
||||
def test_static_session_token_container_set_eacl(
|
||||
self,
|
||||
owner_wallet: WalletInfo,
|
||||
|
|
@ -158,10 +150,7 @@ class TestSessionTokenContainer(ClusterTestBase):
|
|||
assert can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster)
|
||||
|
||||
with allure.step("Deny all operations for other via eACL"):
|
||||
eacl_deny = [
|
||||
EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op)
|
||||
for op in EACLOperation
|
||||
]
|
||||
eacl_deny = [EACLRule(access=EACLAccess.DENY, role=EACLRole.OTHERS, operation=op) for op in EACLOperation]
|
||||
set_eacl(
|
||||
user_wallet.path,
|
||||
cid,
|
||||
|
|
|
|||
|
|
@ -66,8 +66,7 @@ class Shard:
|
|||
|
||||
blobstor_count = Shard._get_blobstor_count_from_section(config_object, shard_id)
|
||||
blobstors = [
|
||||
Blobstor.from_config_object(config_object, shard_id, blobstor_id)
|
||||
for blobstor_id in range(blobstor_count)
|
||||
Blobstor.from_config_object(config_object, shard_id, blobstor_id) for blobstor_id in range(blobstor_count)
|
||||
]
|
||||
|
||||
write_cache_enabled = config_object.as_bool(f"{var_prefix}_WRITECACHE_ENABLED")
|
||||
|
|
@ -81,15 +80,10 @@ class Shard:
|
|||
@staticmethod
|
||||
def from_object(shard):
|
||||
metabase = shard["metabase"]["path"] if "path" in shard["metabase"] else shard["metabase"]
|
||||
writecache = (
|
||||
shard["writecache"]["path"] if "path" in shard["writecache"] else shard["writecache"]
|
||||
)
|
||||
writecache = shard["writecache"]["path"] if "path" in shard["writecache"] else shard["writecache"]
|
||||
|
||||
return Shard(
|
||||
blobstor=[
|
||||
Blobstor(path=blobstor["path"], path_type=blobstor["type"])
|
||||
for blobstor in shard["blobstor"]
|
||||
],
|
||||
blobstor=[Blobstor(path=blobstor["path"], path_type=blobstor["type"]) for blobstor in shard["blobstor"]],
|
||||
metabase=metabase,
|
||||
writecache=writecache,
|
||||
)
|
||||
|
|
@ -111,7 +105,6 @@ def shards_from_env(contents: str) -> list[Shard]:
|
|||
return [Shard.from_config_object(configObj, shard_id) for shard_id in range(num_shards)]
|
||||
|
||||
|
||||
@pytest.mark.sanity
|
||||
@pytest.mark.shard
|
||||
class TestControlShard:
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import os
|
||||
import shutil
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
import allure
|
||||
|
|
@ -35,9 +36,12 @@ class TestLogs:
|
|||
os.makedirs(logs_dir)
|
||||
# Using \b here because 'oom' and 'panic' can sometimes be found in OID or CID
|
||||
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas"
|
||||
exclude_filter = r"too many requests"
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
futures = parallel(
|
||||
self._collect_logs_on_host, cluster.hosts, logs_dir, issues_regex, session_start_time, end_time
|
||||
self._collect_logs_on_host, cluster.hosts, logs_dir, issues_regex, session_start_time, end_time, exclude_filter
|
||||
)
|
||||
|
||||
hosts_with_problems = [
|
||||
|
|
@ -50,9 +54,9 @@ class TestLogs:
|
|||
not hosts_with_problems
|
||||
), f"The following hosts contains contain critical errors in system logs: {', '.join(hosts_with_problems)}"
|
||||
|
||||
def _collect_logs_on_host(self, host: Host, logs_dir: str, regex: str, since: datetime, until: datetime):
|
||||
def _collect_logs_on_host(self, host: Host, logs_dir: str, regex: str, since: datetime, until: datetime, exclude_filter: str):
|
||||
with allure.step(f"Get logs from {host.config.address}"):
|
||||
logs = host.get_filtered_logs(regex, since, until)
|
||||
logs = host.get_filtered_logs(filter_regex=regex, since=since, until=until, exclude_filter=exclude_filter)
|
||||
|
||||
if not logs:
|
||||
return None
|
||||
|
|
|
|||
Loading…
Reference in a new issue