Compare commits

...

25 commits

Author SHA1 Message Date
91cd71de19 [#324] Remove basic-acl part two
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-11-16 14:33:56 +03:00
20c2948818 [#323] Update allure-validator to 1.1.1 version
Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-11-15 16:26:23 +03:00
6f254af8b1 [#322] Remove basic acl part one
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-11-13 17:50:31 +03:00
1ee5f73243 [#321] Add tombstone expiration test
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-11-12 16:19:40 +03:00
aef9d43979 [#319] Remove hardcoded version of botocore
Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-11-11 13:40:04 +03:00
0e72717dcd [#310] Extend bucket testsuite
- Added tests to check correctness of bucket names according to AWS specification
- Added test to check availability of non-empty bucket after attempting to delete it

Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-10-31 12:14:44 +00:00
b33514df3c [#303] add local deny ape tests 2024-10-31 13:19:49 +03:00
09acd6f283 [#316] Move variable under package for single repo runs
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-10-30 14:19:26 +03:00
b7669fc96f [#315] Use relpath for files
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-10-29 13:32:07 +03:00
75508cc70c Added nearest node count calculation for choosing ec policy
Signed-off-by: Dmitry Anurin <danurin@yadro.com>
2024-10-25 08:08:43 +03:00
6b83a89b94 [#312] Extend container metrics tests 2024-10-24 16:00:10 +03:00
77126f2706 [#311] add new pattern 2024-10-17 15:07:53 +03:00
64bc778116 [#308] Fix unique user names
Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-10-15 07:56:04 +00:00
6442a52abd [#307] fix and extend container metrics tests 2024-10-14 10:07:17 +00:00
8dcb3ccf3c [#309] Add marks
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-10-11 09:31:24 +00:00
44ed00f9bc [#306] Refactor tests: use unique_name instead hex + timestamp
Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-10-07 18:19:06 +03:00
d10e5975e7 TrueCloudLab/frostfs-node#1297 update error pattern
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-10-03 08:22:55 +00:00
f6576d4f6f [#302] Fixed logs metrics test 2024-09-27 16:53:52 +03:00
1afadfa363 [#301] Update all tests EC policy
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
2024-09-17 10:19:20 +00:00
7d0fa79fb2 [#300] Move temp dir fixture to testlib
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-09-10 13:29:02 +00:00
64c70948f9 [#299] Small change logic EC policy tests
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
2024-09-10 13:28:45 +03:00
8234a0ece2 [#298] Add set mode shards in teardown test ec
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
2024-09-03 16:53:08 +00:00
9528ff0333 [#295] Update revision allure-validator
Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-09-03 16:21:23 +00:00
ffdfff6ba0 [#297] Refactore APE tests
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-09-02 19:22:59 +03:00
ccdd6ab784 [#296] Add resolve bucket fixture into old resolve func.
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
2024-09-02 13:25:13 +03:00
62 changed files with 2412 additions and 1593 deletions

View file

@ -10,10 +10,14 @@ repos:
- id: isort - id: isort
name: isort (python) name: isort (python)
- repo: https://git.frostfs.info/TrueCloudLab/allure-validator - repo: https://git.frostfs.info/TrueCloudLab/allure-validator
rev: 1.0.1 rev: 1.1.1
hooks: hooks:
- id: allure-validator - id: allure-validator
args: ["pytest_tests/"] args: [
"pytest_tests/",
"--plugins",
"frostfs[-_]testlib*",
]
pass_filenames: false pass_filenames: false
ci: ci:

0
__init__.py Normal file
View file

View file

@ -13,6 +13,8 @@ markers =
# controlling markers # controlling markers
order: manual control of test order order: manual control of test order
logs_after_session: Make the last test in session logs_after_session: Make the last test in session
# parametrizing markers
container: specify container details for container creation
# functional markers # functional markers
maintenance: tests for change mode node maintenance: tests for change mode node
container: tests for container creation container: tests for container creation

View file

@ -0,0 +1,3 @@
import os
TESTS_BASE_PATH = os.path.dirname(os.path.relpath(__file__))

View file

@ -6,7 +6,7 @@ from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses import ape from frostfs_testlib.storage.dataclasses import ape
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from pytest_tests.helpers.object_access import ( from ..helpers.object_access import (
can_delete_object, can_delete_object,
can_get_head_object, can_get_head_object,
can_get_object, can_get_object,

View file

@ -0,0 +1,56 @@
import json
import time
from frostfs_testlib import reporter
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.shell.interfaces import Shell
from frostfs_testlib.steps.cli.container import create_container, search_nodes_with_container
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses import ape
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.utils import datetime_utils
from .container_request import ContainerRequest
def create_container_with_ape(
frostfs_cli: FrostfsCli, wallet: WalletInfo, shell: Shell, cluster: Cluster, endpoint: str, container_request: ContainerRequest
):
with reporter.step("Create container"):
cid = _create_container_by_spec(wallet, shell, cluster, endpoint, container_request)
with reporter.step("Apply APE rules for container"):
if container_request.ape_rules:
_apply_ape_rules(frostfs_cli, endpoint, cid, container_request.ape_rules)
with reporter.step("Wait for one block"):
time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME))
with reporter.step("Search nodes holding the container"):
container_holder_nodes = search_nodes_with_container(wallet, cid, shell, cluster.default_rpc_endpoint, cluster)
report_data = {node.id: node.host_ip for node in container_holder_nodes}
reporter.attach(json.dumps(report_data, indent=2), "container_nodes.json")
return cid
@reporter.step("Create container by spec {container_request}")
def _create_container_by_spec(
wallet: WalletInfo, shell: Shell, cluster: Cluster, endpoint: str, container_request: ContainerRequest
) -> str:
return create_container(wallet, shell, endpoint, container_request.parsed_rule(cluster), wait_for_creation=False)
def _apply_ape_rules(frostfs_cli: FrostfsCli, endpoint: str, cid: str, ape_rules: list[ape.Rule]):
for ape_rule in ape_rules:
rule_str = ape_rule.as_string()
with reporter.step(f"Apply APE rule '{rule_str}' for container {cid}"):
frostfs_cli.ape_manager.add(
endpoint,
ape_rule.chain_id,
target_name=cid,
target_type="container",
rule=rule_str,
)

View file

@ -0,0 +1,58 @@
from dataclasses import dataclass
from frostfs_testlib.steps.cli.container import DEFAULT_PLACEMENT_RULE
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses import ape
APE_EVERYONE_ALLOW_ALL = [ape.Rule(ape.Verb.ALLOW, ape.ObjectOperations.WILDCARD_ALL)]
# In case if we need container operations
# ape.Rule(ape.Verb.ALLOW, ape.ContainerOperations.WILDCARD_ALL)]
APE_OWNER_ALLOW_ALL = [ape.Rule(ape.Verb.ALLOW, ape.ObjectOperations.WILDCARD_ALL, ape.Condition.by_role(ape.Role.OWNER))]
# In case if we need container operations
# ape.Rule(ape.Verb.ALLOW, ape.ContainerOperations.WILDCARD_ALL, ape.Condition.by_role(ape.Role.OWNER))]
@dataclass
class ContainerRequest:
policy: str = None
ape_rules: list[ape.Rule] = None
short_name: str | None = None
def __post_init__(self):
if self.ape_rules is None:
self.ape_rules = []
# For pytest instead of ids=[...] everywhere
self.__name__ = self.short_name
def parsed_rule(self, cluster: Cluster):
if self.policy is None:
return None
substitutions = {"%NODE_COUNT%": str(len(cluster.cluster_nodes))}
parsed_rule = self.policy
for sub, replacement in substitutions.items():
parsed_rule = parsed_rule.replace(sub, replacement)
return parsed_rule
def __repr__(self):
if self.short_name:
return self.short_name
spec_info: list[str] = []
if self.policy:
spec_info.append(f"policy='{self.policy}'")
if self.ape_rules:
ape_rules_list = ", ".join([f"'{rule.as_string()}'" for rule in self.ape_rules])
spec_info.append(f"ape_rules=[{ape_rules_list}]")
return f"({', '.join(spec_info)})"
EVERYONE_ALLOW_ALL = ContainerRequest(policy=DEFAULT_PLACEMENT_RULE, ape_rules=APE_EVERYONE_ALLOW_ALL, short_name="Everyone_Allow_All")
OWNER_ALLOW_ALL = ContainerRequest(policy=DEFAULT_PLACEMENT_RULE, ape_rules=APE_OWNER_ALLOW_ALL, short_name="Owner_Allow_All")
PRIVATE = ContainerRequest(policy=DEFAULT_PLACEMENT_RULE, ape_rules=[], short_name="Private_No_APE")

View file

@ -2,7 +2,7 @@ from typing import Optional
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
from frostfs_testlib.resources.error_patterns import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND from frostfs_testlib.resources.error_patterns import OBJECT_ACCESS_DENIED
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.object import ( from frostfs_testlib.steps.cli.object import (
delete_object, delete_object,
@ -20,10 +20,6 @@ from frostfs_testlib.utils.file_utils import get_file_hash
OPERATION_ERROR_TYPE = RuntimeError OPERATION_ERROR_TYPE = RuntimeError
# TODO: Revert to just OBJECT_ACCESS_DENIED when the issue is fixed
# https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/1297
OBJECT_NO_ACCESS = rf"(?:{OBJECT_NOT_FOUND}|{OBJECT_ACCESS_DENIED})"
def can_get_object( def can_get_object(
wallet: WalletInfo, wallet: WalletInfo,
@ -47,7 +43,7 @@ def can_get_object(
cluster=cluster, cluster=cluster,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}" assert string_utils.is_str_match_pattern(err, OBJECT_ACCESS_DENIED), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
assert get_file_hash(file_name) == get_file_hash(got_file_path) assert get_file_hash(file_name) == get_file_hash(got_file_path)
return True return True
@ -102,7 +98,7 @@ def can_delete_object(
endpoint=endpoint, endpoint=endpoint,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}" assert string_utils.is_str_match_pattern(err, OBJECT_ACCESS_DENIED), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
return True return True
@ -130,7 +126,7 @@ def can_get_head_object(
timeout=timeout, timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}" assert string_utils.is_str_match_pattern(err, OBJECT_ACCESS_DENIED), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
return True return True
@ -159,7 +155,7 @@ def can_get_range_of_object(
timeout=timeout, timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}" assert string_utils.is_str_match_pattern(err, OBJECT_ACCESS_DENIED), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
return True return True
@ -188,7 +184,7 @@ def can_get_range_hash_of_object(
timeout=timeout, timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}" assert string_utils.is_str_match_pattern(err, OBJECT_ACCESS_DENIED), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
return True return True
@ -215,7 +211,7 @@ def can_search_object(
timeout=timeout, timeout=timeout,
) )
except OPERATION_ERROR_TYPE as err: except OPERATION_ERROR_TYPE as err:
assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}" assert string_utils.is_str_match_pattern(err, OBJECT_ACCESS_DENIED), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
return False return False
if oid: if oid:
return oid in oids return oid in oids

View file

@ -35,3 +35,16 @@ def wait_for_gc_pass_on_storage_nodes() -> None:
wait_time = datetime_utils.parse_time(STORAGE_GC_TIME) wait_time = datetime_utils.parse_time(STORAGE_GC_TIME)
with reporter.step(f"Wait {wait_time}s until GC completes on storage nodes"): with reporter.step(f"Wait {wait_time}s until GC completes on storage nodes"):
time.sleep(wait_time) time.sleep(wait_time)
def are_numbers_similar(num1, num2, tolerance_percentage: float = 1.0):
"""
if difference of numbers is less than permissible deviation than numbers are similar
"""
# Calculate the permissible deviation
average = (num1 + num2) / 2
tolerance = average * (tolerance_percentage / 100)
# Calculate the real difference
difference = abs(num1 - num2)
return difference <= tolerance

View file

@ -1,5 +1,8 @@
import os import os
from .. import TESTS_BASE_PATH
TEST_CYCLES_COUNT = int(os.getenv("TEST_CYCLES_COUNT", "1")) TEST_CYCLES_COUNT = int(os.getenv("TEST_CYCLES_COUNT", "1"))
DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "frostfs-dev-env")) DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "frostfs-dev-env"))
S3_POLICY_FILE_LOCATION = os.path.join(TESTS_BASE_PATH, "resources/files/policy.json")

View file

@ -1,123 +0,0 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PRIVATE_ACL_F, PUBLIC_ACL_F, READONLY_ACL_F
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import put_object_to_random_node
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.container_access import assert_full_access_to_container, assert_no_access_to_container, assert_read_only_container
@pytest.mark.sanity
@pytest.mark.smoke
@pytest.mark.acl
class TestACLBasic(ClusterTestBase):
@pytest.fixture(scope="module")
def public_container(self, default_wallet: WalletInfo):
with reporter.step("Create public container"):
cid_public = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint, basic_acl=PUBLIC_ACL_F)
return cid_public
@pytest.fixture(scope="module")
def private_container(self, default_wallet: WalletInfo):
with reporter.step("Create private container"):
cid_private = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint, basic_acl=PRIVATE_ACL_F)
return cid_private
@pytest.fixture(scope="module")
def readonly_container(self, default_wallet: WalletInfo):
with reporter.step("Create public readonly container"):
cid_read_only = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint, basic_acl=READONLY_ACL_F)
return cid_read_only
@allure.title("Operations in public container available to everyone (obj_size={object_size})")
def test_basic_acl_public(
self,
default_wallet: WalletInfo,
other_wallet: WalletInfo,
client_shell: Shell,
public_container: str,
file_path: str,
cluster: Cluster,
):
"""
Test access to object operations in public container.
"""
for wallet, role in ((default_wallet, "owner"), (other_wallet, "others")):
with reporter.step("Put objects to container"):
# We create new objects for each wallet because assert_full_access_to_container
# deletes the object
owner_object_oid = put_object_to_random_node(
default_wallet,
file_path,
public_container,
shell=self.shell,
cluster=self.cluster,
attributes={"created": "owner"},
)
other_object_oid = put_object_to_random_node(
other_wallet,
file_path,
public_container,
shell=self.shell,
cluster=self.cluster,
attributes={"created": "other"},
)
with reporter.step(f"Check {role} has full access to public container"):
assert_full_access_to_container(wallet, public_container, owner_object_oid, file_path, client_shell, cluster)
assert_full_access_to_container(wallet, public_container, other_object_oid, file_path, client_shell, cluster)
@allure.title("Operations in private container only available to owner (obj_size={object_size})")
def test_basic_acl_private(
self,
default_wallet: WalletInfo,
other_wallet: WalletInfo,
client_shell: Shell,
private_container: str,
file_path: str,
cluster: Cluster,
):
"""
Test access to object operations in private container.
"""
with reporter.step("Put object to container"):
owner_object_oid = put_object_to_random_node(default_wallet, file_path, private_container, client_shell, cluster)
with reporter.step("Check no one except owner has access to operations with container"):
assert_no_access_to_container(other_wallet, private_container, owner_object_oid, file_path, client_shell, cluster)
with reporter.step("Check owner has full access to private container"):
assert_full_access_to_container(default_wallet, private_container, owner_object_oid, file_path, self.shell, cluster)
@allure.title("Read operations in readonly container available to others (obj_size={object_size})")
def test_basic_acl_readonly(
self,
default_wallet: WalletInfo,
other_wallet: WalletInfo,
client_shell: Shell,
readonly_container: str,
file_path: str,
cluster: Cluster,
):
"""
Test access to object operations in readonly container.
"""
with reporter.step("Put object to container"):
object_oid = put_object_to_random_node(default_wallet, file_path, readonly_container, client_shell, cluster)
with reporter.step("Check others has read-only access to operations with container"):
assert_read_only_container(other_wallet, readonly_container, object_oid, file_path, client_shell, cluster)
with reporter.step("Check owner has full access to public container"):
assert_full_access_to_container(default_wallet, readonly_container, object_oid, file_path, client_shell, cluster)

View file

@ -2,8 +2,6 @@ import allure
import pytest import pytest
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import put_object_to_random_node from frostfs_testlib.steps.cli.object import put_object_to_random_node
from frostfs_testlib.steps.node_management import drop_object from frostfs_testlib.steps.node_management import drop_object
from frostfs_testlib.storage.dataclasses import ape from frostfs_testlib.storage.dataclasses import ape
@ -13,74 +11,69 @@ from frostfs_testlib.utils import wallet_utils
from frostfs_testlib.utils.failover_utils import wait_object_replication from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_utils import TestFile from frostfs_testlib.utils.file_utils import TestFile
from pytest_tests.helpers.container_access import ( from ....helpers.container_access import (
ALL_OBJECT_OPERATIONS, ALL_OBJECT_OPERATIONS,
assert_access_to_container, assert_access_to_container,
assert_full_access_to_container, assert_full_access_to_container,
assert_no_access_to_container, assert_no_access_to_container,
) )
from ....helpers.container_request import APE_EVERYONE_ALLOW_ALL, OWNER_ALLOW_ALL, ContainerRequest
@pytest.fixture
def denied_wallet(default_wallet: WalletInfo, other_wallet: WalletInfo, role: ape.Role) -> WalletInfo:
return other_wallet if role == ape.Role.OTHERS else default_wallet
@pytest.fixture
def allowed_wallet(default_wallet: WalletInfo, other_wallet: WalletInfo, role: ape.Role) -> WalletInfo:
return default_wallet if role == ape.Role.OTHERS else other_wallet
@pytest.mark.nightly
@pytest.mark.ape @pytest.mark.ape
class TestApeContainer(ClusterTestBase): class TestApeContainer(ClusterTestBase):
@pytest.fixture(scope="function")
def full_placement_container_with_object(self, default_wallet: WalletInfo, file_path: str) -> tuple[str, str, str]:
storage_nodes = self.cluster.storage_nodes
node_count = len(storage_nodes)
with reporter.step("Create public container with full placement rule"):
full_placement_rule = f"REP {node_count} IN X CBF 1 SELECT {node_count} FROM * AS X"
cid = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint, full_placement_rule, PUBLIC_ACL)
with reporter.step("Put object to container"):
oid = put_object_to_random_node(default_wallet, file_path, cid, shell=self.shell, cluster=self.cluster)
wait_object_replication(cid, oid, node_count, shell=self.shell, nodes=storage_nodes)
yield cid, oid
@pytest.mark.sanity @pytest.mark.sanity
@allure.title("Deny operations via APE by role (role={role}, obj_size={object_size})") @allure.title("Deny operations via APE by role (role={role}, obj_size={object_size})")
@pytest.mark.parametrize("role", [ape.Role.OWNER, ape.Role.OTHERS], indirect=True) @pytest.mark.parametrize("role", [ape.Role.OWNER, ape.Role.OTHERS], indirect=True)
def test_deny_operations_via_ape_by_role( def test_deny_operations_via_ape_by_role(
self, self,
default_wallet: WalletInfo, denied_wallet: WalletInfo,
other_wallet: WalletInfo, allowed_wallet: WalletInfo,
frostfs_cli: FrostfsCli, frostfs_cli: FrostfsCli,
container_with_objects: tuple[str, list[str], str], container: str,
objects: list[str],
role: ape.Role, role: ape.Role,
file_path: TestFile, file_path: TestFile,
rpc_endpoint: str,
): ):
denied_wallet = other_wallet if role == ape.Role.OTHERS else default_wallet
allowed_wallet = default_wallet if role == ape.Role.OTHERS else other_wallet
allowed_role = ape.Role.OWNER if role == ape.Role.OTHERS else ape.Role.OTHERS
cid, object_oids, file_path = container_with_objects
endpoint = self.cluster.default_rpc_endpoint
role_condition = ape.Condition.by_role(role.value)
with reporter.step(f"Deny all operations for {role} via APE"): with reporter.step(f"Deny all operations for {role} via APE"):
deny_rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, role_condition) deny_rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, ape.Condition.by_role(role.value))
frostfs_cli.ape_manager.add(endpoint, deny_rule.chain_id, target_name=cid, target_type="container", rule=deny_rule.as_string()) frostfs_cli.ape_manager.add(
rpc_endpoint, deny_rule.chain_id, target_name=container, target_type="container", rule=deny_rule.as_string()
)
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
with reporter.step(f"Assert {role} have no access to public container"): with reporter.step(f"Assert denied role have no access to public container"):
assert_no_access_to_container(denied_wallet, cid, object_oids[0], file_path, self.shell, self.cluster) # access checks will try to remove object, so we use .pop() to ensure we have object before deletion
assert_no_access_to_container(denied_wallet, container, objects.pop(), file_path, self.shell, self.cluster)
with reporter.step(f"Assert {allowed_role} have full access to public container"): with reporter.step(f"Assert allowed role have full access to public container"):
assert_full_access_to_container(allowed_wallet, cid, object_oids.pop(), file_path, self.shell, self.cluster) assert_full_access_to_container(allowed_wallet, container, objects.pop(), file_path, self.shell, self.cluster)
with reporter.step(f"Remove deny rule from APE"): with reporter.step(f"Remove deny rule from APE"):
frostfs_cli.ape_manager.remove(endpoint, deny_rule.chain_id, target_name=cid, target_type="container") frostfs_cli.ape_manager.remove(rpc_endpoint, deny_rule.chain_id, target_name=container, target_type="container")
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
with reporter.step("Assert owner have full access to public container"): with reporter.step("Assert allowed role have full access to public container"):
assert_full_access_to_container(default_wallet, cid, object_oids.pop(), file_path, self.shell, self.cluster) assert_full_access_to_container(allowed_wallet, container, objects.pop(), file_path, self.shell, self.cluster)
with reporter.step("Assert others have full access to public container"): with reporter.step("Assert denied role have full access to public container"):
assert_full_access_to_container(other_wallet, cid, object_oids.pop(), file_path, self.shell, self.cluster) assert_full_access_to_container(denied_wallet, container, objects.pop(), file_path, self.shell, self.cluster)
@allure.title("Deny operations for others via APE excluding single pubkey (obj_size={object_size})") @allure.title("Deny operations for others via APE excluding single pubkey (obj_size={object_size})")
def test_deny_opeartions_excluding_pubkey( def test_deny_opeartions_excluding_pubkey(
@ -89,11 +82,11 @@ class TestApeContainer(ClusterTestBase):
default_wallet: WalletInfo, default_wallet: WalletInfo,
other_wallet: WalletInfo, other_wallet: WalletInfo,
other_wallet_2: WalletInfo, other_wallet_2: WalletInfo,
container_with_objects: tuple[str, list[str], str], container: str,
objects: list[str],
rpc_endpoint: str,
file_path: TestFile,
): ):
endpoint = self.cluster.default_rpc_endpoint
cid, object_oids, file_path = container_with_objects
with reporter.step("Add deny APE rules for others except single wallet"): with reporter.step("Add deny APE rules for others except single wallet"):
rule_conditions = [ rule_conditions = [
ape.Condition.by_role(ape.Role.OTHERS), ape.Condition.by_role(ape.Role.OTHERS),
@ -103,29 +96,40 @@ class TestApeContainer(ClusterTestBase):
), ),
] ]
rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, rule_conditions) rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, rule_conditions)
frostfs_cli.ape_manager.add(endpoint, rule.chain_id, target_name=cid, target_type="container", rule=rule.as_string()) frostfs_cli.ape_manager.add(rpc_endpoint, rule.chain_id, target_name=container, target_type="container", rule=rule.as_string())
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
with reporter.step("Assert others have no access to public container"): with reporter.step("Assert others have no access to public container"):
assert_no_access_to_container(other_wallet, cid, object_oids[0], file_path, self.shell, self.cluster) # access checks will try to remove object, so we use .pop() to ensure we have object before deletion
assert_no_access_to_container(other_wallet, container, objects[0], file_path, self.shell, self.cluster)
with reporter.step("Assert owner have full access to public container"): with reporter.step("Assert owner have full access to public container"):
assert_full_access_to_container(default_wallet, cid, object_oids.pop(), file_path, self.shell, self.cluster) assert_full_access_to_container(default_wallet, container, objects.pop(), file_path, self.shell, self.cluster)
with reporter.step("Assert allowed wallet have full access to public container"): with reporter.step("Assert allowed wallet have full access to public container"):
assert_full_access_to_container(other_wallet_2, cid, object_oids.pop(), file_path, self.shell, self.cluster) assert_full_access_to_container(other_wallet_2, container, objects.pop(), file_path, self.shell, self.cluster)
@allure.title("Replication works with APE deny rules on OWNER and OTHERS (obj_size={object_size})") @allure.title("Replication works with APE deny rules on OWNER and OTHERS (obj_size={object_size})")
@pytest.mark.parametrize(
"container_request",
[ContainerRequest(f"REP %NODE_COUNT% IN X CBF 1 SELECT %NODE_COUNT% FROM * AS X", APE_EVERYONE_ALLOW_ALL, "custom")],
indirect=True,
)
def test_replication_works_with_deny_rules( def test_replication_works_with_deny_rules(
self, self,
default_wallet: WalletInfo,
frostfs_cli: FrostfsCli, frostfs_cli: FrostfsCli,
full_placement_container_with_object: tuple[str, list[str], str], container: str,
rpc_endpoint: str,
file_path: TestFile,
): ):
endpoint = self.cluster.default_rpc_endpoint with reporter.step("Put object to container"):
cid, oid = full_placement_container_with_object oid = put_object_to_random_node(default_wallet, file_path, container, self.shell, self.cluster)
storage_nodes = self.cluster.storage_nodes
with reporter.step("Wait for object replication after upload"):
wait_object_replication(container, oid, len(self.cluster.cluster_nodes), self.shell, self.cluster.storage_nodes)
with reporter.step("Add deny APE rules for owner and others"): with reporter.step("Add deny APE rules for owner and others"):
rule_conditions = [ rule_conditions = [
@ -134,93 +138,96 @@ class TestApeContainer(ClusterTestBase):
] ]
for rule_condition in rule_conditions: for rule_condition in rule_conditions:
rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, rule_condition) rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, rule_condition)
frostfs_cli.ape_manager.add(endpoint, rule.chain_id, target_name=cid, target_type="container", rule=rule.as_string()) frostfs_cli.ape_manager.add(
rpc_endpoint, rule.chain_id, target_name=container, target_type="container", rule=rule.as_string()
)
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
with reporter.step("Drop object"): with reporter.step("Drop object"):
drop_object(storage_nodes[0], cid, oid) drop_object(self.cluster.storage_nodes[0], container, oid)
with reporter.step("Wait for dropped object to be replicated"): with reporter.step("Wait for dropped object to be replicated"):
wait_object_replication(cid, oid, len(storage_nodes), self.shell, storage_nodes) wait_object_replication(container, oid, len(self.cluster.storage_nodes), self.shell, self.cluster.storage_nodes)
@allure.title("Deny operations via APE by role (role=ir, obj_size={object_size})") @allure.title("Deny operations via APE by role (role=ir, obj_size={object_size})")
@pytest.mark.parametrize("container_request", [OWNER_ALLOW_ALL], indirect=True)
def test_deny_operations_via_ape_by_role_ir( def test_deny_operations_via_ape_by_role_ir(
self, frostfs_cli: FrostfsCli, ir_wallet: WalletInfo, container_with_objects: tuple[str, list[str], str] self, frostfs_cli: FrostfsCli, ir_wallet: WalletInfo, container: str, objects: list[str], rpc_endpoint: str, file_path: TestFile
): ):
cid, object_oids, file_path = container_with_objects
endpoint = self.cluster.default_rpc_endpoint
default_ir_access = { default_ir_access = {
ape.ObjectOperations.PUT: False, ape.ObjectOperations.PUT: False,
ape.ObjectOperations.GET: True, ape.ObjectOperations.GET: True,
ape.ObjectOperations.HEAD: True, ape.ObjectOperations.HEAD: True,
ape.ObjectOperations.GET_RANGE: False, ape.ObjectOperations.GET_RANGE: True,
ape.ObjectOperations.GET_RANGE_HASH: True, ape.ObjectOperations.GET_RANGE_HASH: True,
ape.ObjectOperations.SEARCH: True, ape.ObjectOperations.SEARCH: True,
ape.ObjectOperations.DELETE: False, ape.ObjectOperations.DELETE: False,
} }
with reporter.step("Assert IR wallet access in default state"): with reporter.step("Assert IR wallet access in default state"):
assert_access_to_container(default_ir_access, ir_wallet, cid, object_oids[0], file_path, self.shell, self.cluster) assert_access_to_container(default_ir_access, ir_wallet, container, objects[0], file_path, self.shell, self.cluster)
with reporter.step("Add deny APE rule with deny all operations for IR role"): with reporter.step("Add deny APE rule with deny all operations for IR role"):
rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, [ape.Condition.by_role(ape.Role.IR.value)]) rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, [ape.Condition.by_role(ape.Role.IR.value)])
frostfs_cli.ape_manager.add(endpoint, rule.chain_id, target_name=cid, target_type="container", rule=rule.as_string()) frostfs_cli.ape_manager.add(rpc_endpoint, rule.chain_id, target_name=container, target_type="container", rule=rule.as_string())
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
with reporter.step("Assert IR wallet ignores APE rules"): with reporter.step("Assert IR wallet ignores APE rules"):
assert_access_to_container(default_ir_access, ir_wallet, cid, object_oids[0], file_path, self.shell, self.cluster) assert_access_to_container(default_ir_access, ir_wallet, container, objects[0], file_path, self.shell, self.cluster)
with reporter.step("Remove APE rule"): with reporter.step("Remove APE rule"):
frostfs_cli.ape_manager.remove(endpoint, rule.chain_id, target_name=cid, target_type="container") frostfs_cli.ape_manager.remove(rpc_endpoint, rule.chain_id, target_name=container, target_type="container")
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
with reporter.step("Assert IR wallet access is restored"): with reporter.step("Assert IR wallet access is restored"):
assert_access_to_container(default_ir_access, ir_wallet, cid, object_oids[0], file_path, self.shell, self.cluster) assert_access_to_container(default_ir_access, ir_wallet, container, objects[0], file_path, self.shell, self.cluster)
@allure.title("Deny operations via APE by role (role=container, obj_size={object_size})") @allure.title("Deny operations via APE by role (role=container, obj_size={object_size})")
@pytest.mark.parametrize("container_request", [OWNER_ALLOW_ALL], indirect=True)
def test_deny_operations_via_ape_by_role_container( def test_deny_operations_via_ape_by_role_container(
self, frostfs_cli: FrostfsCli, storage_wallet: WalletInfo, container_with_objects: tuple[str, list[str], str] self,
frostfs_cli: FrostfsCli,
container_node_wallet: WalletInfo,
container: str,
objects: list[str],
rpc_endpoint: str,
file_path: TestFile,
): ):
access_matrix = {
cid, object_oids, file_path = container_with_objects
endpoint = self.cluster.default_rpc_endpoint
default_container_access = {
ape.ObjectOperations.PUT: True, ape.ObjectOperations.PUT: True,
ape.ObjectOperations.GET: True, ape.ObjectOperations.GET: True,
ape.ObjectOperations.HEAD: True, ape.ObjectOperations.HEAD: True,
ape.ObjectOperations.GET_RANGE: False, ape.ObjectOperations.GET_RANGE: True,
ape.ObjectOperations.GET_RANGE_HASH: True, ape.ObjectOperations.GET_RANGE_HASH: True,
ape.ObjectOperations.SEARCH: True, ape.ObjectOperations.SEARCH: True,
ape.ObjectOperations.DELETE: False, ape.ObjectOperations.DELETE: True,
} }
with reporter.step("Assert CONTAINER wallet access in default state"): with reporter.step("Assert CONTAINER wallet access in default state"):
assert_access_to_container(default_container_access, storage_wallet, cid, object_oids[0], file_path, self.shell, self.cluster) assert_access_to_container(access_matrix, container_node_wallet, container, objects[0], file_path, self.shell, self.cluster)
rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, ape.Condition.by_role(ape.Role.CONTAINER.value)) rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, ape.Condition.by_role(ape.Role.CONTAINER.value))
with reporter.step(f"Add APE rule with deny all operations for CONTAINER and IR roles"): with reporter.step(f"Add APE rule with deny all operations for CONTAINER and IR roles"):
frostfs_cli.ape_manager.add(endpoint, rule.chain_id, target_name=cid, target_type="container", rule=rule.as_string()) frostfs_cli.ape_manager.add(rpc_endpoint, rule.chain_id, target_name=container, target_type="container", rule=rule.as_string())
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
with reporter.step("Assert CONTAINER wallet ignores APE rule"): with reporter.step("Assert CONTAINER wallet ignores APE rule"):
assert_access_to_container(default_container_access, storage_wallet, cid, object_oids[0], file_path, self.shell, self.cluster) assert_access_to_container(access_matrix, container_node_wallet, container, objects[1], file_path, self.shell, self.cluster)
with reporter.step("Remove APE rules"): with reporter.step("Remove APE rule"):
frostfs_cli.ape_manager.remove(endpoint, rule.chain_id, target_name=cid, target_type="container") frostfs_cli.ape_manager.remove(rpc_endpoint, rule.chain_id, target_name=container, target_type="container")
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
with reporter.step("Assert CONTAINER wallet access is restored"): with reporter.step("Assert CONTAINER wallet access after rule was removed"):
assert_access_to_container(default_container_access, storage_wallet, cid, object_oids[0], file_path, self.shell, self.cluster) assert_access_to_container(access_matrix, container_node_wallet, container, objects[2], file_path, self.shell, self.cluster)

View file

@ -2,28 +2,27 @@ import allure
import pytest import pytest
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
from frostfs_testlib.resources.error_patterns import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND from frostfs_testlib.resources.error_patterns import OBJECT_ACCESS_DENIED
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import get_object_from_random_node, head_object, put_object_to_random_node from frostfs_testlib.steps.cli.object import get_object_from_random_node, head_object, put_object_to_random_node
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses import ape from frostfs_testlib.storage.dataclasses import ape
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils.file_utils import TestFile from frostfs_testlib.utils.file_utils import TestFile
from pytest_tests.helpers.bearer_token import create_bearer_token from ....helpers.bearer_token import create_bearer_token
from pytest_tests.helpers.container_access import ( from ....helpers.container_access import (
ALL_OBJECT_OPERATIONS, ALL_OBJECT_OPERATIONS,
FULL_ACCESS, FULL_ACCESS,
assert_access_to_container, assert_access_to_container,
assert_full_access_to_container, assert_full_access_to_container,
assert_no_access_to_container, assert_no_access_to_container,
) )
from pytest_tests.helpers.object_access import OBJECT_NO_ACCESS from ....helpers.container_request import OWNER_ALLOW_ALL
from ....helpers.object_access import OBJECT_ACCESS_DENIED
@pytest.mark.nightly
@pytest.mark.ape @pytest.mark.ape
class TestApeFilters(ClusterTestBase): class TestApeFilters(ClusterTestBase):
# SPEC: https://github.com/nspcc-dev/neofs-spec/blob/master/01-arch/07-acl.md # SPEC: https://github.com/nspcc-dev/neofs-spec/blob/master/01-arch/07-acl.md
@ -47,61 +46,27 @@ class TestApeFilters(ClusterTestBase):
ape.ObjectOperations.PUT, ape.ObjectOperations.PUT,
] ]
@pytest.fixture(scope="function") @pytest.fixture
def public_container_with_objects(self, default_wallet: WalletInfo, file_path: TestFile): def objects_with_attributes(self, default_wallet: WalletInfo, file_path: TestFile, container: str):
with reporter.step("Create public container"): return [
cid = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint, basic_acl=PUBLIC_ACL) put_object_to_random_node(
default_wallet, file_path, container, self.shell, self.cluster, attributes={**self.ATTRIBUTES, "key": val}
objects_with_header, objects_with_other_header, objects_without_header = self._fill_container(default_wallet, file_path, cid)
return cid, objects_with_header, objects_with_other_header, objects_without_header, file_path
@pytest.fixture(scope="function")
def private_container(self, default_wallet: WalletInfo, frostfs_cli: FrostfsCli, cluster: Cluster):
with reporter.step("Create private container"):
cid = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint, basic_acl="0")
with reporter.step("Create allow APE rule for container owner"):
role_condition = ape.Condition.by_role(ape.Role.OWNER)
deny_rule = ape.Rule(ape.Verb.ALLOW, ape.ObjectOperations.WILDCARD_ALL, role_condition)
frostfs_cli.ape_manager.add(
cluster.default_rpc_endpoint,
deny_rule.chain_id,
target_name=cid,
target_type="container",
rule=deny_rule.as_string(),
) )
with reporter.step("Wait for one block"):
self.wait_for_blocks()
return cid
@pytest.fixture(scope="function")
def container_with_objects(self, private_container: str, default_wallet: WalletInfo, file_path: TestFile):
objects_with_header, objects_with_other_header, objects_without_header = self._fill_container(
default_wallet, file_path, private_container
)
return private_container, objects_with_header, objects_with_other_header, objects_without_header, file_path
@reporter.step("Add objects to container")
def _fill_container(self, wallet: WalletInfo, test_file: TestFile, cid: str):
objects_with_header = [
put_object_to_random_node(wallet, test_file, cid, self.shell, self.cluster, attributes={**self.ATTRIBUTES, "key": val})
for val in range(self.OBJECT_COUNT) for val in range(self.OBJECT_COUNT)
] ]
objects_with_other_header = [ @pytest.fixture
put_object_to_random_node(wallet, test_file, cid, self.shell, self.cluster, attributes={**self.OTHER_ATTRIBUTES, "key": val}) def objects_with_other_attributes(self, default_wallet: WalletInfo, file_path: TestFile, container: str):
return [
put_object_to_random_node(
default_wallet, file_path, container, self.shell, self.cluster, attributes={**self.OTHER_ATTRIBUTES, "key": val}
)
for val in range(self.OBJECT_COUNT) for val in range(self.OBJECT_COUNT)
] ]
objects_without_header = [ @pytest.fixture
put_object_to_random_node(wallet, test_file, cid, self.shell, self.cluster) for _ in range(self.OBJECT_COUNT) def objects_without_attributes(self, default_wallet: WalletInfo, file_path: TestFile, container: str):
] return [put_object_to_random_node(default_wallet, file_path, container, self.shell, self.cluster) for _ in range(self.OBJECT_COUNT)]
return objects_with_header, objects_with_other_header, objects_without_header
@pytest.mark.sanity @pytest.mark.sanity
@allure.title("Operations with request filter (match_type={match_type}, obj_size={object_size})") @allure.title("Operations with request filter (match_type={match_type}, obj_size={object_size})")
@ -112,24 +77,22 @@ class TestApeFilters(ClusterTestBase):
frostfs_cli: FrostfsCli, frostfs_cli: FrostfsCli,
temp_directory: str, temp_directory: str,
other_wallet: WalletInfo, other_wallet: WalletInfo,
public_container_with_objects: tuple[str, list[str], str], container: str,
objects_with_attributes: list[str],
objects_with_other_attributes: list[str],
objects_without_attributes: list[str],
match_type: ape.MatchType, match_type: ape.MatchType,
file_path: TestFile,
rpc_endpoint: str,
): ):
(
cid,
objects_with_header,
objects_with_other_header,
objects_without_header,
file_path,
) = public_container_with_objects
endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Deny all operations for others via APE with request condition"): with reporter.step("Deny all operations for others via APE with request condition"):
request_condition = ape.Condition('"frostfs:xheader/check_key"', '"check_value"', ape.ConditionType.REQUEST, match_type) request_condition = ape.Condition('"frostfs:xheader/check_key"', '"check_value"', ape.ConditionType.REQUEST, match_type)
role_condition = ape.Condition.by_role(ape.Role.OTHERS) role_condition = ape.Condition.by_role(ape.Role.OTHERS)
deny_rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, [request_condition, role_condition]) deny_rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, [request_condition, role_condition])
frostfs_cli.ape_manager.add(endpoint, deny_rule.chain_id, target_name=cid, target_type="container", rule=deny_rule.as_string()) frostfs_cli.ape_manager.add(
rpc_endpoint, deny_rule.chain_id, target_name=container, target_type="container", rule=deny_rule.as_string()
)
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
@ -137,7 +100,7 @@ class TestApeFilters(ClusterTestBase):
with reporter.step("Create bearer token with everything allowed for others role"): with reporter.step("Create bearer token with everything allowed for others role"):
role_condition = ape.Condition.by_role(ape.Role.OTHERS) role_condition = ape.Condition.by_role(ape.Role.OTHERS)
rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, role_condition) rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, role_condition)
bearer = create_bearer_token(frostfs_cli, temp_directory, cid, rule, endpoint) bearer = create_bearer_token(frostfs_cli, temp_directory, container, rule, rpc_endpoint)
# Filter denies requests where "check_key {match_type} ATTRIBUTE", so when match_type # Filter denies requests where "check_key {match_type} ATTRIBUTE", so when match_type
# is STRING_EQUAL, then requests with "check_key=OTHER_ATTRIBUTE" will be allowed while # is STRING_EQUAL, then requests with "check_key=OTHER_ATTRIBUTE" will be allowed while
@ -147,18 +110,22 @@ class TestApeFilters(ClusterTestBase):
# We test on 3 groups of objects with various headers, # We test on 3 groups of objects with various headers,
# but APE rule should ignore object headers and only work based on request headers # but APE rule should ignore object headers and only work based on request headers
for oids in [objects_with_header, objects_with_other_header, objects_without_header]: for oids in [objects_with_attributes, objects_with_other_attributes, objects_without_attributes]:
with reporter.step("Check others has full access when sending request without headers"): with reporter.step("Check others has full access when sending request without headers"):
assert_full_access_to_container(other_wallet, cid, oids.pop(), file_path, self.shell, self.cluster) assert_full_access_to_container(other_wallet, container, oids.pop(), file_path, self.shell, self.cluster)
with reporter.step("Check others has full access when sending request with allowed headers"): with reporter.step("Check others has full access when sending request with allowed headers"):
assert_full_access_to_container(other_wallet, cid, oids.pop(), file_path, self.shell, self.cluster, xhdr=allow_headers) assert_full_access_to_container(
other_wallet, container, oids.pop(), file_path, self.shell, self.cluster, xhdr=allow_headers
)
with reporter.step("Check others has no access when sending request with denied headers"): with reporter.step("Check others has no access when sending request with denied headers"):
assert_no_access_to_container(other_wallet, cid, oids.pop(), file_path, self.shell, self.cluster, xhdr=deny_headers) assert_no_access_to_container(other_wallet, container, oids.pop(), file_path, self.shell, self.cluster, xhdr=deny_headers)
with reporter.step("Check others has full access when sending request with denied headers and using bearer token"): with reporter.step("Check others has full access when sending request with denied headers and using bearer token"):
assert_full_access_to_container(other_wallet, cid, oids.pop(), file_path, self.shell, self.cluster, bearer, deny_headers) assert_full_access_to_container(
other_wallet, container, oids.pop(), file_path, self.shell, self.cluster, bearer, deny_headers
)
@allure.title("Operations with deny user headers filter (match_type={match_type}, obj_size={object_size})") @allure.title("Operations with deny user headers filter (match_type={match_type}, obj_size={object_size})")
@pytest.mark.parametrize("match_type", [ape.MatchType.EQUAL, ape.MatchType.NOT_EQUAL]) @pytest.mark.parametrize("match_type", [ape.MatchType.EQUAL, ape.MatchType.NOT_EQUAL])
@ -168,19 +135,14 @@ class TestApeFilters(ClusterTestBase):
frostfs_cli: FrostfsCli, frostfs_cli: FrostfsCli,
temp_directory: str, temp_directory: str,
other_wallet: WalletInfo, other_wallet: WalletInfo,
public_container_with_objects: tuple[str, list[str], str], container: str,
objects_with_attributes: list[str],
objects_with_other_attributes: list[str],
objects_without_attributes: list[str],
match_type: ape.MatchType, match_type: ape.MatchType,
rpc_endpoint: str,
file_path: TestFile,
): ):
# TODO: Refactor this to be fixtures, not test logic
(
cid,
objects_with_attributes,
objects_with_other_attributes,
objs_without_attributes,
file_path,
) = public_container_with_objects
endpoint = self.cluster.default_rpc_endpoint
allow_objects = objects_with_other_attributes if match_type == ape.MatchType.EQUAL else objects_with_attributes allow_objects = objects_with_other_attributes if match_type == ape.MatchType.EQUAL else objects_with_attributes
deny_objects = objects_with_attributes if match_type == ape.MatchType.EQUAL else objects_with_other_attributes deny_objects = objects_with_attributes if match_type == ape.MatchType.EQUAL else objects_with_other_attributes
@ -210,15 +172,15 @@ class TestApeFilters(ClusterTestBase):
ape.ObjectOperations.DELETE: False, # Because delete needs to put a tombstone without attributes ape.ObjectOperations.DELETE: False, # Because delete needs to put a tombstone without attributes
}, },
} }
# End of refactor
with reporter.step("Deny operations for others via APE with resource condition"): with reporter.step("Deny operations for others via APE with resource condition"):
resource_condition = ape.Condition('"check_key"', '"check_value"', ape.ConditionType.RESOURCE, match_type) resource_condition = ape.Condition('"check_key"', '"check_value"', ape.ConditionType.RESOURCE, match_type)
not_a_tombstone_condition = ape.Condition.by_object_type("TOMBSTONE", ape.ConditionType.RESOURCE, ape.MatchType.NOT_EQUAL)
role_condition = ape.Condition.by_role(ape.Role.OTHERS) role_condition = ape.Condition.by_role(ape.Role.OTHERS)
deny_rule = ape.Rule(ape.Verb.DENY, self.RESOURCE_OPERATIONS, [resource_condition, role_condition]) deny_rule = ape.Rule(ape.Verb.DENY, self.RESOURCE_OPERATIONS, [resource_condition, role_condition])
frostfs_cli.ape_manager.add(endpoint, deny_rule.chain_id, target_name=cid, target_type="container", rule=deny_rule.as_string()) frostfs_cli.ape_manager.add(
rpc_endpoint, deny_rule.chain_id, target_name=container, target_type="container", rule=deny_rule.as_string()
)
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
@ -226,12 +188,12 @@ class TestApeFilters(ClusterTestBase):
with reporter.step("Create bearer token with everything allowed for others role"): with reporter.step("Create bearer token with everything allowed for others role"):
role_condition = ape.Condition.by_role(ape.Role.OTHERS) role_condition = ape.Condition.by_role(ape.Role.OTHERS)
rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, role_condition) rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, role_condition)
bearer = create_bearer_token(frostfs_cli, temp_directory, cid, rule, endpoint) bearer = create_bearer_token(frostfs_cli, temp_directory, container, rule, rpc_endpoint)
with reporter.step("Create bearer token with allowed put for others role"): with reporter.step("Create bearer token with allowed put for others role"):
role_condition = ape.Condition.by_role(ape.Role.OTHERS) role_condition = ape.Condition.by_role(ape.Role.OTHERS)
rule = ape.Rule(ape.Verb.ALLOW, ape.ObjectOperations.PUT, role_condition) rule = ape.Rule(ape.Verb.ALLOW, ape.ObjectOperations.PUT, role_condition)
bearer_put = create_bearer_token(frostfs_cli, temp_directory, cid, rule, endpoint) bearer_put = create_bearer_token(frostfs_cli, temp_directory, container, rule, rpc_endpoint)
# We will attempt requests with various headers, # We will attempt requests with various headers,
# but APE rule should ignore request headers and validate only object headers # but APE rule should ignore request headers and validate only object headers
@ -240,8 +202,8 @@ class TestApeFilters(ClusterTestBase):
assert_access_to_container( assert_access_to_container(
no_attributes_access[match_type], no_attributes_access[match_type],
other_wallet, other_wallet,
cid, container,
objs_without_attributes.pop(), objects_without_attributes.pop(),
file_path, file_path,
self.shell, self.shell,
self.cluster, self.cluster,
@ -250,72 +212,70 @@ class TestApeFilters(ClusterTestBase):
with reporter.step("Check others have full access to objects without deny attribute"): with reporter.step("Check others have full access to objects without deny attribute"):
assert_access_to_container( assert_access_to_container(
allowed_access[match_type], other_wallet, cid, allow_objects.pop(), file_path, self.shell, self.cluster, xhdr=xhdr allowed_access[match_type], other_wallet, container, allow_objects.pop(), file_path, self.shell, self.cluster, xhdr=xhdr
) )
with reporter.step("Check others have no access to objects with deny attribute"): with reporter.step("Check others have no access to objects with deny attribute"):
with pytest.raises(Exception, match=OBJECT_NO_ACCESS): with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
head_object(other_wallet, cid, deny_objects[0], self.shell, endpoint, xhdr=xhdr) head_object(other_wallet, container, deny_objects[0], self.shell, rpc_endpoint, xhdr=xhdr)
with pytest.raises(Exception, match=OBJECT_NO_ACCESS): with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
get_object_from_random_node(other_wallet, cid, deny_objects[0], self.shell, self.cluster, xhdr=xhdr) get_object_from_random_node(other_wallet, container, deny_objects[0], self.shell, self.cluster, xhdr=xhdr)
with reporter.step("Check others have access to objects with deny attribute and using bearer token"): with reporter.step("Check others have access to objects with deny attribute and using bearer token"):
assert_full_access_to_container(other_wallet, cid, deny_objects.pop(), file_path, self.shell, self.cluster, bearer, xhdr) assert_full_access_to_container(
other_wallet, container, deny_objects.pop(), file_path, self.shell, self.cluster, bearer, xhdr
)
allow_attribute = self.OTHER_HEADER if match_type == ape.MatchType.EQUAL else self.HEADER allow_attribute = self.OTHER_HEADER if match_type == ape.MatchType.EQUAL else self.HEADER
with reporter.step("Check others can PUT objects without denied attribute"): with reporter.step("Check others can PUT objects without denied attribute"):
put_object_to_random_node(other_wallet, file_path, cid, self.shell, self.cluster, attributes=allow_attribute) put_object_to_random_node(other_wallet, file_path, container, self.shell, self.cluster, attributes=allow_attribute)
deny_attribute = self.HEADER if match_type == ape.MatchType.EQUAL else self.OTHER_HEADER deny_attribute = self.HEADER if match_type == ape.MatchType.EQUAL else self.OTHER_HEADER
with reporter.step("Check others can not PUT objects with denied attribute"): with reporter.step("Check others can not PUT objects with denied attribute"):
with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED): with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
put_object_to_random_node(other_wallet, file_path, cid, self.shell, self.cluster, attributes=deny_attribute) put_object_to_random_node(other_wallet, file_path, container, self.shell, self.cluster, attributes=deny_attribute)
with reporter.step("Check others can PUT objects with denied attribute and using bearer token"): with reporter.step("Check others can PUT objects with denied attribute and using bearer token"):
put_object_to_random_node(other_wallet, file_path, cid, self.shell, self.cluster, bearer_put, attributes=deny_attribute) put_object_to_random_node(other_wallet, file_path, container, self.shell, self.cluster, bearer_put, attributes=deny_attribute)
@allure.title("Operations with allow APE rule with resource filters (match_type={match_type}, obj_size={object_size})") @allure.title("Operations with allow APE rule with resource filters (match_type={match_type}, obj_size={object_size})")
@pytest.mark.parametrize("match_type", [ape.MatchType.EQUAL, ape.MatchType.NOT_EQUAL]) @pytest.mark.parametrize("match_type", [ape.MatchType.EQUAL, ape.MatchType.NOT_EQUAL])
@pytest.mark.parametrize("object_size", ["simple"], indirect=True) @pytest.mark.parametrize("object_size, container_request", [("simple", OWNER_ALLOW_ALL)], indirect=True)
def test_ape_allow_filters_object( def test_ape_allow_filters_object(
self, self,
frostfs_cli: FrostfsCli, frostfs_cli: FrostfsCli,
other_wallet: WalletInfo, other_wallet: WalletInfo,
container_with_objects: tuple[str, list[str], str], container: str,
temp_directory: str, objects_with_attributes: list[str],
objects_with_other_attributes: list[str],
objects_without_attributes: list[str],
match_type: ape.MatchType, match_type: ape.MatchType,
rpc_endpoint: str,
file_path: TestFile,
temp_directory: str,
): ):
# TODO: Refactor this to be fixtures, not test logic!
(
cid,
objects_with_attributes,
objects_with_other_attributes,
objects_without_attributes,
file_path,
) = container_with_objects
endpoint = self.cluster.default_rpc_endpoint
if match_type == ape.MatchType.EQUAL: if match_type == ape.MatchType.EQUAL:
allow_objects = objects_with_attributes allow_objects = objects_with_attributes
deny_objects = objects_with_other_attributes deny_objects = objects_with_other_attributes
allow_attribute = self.HEADER allow_attribute = self.HEADER
deny_attribute = self.OTHER_HEADER deny_attribute = self.OTHER_HEADER
no_attributes_match_context = pytest.raises(Exception, match=OBJECT_NO_ACCESS) no_attributes_match_context = pytest.raises(Exception, match=OBJECT_ACCESS_DENIED)
else: else:
allow_objects = objects_with_other_attributes allow_objects = objects_with_other_attributes
deny_objects = objects_with_attributes deny_objects = objects_with_attributes
allow_attribute = self.OTHER_HEADER allow_attribute = self.OTHER_HEADER
deny_attribute = self.HEADER deny_attribute = self.HEADER
no_attributes_match_context = expect_not_raises() no_attributes_match_context = expect_not_raises()
# End of refactor block
with reporter.step("Allow operations for others except few operations by resource condition via APE"): with reporter.step("Allow operations for others except few operations by resource condition via APE"):
resource_condition = ape.Condition('"check_key"', '"check_value"', ape.ConditionType.RESOURCE, match_type) resource_condition = ape.Condition('"check_key"', '"check_value"', ape.ConditionType.RESOURCE, match_type)
role_condition = ape.Condition.by_role(ape.Role.OTHERS) role_condition = ape.Condition.by_role(ape.Role.OTHERS)
deny_rule = ape.Rule(ape.Verb.ALLOW, self.RESOURCE_OPERATIONS, [resource_condition, role_condition]) deny_rule = ape.Rule(ape.Verb.ALLOW, self.RESOURCE_OPERATIONS, [resource_condition, role_condition])
frostfs_cli.ape_manager.add(endpoint, deny_rule.chain_id, target_name=cid, target_type="container", rule=deny_rule.as_string()) frostfs_cli.ape_manager.add(
rpc_endpoint, deny_rule.chain_id, target_name=container, target_type="container", rule=deny_rule.as_string()
)
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
@ -323,113 +283,115 @@ class TestApeFilters(ClusterTestBase):
with reporter.step("Check GET, PUT and HEAD operations with objects without attributes for OTHERS role"): with reporter.step("Check GET, PUT and HEAD operations with objects without attributes for OTHERS role"):
oid = objects_without_attributes.pop() oid = objects_without_attributes.pop()
with no_attributes_match_context: with no_attributes_match_context:
assert head_object(other_wallet, cid, oid, self.shell, endpoint) assert head_object(other_wallet, container, oid, self.shell, rpc_endpoint)
with no_attributes_match_context: with no_attributes_match_context:
assert get_object_from_random_node(other_wallet, cid, oid, self.shell, self.cluster) assert get_object_from_random_node(other_wallet, container, oid, self.shell, self.cluster)
with no_attributes_match_context: with no_attributes_match_context:
assert put_object_to_random_node(other_wallet, file_path, cid, self.shell, self.cluster) assert put_object_to_random_node(other_wallet, file_path, container, self.shell, self.cluster)
with reporter.step("Create bearer token with everything allowed for others role"): with reporter.step("Create bearer token with everything allowed for others role"):
role_condition = ape.Condition.by_role(ape.Role.OTHERS) role_condition = ape.Condition.by_role(ape.Role.OTHERS)
rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, role_condition) rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, role_condition)
bearer = create_bearer_token(frostfs_cli, temp_directory, cid, rule, endpoint) bearer = create_bearer_token(frostfs_cli, temp_directory, container, rule, rpc_endpoint)
with reporter.step("Check others can get and put objects without attributes and using bearer token"): with reporter.step("Check others can get and put objects without attributes and using bearer token"):
oid = objects_without_attributes[0] oid = objects_without_attributes[0]
with expect_not_raises(): with expect_not_raises():
head_object(other_wallet, cid, oid, self.shell, endpoint, bearer) head_object(other_wallet, container, oid, self.shell, rpc_endpoint, bearer)
with expect_not_raises(): with expect_not_raises():
get_object_from_random_node(other_wallet, cid, oid, self.shell, self.cluster, bearer) get_object_from_random_node(other_wallet, container, oid, self.shell, self.cluster, bearer)
with expect_not_raises(): with expect_not_raises():
put_object_to_random_node(other_wallet, file_path, cid, self.shell, self.cluster, bearer) put_object_to_random_node(other_wallet, file_path, container, self.shell, self.cluster, bearer)
with reporter.step("Check others can get and put objects with attributes matching the filter"): with reporter.step("Check others can get and put objects with attributes matching the filter"):
oid = allow_objects.pop() oid = allow_objects.pop()
with expect_not_raises(): with expect_not_raises():
head_object(other_wallet, cid, oid, self.shell, endpoint) head_object(other_wallet, container, oid, self.shell, rpc_endpoint)
with expect_not_raises(): with expect_not_raises():
get_object_from_random_node(other_wallet, cid, oid, self.shell, self.cluster) get_object_from_random_node(other_wallet, container, oid, self.shell, self.cluster)
with expect_not_raises(): with expect_not_raises():
put_object_to_random_node(other_wallet, file_path, cid, self.shell, self.cluster, attributes=allow_attribute) put_object_to_random_node(other_wallet, file_path, container, self.shell, self.cluster, attributes=allow_attribute)
with reporter.step("Check others cannot get and put objects without attributes matching the filter"): with reporter.step("Check others cannot get and put objects without attributes matching the filter"):
oid = deny_objects[0] oid = deny_objects[0]
with pytest.raises(Exception, match=OBJECT_NO_ACCESS): with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
head_object(other_wallet, cid, oid, self.shell, endpoint) head_object(other_wallet, container, oid, self.shell, rpc_endpoint)
with pytest.raises(Exception, match=OBJECT_NO_ACCESS): with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
assert get_object_from_random_node(other_wallet, cid, oid, self.shell, self.cluster) assert get_object_from_random_node(other_wallet, container, oid, self.shell, self.cluster)
with pytest.raises(Exception, match=OBJECT_NO_ACCESS): with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
assert put_object_to_random_node(other_wallet, file_path, cid, self.shell, self.cluster, attributes=deny_attribute) assert put_object_to_random_node(other_wallet, file_path, container, self.shell, self.cluster, attributes=deny_attribute)
with reporter.step("Check others can get and put objects without attributes matching the filter with bearer token"): with reporter.step("Check others can get and put objects without attributes matching the filter with bearer token"):
oid = deny_objects.pop() oid = deny_objects.pop()
with expect_not_raises(): with expect_not_raises():
head_object(other_wallet, cid, oid, self.shell, endpoint, bearer) head_object(other_wallet, container, oid, self.shell, rpc_endpoint, bearer)
with expect_not_raises(): with expect_not_raises():
get_object_from_random_node(other_wallet, cid, oid, self.shell, self.cluster, bearer) get_object_from_random_node(other_wallet, container, oid, self.shell, self.cluster, bearer)
with expect_not_raises(): with expect_not_raises():
put_object_to_random_node(other_wallet, file_path, cid, self.shell, self.cluster, bearer, attributes=allow_attribute) put_object_to_random_node(other_wallet, file_path, container, self.shell, self.cluster, bearer, attributes=allow_attribute)
@allure.title("PUT and GET object using bearer with objectID in filter (obj_size={object_size}, match_type=NOT_EQUAL)") @allure.title("PUT and GET object using bearer with objectID in filter (obj_size={object_size}, match_type=NOT_EQUAL)")
@pytest.mark.parametrize("container_request", [OWNER_ALLOW_ALL], indirect=True)
def test_ape_filter_object_id_not_equals( def test_ape_filter_object_id_not_equals(
self, self,
frostfs_cli: FrostfsCli, frostfs_cli: FrostfsCli,
default_wallet: WalletInfo, default_wallet: WalletInfo,
other_wallet: WalletInfo, other_wallet: WalletInfo,
private_container: str, container: str,
temp_directory: str, temp_directory: str,
file_path: TestFile, file_path: TestFile,
): ):
with reporter.step("Put object to container"): with reporter.step("Put object to container"):
oid = put_object_to_random_node(default_wallet, file_path, private_container, self.shell, self.cluster) oid = put_object_to_random_node(default_wallet, file_path, container, self.shell, self.cluster)
with reporter.step("Create bearer token with objectID filter"): with reporter.step("Create bearer token with objectID filter"):
role_condition = ape.Condition.by_role(ape.Role.OTHERS) role_condition = ape.Condition.by_role(ape.Role.OTHERS)
object_condition = ape.Condition.by_object_id(oid, ape.ConditionType.RESOURCE, ape.MatchType.NOT_EQUAL) object_condition = ape.Condition.by_object_id(oid, ape.ConditionType.RESOURCE, ape.MatchType.NOT_EQUAL)
rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, [role_condition, object_condition]) rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, [role_condition, object_condition])
bearer = create_bearer_token(frostfs_cli, temp_directory, private_container, rule, self.cluster.default_rpc_endpoint) bearer = create_bearer_token(frostfs_cli, temp_directory, container, rule, self.cluster.default_rpc_endpoint)
with reporter.step("Others should be able to put object using bearer token"): with reporter.step("Others should be able to put object using bearer token"):
with expect_not_raises(): with expect_not_raises():
put_object_to_random_node(other_wallet, file_path, private_container, self.shell, self.cluster, bearer) put_object_to_random_node(other_wallet, file_path, container, self.shell, self.cluster, bearer)
with reporter.step("Others should not be able to get object matching the filter"): with reporter.step("Others should not be able to get object matching the filter"):
with pytest.raises(Exception, match=OBJECT_NO_ACCESS): with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
get_object_from_random_node(other_wallet, private_container, oid, self.shell, self.cluster, bearer) get_object_from_random_node(other_wallet, container, oid, self.shell, self.cluster, bearer)
@allure.title("PUT and GET object using bearer with objectID in filter (obj_size={object_size}, match_type=EQUAL)") @allure.title("PUT and GET object using bearer with objectID in filter (obj_size={object_size}, match_type=EQUAL)")
@pytest.mark.parametrize("container_request", [OWNER_ALLOW_ALL], indirect=True)
def test_ape_filter_object_id_equals( def test_ape_filter_object_id_equals(
self, self,
frostfs_cli: FrostfsCli, frostfs_cli: FrostfsCli,
default_wallet: WalletInfo, default_wallet: WalletInfo,
other_wallet: WalletInfo, other_wallet: WalletInfo,
private_container: str, container: str,
temp_directory: str, temp_directory: str,
file_path: TestFile, file_path: TestFile,
): ):
with reporter.step("Put object to container"): with reporter.step("Put object to container"):
oid = put_object_to_random_node(default_wallet, file_path, private_container, self.shell, self.cluster) oid = put_object_to_random_node(default_wallet, file_path, container, self.shell, self.cluster)
with reporter.step("Create bearer token with objectID filter"): with reporter.step("Create bearer token with objectID filter"):
role_condition = ape.Condition.by_role(ape.Role.OTHERS) role_condition = ape.Condition.by_role(ape.Role.OTHERS)
object_condition = ape.Condition.by_object_id(oid, ape.ConditionType.RESOURCE, ape.MatchType.EQUAL) object_condition = ape.Condition.by_object_id(oid, ape.ConditionType.RESOURCE, ape.MatchType.EQUAL)
rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, [role_condition, object_condition]) rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, [role_condition, object_condition])
bearer = create_bearer_token(frostfs_cli, temp_directory, private_container, rule, self.cluster.default_rpc_endpoint) bearer = create_bearer_token(frostfs_cli, temp_directory, container, rule, self.cluster.default_rpc_endpoint)
with reporter.step("Others should not be able to put object using bearer token"): with reporter.step("Others should not be able to put object using bearer token"):
with pytest.raises(Exception, match=OBJECT_NO_ACCESS): with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
put_object_to_random_node(other_wallet, file_path, private_container, self.shell, self.cluster, bearer) put_object_to_random_node(other_wallet, file_path, container, self.shell, self.cluster, bearer)
with reporter.step("Others should be able to get object matching the filter"): with reporter.step("Others should be able to get object matching the filter"):
with expect_not_raises(): with expect_not_raises():
get_object_from_random_node(other_wallet, private_container, oid, self.shell, self.cluster, bearer) get_object_from_random_node(other_wallet, container, oid, self.shell, self.cluster, bearer)

View file

@ -5,9 +5,10 @@ from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
from frostfs_testlib.storage.dataclasses import ape from frostfs_testlib.storage.dataclasses import ape
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import TestFile
from pytest_tests.helpers.bearer_token import create_bearer_token from ....helpers.bearer_token import create_bearer_token
from pytest_tests.helpers.container_access import ( from ....helpers.container_access import (
ALL_OBJECT_OPERATIONS, ALL_OBJECT_OPERATIONS,
assert_access_to_container, assert_access_to_container,
assert_full_access_to_container, assert_full_access_to_container,
@ -15,6 +16,7 @@ from pytest_tests.helpers.container_access import (
) )
@pytest.mark.nightly
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.bearer @pytest.mark.bearer
@pytest.mark.ape @pytest.mark.ape
@ -23,21 +25,21 @@ class TestApeBearer(ClusterTestBase):
@pytest.mark.parametrize("role", [ape.Role.OWNER, ape.Role.OTHERS], indirect=True) @pytest.mark.parametrize("role", [ape.Role.OWNER, ape.Role.OTHERS], indirect=True)
def test_bearer_token_operations( def test_bearer_token_operations(
self, self,
container_with_objects: tuple[str, list[str], str], container: str,
objects: list[str],
frostfs_cli: FrostfsCli, frostfs_cli: FrostfsCli,
temp_directory: str, temp_directory: str,
test_wallet: WalletInfo, test_wallet: WalletInfo,
role: ape.Role, role: ape.Role,
file_path: TestFile,
rpc_endpoint: str,
): ):
cid, objects_oids, file_path = container_with_objects
endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Check {role} has full access to container without bearer token"): with reporter.step(f"Check {role} has full access to container without bearer token"):
assert_full_access_to_container(test_wallet, cid, objects_oids.pop(), file_path, self.shell, self.cluster) assert_full_access_to_container(test_wallet, container, objects.pop(), file_path, self.shell, self.cluster)
with reporter.step(f"Deny all operations for everyone via APE"): with reporter.step(f"Deny all operations for everyone via APE"):
rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS) rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS)
frostfs_cli.ape_manager.add(endpoint, rule.chain_id, target_name=cid, target_type="container", rule=rule.as_string()) frostfs_cli.ape_manager.add(rpc_endpoint, rule.chain_id, target_name=container, target_type="container", rule=rule.as_string())
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
@ -46,25 +48,25 @@ class TestApeBearer(ClusterTestBase):
bearer = create_bearer_token( bearer = create_bearer_token(
frostfs_cli, frostfs_cli,
temp_directory, temp_directory,
cid, container,
rule=ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS), rule=ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS),
endpoint=self.cluster.default_rpc_endpoint, endpoint=rpc_endpoint,
) )
with reporter.step(f"Check {role} without token has no access to all operations with container"): with reporter.step(f"Check {role} without token has no access to all operations with container"):
assert_no_access_to_container(test_wallet, cid, objects_oids.pop(), file_path, self.shell, self.cluster) assert_no_access_to_container(test_wallet, container, objects.pop(), file_path, self.shell, self.cluster)
with reporter.step(f"Check {role} with token has access to all operations with container"): with reporter.step(f"Check {role} with token has access to all operations with container"):
assert_full_access_to_container(test_wallet, cid, objects_oids.pop(), file_path, self.shell, self.cluster, bearer) assert_full_access_to_container(test_wallet, container, objects.pop(), file_path, self.shell, self.cluster, bearer)
with reporter.step(f"Remove deny rule from APE"): with reporter.step(f"Remove deny rule from APE"):
frostfs_cli.ape_manager.remove(endpoint, rule.chain_id, target_name=cid, target_type="container") frostfs_cli.ape_manager.remove(rpc_endpoint, rule.chain_id, target_name=container, target_type="container")
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
with reporter.step(f"Check {role} without token has access to all operations with container"): with reporter.step(f"Check {role} without token has access to all operations with container"):
assert_full_access_to_container(test_wallet, cid, objects_oids.pop(), file_path, self.shell, self.cluster) assert_full_access_to_container(test_wallet, container, objects.pop(), file_path, self.shell, self.cluster)
@allure.title("BearerToken for compound operations (obj_size={object_size})") @allure.title("BearerToken for compound operations (obj_size={object_size})")
def test_bearer_token_compound_operations( def test_bearer_token_compound_operations(
@ -73,16 +75,16 @@ class TestApeBearer(ClusterTestBase):
temp_directory: str, temp_directory: str,
default_wallet: WalletInfo, default_wallet: WalletInfo,
other_wallet: WalletInfo, other_wallet: WalletInfo,
container_with_objects: tuple[str, list[str], str], container: str,
objects: list[str],
rpc_endpoint: str,
file_path: TestFile,
): ):
""" """
Bearer Token COMPLETLY overrides chains set for the specific target. Bearer Token COMPLETLY overrides chains set for the specific target.
Thus, any restictions or permissions should be explicitly defined in BT. Thus, any restictions or permissions should be explicitly defined in BT.
""" """
endpoint = self.cluster.default_rpc_endpoint
cid, objects_oids, file_path = container_with_objects
wallets_map = { wallets_map = {
ape.Role.OWNER: default_wallet, ape.Role.OWNER: default_wallet,
ape.Role.OTHERS: other_wallet, ape.Role.OTHERS: other_wallet,
@ -111,25 +113,24 @@ class TestApeBearer(ClusterTestBase):
bt_access_map = { bt_access_map = {
ape.Role.OWNER: { ape.Role.OWNER: {
ape.ObjectOperations.PUT: True,
ape.ObjectOperations.GET: True,
ape.ObjectOperations.HEAD: True,
ape.ObjectOperations.GET_RANGE: True,
ape.ObjectOperations.GET_RANGE_HASH: True,
ape.ObjectOperations.SEARCH: True,
ape.ObjectOperations.DELETE: True,
},
ape.Role.OTHERS: {
ape.ObjectOperations.PUT: True, ape.ObjectOperations.PUT: True,
ape.ObjectOperations.GET: False, ape.ObjectOperations.GET: False,
ape.ObjectOperations.HEAD: True, ape.ObjectOperations.HEAD: True,
ape.ObjectOperations.GET_RANGE: True,
ape.ObjectOperations.GET_RANGE_HASH: False,
ape.ObjectOperations.SEARCH: False,
ape.ObjectOperations.DELETE: True,
},
# Bearer Token COMPLETLY overrides chains set for the specific target.
# Thus, any restictions or permissions should be explicitly defined in BT.
ape.Role.OTHERS: {
ape.ObjectOperations.PUT: False,
ape.ObjectOperations.GET: False,
ape.ObjectOperations.HEAD: False,
ape.ObjectOperations.GET_RANGE: False, ape.ObjectOperations.GET_RANGE: False,
ape.ObjectOperations.GET_RANGE_HASH: False, ape.ObjectOperations.GET_RANGE_HASH: False,
# Although SEARCH is denied by the APE chain defined in Policy contract, ape.ObjectOperations.SEARCH: False,
# Bearer Token COMPLETLY overrides chains set for the specific target. ape.ObjectOperations.DELETE: False,
# Thus, any restictions or permissions should be explicitly defined in BT.
ape.ObjectOperations.SEARCH: True,
ape.ObjectOperations.DELETE: True,
}, },
} }
@ -146,9 +147,11 @@ class TestApeBearer(ClusterTestBase):
# Operations that we will allow for each role with bearer token # Operations that we will allow for each role with bearer token
bearer_map = { bearer_map = {
ape.Role.OWNER: [ ape.Role.OWNER: [
ape.ObjectOperations.DELETE,
ape.ObjectOperations.PUT, ape.ObjectOperations.PUT,
ape.ObjectOperations.HEAD,
ape.ObjectOperations.GET_RANGE, ape.ObjectOperations.GET_RANGE,
# Delete also requires PUT (to make tobstone) and HEAD (to get simple objects header)
ape.ObjectOperations.DELETE,
], ],
ape.Role.OTHERS: [ ape.Role.OTHERS: [
ape.ObjectOperations.GET, ape.ObjectOperations.GET,
@ -167,24 +170,26 @@ class TestApeBearer(ClusterTestBase):
for role, operations in deny_map.items(): for role, operations in deny_map.items():
with reporter.step(f"Add APE deny rule for {role}"): with reporter.step(f"Add APE deny rule for {role}"):
rule = ape.Rule(ape.Verb.DENY, operations, conditions_map[role]) rule = ape.Rule(ape.Verb.DENY, operations, conditions_map[role])
frostfs_cli.ape_manager.add(endpoint, rule.chain_id, target_name=cid, target_type="container", rule=rule.as_string()) frostfs_cli.ape_manager.add(
rpc_endpoint, rule.chain_id, target_name=container, target_type="container", rule=rule.as_string()
)
with reporter.step("Wait for one block"): with reporter.step("Wait for one block"):
self.wait_for_blocks() self.wait_for_blocks()
for role, wallet in wallets_map.items(): for role, wallet in wallets_map.items():
with reporter.step(f"Assert access to container without bearer token for {role}"): with reporter.step(f"Assert access to container without bearer token for {role}"):
assert_access_to_container(access_map[role], wallet, cid, objects_oids.pop(), file_path, self.shell, self.cluster) assert_access_to_container(access_map[role], wallet, container, objects.pop(), file_path, self.shell, self.cluster)
bearer_tokens = {} bearer_tokens = {}
for role in wallets_map.keys(): for role in wallets_map.keys():
with reporter.step(f"Create bearer token for {role}"): with reporter.step(f"Create bearer token for {role}"):
rule = ape.Rule(verb_map[role], bearer_map[role], conditions_map[role]) rule = ape.Rule(verb_map[role], bearer_map[role], conditions_map[role])
bt = create_bearer_token(frostfs_cli, temp_directory, cid, rule, endpoint) bt = create_bearer_token(frostfs_cli, temp_directory, container, rule, rpc_endpoint)
bearer_tokens[role] = bt bearer_tokens[role] = bt
for role, wallet in wallets_map.items(): for role, wallet in wallets_map.items():
with reporter.step(f"Assert access to container with bearer token for {role}"): with reporter.step(f"Assert access to container with bearer token for {role}"):
assert_access_to_container( assert_access_to_container(
bt_access_map[role], wallet, cid, objects_oids.pop(), file_path, self.shell, self.cluster, bearer_tokens[role] bt_access_map[role], wallet, container, objects.pop(), file_path, self.shell, self.cluster, bearer_tokens[role]
) )

View file

@ -1,10 +1,11 @@
import json
import pytest import pytest
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import create_container from frostfs_testlib.steps.cli.container import search_nodes_with_container
from frostfs_testlib.steps.cli.object import put_object_to_random_node from frostfs_testlib.steps.cli.object import put_object_to_random_node
from frostfs_testlib.storage.cluster import Cluster from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses import ape from frostfs_testlib.storage.dataclasses import ape
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.parallel import parallel from frostfs_testlib.testing.parallel import parallel
@ -39,26 +40,33 @@ def test_wallet(default_wallet: WalletInfo, other_wallet: WalletInfo, role: ape.
return role_to_wallet_map[role] return role_to_wallet_map[role]
@pytest.fixture(scope="function") @pytest.fixture
def container_with_objects(default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, file_path: str) -> tuple[str, list[str], str]: def objects(container: str, default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, file_path: str):
with reporter.step("Create public container"):
cid = create_container(
default_wallet,
shell=client_shell,
endpoint=cluster.default_rpc_endpoint,
basic_acl=PUBLIC_ACL,
)
with reporter.step("Add test objects to container"): with reporter.step("Add test objects to container"):
put_results = parallel( put_results = parallel(
[put_object_to_random_node] * OBJECT_COUNT, [put_object_to_random_node] * OBJECT_COUNT,
wallet=default_wallet, wallet=default_wallet,
path=file_path, path=file_path,
cid=cid, cid=container,
shell=client_shell, shell=client_shell,
cluster=cluster, cluster=cluster,
) )
objects_oids = [put_result.result() for put_result in put_results] objects_oids = [put_result.result() for put_result in put_results]
return cid, objects_oids, file_path return objects_oids
@pytest.fixture
def container_nodes(default_wallet: WalletInfo, container: str, client_shell: Shell, cluster: Cluster) -> list[ClusterNode]:
cid = container
container_holder_nodes = search_nodes_with_container(default_wallet, cid, client_shell, cluster.default_rpc_endpoint, cluster)
report_data = {node.id: node.host_ip for node in container_holder_nodes}
reporter.attach(json.dumps(report_data, indent=2), "container_nodes.json")
return container_holder_nodes
@pytest.fixture
def container_node_wallet(container_nodes: list[ClusterNode]) -> WalletInfo:
return WalletInfo.from_node(container_nodes[0].storage_node)

File diff suppressed because it is too large Load diff

View file

@ -1,7 +1,5 @@
import logging import logging
import os
import random import random
import shutil
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Optional from typing import Optional
@ -14,11 +12,13 @@ from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.healthcheck.interfaces import Healthcheck from frostfs_testlib.healthcheck.interfaces import Healthcheck
from frostfs_testlib.hosting import Hosting from frostfs_testlib.hosting import Hosting
from frostfs_testlib.resources import optionals from frostfs_testlib.resources import optionals
from frostfs_testlib.resources.common import ASSETS_DIR, COMPLEX_OBJECT_CHUNKS_COUNT, COMPLEX_OBJECT_TAIL_SIZE, SIMPLE_OBJECT_SIZE from frostfs_testlib.resources.common import COMPLEX_OBJECT_CHUNKS_COUNT, COMPLEX_OBJECT_TAIL_SIZE, SIMPLE_OBJECT_SIZE
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.shell import LocalShell, Shell from frostfs_testlib.shell import LocalShell, Shell
from frostfs_testlib.steps.cli.container import DEFAULT_EC_PLACEMENT_RULE, DEFAULT_PLACEMENT_RULE, FROSTFS_CLI_EXEC from frostfs_testlib.steps.cli.container import DEFAULT_EC_PLACEMENT_RULE, DEFAULT_PLACEMENT_RULE, FROSTFS_CLI_EXEC
from frostfs_testlib.steps.cli.object import get_netmap_netinfo from frostfs_testlib.steps.cli.object import get_netmap_netinfo
from frostfs_testlib.steps.epoch import ensure_fresh_epoch
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.cluster import Cluster, ClusterNode from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
@ -26,19 +26,24 @@ from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.storage.grpc_operations.client_wrappers import CliClientWrapper
from frostfs_testlib.storage.grpc_operations.interfaces import GrpcClientWrapper
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.parallel import parallel from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.testing.test_control import run_optionally, wait_for_success from frostfs_testlib.testing.test_control import cached_fixture, run_optionally, wait_for_success
from frostfs_testlib.utils import env_utils, string_utils, version_utils from frostfs_testlib.utils import env_utils, string_utils, version_utils
from frostfs_testlib.utils.file_utils import TestFile, generate_file from frostfs_testlib.utils.file_utils import TestFile, generate_file
from pytest_tests.resources.common import TEST_CYCLES_COUNT from ..helpers.container_creation import create_container_with_ape
from ..helpers.container_request import EVERYONE_ALLOW_ALL, ContainerRequest
from ..resources.common import TEST_CYCLES_COUNT
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
SERVICE_ACTIVE_TIME = 20 SERVICE_ACTIVE_TIME = 20
WALLTETS_IN_POOL = 2 WALLTETS_IN_POOL = 2
# Add logs check test even if it's not fit to mark selectors # Add logs check test even if it's not fit to mark selectors
def pytest_configure(config: pytest.Config): def pytest_configure(config: pytest.Config):
markers = config.option.markexpr markers = config.option.markexpr
@ -49,6 +54,8 @@ def pytest_configure(config: pytest.Config):
number_key = pytest.StashKey[str]() number_key = pytest.StashKey[str]()
start_time = pytest.StashKey[int]() start_time = pytest.StashKey[int]()
test_outcome = pytest.StashKey[str]() test_outcome = pytest.StashKey[str]()
# pytest hook. Do not rename # pytest hook. Do not rename
def pytest_collection_modifyitems(items: list[pytest.Item]): def pytest_collection_modifyitems(items: list[pytest.Item]):
# Change order of tests based on @pytest.mark.order(<int>) marker # Change order of tests based on @pytest.mark.order(<int>) marker
@ -106,11 +113,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc):
return return
metafunc.fixturenames.append("cycle") metafunc.fixturenames.append("cycle")
metafunc.parametrize( metafunc.parametrize("cycle", range(1, TEST_CYCLES_COUNT + 1), ids=[f"cycle {cycle}" for cycle in range(1, TEST_CYCLES_COUNT + 1)])
"cycle",
range(1, TEST_CYCLES_COUNT + 1),
ids=[f"cycle {cycle}" for cycle in range(1, TEST_CYCLES_COUNT + 1)],
)
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
@ -139,19 +142,16 @@ def require_multiple_interfaces(cluster: Cluster):
interfaces = cluster.cluster_nodes[0].host.config.interfaces interfaces = cluster.cluster_nodes[0].host.config.interfaces
if "internal1" not in interfaces or "data1" not in interfaces: if "internal1" not in interfaces or "data1" not in interfaces:
pytest.skip("This test requires multiple internal and data interfaces") pytest.skip("This test requires multiple internal and data interfaces")
yield return
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
@cached_fixture(optionals.OPTIONAL_CACHE_FIXTURES)
def max_object_size(cluster: Cluster, client_shell: Shell) -> int: def max_object_size(cluster: Cluster, client_shell: Shell) -> int:
storage_node = cluster.storage_nodes[0] storage_node = cluster.storage_nodes[0]
wallet = WalletInfo.from_node(storage_node) wallet = WalletInfo.from_node(storage_node)
net_info = get_netmap_netinfo( net_info = get_netmap_netinfo(wallet=wallet, endpoint=storage_node.get_rpc_endpoint(), shell=client_shell)
wallet=wallet, return net_info["maximum_object_size"]
endpoint=storage_node.get_rpc_endpoint(),
shell=client_shell,
)
yield net_info["maximum_object_size"]
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
@ -160,11 +160,6 @@ def simple_object_size(max_object_size: int) -> ObjectSize:
return ObjectSize("simple", size) return ObjectSize("simple", size)
@pytest.fixture()
def file_path(object_size: ObjectSize) -> TestFile:
return generate_file(object_size.value)
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def complex_object_size(max_object_size: int) -> ObjectSize: def complex_object_size(max_object_size: int) -> ObjectSize:
size = max_object_size * int(COMPLEX_OBJECT_CHUNKS_COUNT) + int(COMPLEX_OBJECT_TAIL_SIZE) size = max_object_size * int(COMPLEX_OBJECT_CHUNKS_COUNT) + int(COMPLEX_OBJECT_TAIL_SIZE)
@ -174,8 +169,7 @@ def complex_object_size(max_object_size: int) -> ObjectSize:
# By default we want all tests to be executed with both object sizes # By default we want all tests to be executed with both object sizes
# This can be overriden in choosen tests if needed # This can be overriden in choosen tests if needed
@pytest.fixture( @pytest.fixture(
scope="session", scope="session", params=[pytest.param("simple", marks=pytest.mark.simple), pytest.param("complex", marks=pytest.mark.complex)]
params=[pytest.param("simple", marks=pytest.mark.simple), pytest.param("complex", marks=pytest.mark.complex)],
) )
def object_size(simple_object_size: ObjectSize, complex_object_size: ObjectSize, request: pytest.FixtureRequest) -> ObjectSize: def object_size(simple_object_size: ObjectSize, complex_object_size: ObjectSize, request: pytest.FixtureRequest) -> ObjectSize:
if request.param == "simple": if request.param == "simple":
@ -184,6 +178,22 @@ def object_size(simple_object_size: ObjectSize, complex_object_size: ObjectSize,
return complex_object_size return complex_object_size
@pytest.fixture()
def test_file(object_size: ObjectSize) -> TestFile:
return generate_file(object_size.value)
@pytest.fixture(scope="module")
def test_file_module(object_size: ObjectSize) -> TestFile:
return generate_file(object_size.value)
# Deprecated. Please migrate all to test_file
@pytest.fixture()
def file_path(test_file: TestFile) -> TestFile:
return test_file
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def rep_placement_policy() -> PlacementPolicy: def rep_placement_policy() -> PlacementPolicy:
return PlacementPolicy("rep", DEFAULT_PLACEMENT_RULE) return PlacementPolicy("rep", DEFAULT_PLACEMENT_RULE)
@ -200,19 +210,24 @@ def frostfs_cli(client_shell: Shell, default_wallet: WalletInfo) -> FrostfsCli:
return FrostfsCli(client_shell, FROSTFS_CLI_EXEC, default_wallet.config_path) return FrostfsCli(client_shell, FROSTFS_CLI_EXEC, default_wallet.config_path)
@pytest.fixture(scope="session")
@allure.title("Init GrpcClientWrapper with local Frostfs CLI")
def grpc_client(frostfs_cli: FrostfsCli) -> GrpcClientWrapper:
return CliClientWrapper(frostfs_cli)
# By default we want all tests to be executed with both storage policies. # By default we want all tests to be executed with both storage policies.
# This can be overriden in choosen tests if needed. # This can be overriden in choosen tests if needed.
@pytest.fixture( @pytest.fixture(scope="session", params=[pytest.param("rep", marks=pytest.mark.rep), pytest.param("ec", marks=pytest.mark.ec)])
scope="session",
params=[pytest.param("rep", marks=pytest.mark.rep), pytest.param("ec", marks=pytest.mark.ec)],
)
def placement_policy( def placement_policy(
rep_placement_policy: PlacementPolicy, ec_placement_policy: PlacementPolicy, request: pytest.FixtureRequest rep_placement_policy: PlacementPolicy, ec_placement_policy: PlacementPolicy, request: pytest.FixtureRequest
) -> PlacementPolicy: ) -> PlacementPolicy:
if request.param == "rep": if request.param == "rep":
return rep_placement_policy return rep_placement_policy
elif request.param == "ec":
return ec_placement_policy
return ec_placement_policy return request.param
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
@ -371,20 +386,6 @@ def collect_binary_versions(hosting: Hosting, client_shell: Shell, request: pyte
env_utils.save_env_properties(file_path, all_versions) env_utils.save_env_properties(file_path, all_versions)
@reporter.step("Prepare tmp directory")
@pytest.fixture(scope="session")
def temp_directory(configure_testlib):
with reporter.step("Prepare tmp directory"):
full_path = os.path.join(os.getcwd(), ASSETS_DIR)
shutil.rmtree(full_path, ignore_errors=True)
os.mkdir(full_path)
yield full_path
with reporter.step("Remove tmp directory"):
shutil.rmtree(full_path)
@reporter.step("[Autouse/Session] Test session start time") @reporter.step("[Autouse/Session] Test session start time")
@pytest.fixture(scope="session", autouse=True) @pytest.fixture(scope="session", autouse=True)
def session_start_time(configure_testlib): def session_start_time(configure_testlib):
@ -400,6 +401,11 @@ def after_deploy_healthcheck(cluster: Cluster):
parallel(readiness_on_node, cluster.cluster_nodes) parallel(readiness_on_node, cluster.cluster_nodes)
@pytest.fixture(scope="session")
def rpc_endpoint(cluster: Cluster):
return cluster.default_rpc_endpoint
@wait_for_success(60 * SERVICE_ACTIVE_TIME * 3, 60, title="Wait for {cluster_node} readiness") @wait_for_success(60 * SERVICE_ACTIVE_TIME * 3, 60, title="Wait for {cluster_node} readiness")
def readiness_on_node(cluster_node: ClusterNode): def readiness_on_node(cluster_node: ClusterNode):
if "skip_readiness_check" in cluster_node.host.config.attributes and cluster_node.host.config.attributes["skip_readiness_check"]: if "skip_readiness_check" in cluster_node.host.config.attributes and cluster_node.host.config.attributes["skip_readiness_check"]:
@ -427,6 +433,7 @@ def readiness_on_node(cluster_node: ClusterNode):
@reporter.step("Prepare default user with wallet") @reporter.step("Prepare default user with wallet")
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
@cached_fixture(optionals.OPTIONAL_CACHE_FIXTURES)
def default_user(credentials_provider: CredentialsProvider, cluster: Cluster) -> User: def default_user(credentials_provider: CredentialsProvider, cluster: Cluster) -> User:
user = User(string_utils.unique_name("user-")) user = User(string_utils.unique_name("user-"))
node = cluster.cluster_nodes[0] node = cluster.cluster_nodes[0]
@ -443,6 +450,7 @@ def default_wallet(default_user: User) -> WalletInfo:
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
@cached_fixture(optionals.OPTIONAL_CACHE_FIXTURES)
def wallets_pool(credentials_provider: CredentialsProvider, cluster: Cluster) -> list[WalletInfo]: def wallets_pool(credentials_provider: CredentialsProvider, cluster: Cluster) -> list[WalletInfo]:
users = [User(string_utils.unique_name("user-")) for _ in range(WALLTETS_IN_POOL)] users = [User(string_utils.unique_name("user-")) for _ in range(WALLTETS_IN_POOL)]
parallel(credentials_provider.GRPC.provide, users, cluster_node=cluster.cluster_nodes[0]) parallel(credentials_provider.GRPC.provide, users, cluster_node=cluster.cluster_nodes[0])
@ -470,3 +478,56 @@ def node_under_test(cluster: Cluster) -> ClusterNode:
selected_node = random.choice(cluster.cluster_nodes) selected_node = random.choice(cluster.cluster_nodes)
reporter.attach(f"{selected_node}", "Selected node") reporter.attach(f"{selected_node}", "Selected node")
return selected_node return selected_node
@allure.title("Init bucket container resolver")
@pytest.fixture()
def bucket_container_resolver(node_under_test: ClusterNode) -> BucketContainerResolver:
resolver_cls = plugins.load_plugin("frostfs.testlib.bucket_cid_resolver", node_under_test.host.config.product)
resolver: BucketContainerResolver = resolver_cls()
return resolver
@pytest.fixture(scope="session", params=[pytest.param(EVERYONE_ALLOW_ALL)])
def container_request(request: pytest.FixtureRequest) -> ContainerRequest:
if "param" in request.__dict__:
return request.param
container_marker = request.node.get_closest_marker("container")
# let default container to be public at the moment
container_request = EVERYONE_ALLOW_ALL
if container_marker:
if len(container_marker.args) != 1:
raise RuntimeError(f"Something wrong with container marker: {container_marker}")
container_request = container_marker.args[0]
if not container_request:
raise RuntimeError(
f"""Container specification is empty.
Add @pytest.mark.parametrize("container_request", [ContainerRequest(...)], indirect=True) decorator."""
)
return container_request
@pytest.fixture
def container(
default_wallet: WalletInfo,
frostfs_cli: FrostfsCli,
client_shell: Shell,
cluster: Cluster,
rpc_endpoint: str,
container_request: ContainerRequest,
) -> str:
return create_container_with_ape(frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint, container_request)
@pytest.fixture()
def new_epoch(client_shell: Shell, cluster: Cluster) -> int:
return ensure_fresh_epoch(client_shell, cluster)
@pytest.fixture(scope="module")
def new_epoch_module_scope(client_shell: Shell, cluster: Cluster) -> int:
return ensure_fresh_epoch(client_shell, cluster)

View file

@ -13,37 +13,27 @@ from frostfs_testlib.steps.cli.container import (
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from pytest_tests.helpers.utility import placement_policy_from_container from ...helpers.utility import placement_policy_from_container
@pytest.mark.container @pytest.mark.nightly
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.container
class TestContainer(ClusterTestBase): class TestContainer(ClusterTestBase):
PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X"
@allure.title("Create container (name={name})") @allure.title("Create container (name={name})")
@pytest.mark.parametrize("name", ["", "test-container"], ids=["No name", "Set particular name"]) @pytest.mark.parametrize("name", ["", "test-container"], ids=["No name", "Set particular name"])
@pytest.mark.smoke @pytest.mark.smoke
def test_container_creation(self, default_wallet: WalletInfo, name: str): def test_container_creation(self, default_wallet: WalletInfo, name: str, rpc_endpoint: str):
wallet = default_wallet wallet = default_wallet
placement_rule = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X" cid = create_container(wallet, self.shell, rpc_endpoint, self.PLACEMENT_RULE, name=name)
cid = create_container(
wallet,
rule=placement_rule,
name=name,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
containers = list_containers(wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint) containers = list_containers(wallet, self.shell, rpc_endpoint)
assert cid in containers, f"Expected container {cid} in containers: {containers}" assert cid in containers, f"Expected container {cid} in containers: {containers}"
container_info: str = get_container( container_info: str = get_container(wallet, cid, self.shell, rpc_endpoint, False)
wallet,
cid,
json_mode=False,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
container_info = container_info.casefold() # To ignore case when comparing with expected values container_info = container_info.casefold() # To ignore case when comparing with expected values
info_to_check = { info_to_check = {
@ -55,7 +45,7 @@ class TestContainer(ClusterTestBase):
info_to_check.add(f"Name={name}") info_to_check.add(f"Name={name}")
with reporter.step("Check container has correct information"): with reporter.step("Check container has correct information"):
expected_policy = placement_rule.casefold() expected_policy = self.PLACEMENT_RULE.casefold()
actual_policy = placement_policy_from_container(container_info) actual_policy = placement_policy_from_container(container_info)
assert actual_policy == expected_policy, f"Expected policy\n{expected_policy} but got policy\n{actual_policy}" assert actual_policy == expected_policy, f"Expected policy\n{expected_policy} but got policy\n{actual_policy}"
@ -64,50 +54,42 @@ class TestContainer(ClusterTestBase):
assert expected_info in container_info, f"Expected {expected_info} in container info:\n{container_info}" assert expected_info in container_info, f"Expected {expected_info} in container info:\n{container_info}"
with reporter.step("Delete container and check it was deleted"): with reporter.step("Delete container and check it was deleted"):
delete_container( # Force to skip frostfs-cli verifictions before delete.
wallet, # Because no APE rules assigned to container, those verifications will fail due to APE requests denial.
cid, delete_container(wallet, cid, self.shell, rpc_endpoint, force=True, await_mode=True)
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
await_mode=True,
)
self.tick_epoch() self.tick_epoch()
wait_for_container_deletion(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint) wait_for_container_deletion(wallet, cid, self.shell, rpc_endpoint)
@allure.title("Delete container without force (name={name})")
@pytest.mark.smoke
def test_container_deletion_no_force(self, container: str, default_wallet: WalletInfo, rpc_endpoint: str):
with reporter.step("Delete container and check it was deleted"):
delete_container(default_wallet, container, self.shell, rpc_endpoint, await_mode=True)
self.tick_epoch()
wait_for_container_deletion(default_wallet, container, self.shell, rpc_endpoint)
@allure.title("Parallel container creation and deletion") @allure.title("Parallel container creation and deletion")
def test_container_creation_deletion_parallel(self, default_wallet: WalletInfo): def test_container_creation_deletion_parallel(self, default_wallet: WalletInfo, rpc_endpoint: str):
containers_count = 3 containers_count = 3
wallet = default_wallet wallet = default_wallet
placement_rule = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X"
iteration_count = 10 iteration_count = 10
for iteration in range(iteration_count): for _ in range(iteration_count):
cids: list[str] = [] cids: list[str] = []
with reporter.step(f"Create {containers_count} containers"): with reporter.step(f"Create {containers_count} containers"):
for _ in range(containers_count): for _ in range(containers_count):
cids.append( cids.append(
create_container( create_container(wallet, self.shell, rpc_endpoint, self.PLACEMENT_RULE, await_mode=False, wait_for_creation=False)
wallet,
rule=placement_rule,
await_mode=False,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
wait_for_creation=False,
)
) )
with reporter.step("Wait for containers occur in container list"): with reporter.step("Wait for containers occur in container list"):
for cid in cids: for cid in cids:
wait_for_container_creation( wait_for_container_creation(wallet, cid, self.shell, rpc_endpoint, sleep_interval=containers_count)
wallet,
cid,
sleep_interval=containers_count,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
with reporter.step("Delete containers and check they were deleted"): with reporter.step("Delete containers and check they were deleted"):
for cid in cids: for cid in cids:
delete_container(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, await_mode=True) # Force to skip frostfs-cli verifictions before delete.
containers_list = list_containers(wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint) # Because no APE rules assigned to container, those verifications will fail due to APE requests denial.
delete_container(wallet, cid, self.shell, rpc_endpoint, force=True, await_mode=True)
containers_list = list_containers(wallet, self.shell, rpc_endpoint)
assert cid not in containers_list, "Container not deleted" assert cid not in containers_list, "Container not deleted"

View file

@ -16,16 +16,11 @@ from frostfs_testlib.testing.test_control import wait_for_success
from frostfs_testlib.utils.cli_utils import parse_netmap_output from frostfs_testlib.utils.cli_utils import parse_netmap_output
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
from pytest_tests.helpers.utility import placement_policy_from_container from ...helpers.utility import placement_policy_from_container
from pytest_tests.resources.policy_error_patterns import ( from ...resources.policy_error_patterns import NOT_ENOUGH_TO_SELECT, NOT_FOUND_FILTER, NOT_FOUND_SELECTOR, NOT_PARSE_POLICY
NOT_ENOUGH_TO_SELECT,
NOT_FOUND_FILTER,
NOT_FOUND_SELECTOR,
NOT_PARSE_POLICY,
)
@pytest.mark.container @pytest.mark.nightly
@pytest.mark.policy @pytest.mark.policy
class TestPolicy(ClusterTestBase): class TestPolicy(ClusterTestBase):
@wait_for_success(1200, 60, title="Wait for full field price on node", expected_result=True) @wait_for_success(1200, 60, title="Wait for full field price on node", expected_result=True)
@ -93,7 +88,9 @@ class TestPolicy(ClusterTestBase):
""" """
Negative test for placement policy: Filter not found. Negative test for placement policy: Filter not found.
""" """
placement_rule = "REP 2 IN HALF CBF 1 SELECT 2 FROM GT15 AS HALF FILTER @NOTRU AND Price GT 15 AS GT15 FILTER CountryCode NE RU AS NOTRU" placement_rule = (
"REP 2 IN HALF CBF 1 SELECT 2 FROM GT15 AS HALF FILTER @NOTRU AND Price GT 15 AS GT15 FILTER CountryCode NE RU AS NOTRU"
)
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
with pytest.raises(Exception, match=NOT_FOUND_FILTER): with pytest.raises(Exception, match=NOT_FOUND_FILTER):
@ -137,9 +134,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -149,9 +144,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -174,9 +167,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -186,9 +177,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -212,9 +201,7 @@ class TestPolicy(ClusterTestBase):
placement_params = {"country": "Russia"} placement_params = {"country": "Russia"}
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -224,9 +211,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -259,9 +244,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -271,9 +254,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -309,9 +290,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -321,9 +300,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -359,9 +336,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -371,9 +346,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -406,9 +379,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -418,9 +389,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -443,9 +412,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -455,9 +422,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -482,9 +447,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -494,9 +457,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -530,9 +491,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -542,9 +501,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -580,9 +537,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -592,9 +547,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -628,9 +581,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -640,9 +591,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -665,9 +614,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -677,9 +624,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -704,9 +649,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -716,9 +659,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -746,16 +687,16 @@ class TestPolicy(ClusterTestBase):
""" """
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 50% of available nodes. This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 50% of available nodes.
""" """
placement_rule = "REP 2 IN HALF CBF 2 SELECT 2 FROM GE15 AS HALF FILTER CountryCode NE RU AS NOTRU FILTER @NOTRU AND Price GE 15 AS GE15" placement_rule = (
"REP 2 IN HALF CBF 2 SELECT 2 FROM GE15 AS HALF FILTER CountryCode NE RU AS NOTRU FILTER @NOTRU AND Price GE 15 AS GE15"
)
placement_params = {"Price": 15, "country_code": "RU"} placement_params = {"Price": 15, "country_code": "RU"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 2 expected_copies = 2
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -765,16 +706,12 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
netmap = self.get_netmap_param(netmap) netmap = self.get_netmap_param(netmap)
with reporter.step( with reporter.step(f"Check two nodes are selected not with country code '{placement_params['country_code']}'"):
f"Check two nodes are selected not with country code '{placement_params['country_code']}'"
):
for node in resulting_copies: for node in resulting_copies:
node_address = node.get_rpc_endpoint().split(":")[0] node_address = node.get_rpc_endpoint().split(":")[0]
assert ( assert (
@ -806,9 +743,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -818,9 +753,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -862,9 +795,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -874,9 +805,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -899,9 +828,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -911,9 +838,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -938,9 +863,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -950,9 +873,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -980,16 +901,16 @@ class TestPolicy(ClusterTestBase):
""" """
This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 75% of available nodes. This test checks object's copies based on container's placement policy with SELECT and Complex FILTER results with 75% of available nodes.
""" """
placement_rule = "REP 2 IN NODES75 SELECT 2 FROM LT65 AS NODES75 FILTER Continent NE America AS NOAM FILTER @NOAM AND Price LT 65 AS LT65" placement_rule = (
"REP 2 IN NODES75 SELECT 2 FROM LT65 AS NODES75 FILTER Continent NE America AS NOAM FILTER @NOAM AND Price LT 65 AS LT65"
)
placement_params = {"Price": 65, "continent": "America"} placement_params = {"Price": 65, "continent": "America"}
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
expected_copies = 2 expected_copies = 2
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -999,9 +920,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -1039,9 +958,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -1051,9 +968,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -1094,9 +1009,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -1106,9 +1019,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1131,9 +1042,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -1143,9 +1052,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1170,9 +1077,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -1182,9 +1087,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -1217,9 +1120,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -1229,9 +1130,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -1268,9 +1167,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -1280,9 +1177,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -1315,9 +1210,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -1327,9 +1220,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1352,9 +1243,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -1364,9 +1253,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Delete the object from the container"): with reporter.step(f"Delete the object from the container"):
delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint) delete_object(wallet=default_wallet, cid=cid, oid=oid, shell=self.shell, endpoint=endpoint)
@ -1390,9 +1277,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -1402,9 +1287,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -1440,9 +1323,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -1452,9 +1333,7 @@ class TestPolicy(ClusterTestBase):
with reporter.step(f"Check object expected copies"): with reporter.step(f"Check object expected copies"):
resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes) resulting_copies = get_nodes_with_object(cid, oid, shell=self.shell, nodes=self.cluster.storage_nodes)
assert ( assert len(resulting_copies) == expected_copies, f"Expected {expected_copies} copies, got {len(resulting_copies)}"
len(resulting_copies) == expected_copies
), f"Expected {expected_copies} copies, got {len(resulting_copies)}"
with reporter.step(f"Check the object appearance"): with reporter.step(f"Check the object appearance"):
netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell)) netmap = parse_netmap_output(get_netmap_snapshot(node=resulting_copies[0], shell=self.shell))
@ -1490,9 +1369,7 @@ class TestPolicy(ClusterTestBase):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
with reporter.step(f"Create container with policy {placement_rule}"): with reporter.step(f"Create container with policy {placement_rule}"):
cid = create_container( cid = create_container(wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint)
wallet=default_wallet, rule=placement_rule, basic_acl=PUBLIC_ACL, shell=self.shell, endpoint=endpoint
)
with reporter.step(f"Check container policy"): with reporter.step(f"Check container policy"):
self.validate_object_policy(default_wallet, placement_rule, cid, endpoint) self.validate_object_policy(default_wallet, placement_rule, cid, endpoint)
@ -1528,9 +1405,7 @@ class TestPolicy(ClusterTestBase):
delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint) delete_container(wallet=default_wallet, cid=cid, shell=self.shell, endpoint=endpoint)
def validate_object_policy(self, wallet: str, placement_rule: str, cid: str, endpoint: str): def validate_object_policy(self, wallet: str, placement_rule: str, cid: str, endpoint: str):
got_policy = placement_policy_from_container( got_policy = placement_policy_from_container(get_container(wallet, cid, json_mode=False, shell=self.shell, endpoint=endpoint))
get_container(wallet, cid, json_mode=False, shell=self.shell, endpoint=endpoint)
)
assert got_policy.replace("'", "") == placement_rule.replace( assert got_policy.replace("'", "") == placement_rule.replace(
"'", "" "'", ""
), f"Expected \n{placement_rule} and got policy \n{got_policy} are the same" ), f"Expected \n{placement_rule} and got policy \n{got_policy} are the same"

View file

@ -9,6 +9,7 @@ from frostfs_testlib import reporter
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
from frostfs_testlib.steps.cli.object import get_object, put_object_to_random_node from frostfs_testlib.steps.cli.object import get_object, put_object_to_random_node
from frostfs_testlib.steps.node_management import ( from frostfs_testlib.steps.node_management import (
@ -33,6 +34,8 @@ from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_keeper import FileKeeper from frostfs_testlib.utils.file_keeper import FileKeeper
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
from ...resources.common import S3_POLICY_FILE_LOCATION
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
stopped_nodes: list[StorageNode] = [] stopped_nodes: list[StorageNode] = []
@ -94,9 +97,7 @@ class TestFailoverStorage(ClusterTestBase):
) )
with reporter.step("Check object data is not corrupted"): with reporter.step("Check object data is not corrupted"):
got_file_path = get_object( got_file_path = get_object(wallet, cid, oid, endpoint=replicated_nodes[0].get_rpc_endpoint(), shell=self.shell)
wallet, cid, oid, endpoint=replicated_nodes[0].get_rpc_endpoint(), shell=self.shell
)
assert get_file_hash(source_file_path) == get_file_hash(got_file_path) assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
with reporter.step("Return all hosts"): with reporter.step("Return all hosts"):
@ -104,12 +105,10 @@ class TestFailoverStorage(ClusterTestBase):
with reporter.step("Check object data is not corrupted"): with reporter.step("Check object data is not corrupted"):
replicated_nodes = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.cluster.storage_nodes) replicated_nodes = wait_object_replication(cid, oid, 2, shell=self.shell, nodes=self.cluster.storage_nodes)
got_file_path = get_object( got_file_path = get_object(wallet, cid, oid, shell=self.shell, endpoint=replicated_nodes[0].get_rpc_endpoint())
wallet, cid, oid, shell=self.shell, endpoint=replicated_nodes[0].get_rpc_endpoint()
)
assert get_file_hash(source_file_path) == get_file_hash(got_file_path) assert get_file_hash(source_file_path) == get_file_hash(got_file_path)
@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True) @pytest.mark.parametrize("s3_policy", [S3_POLICY_FILE_LOCATION], indirect=True)
@allure.title("Do not ignore unhealthy tree endpoints (s3_client={s3_client})") @allure.title("Do not ignore unhealthy tree endpoints (s3_client={s3_client})")
def test_unhealthy_tree( def test_unhealthy_tree(
self, self,
@ -117,6 +116,7 @@ class TestFailoverStorage(ClusterTestBase):
default_wallet: WalletInfo, default_wallet: WalletInfo,
simple_object_size: ObjectSize, simple_object_size: ObjectSize,
cluster_state_controller: ClusterStateController, cluster_state_controller: ClusterStateController,
bucket_container_resolver: BucketContainerResolver,
): ):
default_node = self.cluster.cluster_nodes[0] default_node = self.cluster.cluster_nodes[0]
@ -149,6 +149,7 @@ class TestFailoverStorage(ClusterTestBase):
wallet=default_wallet, wallet=default_wallet,
shell=self.shell, shell=self.shell,
endpoint=self.cluster.storage_nodes[0].get_rpc_endpoint(), endpoint=self.cluster.storage_nodes[0].get_rpc_endpoint(),
bucket_container_resolver=bucket_container_resolver,
)[0] )[0]
with reporter.step("Turn off all storage nodes except bucket node"): with reporter.step("Turn off all storage nodes except bucket node"):
@ -279,9 +280,7 @@ class TestEmptyMap(ClusterTestBase):
cluster_state_controller.stop_services_of_type(StorageNode) cluster_state_controller.stop_services_of_type(StorageNode)
with reporter.step("Remove all nodes from network map"): with reporter.step("Remove all nodes from network map"):
remove_nodes_from_map_morph( remove_nodes_from_map_morph(shell=self.shell, cluster=self.cluster, remove_nodes=self.cluster.services(StorageNode))
shell=self.shell, cluster=self.cluster, remove_nodes=self.cluster.services(StorageNode)
)
with reporter.step("Return all storage nodes to network map"): with reporter.step("Return all storage nodes to network map"):
self.return_nodes_after_stop_with_check_empty_map(cluster_state_controller) self.return_nodes_after_stop_with_check_empty_map(cluster_state_controller)
@ -462,9 +461,7 @@ class TestStorageDataLoss(ClusterTestBase):
s3_client.put_object(bucket, complex_object_path) s3_client.put_object(bucket, complex_object_path)
with reporter.step("Check objects are in bucket"): with reporter.step("Check objects are in bucket"):
s3_helper.check_objects_in_bucket( s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[simple_object_key, complex_object_key])
s3_client, bucket, expected_objects=[simple_object_key, complex_object_key]
)
with reporter.step("Stop storage services on all nodes"): with reporter.step("Stop storage services on all nodes"):
cluster_state_controller.stop_services_of_type(StorageNode) cluster_state_controller.stop_services_of_type(StorageNode)
@ -578,17 +575,13 @@ class TestStorageDataLoss(ClusterTestBase):
exception_messages.append(f"Shard {shard} changed status to {status}") exception_messages.append(f"Shard {shard} changed status to {status}")
with reporter.step("No related errors should be in log"): with reporter.step("No related errors should be in log"):
if node_under_test.host.is_message_in_logs( if node_under_test.host.is_message_in_logs(message_regex=r"\Wno such file or directory\W", since=test_start_time):
message_regex=r"\Wno such file or directory\W", since=test_start_time
):
exception_messages.append(f"Node {node_under_test} have shard errors in logs") exception_messages.append(f"Node {node_under_test} have shard errors in logs")
with reporter.step("Pass test if no errors found"): with reporter.step("Pass test if no errors found"):
assert not exception_messages, "\n".join(exception_messages) assert not exception_messages, "\n".join(exception_messages)
@allure.title( @allure.title("Loss of one node should trigger use of tree and storage service in another node (s3_client={s3_client})")
"Loss of one node should trigger use of tree and storage service in another node (s3_client={s3_client})"
)
def test_s3_one_endpoint_loss( def test_s3_one_endpoint_loss(
self, self,
bucket, bucket,
@ -610,7 +603,7 @@ class TestStorageDataLoss(ClusterTestBase):
put_object = s3_client.put_object(bucket, file_path) put_object = s3_client.put_object(bucket, file_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name]) s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True) @pytest.mark.parametrize("s3_policy", [S3_POLICY_FILE_LOCATION], indirect=True)
@allure.title("After Pilorama.db loss on one node object is retrievable (s3_client={s3_client})") @allure.title("After Pilorama.db loss on one node object is retrievable (s3_client={s3_client})")
def test_s3_one_pilorama_loss( def test_s3_one_pilorama_loss(
self, self,

View file

@ -44,7 +44,7 @@ from frostfs_testlib.utils import string_utils
from frostfs_testlib.utils.failover_utils import wait_object_replication from frostfs_testlib.utils.failover_utils import wait_object_replication
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes from ...helpers.utility import wait_for_gc_pass_on_storage_nodes
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
check_nodes: list[StorageNode] = [] check_nodes: list[StorageNode] = []
@ -364,7 +364,7 @@ class TestMaintenanceMode(ClusterTestBase):
cluster_state_controller: ClusterStateController, cluster_state_controller: ClusterStateController,
restore_node_status: list[ClusterNode], restore_node_status: list[ClusterNode],
): ):
with reporter.step("Create container and create\put object"): with reporter.step("Create container and put object"):
cid = create_container( cid = create_container(
wallet=default_wallet, wallet=default_wallet,
shell=self.shell, shell=self.shell,

View file

@ -1,22 +1,38 @@
import math import math
import time
import allure import allure
import pytest import pytest
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.steps.cli.container import create_container, search_nodes_with_container from frostfs_testlib.steps.cli.container import create_container, delete_container, search_nodes_with_container, wait_for_container_deletion
from frostfs_testlib.steps.cli.object import delete_object, head_object, put_object_to_random_node from frostfs_testlib.steps.cli.object import delete_object, head_object, put_object_to_random_node
from frostfs_testlib.steps.metrics import check_metrics_counter, get_metrics_value from frostfs_testlib.steps.metrics import calc_metrics_count_from_stdout, check_metrics_counter, get_metrics_value
from frostfs_testlib.steps.storage_policy import get_nodes_with_object from frostfs_testlib.steps.storage_policy import get_nodes_with_object
from frostfs_testlib.storage.cluster import Cluster from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
from ...helpers.utility import are_numbers_similar
@pytest.mark.container
@pytest.mark.nightly
@pytest.mark.metrics
class TestContainerMetrics(ClusterTestBase): class TestContainerMetrics(ClusterTestBase):
@reporter.step("Put object to container: {cid}")
def put_object_parallel(self, file_path: str, wallet: WalletInfo, cid: str):
oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)
return oid
@reporter.step("Get metrics value from node")
def get_metrics_search_by_greps_parallel(self, node: ClusterNode, **greps):
try:
content_stdout = node.metrics.storage.get_metrics_search_by_greps(greps)
return calc_metrics_count_from_stdout(content_stdout)
except Exception as e:
return None
@allure.title("Container metrics (obj_size={object_size},policy={policy})") @allure.title("Container metrics (obj_size={object_size},policy={policy})")
@pytest.mark.parametrize("placement_policy, policy", [("REP 2 IN X CBF 2 SELECT 2 FROM * AS X", "REP"), ("EC 1.1 CBF 1", "EC")]) @pytest.mark.parametrize("placement_policy, policy", [("REP 2 IN X CBF 2 SELECT 2 FROM * AS X", "REP"), ("EC 1.1 CBF 1", "EC")])
def test_container_metrics( def test_container_metrics(
@ -120,3 +136,72 @@ class TestContainerMetrics(ClusterTestBase):
with reporter.step(f"Check container size metrics"): with reporter.step(f"Check container size metrics"):
act_metric = get_metrics_value(object_nodes[0], command="frostfs_node_engine_container_size_bytes", cid=cid) act_metric = get_metrics_value(object_nodes[0], command="frostfs_node_engine_container_size_bytes", cid=cid)
assert act_metric == int(tombstone["header"]["payloadLength"]) assert act_metric == int(tombstone["header"]["payloadLength"])
@allure.title("Container size metrics put {objects_count} objects (obj_size={object_size})")
@pytest.mark.parametrize("objects_count", [5, 10, 20])
def test_container_size_metrics_more_objects(self, object_size: ObjectSize, default_wallet: WalletInfo, objects_count: int):
with reporter.step(f"Create container"):
cid = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step(f"Put {objects_count} objects"):
files_path = [generate_file(object_size.value) for _ in range(objects_count)]
futures = parallel(self.put_object_parallel, files_path, wallet=default_wallet, cid=cid)
oids = [future.result() for future in futures]
with reporter.step("Check metric appears in all nodes"):
metric_values = [
get_metrics_value(node, command="frostfs_node_engine_container_size_bytes", cid=cid) for node in self.cluster.cluster_nodes
]
actual_value = sum(metric_values) // 2 # for policy REP 2, value divide by 2
expected_value = object_size.value * objects_count
assert are_numbers_similar(
actual_value, expected_value, tolerance_percentage=2
), "metric container size bytes value not correct"
with reporter.step("Delete file, wait until gc remove object"):
tombstones_size = 0
for oid in oids:
tombstone_id = delete_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint)
tombstone = head_object(default_wallet, cid, tombstone_id, self.shell, self.cluster.default_rpc_endpoint)
tombstones_size += int(tombstone["header"]["payloadLength"])
with reporter.step(f"Check container size metrics, 'should be positive in all nodes'"):
futures = parallel(get_metrics_value, self.cluster.cluster_nodes, command="frostfs_node_engine_container_size_bytes", cid=cid)
metrics_value_nodes = [future.result() for future in futures]
for act_metric in metrics_value_nodes:
assert act_metric >= 0, "Metrics value is negative"
assert sum(metrics_value_nodes) // len(self.cluster.cluster_nodes) == tombstones_size, "tomstone size of objects not correct"
@allure.title("Container metrics (policy={policy})")
@pytest.mark.parametrize("placement_policy, policy", [("REP 2 IN X CBF 2 SELECT 2 FROM * AS X", "REP"), ("EC 1.1 CBF 1", "EC")])
def test_container_metrics_delete_complex_objects(
self, complex_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster, placement_policy: str, policy: str
):
copies = 2 if policy == "REP" else 1
objects_count = 2
metric_name = "frostfs_node_engine_container_objects_total"
with reporter.step(f"Create container"):
cid = create_container(default_wallet, self.shell, cluster.default_rpc_endpoint, rule=placement_policy)
with reporter.step(f"Put {objects_count} objects"):
files_path = [generate_file(complex_object_size.value) for _ in range(objects_count)]
futures = parallel(self.put_object_parallel, files_path, wallet=default_wallet, cid=cid)
oids = [future.result() for future in futures]
with reporter.step(f"Check metrics value in each nodes, should be {objects_count} for 'user'"):
check_metrics_counter(cluster.cluster_nodes, counter_exp=objects_count * copies, command=metric_name, cid=cid, type="user")
with reporter.step("Delete objects and container"):
for oid in oids:
delete_object(default_wallet, cid, oid, self.shell, cluster.default_rpc_endpoint)
delete_container(default_wallet, cid, self.shell, cluster.default_rpc_endpoint)
with reporter.step("Tick epoch and check container was deleted"):
self.tick_epoch()
wait_for_container_deletion(default_wallet, cid, shell=self.shell, endpoint=cluster.default_rpc_endpoint)
with reporter.step(f"Check metrics value in each nodes, should not be show any result"):
futures = parallel(self.get_metrics_search_by_greps_parallel, cluster.cluster_nodes, command=metric_name, cid=cid)
metrics_results = [future.result() for future in futures if future.result() is not None]
assert len(metrics_results) == 0, f"Metrics value is not empty in Prometheus, actual value in nodes: {metrics_results}"

View file

@ -16,6 +16,7 @@ from frostfs_testlib.testing.test_control import wait_for_success
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.nightly
class TestGarbageCollectorMetrics(ClusterTestBase): class TestGarbageCollectorMetrics(ClusterTestBase):
@wait_for_success(interval=10) @wait_for_success(interval=10)
def check_metrics_in_node(self, cluster_node: ClusterNode, counter_exp: int, **metrics_greps: str): def check_metrics_in_node(self, cluster_node: ClusterNode, counter_exp: int, **metrics_greps: str):

View file

@ -18,6 +18,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.nightly
class TestGRPCMetrics(ClusterTestBase): class TestGRPCMetrics(ClusterTestBase):
@pytest.fixture @pytest.fixture
def disable_policer(self, cluster_state_controller: ClusterStateController): def disable_policer(self, cluster_state_controller: ClusterStateController):
@ -34,9 +35,7 @@ class TestGRPCMetrics(ClusterTestBase):
node = random.choice(cluster.cluster_nodes) node = random.choice(cluster.cluster_nodes)
with reporter.step("Get current gRPC metrics for method 'Put'"): with reporter.step("Get current gRPC metrics for method 'Put'"):
metrics_counter_put = get_metrics_value( metrics_counter_put = get_metrics_value(node, command="grpc_server_handled_total", service="ContainerService", method="Put")
node, command="grpc_server_handled_total", service="ContainerService", method="Put"
)
with reporter.step(f"Create container with policy {placement_policy}"): with reporter.step(f"Create container with policy {placement_policy}"):
cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy) cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
@ -52,9 +51,7 @@ class TestGRPCMetrics(ClusterTestBase):
) )
with reporter.step("Get current gRPC metrics for method 'Get'"): with reporter.step("Get current gRPC metrics for method 'Get'"):
metrics_counter_get = get_metrics_value( metrics_counter_get = get_metrics_value(node, command="grpc_server_handled_total", service="ContainerService", method="Get")
node, command="grpc_server_handled_total", service="ContainerService", method="Get"
)
with reporter.step(f"Get container"): with reporter.step(f"Get container"):
get_container(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint()) get_container(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint())
@ -70,9 +67,7 @@ class TestGRPCMetrics(ClusterTestBase):
) )
with reporter.step("Get current gRPC metrics for method 'List'"): with reporter.step("Get current gRPC metrics for method 'List'"):
metrics_counter_list = get_metrics_value( metrics_counter_list = get_metrics_value(node, command="grpc_server_handled_total", service="ContainerService", method="List")
node, command="grpc_server_handled_total", service="ContainerService", method="List"
)
with reporter.step(f"Get container list"): with reporter.step(f"Get container list"):
list_containers(default_wallet, self.shell, node.storage_node.get_rpc_endpoint()) list_containers(default_wallet, self.shell, node.storage_node.get_rpc_endpoint())
@ -101,9 +96,7 @@ class TestGRPCMetrics(ClusterTestBase):
cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy) cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
with reporter.step("Get current gRPC metrics for method 'Put'"): with reporter.step("Get current gRPC metrics for method 'Put'"):
metrics_counter_put = get_metrics_value( metrics_counter_put = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Put")
node, command="grpc_server_handled_total", service="ObjectService", method="Put"
)
with reporter.step("Put object to selected node"): with reporter.step("Put object to selected node"):
oid = put_object(default_wallet, file_path, cid, self.shell, node.storage_node.get_rpc_endpoint()) oid = put_object(default_wallet, file_path, cid, self.shell, node.storage_node.get_rpc_endpoint())
@ -119,9 +112,7 @@ class TestGRPCMetrics(ClusterTestBase):
) )
with reporter.step("Get current gRPC metrics for method 'Get'"): with reporter.step("Get current gRPC metrics for method 'Get'"):
metrics_counter_get = get_metrics_value( metrics_counter_get = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Get")
node, command="grpc_server_handled_total", service="ObjectService", method="Get"
)
with reporter.step(f"Get object"): with reporter.step(f"Get object"):
get_object(default_wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint()) get_object(default_wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())
@ -137,9 +128,7 @@ class TestGRPCMetrics(ClusterTestBase):
) )
with reporter.step("Get current gRPC metrics for method 'Search'"): with reporter.step("Get current gRPC metrics for method 'Search'"):
metrics_counter_search = get_metrics_value( metrics_counter_search = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Search")
node, command="grpc_server_handled_total", service="ObjectService", method="Search"
)
with reporter.step(f"Search object"): with reporter.step(f"Search object"):
search_object(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint()) search_object(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint())
@ -155,9 +144,7 @@ class TestGRPCMetrics(ClusterTestBase):
) )
with reporter.step("Get current gRPC metrics for method 'Head'"): with reporter.step("Get current gRPC metrics for method 'Head'"):
metrics_counter_head = get_metrics_value( metrics_counter_head = get_metrics_value(node, command="grpc_server_handled_total", service="ObjectService", method="Head")
node, command="grpc_server_handled_total", service="ObjectService", method="Head"
)
with reporter.step(f"Head object"): with reporter.step(f"Head object"):
head_object(default_wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint()) head_object(default_wallet, cid, oid, self.shell, node.storage_node.get_rpc_endpoint())
@ -178,9 +165,7 @@ class TestGRPCMetrics(ClusterTestBase):
node = random.choice(cluster.cluster_nodes) node = random.choice(cluster.cluster_nodes)
with reporter.step("Get current gRPC metrics for Healthcheck"): with reporter.step("Get current gRPC metrics for Healthcheck"):
metrics_counter = get_metrics_value( metrics_counter = get_metrics_value(node, command="grpc_server_handled_total", service="TreeService", method="Healthcheck")
node, command="grpc_server_handled_total", service="TreeService", method="Healthcheck"
)
with reporter.step("Query Tree healthcheck status"): with reporter.step("Query Tree healthcheck status"):
healthcheck.tree_healthcheck(node) healthcheck.tree_healthcheck(node)
@ -206,9 +191,7 @@ class TestGRPCMetrics(ClusterTestBase):
cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy) cid = create_container(default_wallet, self.shell, node.storage_node.get_rpc_endpoint(), placement_policy)
with reporter.step("Get current gRPC metrics for Tree List"): with reporter.step("Get current gRPC metrics for Tree List"):
metrics_counter = get_metrics_value( metrics_counter = get_metrics_value(node, command="grpc_server_handled_total", service="TreeService", method="TreeList")
node, command="grpc_server_handled_total", service="TreeService", method="TreeList"
)
with reporter.step("Query Tree List"): with reporter.step("Query Tree List"):
get_tree_list(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint()) get_tree_list(default_wallet, cid, self.shell, node.storage_node.get_rpc_endpoint())

View file

@ -14,6 +14,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import wait_for_success from frostfs_testlib.testing.test_control import wait_for_success
@pytest.mark.nightly
class TestLogsMetrics(ClusterTestBase): class TestLogsMetrics(ClusterTestBase):
@pytest.fixture @pytest.fixture
def revert_all(self, cluster_state_controller: ClusterStateController): def revert_all(self, cluster_state_controller: ClusterStateController):
@ -41,7 +42,7 @@ class TestLogsMetrics(ClusterTestBase):
logs = cluster_node.host.get_filtered_logs( logs = cluster_node.host.get_filtered_logs(
log_level, unit="frostfs-storage", since=after_time, until=until_time, priority=log_priority log_level, unit="frostfs-storage", since=after_time, until=until_time, priority=log_priority
) )
result = re.findall(rf"Z\s+{log_level}\s+", logs) result = re.findall(rf":\s+{log_level}\s+", logs)
count_logs += len(result) count_logs += len(result)
except RuntimeError as e: except RuntimeError as e:
... ...

View file

@ -16,6 +16,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.nightly
class TestObjectMetrics(ClusterTestBase): class TestObjectMetrics(ClusterTestBase):
@allure.title("Object metrics of removed container (obj_size={object_size})") @allure.title("Object metrics of removed container (obj_size={object_size})")
def test_object_metrics_removed_container(self, object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster): def test_object_metrics_removed_container(self, object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster):

View file

@ -19,6 +19,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.nightly
class TestShardMetrics(ClusterTestBase): class TestShardMetrics(ClusterTestBase):
@pytest.fixture() @pytest.fixture()
@allure.title("Get two shards for set mode") @allure.title("Get two shards for set mode")
@ -73,9 +74,7 @@ class TestShardMetrics(ClusterTestBase):
data_path = node.storage_node.get_data_directory() data_path = node.storage_node.get_data_directory()
all_datas = node_shell.exec(f"ls -la {data_path}/data | awk '{{ print $9 }}'").stdout.strip() all_datas = node_shell.exec(f"ls -la {data_path}/data | awk '{{ print $9 }}'").stdout.strip()
for data_dir in all_datas.replace(".", "").strip().split("\n"): for data_dir in all_datas.replace(".", "").strip().split("\n"):
check_dir = node_shell.exec( check_dir = node_shell.exec(f" [ -d {data_path}/data/{data_dir}/data/{oid_path} ] && echo 1 || echo 0").stdout
f" [ -d {data_path}/data/{data_dir}/data/{oid_path} ] && echo 1 || echo 0"
).stdout
if "1" in check_dir: if "1" in check_dir:
object_path = f"{data_path}/data/{data_dir}/data/{oid_path}" object_path = f"{data_path}/data/{data_dir}/data/{oid_path}"
object_name = f"{oid[4:]}.{cid}" object_name = f"{oid[4:]}.{cid}"
@ -128,9 +127,7 @@ class TestShardMetrics(ClusterTestBase):
) )
@allure.title("Metric for error count on shard") @allure.title("Metric for error count on shard")
def test_shard_metrics_error_count( def test_shard_metrics_error_count(self, max_object_size: int, default_wallet: WalletInfo, cluster: Cluster, revert_all_shards_mode):
self, max_object_size: int, default_wallet: WalletInfo, cluster: Cluster, revert_all_shards_mode
):
file_path = generate_file(round(max_object_size * 0.8)) file_path = generate_file(round(max_object_size * 0.8))
with reporter.step(f"Create container"): with reporter.step(f"Create container"):
@ -147,11 +144,7 @@ class TestShardMetrics(ClusterTestBase):
with reporter.step("Get object nodes"): with reporter.step("Get object nodes"):
object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, cluster.storage_nodes) object_storage_nodes = get_nodes_with_object(cid, oid, self.shell, cluster.storage_nodes)
object_nodes = [ object_nodes = [cluster_node for cluster_node in cluster.cluster_nodes if cluster_node.storage_node in object_storage_nodes]
cluster_node
for cluster_node in cluster.cluster_nodes
if cluster_node.storage_node in object_storage_nodes
]
node = random.choice(object_nodes) node = random.choice(object_nodes)
with reporter.step("Search object in system."): with reporter.step("Search object in system."):

View file

@ -5,6 +5,7 @@ import sys
import allure import allure
import pytest import pytest
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
from frostfs_testlib.resources.error_patterns import ( from frostfs_testlib.resources.error_patterns import (
INVALID_LENGTH_SPECIFIER, INVALID_LENGTH_SPECIFIER,
INVALID_OFFSET_SPECIFIER, INVALID_OFFSET_SPECIFIER,
@ -14,7 +15,7 @@ from frostfs_testlib.resources.error_patterns import (
OUT_OF_RANGE, OUT_OF_RANGE,
) )
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import create_container, search_nodes_with_container from frostfs_testlib.steps.cli.container import DEFAULT_EC_PLACEMENT_RULE, DEFAULT_PLACEMENT_RULE, search_nodes_with_container
from frostfs_testlib.steps.cli.object import ( from frostfs_testlib.steps.cli.object import (
get_object_from_random_node, get_object_from_random_node,
get_range, get_range,
@ -33,12 +34,16 @@ from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file, get_file_content, get_file_hash from frostfs_testlib.utils.file_utils import TestFile, get_file_content, get_file_hash
from ...helpers.container_creation import create_container_with_ape
from ...helpers.container_request import APE_EVERYONE_ALLOW_ALL, ContainerRequest
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
CLEANUP_TIMEOUT = 10 CLEANUP_TIMEOUT = 10
COMMON_ATTRIBUTE = {"common_key": "common_value"} COMMON_ATTRIBUTE = {"common_key": "common_value"}
COMMON_CONTAINER_RULE = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
# Will upload object for each attribute set # Will upload object for each attribute set
OBJECT_ATTRIBUTES = [ OBJECT_ATTRIBUTES = [
None, None,
@ -54,9 +59,7 @@ RANGE_MAX_LEN = 500
STATIC_RANGES = {} STATIC_RANGES = {}
def generate_ranges( def generate_ranges(storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, cluster: Cluster) -> list[(int, int)]:
storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, cluster: Cluster
) -> list[(int, int)]:
file_range_step = storage_object.size / RANGES_COUNT file_range_step = storage_object.size / RANGES_COUNT
file_ranges = [] file_ranges = []
@ -82,7 +85,7 @@ def generate_ranges(
for offset, length in file_ranges: for offset, length in file_ranges:
range_length = random.randint(RANGE_MIN_LEN, RANGE_MAX_LEN) range_length = random.randint(RANGE_MIN_LEN, RANGE_MAX_LEN)
range_start = random.randint(offset, offset + length) range_start = random.randint(offset, offset + length - 1)
file_ranges_to_test.append((range_start, min(range_length, storage_object.size - range_start))) file_ranges_to_test.append((range_start, min(range_length, storage_object.size - range_start)))
@ -92,21 +95,20 @@ def generate_ranges(
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def common_container(default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster) -> str: def common_container(
rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" frostfs_cli: FrostfsCli,
with reporter.step(f"Create container with {rule} and put object"): default_wallet: WalletInfo,
cid = create_container(default_wallet, client_shell, cluster.default_rpc_endpoint, rule) client_shell: Shell,
cluster: Cluster,
return cid rpc_endpoint: str,
) -> str:
container_request = ContainerRequest(COMMON_CONTAINER_RULE, APE_EVERYONE_ALLOW_ALL)
return create_container_with_ape(frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint, container_request)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def container_nodes( def container_nodes(default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, common_container: str) -> list[ClusterNode]:
default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, common_container: str return search_nodes_with_container(default_wallet, common_container, client_shell, cluster.default_rpc_endpoint, cluster)
) -> list[ClusterNode]:
return search_nodes_with_container(
default_wallet, common_container, client_shell, cluster.default_rpc_endpoint, cluster
)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -123,35 +125,35 @@ def storage_objects(
client_shell: Shell, client_shell: Shell,
cluster: Cluster, cluster: Cluster,
object_size: ObjectSize, object_size: ObjectSize,
test_file_module: TestFile,
frostfs_cli: FrostfsCli,
placement_policy: PlacementPolicy, placement_policy: PlacementPolicy,
rpc_endpoint: str,
) -> list[StorageObjectInfo]: ) -> list[StorageObjectInfo]:
wallet = default_wallet cid = create_container_with_ape(
# Separate containers for complex/simple objects to avoid side-effects frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint, ContainerRequest(placement_policy.value, APE_EVERYONE_ALLOW_ALL)
cid = create_container(
wallet, shell=client_shell, rule=placement_policy.value, endpoint=cluster.default_rpc_endpoint
) )
file_path = generate_file(object_size.value) with reporter.step("Generate file"):
file_hash = get_file_hash(file_path) file_hash = get_file_hash(test_file_module.path)
storage_objects = [] storage_objects = []
with reporter.step("Put objects"): with reporter.step("Put objects"):
# We need to upload objects multiple times with different attributes # We need to upload objects multiple times with different attributes
for attributes in OBJECT_ATTRIBUTES: for attributes in OBJECT_ATTRIBUTES:
storage_object_id = put_object_to_random_node( storage_object_id = put_object_to_random_node(
wallet=wallet, default_wallet,
path=file_path, test_file_module.path,
cid=cid, cid,
shell=client_shell, client_shell,
cluster=cluster, cluster,
attributes=attributes, attributes=attributes,
) )
storage_object = StorageObjectInfo(cid, storage_object_id) storage_object = StorageObjectInfo(cid, storage_object_id)
storage_object.size = object_size.value storage_object.size = object_size.value
storage_object.wallet = wallet storage_object.wallet = default_wallet
storage_object.file_path = file_path storage_object.file_path = test_file_module.path
storage_object.file_hash = file_hash storage_object.file_hash = file_hash
storage_object.attributes = attributes storage_object.attributes = attributes
@ -170,6 +172,7 @@ def expected_object_copies(placement_policy: PlacementPolicy) -> int:
return 4 return 4
@pytest.mark.nightly
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.grpc_api @pytest.mark.grpc_api
class TestObjectApi(ClusterTestBase): class TestObjectApi(ClusterTestBase):
@ -248,23 +251,26 @@ class TestObjectApi(ClusterTestBase):
) )
self.check_header_is_presented(head_info, storage_object_2.attributes) self.check_header_is_presented(head_info, storage_object_2.attributes)
@allure.title("Head deleted object with --raw arg (obj_size={object_size}, policy={placement_policy})") @allure.title("Head deleted object with --raw arg (obj_size={object_size}, policy={container_request})")
def test_object_head_raw(self, default_wallet: str, object_size: ObjectSize, placement_policy: PlacementPolicy): @pytest.mark.parametrize(
with reporter.step("Create container"): "container_request",
cid = create_container( [
default_wallet, self.shell, self.cluster.default_rpc_endpoint, placement_policy.value ContainerRequest(DEFAULT_PLACEMENT_RULE, APE_EVERYONE_ALLOW_ALL, "rep"),
) ContainerRequest(DEFAULT_EC_PLACEMENT_RULE, APE_EVERYONE_ALLOW_ALL, "ec"),
],
indirect=True,
ids=["rep", "ec"],
)
def test_object_head_raw(self, default_wallet: str, container: str, test_file: TestFile):
with reporter.step("Upload object"): with reporter.step("Upload object"):
file_path = generate_file(object_size.value) oid = put_object_to_random_node(default_wallet, test_file.path, container, self.shell, self.cluster)
oid = put_object_to_random_node(default_wallet, file_path, cid, self.shell, self.cluster)
with reporter.step("Delete object"): with reporter.step("Delete object"):
delete_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint) delete_object(default_wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Call object head --raw and expect error"): with reporter.step("Call object head --raw and expect error"):
with pytest.raises(Exception, match=OBJECT_ALREADY_REMOVED): with pytest.raises(Exception, match=OBJECT_ALREADY_REMOVED):
head_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint, is_raw=True) head_object(default_wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint, is_raw=True)
@allure.title("Search objects by native API (obj_size={object_size}, policy={placement_policy})") @allure.title("Search objects by native API (obj_size={object_size}, policy={placement_policy})")
def test_search_object_api(self, storage_objects: list[StorageObjectInfo]): def test_search_object_api(self, storage_objects: list[StorageObjectInfo]):
@ -308,57 +314,51 @@ class TestObjectApi(ClusterTestBase):
assert sorted(expected_oids) == sorted(result) assert sorted(expected_oids) == sorted(result)
@allure.title("Search objects with removed items (obj_size={object_size})") @allure.title("Search objects with removed items (obj_size={object_size})")
def test_object_search_should_return_tombstone_items(self, default_wallet: WalletInfo, object_size: ObjectSize): def test_object_search_should_return_tombstone_items(
self,
default_wallet: WalletInfo,
container: str,
object_size: ObjectSize,
rpc_endpoint: str,
test_file: TestFile,
):
""" """
Validate object search with removed items Validate object search with removed items
""" """
wallet = default_wallet with reporter.step("Put object to container"):
cid = create_container(wallet, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Upload file"):
file_path = generate_file(object_size.value)
file_hash = get_file_hash(file_path)
storage_object = StorageObjectInfo( storage_object = StorageObjectInfo(
cid=cid, cid=container,
oid=put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster), oid=put_object_to_random_node(default_wallet, test_file.path, container, self.shell, self.cluster),
size=object_size.value, size=object_size.value,
wallet=wallet, wallet=default_wallet,
file_path=file_path, file_path=test_file.path,
file_hash=file_hash, file_hash=get_file_hash(test_file.path),
) )
with reporter.step("Search object"): with reporter.step("Search object"):
# Root Search object should return root object oid # Root Search object should return root object oid
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True) result = search_object(default_wallet, container, self.shell, rpc_endpoint, root=True)
assert result == [storage_object.oid] objects_before_deletion = len(result)
assert storage_object.oid in result
with reporter.step("Delete file"): with reporter.step("Delete object"):
delete_objects([storage_object], self.shell, self.cluster) delete_objects([storage_object], self.shell, self.cluster)
with reporter.step("Search deleted object with --root"): with reporter.step("Search deleted object with --root"):
# Root Search object should return nothing # Root Search object should return nothing
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True) result = search_object(default_wallet, container, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True)
assert len(result) == 0 assert len(result) == 0
with reporter.step("Search deleted object with --phy should return only tombstones"): with reporter.step("Search deleted object with --phy should return only tombstones"):
# Physical Search object should return only tombstones # Physical Search object should return only tombstones
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, phy=True) result = search_object(default_wallet, container, self.shell, rpc_endpoint, phy=True)
assert storage_object.tombstone in result, "Search result should contain tombstone of removed object" assert storage_object.tombstone in result, "Search result should contain tombstone of removed object"
assert storage_object.oid not in result, "Search result should not contain ObjectId of removed object" assert storage_object.oid not in result, "Search result should not contain ObjectId of removed object"
for tombstone_oid in result: for tombstone_oid in result:
header = head_object( head_info = head_object(default_wallet, container, tombstone_oid, self.shell, rpc_endpoint)
wallet, object_type = head_info["header"]["objectType"]
cid, assert object_type == "TOMBSTONE", f"Object wasn't deleted properly. Found object {tombstone_oid} with type {object_type}"
tombstone_oid,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)["header"]
object_type = header["objectType"]
assert (
object_type == "TOMBSTONE"
), f"Object wasn't deleted properly. Found object {tombstone_oid} with type {object_type}"
@allure.title("Get range hash by native API (obj_size={object_size}, policy={placement_policy})") @allure.title("Get range hash by native API (obj_size={object_size}, policy={placement_policy})")
@pytest.mark.grpc_api @pytest.mark.grpc_api
@ -419,8 +419,7 @@ class TestObjectApi(ClusterTestBase):
range_cut=range_cut, range_cut=range_cut,
) )
assert ( assert (
get_file_content(file_path, content_len=range_len, mode="rb", offset=range_start) get_file_content(file_path, content_len=range_len, mode="rb", offset=range_start) == range_content
== range_content
), f"Expected range content to match {range_cut} slice of file payload" ), f"Expected range content to match {range_cut} slice of file payload"
@allure.title("[NEGATIVE] Get invalid range by native API (obj_size={object_size}, policy={placement_policy})") @allure.title("[NEGATIVE] Get invalid range by native API (obj_size={object_size}, policy={placement_policy})")
@ -438,9 +437,7 @@ class TestObjectApi(ClusterTestBase):
oids = [storage_object.oid for storage_object in storage_objects[:2]] oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_size = storage_objects[0].size file_size = storage_objects[0].size
assert ( assert RANGE_MIN_LEN < file_size, f"Incorrect test setup. File size ({file_size}) is less than RANGE_MIN_LEN ({RANGE_MIN_LEN})"
RANGE_MIN_LEN < file_size
), f"Incorrect test setup. File size ({file_size}) is less than RANGE_MIN_LEN ({RANGE_MIN_LEN})"
file_ranges_to_test: list[tuple(int, int, str)] = [ file_ranges_to_test: list[tuple(int, int, str)] = [
# Offset is bigger than the file size, the length is small. # Offset is bigger than the file size, the length is small.
@ -485,9 +482,7 @@ class TestObjectApi(ClusterTestBase):
oids = [storage_object.oid for storage_object in storage_objects[:2]] oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_size = storage_objects[0].size file_size = storage_objects[0].size
assert ( assert RANGE_MIN_LEN < file_size, f"Incorrect test setup. File size ({file_size}) is less than RANGE_MIN_LEN ({RANGE_MIN_LEN})"
RANGE_MIN_LEN < file_size
), f"Incorrect test setup. File size ({file_size}) is less than RANGE_MIN_LEN ({RANGE_MIN_LEN})"
file_ranges_to_test: list[tuple(int, int, str)] = [ file_ranges_to_test: list[tuple(int, int, str)] = [
# Offset is bigger than the file size, the length is small. # Offset is bigger than the file size, the length is small.
@ -530,9 +525,7 @@ class TestObjectApi(ClusterTestBase):
with reporter.step("Put object to container"): with reporter.step("Put object to container"):
container_node = random.choice(container_nodes) container_node = random.choice(container_nodes)
oid = put_object( oid = put_object(default_wallet, file_path, common_container, self.shell, container_node.storage_node.get_rpc_endpoint())
default_wallet, file_path, common_container, self.shell, container_node.storage_node.get_rpc_endpoint()
)
with reporter.step("Get range from container node endpoint"): with reporter.step("Get range from container node endpoint"):
get_range( get_range(
@ -567,9 +560,7 @@ class TestObjectApi(ClusterTestBase):
with reporter.step("Put object to container"): with reporter.step("Put object to container"):
container_node = random.choice(container_nodes) container_node = random.choice(container_nodes)
oid = put_object( oid = put_object(default_wallet, file_path, common_container, self.shell, container_node.storage_node.get_rpc_endpoint())
default_wallet, file_path, common_container, self.shell, container_node.storage_node.get_rpc_endpoint()
)
with reporter.step("Get range hash from container node endpoint"): with reporter.step("Get range hash from container node endpoint"):
get_range_hash( get_range_hash(
@ -595,6 +586,4 @@ class TestObjectApi(ClusterTestBase):
def check_header_is_presented(self, head_info: dict, object_header: dict) -> None: def check_header_is_presented(self, head_info: dict, object_header: dict) -> None:
for key_to_check, val_to_check in object_header.items(): for key_to_check, val_to_check in object_header.items():
assert key_to_check in head_info["header"]["attributes"], f"Key {key_to_check} is found in {head_object}" assert key_to_check in head_info["header"]["attributes"], f"Key {key_to_check} is found in {head_object}"
assert head_info["header"]["attributes"].get(key_to_check) == str( assert head_info["header"]["attributes"].get(key_to_check) == str(val_to_check), f"Value {val_to_check} is equal"
val_to_check
), f"Value {val_to_check} is equal"

View file

@ -2,14 +2,12 @@ import allure
import pytest import pytest
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import ( from frostfs_testlib.steps.cli.container import (
REP_2_FOR_3_NODES_PLACEMENT_RULE, REP_2_FOR_3_NODES_PLACEMENT_RULE,
SINGLE_PLACEMENT_RULE, SINGLE_PLACEMENT_RULE,
StorageContainer, StorageContainer,
StorageContainerInfo, StorageContainerInfo,
create_container,
) )
from frostfs_testlib.steps.cli.object import delete_object, get_object from frostfs_testlib.steps.cli.object import delete_object, get_object
from frostfs_testlib.steps.storage_object import StorageObjectInfo from frostfs_testlib.steps.storage_object import StorageObjectInfo
@ -21,15 +19,20 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.testing.test_control import expect_not_raises
from pytest import FixtureRequest from pytest import FixtureRequest
from pytest_tests.helpers.bearer_token import create_bearer_token from ...helpers.bearer_token import create_bearer_token
from pytest_tests.helpers.container_access import assert_full_access_to_container from ...helpers.container_access import assert_full_access_to_container
from ...helpers.container_creation import create_container_with_ape
from ...helpers.container_request import ContainerRequest
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
@allure.title("Create user container for bearer token usage") @allure.title("Create user container for bearer token usage")
def user_container(default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, request: FixtureRequest) -> StorageContainer: def user_container(
rule = request.param if "param" in request.__dict__ else SINGLE_PLACEMENT_RULE frostfs_cli: FrostfsCli, default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, rpc_endpoint: str, request: FixtureRequest
container_id = create_container(default_wallet, client_shell, cluster.default_rpc_endpoint, rule, PUBLIC_ACL) ) -> StorageContainer:
policy = request.param if "param" in request.__dict__ else SINGLE_PLACEMENT_RULE
container_request = ContainerRequest(policy)
container_id = create_container_with_ape(frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint, container_request)
# Deliberately using s3gate wallet here to test bearer token # Deliberately using s3gate wallet here to test bearer token
s3_gate_wallet = WalletInfo.from_node(cluster.s3_gates[0]) s3_gate_wallet = WalletInfo.from_node(cluster.s3_gates[0])
@ -62,9 +65,10 @@ def storage_objects(
return storage_objects return storage_objects
@pytest.mark.smoke @pytest.mark.nightly
@pytest.mark.bearer @pytest.mark.bearer
@pytest.mark.ape @pytest.mark.ape
@pytest.mark.grpc_api
class TestObjectApiWithBearerToken(ClusterTestBase): class TestObjectApiWithBearerToken(ClusterTestBase):
@allure.title("Object can be deleted from any node using s3gate wallet with bearer token (obj_size={object_size})") @allure.title("Object can be deleted from any node using s3gate wallet with bearer token (obj_size={object_size})")
@pytest.mark.parametrize( @pytest.mark.parametrize(

View file

@ -4,52 +4,47 @@ import allure
import pytest import pytest
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import get_object_from_random_node, head_object, put_object_to_random_node from frostfs_testlib.steps.cli.object import get_object_from_random_node, head_object, put_object_to_random_node
from frostfs_testlib.steps.epoch import get_epoch from frostfs_testlib.steps.epoch import get_epoch
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils.file_utils import TestFile
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes from ...helpers.utility import wait_for_gc_pass_on_storage_nodes
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@pytest.mark.nightly
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.grpc_api @pytest.mark.grpc_api
class TestObjectApiLifetime(ClusterTestBase): class TestObjectApiLifetime(ClusterTestBase):
@allure.title("Object is removed when lifetime expired (obj_size={object_size})") @allure.title("Object is removed when lifetime expired (obj_size={object_size})")
def test_object_api_lifetime(self, default_wallet: WalletInfo, object_size: ObjectSize): def test_object_api_lifetime(self, container: str, test_file: TestFile, default_wallet: WalletInfo):
""" """
Test object deleted after expiration epoch. Test object deleted after expiration epoch.
""" """
wallet = default_wallet wallet = default_wallet
endpoint = self.cluster.default_rpc_endpoint
cid = create_container(wallet, self.shell, endpoint)
file_path = generate_file(object_size.value)
file_hash = get_file_hash(file_path)
epoch = get_epoch(self.shell, self.cluster) epoch = get_epoch(self.shell, self.cluster)
oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster, expire_at=epoch + 1) oid = put_object_to_random_node(wallet, test_file.path, container, self.shell, self.cluster, expire_at=epoch + 1)
got_file = get_object_from_random_node(wallet, cid, oid, self.shell, self.cluster) with expect_not_raises():
assert get_file_hash(got_file) == file_hash head_object(wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint)
with reporter.step("Tick two epochs"): with reporter.step("Tick two epochs"):
for _ in range(2): self.tick_epochs(2)
self.tick_epoch()
# Wait for GC, because object with expiration is counted as alive until GC removes it # Wait for GC, because object with expiration is counted as alive until GC removes it
wait_for_gc_pass_on_storage_nodes() wait_for_gc_pass_on_storage_nodes()
with reporter.step("Check object deleted because it expires on epoch"): with reporter.step("Check object deleted because it expires on epoch"):
with pytest.raises(Exception, match=OBJECT_NOT_FOUND): with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object(wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint) head_object(wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint)
with pytest.raises(Exception, match=OBJECT_NOT_FOUND): with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node(wallet, cid, oid, self.shell, self.cluster) get_object_from_random_node(wallet, container, oid, self.shell, self.cluster)
with reporter.step("Tick additional epoch"): with reporter.step("Tick additional epoch"):
self.tick_epoch() self.tick_epoch()
@ -58,6 +53,6 @@ class TestObjectApiLifetime(ClusterTestBase):
with reporter.step("Check object deleted because it expires on previous epoch"): with reporter.step("Check object deleted because it expires on previous epoch"):
with pytest.raises(Exception, match=OBJECT_NOT_FOUND): with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object(wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint) head_object(wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint)
with pytest.raises(Exception, match=OBJECT_NOT_FOUND): with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
get_object_from_random_node(wallet, cid, oid, self.shell, self.cluster) get_object_from_random_node(wallet, container, oid, self.shell, self.cluster)

View file

@ -1,26 +1,23 @@
import logging import logging
import re
from datetime import datetime
import allure import allure
import pytest import pytest
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
from frostfs_testlib.resources.common import STORAGE_GC_TIME from frostfs_testlib.resources.common import STORAGE_GC_TIME
from frostfs_testlib.resources.error_patterns import ( from frostfs_testlib.resources.error_patterns import (
LIFETIME_REQUIRED, LIFETIME_REQUIRED,
LOCK_NON_REGULAR_OBJECT, LOCK_NON_REGULAR_OBJECT,
LOCK_OBJECT_EXPIRATION, LOCK_OBJECT_EXPIRATION,
LOCK_OBJECT_REMOVAL, LOCK_OBJECT_REMOVAL,
OBJECT_ALREADY_REMOVED,
OBJECT_IS_LOCKED, OBJECT_IS_LOCKED,
OBJECT_NOT_FOUND, OBJECT_NOT_FOUND,
) )
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo
from frostfs_testlib.steps.cli.object import delete_object, head_object, lock_object from frostfs_testlib.steps.cli.object import delete_object, head_object, lock_object
from frostfs_testlib.steps.complex_object_actions import get_link_object, get_storage_object_chunks from frostfs_testlib.steps.complex_object_actions import get_link_object, get_storage_object_chunks
from frostfs_testlib.steps.epoch import ensure_fresh_epoch, get_epoch, tick_epoch from frostfs_testlib.steps.epoch import ensure_fresh_epoch
from frostfs_testlib.steps.node_management import drop_object from frostfs_testlib.steps.node_management import drop_object
from frostfs_testlib.steps.storage_object import delete_objects from frostfs_testlib.steps.storage_object import delete_objects
from frostfs_testlib.steps.storage_policy import get_nodes_with_object from frostfs_testlib.steps.storage_policy import get_nodes_with_object
@ -32,7 +29,9 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success
from frostfs_testlib.utils import datetime_utils from frostfs_testlib.utils import datetime_utils
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes from ...helpers.container_creation import create_container_with_ape
from ...helpers.container_request import EVERYONE_ALLOW_ALL
from ...helpers.utility import wait_for_gc_pass_on_storage_nodes
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@ -41,25 +40,15 @@ FIXTURE_OBJECT_LIFETIME = 10
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def user_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo: def user_container(
with reporter.step("Create user wallet with container"): frostfs_cli: FrostfsCli, default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, rpc_endpoint: str
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}") ) -> StorageContainer:
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0]) cid = create_container_with_ape(frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint, EVERYONE_ALLOW_ALL)
return StorageContainer(StorageContainerInfo(cid, default_wallet), client_shell, cluster)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def user_container(user_wallet: WalletInfo, client_shell: Shell, cluster: Cluster): def locked_storage_object(user_container: StorageContainer, client_shell: Shell, cluster: Cluster, object_size: ObjectSize):
container_id = create_container(user_wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
return StorageContainer(StorageContainerInfo(container_id, user_wallet), client_shell, cluster)
@pytest.fixture(scope="module")
def locked_storage_object(
user_container: StorageContainer,
client_shell: Shell,
cluster: Cluster,
object_size: ObjectSize,
):
""" """
Intention of this fixture is to provide storage object which is NOT expected to be deleted during test act phase Intention of this fixture is to provide storage object which is NOT expected to be deleted during test act phase
""" """
@ -67,9 +56,7 @@ def locked_storage_object(
current_epoch = ensure_fresh_epoch(client_shell, cluster) current_epoch = ensure_fresh_epoch(client_shell, cluster)
expiration_epoch = current_epoch + FIXTURE_LOCK_LIFETIME expiration_epoch = current_epoch + FIXTURE_LOCK_LIFETIME
storage_object = user_container.generate_object( storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + FIXTURE_OBJECT_LIFETIME)
object_size.value, expire_at=current_epoch + FIXTURE_OBJECT_LIFETIME
)
lock_object_id = lock_object( lock_object_id = lock_object(
storage_object.wallet, storage_object.wallet,
storage_object.cid, storage_object.cid,
@ -78,59 +65,24 @@ def locked_storage_object(
cluster.default_rpc_endpoint, cluster.default_rpc_endpoint,
lifetime=FIXTURE_LOCK_LIFETIME, lifetime=FIXTURE_LOCK_LIFETIME,
) )
storage_object.locks = [ storage_object.locks = [LockObjectInfo(storage_object.cid, lock_object_id, FIXTURE_LOCK_LIFETIME, expiration_epoch)]
LockObjectInfo(storage_object.cid, lock_object_id, FIXTURE_LOCK_LIFETIME, expiration_epoch)
]
yield storage_object return storage_object
with reporter.step("Delete created locked object"):
current_epoch = get_epoch(client_shell, cluster)
epoch_diff = expiration_epoch - current_epoch + 1
if epoch_diff > 0:
with reporter.step(f"Tick {epoch_diff} epochs"):
for _ in range(epoch_diff):
tick_epoch(client_shell, cluster)
try:
delete_object(
storage_object.wallet,
storage_object.cid,
storage_object.oid,
client_shell,
cluster.default_rpc_endpoint,
)
except Exception as ex:
ex_message = str(ex)
# It's okay if object already removed
if not re.search(OBJECT_NOT_FOUND, ex_message) and not re.search(OBJECT_ALREADY_REMOVED, ex_message):
raise ex
logger.debug(ex_message)
@wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME)) @wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME))
def check_object_not_found(wallet: WalletInfo, cid: str, oid: str, shell: Shell, rpc_endpoint: str): def check_object_not_found(wallet: WalletInfo, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
with pytest.raises(Exception, match=OBJECT_NOT_FOUND): with pytest.raises(Exception, match=OBJECT_NOT_FOUND):
head_object( head_object(wallet, cid, oid, shell, rpc_endpoint)
wallet,
cid,
oid,
shell,
rpc_endpoint,
)
def verify_object_available(wallet: WalletInfo, cid: str, oid: str, shell: Shell, rpc_endpoint: str): def verify_object_available(wallet: WalletInfo, cid: str, oid: str, shell: Shell, rpc_endpoint: str):
with expect_not_raises(): with expect_not_raises():
head_object( head_object(wallet, cid, oid, shell, rpc_endpoint)
wallet,
cid,
oid,
shell,
rpc_endpoint,
)
@pytest.mark.nightly
@pytest.mark.grpc_api
@pytest.mark.grpc_object_lock @pytest.mark.grpc_object_lock
class TestObjectLockWithGrpc(ClusterTestBase): class TestObjectLockWithGrpc(ClusterTestBase):
@pytest.fixture() @pytest.fixture()
@ -142,9 +94,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Creating locked object"): with reporter.step("Creating locked object"):
current_epoch = self.get_epoch() current_epoch = self.get_epoch()
storage_object = user_container.generate_object( storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + FIXTURE_OBJECT_LIFETIME)
object_size.value, expire_at=current_epoch + FIXTURE_OBJECT_LIFETIME
)
lock_object( lock_object(
storage_object.wallet, storage_object.wallet,
storage_object.cid, storage_object.cid,
@ -220,9 +170,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
1, 1,
) )
@allure.title( @allure.title("Lock must contain valid lifetime or expire_at field: (lifetime={wrong_lifetime}, expire-at={wrong_expire_at})")
"Lock must contain valid lifetime or expire_at field: (lifetime={wrong_lifetime}, expire-at={wrong_expire_at})"
)
# We operate with only lock object here so no complex object needed in this test # We operate with only lock object here so no complex object needed in this test
@pytest.mark.parametrize("object_size", ["simple"], indirect=True) @pytest.mark.parametrize("object_size", ["simple"], indirect=True)
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -675,9 +623,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
with reporter.step("Generate two objects"): with reporter.step("Generate two objects"):
for epoch_i in range(2): for epoch_i in range(2):
storage_objects.append( storage_objects.append(user_container.generate_object(object_size.value, expire_at=current_epoch + epoch_i + 3))
user_container.generate_object(object_size.value, expire_at=current_epoch + epoch_i + 3)
)
self.tick_epoch() self.tick_epoch()

View file

@ -0,0 +1,62 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import EXPIRATION_EPOCH_ATTRIBUTE
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
from frostfs_testlib.storage.grpc_operations.interfaces import GrpcClientWrapper
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import TestFile
@pytest.mark.nightly
@pytest.mark.grpc_api
class TestObjectTombstone(ClusterTestBase):
@pytest.fixture()
@allure.title("Change tombstone lifetime")
def tombstone_lifetime(self, cluster_state_controller: ClusterStateController, request: pytest.FixtureRequest):
config_manager = cluster_state_controller.manager(ConfigStateManager)
config_manager.set_on_all_nodes(StorageNode, {"object:delete:tombstone_lifetime": request.param}, True)
yield f"Tombstone lifetime was changed to {request.param}"
config_manager.revert_all(True)
@pytest.mark.parametrize("object_size, tombstone_lifetime", [("simple", 2)], indirect=True)
@allure.title("Tombstone object should be removed after expiration")
def test_tombstone_lifetime(
self,
new_epoch: int,
container: str,
grpc_client: GrpcClientWrapper,
test_file: TestFile,
rpc_endpoint: str,
tombstone_lifetime: str,
):
allure.dynamic.description(tombstone_lifetime)
with reporter.step("Put object"):
oid = grpc_client.object.put(test_file.path, container, rpc_endpoint)
with reporter.step("Remove object"):
tombstone_oid = grpc_client.object.delete(container, oid, rpc_endpoint)
with reporter.step("Get tombstone object lifetime"):
tombstone_info = grpc_client.object.head(container, tombstone_oid, rpc_endpoint)
tombstone_expiration_epoch = tombstone_info["header"]["attributes"][EXPIRATION_EPOCH_ATTRIBUTE]
with reporter.step("Tombstone lifetime should be <= 3"):
epochs_to_skip = int(tombstone_expiration_epoch) - new_epoch + 1
assert epochs_to_skip <= 3
with reporter.step("Wait for tombstone expiration"):
self.tick_epochs(epochs_to_skip)
with reporter.step("Tombstone should be removed after expiration"):
with pytest.raises(RuntimeError, match=OBJECT_NOT_FOUND):
grpc_client.object.head(container, tombstone_oid, rpc_endpoint)
with pytest.raises(RuntimeError, match=OBJECT_NOT_FOUND):
grpc_client.object.get(container, tombstone_oid, rpc_endpoint)

View file

@ -7,9 +7,7 @@ from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC
from frostfs_testlib.resources.error_patterns import OBJECT_IS_LOCKED from frostfs_testlib.resources.error_patterns import OBJECT_IS_LOCKED
from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL_F
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.testing.test_control import expect_not_raises
@ -18,6 +16,8 @@ from frostfs_testlib.utils.file_utils import TestFile, get_file_hash
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@pytest.mark.nightly
@pytest.mark.grpc_api
@pytest.mark.grpc_without_user @pytest.mark.grpc_without_user
class TestObjectApiWithoutUser(ClusterTestBase): class TestObjectApiWithoutUser(ClusterTestBase):
def _parse_oid(self, stdout: str) -> str: def _parse_oid(self, stdout: str) -> str:
@ -30,96 +30,72 @@ class TestObjectApiWithoutUser(ClusterTestBase):
tombstone = id_str.split(":")[1] tombstone = id_str.split(":")[1]
return tombstone.strip() return tombstone.strip()
@pytest.fixture(scope="function")
def public_container(self, default_wallet: WalletInfo) -> str:
with reporter.step("Create public container"):
cid_public = create_container(
default_wallet,
self.shell,
self.cluster.default_rpc_endpoint,
basic_acl=PUBLIC_ACL_F,
)
return cid_public
@pytest.fixture(scope="class") @pytest.fixture(scope="class")
def frostfs_cli(self, client_shell: Shell) -> FrostfsCli: def cli_without_wallet(self, client_shell: Shell) -> FrostfsCli:
return FrostfsCli(client_shell, FROSTFS_CLI_EXEC) return FrostfsCli(client_shell, FROSTFS_CLI_EXEC)
@allure.title("Get public container by native API with generate private key") @allure.title("Get public container by native API with generate private key")
def test_get_container_with_generated_key(self, frostfs_cli: FrostfsCli, public_container: str): def test_get_container_with_generated_key(self, cli_without_wallet: FrostfsCli, container: str, rpc_endpoint: str):
""" """
Validate `container get` native API with flag `--generate-key`. Validate `container get` native API with flag `--generate-key`.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Get container with generate key"): with reporter.step("Get container with generate key"):
with expect_not_raises(): with expect_not_raises():
frostfs_cli.container.get(rpc_endpoint, cid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) cli_without_wallet.container.get(rpc_endpoint, container, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT)
@allure.title("Get list containers by native API with generate private key") @allure.title("Get list containers by native API with generate private key")
def test_list_containers_with_generated_key(self, frostfs_cli: FrostfsCli, default_wallet: WalletInfo, public_container: str): def test_list_containers_with_generated_key(
self, cli_without_wallet: FrostfsCli, default_wallet: WalletInfo, container: str, rpc_endpoint: str
):
""" """
Validate `container list` native API with flag `--generate-key`. Validate `container list` native API with flag `--generate-key`.
""" """
rpc_endpoint = self.cluster.default_rpc_endpoint
owner = default_wallet.get_address_from_json(0) owner = default_wallet.get_address_from_json(0)
with reporter.step("List containers with generate key"): with reporter.step("List containers with generate key"):
with expect_not_raises(): with expect_not_raises():
result = frostfs_cli.container.list(rpc_endpoint, owner=owner, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) result = cli_without_wallet.container.list(rpc_endpoint, owner=owner, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT)
with reporter.step("Expect container in received containers list"): with reporter.step("Expect container in received containers list"):
containers = result.stdout.split() containers = result.stdout.split()
assert public_container in containers assert container in containers
@allure.title("Get list of public container objects by native API with generate private key") @allure.title("Get list of public container objects by native API with generate private key")
def test_list_objects_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str): def test_list_objects_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, rpc_endpoint: str):
""" """
Validate `container list_objects` native API with flag `--generate-key`. Validate `container list_objects` native API with flag `--generate-key`.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
with reporter.step("List objects with generate key"): with reporter.step("List objects with generate key"):
with expect_not_raises(): with expect_not_raises():
result = frostfs_cli.container.list_objects(rpc_endpoint, cid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) result = cli_without_wallet.container.list_objects(rpc_endpoint, container, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT)
with reporter.step("Expect empty objects list"): with reporter.step("Expect empty objects list"):
objects = result.stdout.split() objects = result.stdout.split()
assert len(objects) == 0, objects assert len(objects) == 0, objects
@allure.title("Search public container nodes by native API with generate private key") @allure.title("Search public container nodes by native API with generate private key")
def test_search_nodes_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str): def test_search_nodes_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, rpc_endpoint: str):
""" """
Validate `container search_node` native API with flag `--generate-key`. Validate `container search_node` native API with flag `--generate-key`.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Search nodes with generate key"): with reporter.step("Search nodes with generate key"):
with expect_not_raises(): with expect_not_raises():
frostfs_cli.container.search_node(rpc_endpoint, cid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) cli_without_wallet.container.search_node(rpc_endpoint, container, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT)
@allure.title("Put object into public container by native API with generate private key (obj_size={object_size})") @allure.title("Put object into public container by native API with generate private key (obj_size={object_size})")
def test_put_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): def test_put_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str):
""" """
Validate `object put` into container with public ACL and flag `--generate-key`. Validate `object put` into container with public ACL and flag `--generate-key`.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Put object with generate key"): with reporter.step("Put object with generate key"):
with expect_not_raises(): with expect_not_raises():
result = frostfs_cli.object.put( result = cli_without_wallet.object.put(
rpc_endpoint, rpc_endpoint,
cid, container,
file_path, file_path,
generate_key=True, generate_key=True,
no_progress=True, no_progress=True,
@ -129,26 +105,24 @@ class TestObjectApiWithoutUser(ClusterTestBase):
oid = self._parse_oid(result.stdout) oid = self._parse_oid(result.stdout)
with reporter.step("List objects with generate key"): with reporter.step("List objects with generate key"):
result = frostfs_cli.container.list_objects(rpc_endpoint, cid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) result = cli_without_wallet.container.list_objects(rpc_endpoint, container, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT)
with reporter.step("Expect object in received objects list"): with reporter.step("Expect object in received objects list"):
objects = result.stdout.split() objects = result.stdout.split()
assert oid in objects, objects assert oid in objects, objects
@allure.title("Get public container object by native API with generate private key (obj_size={object_size})") @allure.title("Get public container object by native API with generate private key (obj_size={object_size})")
def test_get_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): def test_get_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str):
""" """
Validate `object get` for container with public ACL and flag `--generate-key`. Validate `object get` for container with public ACL and flag `--generate-key`.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
expected_hash = get_file_hash(file_path) expected_hash = get_file_hash(file_path)
with reporter.step("Put object with generate key"): with reporter.step("Put object with generate key"):
result = frostfs_cli.object.put( result = cli_without_wallet.object.put(
rpc_endpoint, rpc_endpoint,
cid, container,
file_path, file_path,
generate_key=True, generate_key=True,
no_progress=True, no_progress=True,
@ -159,9 +133,9 @@ class TestObjectApiWithoutUser(ClusterTestBase):
with reporter.step("Get object with generate key"): with reporter.step("Get object with generate key"):
with expect_not_raises(): with expect_not_raises():
frostfs_cli.object.get( cli_without_wallet.object.get(
rpc_endpoint, rpc_endpoint,
cid, container,
oid, oid,
file=file_path, file=file_path,
generate_key=True, generate_key=True,
@ -175,18 +149,15 @@ class TestObjectApiWithoutUser(ClusterTestBase):
assert expected_hash == downloaded_hash assert expected_hash == downloaded_hash
@allure.title("Head public container object by native API with generate private key (obj_size={object_size})") @allure.title("Head public container object by native API with generate private key (obj_size={object_size})")
def test_head_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): def test_head_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str):
""" """
Validate `object head` for container with public ACL and flag `--generate-key`. Validate `object head` for container with public ACL and flag `--generate-key`.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Put object with generate key"): with reporter.step("Put object with generate key"):
result = frostfs_cli.object.put( result = cli_without_wallet.object.put(
rpc_endpoint, rpc_endpoint,
cid, container,
file_path, file_path,
generate_key=True, generate_key=True,
no_progress=True, no_progress=True,
@ -197,21 +168,18 @@ class TestObjectApiWithoutUser(ClusterTestBase):
with reporter.step("Head object with generate key"): with reporter.step("Head object with generate key"):
with expect_not_raises(): with expect_not_raises():
frostfs_cli.object.head(rpc_endpoint, cid, oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) cli_without_wallet.object.head(rpc_endpoint, container, oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT)
@allure.title("Delete public container object by native API with generate private key (obj_size={object_size})") @allure.title("Delete public container object by native API with generate private key (obj_size={object_size})")
def test_delete_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): def test_delete_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str):
""" """
Validate `object delete` for container with public ACL and flag `--generate key`. Validate `object delete` for container with public ACL and flag `--generate key`.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Put object with generate key"): with reporter.step("Put object with generate key"):
result = frostfs_cli.object.put( result = cli_without_wallet.object.put(
rpc_endpoint, rpc_endpoint,
cid, container,
file_path, file_path,
generate_key=True, generate_key=True,
no_progress=True, no_progress=True,
@ -222,14 +190,14 @@ class TestObjectApiWithoutUser(ClusterTestBase):
with reporter.step("Delete object with generate key"): with reporter.step("Delete object with generate key"):
with expect_not_raises(): with expect_not_raises():
result = frostfs_cli.object.delete(rpc_endpoint, cid, oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) result = cli_without_wallet.object.delete(rpc_endpoint, container, oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT)
oid = self._parse_tombstone_oid(result.stdout) oid = self._parse_tombstone_oid(result.stdout)
with reporter.step("Head object with generate key"): with reporter.step("Head object with generate key"):
result = frostfs_cli.object.head( result = cli_without_wallet.object.head(
rpc_endpoint, rpc_endpoint,
cid, container,
oid, oid,
generate_key=True, generate_key=True,
timeout=CLI_DEFAULT_TIMEOUT, timeout=CLI_DEFAULT_TIMEOUT,
@ -240,19 +208,16 @@ class TestObjectApiWithoutUser(ClusterTestBase):
assert object_type == "TOMBSTONE", object_type assert object_type == "TOMBSTONE", object_type
@allure.title("Lock public container object by native API with generate private key (obj_size={object_size})") @allure.title("Lock public container object by native API with generate private key (obj_size={object_size})")
def test_lock_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): def test_lock_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str):
""" """
Validate `object lock` for container with public ACL and flag `--generate-key`. Validate `object lock` for container with public ACL and flag `--generate-key`.
Attempt to delete the locked object. Attempt to delete the locked object.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Put object with generate key"): with reporter.step("Put object with generate key"):
result = frostfs_cli.object.put( result = cli_without_wallet.object.put(
rpc_endpoint, rpc_endpoint,
cid, container,
file_path, file_path,
generate_key=True, generate_key=True,
no_progress=True, no_progress=True,
@ -263,9 +228,9 @@ class TestObjectApiWithoutUser(ClusterTestBase):
with reporter.step("Lock object with generate key"): with reporter.step("Lock object with generate key"):
with expect_not_raises(): with expect_not_raises():
frostfs_cli.object.lock( cli_without_wallet.object.lock(
rpc_endpoint, rpc_endpoint,
cid, container,
oid, oid,
generate_key=True, generate_key=True,
timeout=CLI_DEFAULT_TIMEOUT, timeout=CLI_DEFAULT_TIMEOUT,
@ -274,27 +239,24 @@ class TestObjectApiWithoutUser(ClusterTestBase):
with reporter.step("Delete locked object with generate key and expect error"): with reporter.step("Delete locked object with generate key and expect error"):
with pytest.raises(Exception, match=OBJECT_IS_LOCKED): with pytest.raises(Exception, match=OBJECT_IS_LOCKED):
frostfs_cli.object.delete( cli_without_wallet.object.delete(
rpc_endpoint, rpc_endpoint,
cid, container,
oid, oid,
generate_key=True, generate_key=True,
timeout=CLI_DEFAULT_TIMEOUT, timeout=CLI_DEFAULT_TIMEOUT,
) )
@allure.title("Search public container objects by native API with generate private key (obj_size={object_size})") @allure.title("Search public container objects by native API with generate private key (obj_size={object_size})")
def test_search_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): def test_search_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str):
""" """
Validate `object search` for container with public ACL and flag `--generate-key`. Validate `object search` for container with public ACL and flag `--generate-key`.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Put object with generate key"): with reporter.step("Put object with generate key"):
result = frostfs_cli.object.put( result = cli_without_wallet.object.put(
rpc_endpoint, rpc_endpoint,
cid, container,
file_path, file_path,
generate_key=True, generate_key=True,
no_progress=True, no_progress=True,
@ -305,25 +267,22 @@ class TestObjectApiWithoutUser(ClusterTestBase):
with reporter.step("Object search with generate key"): with reporter.step("Object search with generate key"):
with expect_not_raises(): with expect_not_raises():
result = frostfs_cli.object.search(rpc_endpoint, cid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) result = cli_without_wallet.object.search(rpc_endpoint, container, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT)
with reporter.step("Expect object in received objects list of container"): with reporter.step("Expect object in received objects list of container"):
object_ids = re.findall(r"(\w{43,44})", result.stdout) object_ids = re.findall(r"(\w{43,44})", result.stdout)
assert oid in object_ids assert oid in object_ids
@allure.title("Get range of public container object by native API with generate private key (obj_size={object_size})") @allure.title("Get range of public container object by native API with generate private key (obj_size={object_size})")
def test_range_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): def test_range_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str):
""" """
Validate `object range` for container with public ACL and `--generate-key`. Validate `object range` for container with public ACL and `--generate-key`.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Put object with generate key"): with reporter.step("Put object with generate key"):
result = frostfs_cli.object.put( result = cli_without_wallet.object.put(
rpc_endpoint, rpc_endpoint,
cid, container,
file_path, file_path,
generate_key=True, generate_key=True,
no_progress=True, no_progress=True,
@ -334,9 +293,9 @@ class TestObjectApiWithoutUser(ClusterTestBase):
with reporter.step("Get range of object with generate key"): with reporter.step("Get range of object with generate key"):
with expect_not_raises(): with expect_not_raises():
frostfs_cli.object.range( cli_without_wallet.object.range(
rpc_endpoint, rpc_endpoint,
cid, container,
oid, oid,
"0:10", "0:10",
file=file_path, file=file_path,
@ -345,18 +304,15 @@ class TestObjectApiWithoutUser(ClusterTestBase):
) )
@allure.title("Get hash of public container object by native API with generate private key (obj_size={object_size})") @allure.title("Get hash of public container object by native API with generate private key (obj_size={object_size})")
def test_hash_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): def test_hash_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str):
""" """
Validate `object hash` for container with public ACL and `--generate-key`. Validate `object hash` for container with public ACL and `--generate-key`.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Put object with generate key"): with reporter.step("Put object with generate key"):
result = frostfs_cli.object.put( result = cli_without_wallet.object.put(
rpc_endpoint, rpc_endpoint,
cid, container,
file_path, file_path,
generate_key=True, generate_key=True,
no_progress=True, no_progress=True,
@ -367,9 +323,9 @@ class TestObjectApiWithoutUser(ClusterTestBase):
with reporter.step("Get range hash of object with generate key"): with reporter.step("Get range hash of object with generate key"):
with expect_not_raises(): with expect_not_raises():
frostfs_cli.object.hash( cli_without_wallet.object.hash(
rpc_endpoint, rpc_endpoint,
cid, container,
oid, oid,
range="0:10", range="0:10",
generate_key=True, generate_key=True,
@ -377,18 +333,15 @@ class TestObjectApiWithoutUser(ClusterTestBase):
) )
@allure.title("Get public container object nodes by native API with generate private key (obj_size={object_size})") @allure.title("Get public container object nodes by native API with generate private key (obj_size={object_size})")
def test_nodes_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): def test_nodes_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str):
""" """
Validate `object nodes` for container with public ACL and `--generate-key`. Validate `object nodes` for container with public ACL and `--generate-key`.
""" """
cid = public_container
rpc_endpoint = self.cluster.default_rpc_endpoint
with reporter.step("Put object with generate key"): with reporter.step("Put object with generate key"):
result = frostfs_cli.object.put( result = cli_without_wallet.object.put(
rpc_endpoint, rpc_endpoint,
cid, container,
file_path, file_path,
no_progress=True, no_progress=True,
generate_key=True, generate_key=True,
@ -400,14 +353,14 @@ class TestObjectApiWithoutUser(ClusterTestBase):
with reporter.step("Configure frostfs-cli for alive remote node"): with reporter.step("Configure frostfs-cli for alive remote node"):
alive_node = self.cluster.cluster_nodes[0] alive_node = self.cluster.cluster_nodes[0]
node_shell = alive_node.host.get_shell() node_shell = alive_node.host.get_shell()
rpc_endpoint = alive_node.storage_node.get_rpc_endpoint() alive_endpoint = alive_node.storage_node.get_rpc_endpoint()
node_frostfs_cli = FrostfsCli(node_shell, FROSTFS_CLI_EXEC) node_frostfs_cli_wo_wallet = FrostfsCli(node_shell, FROSTFS_CLI_EXEC)
with reporter.step("Get object nodes with generate key"): with reporter.step("Get object nodes with generate key"):
with expect_not_raises(): with expect_not_raises():
node_frostfs_cli.object.nodes( node_frostfs_cli_wo_wallet.object.nodes(
rpc_endpoint, alive_endpoint,
cid, container,
oid=oid, oid=oid,
generate_key=True, generate_key=True,
timeout=CLI_DEFAULT_TIMEOUT, timeout=CLI_DEFAULT_TIMEOUT,

File diff suppressed because it is too large Load diff

View file

@ -15,7 +15,7 @@ from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
from pytest_tests.helpers.bearer_token import create_bearer_token from ....helpers.bearer_token import create_bearer_token
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")

View file

@ -20,7 +20,7 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file, get_file_hash from frostfs_testlib.utils.file_utils import generate_file, get_file_hash
from pytest_tests.helpers.utility import wait_for_gc_pass_on_storage_nodes from ....helpers.utility import wait_for_gc_pass_on_storage_nodes
OBJECT_NOT_FOUND_ERROR = "not found" OBJECT_NOT_FOUND_ERROR = "not found"
@ -31,6 +31,7 @@ OBJECT_NOT_FOUND_ERROR = "not found"
) )
@allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#uploading", name="uploading") @allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#uploading", name="uploading")
@allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#downloading", name="downloading") @allure.link("https://git.frostfs.info/TrueCloudLab/frostfs-http-gw#downloading", name="downloading")
@pytest.mark.nightly
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.http_gate @pytest.mark.http_gate
class TestHttpGate(ClusterTestBase): class TestHttpGate(ClusterTestBase):
@ -133,12 +134,8 @@ class TestHttpPut(ClusterTestBase):
file_path_large = generate_file(complex_object_size.value) file_path_large = generate_file(complex_object_size.value)
with reporter.step("Put objects using HTTP"): with reporter.step("Put objects using HTTP"):
oid_simple = upload_via_http_gate( oid_simple = upload_via_http_gate(cid=cid, path=file_path_simple, endpoint=self.cluster.default_http_gate_endpoint)
cid=cid, path=file_path_simple, endpoint=self.cluster.default_http_gate_endpoint oid_large = upload_via_http_gate(cid=cid, path=file_path_large, endpoint=self.cluster.default_http_gate_endpoint)
)
oid_large = upload_via_http_gate(
cid=cid, path=file_path_large, endpoint=self.cluster.default_http_gate_endpoint
)
for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)): for oid, file_path in ((oid_simple, file_path_simple), (oid_large, file_path_large)):
verify_object_hash( verify_object_hash(
@ -353,9 +350,7 @@ class TestHttpPut(ClusterTestBase):
file_path_large = generate_file(complex_object_size.value) file_path_large = generate_file(complex_object_size.value)
with reporter.step("Put objects using curl utility"): with reporter.step("Put objects using curl utility"):
oid_simple = upload_via_http_gate_curl( oid_simple = upload_via_http_gate_curl(cid=cid, filepath=file_path_simple, endpoint=self.cluster.default_http_gate_endpoint)
cid=cid, filepath=file_path_simple, endpoint=self.cluster.default_http_gate_endpoint
)
oid_large = upload_via_http_gate_curl( oid_large = upload_via_http_gate_curl(
cid=cid, cid=cid,
filepath=file_path_large, filepath=file_path_large,

View file

@ -22,6 +22,7 @@ from frostfs_testlib.utils.file_utils import generate_file
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
@pytest.mark.nightly
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.http_gate @pytest.mark.http_gate
class Test_http_object(ClusterTestBase): class Test_http_object(ClusterTestBase):

View file

@ -23,7 +23,10 @@ from frostfs_testlib.utils.file_utils import generate_file
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
EXPIRATION_TIMESTAMP_HEADER = "__SYSTEM__EXPIRATION_TIMESTAMP" EXPIRATION_TIMESTAMP_HEADER = "__SYSTEM__EXPIRATION_TIMESTAMP"
# TODO: Depreacated. Use EXPIRATION_EPOCH_ATTRIBUTE from testlib
EXPIRATION_EPOCH_HEADER = "__SYSTEM__EXPIRATION_EPOCH" EXPIRATION_EPOCH_HEADER = "__SYSTEM__EXPIRATION_EPOCH"
EXPIRATION_DURATION_HEADER = "__SYSTEM__EXPIRATION_DURATION" EXPIRATION_DURATION_HEADER = "__SYSTEM__EXPIRATION_DURATION"
EXPIRATION_EXPIRATION_RFC = "__SYSTEM__EXPIRATION_RFC3339" EXPIRATION_EXPIRATION_RFC = "__SYSTEM__EXPIRATION_RFC3339"
SYSTEM_EXPIRATION_EPOCH = "System-Expiration-Epoch" SYSTEM_EXPIRATION_EPOCH = "System-Expiration-Epoch"
@ -151,9 +154,7 @@ class Test_http_system_header(ClusterTestBase):
def test_unable_put_negative_duration(self, user_container: str, simple_object_size: ObjectSize): def test_unable_put_negative_duration(self, user_container: str, simple_object_size: ObjectSize):
headers = attr_into_str_header_curl({"System-Expiration-Duration": "-1h"}) headers = attr_into_str_header_curl({"System-Expiration-Duration": "-1h"})
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
with reporter.step( with reporter.step("Put object using HTTP with attribute System-Expiration-Duration where duration is negative"):
"Put object using HTTP with attribute System-Expiration-Duration where duration is negative"
):
upload_via_http_gate_curl( upload_via_http_gate_curl(
cid=user_container, cid=user_container,
filepath=file_path, filepath=file_path,
@ -166,9 +167,7 @@ class Test_http_system_header(ClusterTestBase):
def test_unable_put_expired_timestamp(self, user_container: str, simple_object_size: ObjectSize): def test_unable_put_expired_timestamp(self, user_container: str, simple_object_size: ObjectSize):
headers = attr_into_str_header_curl({"System-Expiration-Timestamp": "1635075727"}) headers = attr_into_str_header_curl({"System-Expiration-Timestamp": "1635075727"})
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
with reporter.step( with reporter.step("Put object using HTTP with attribute System-Expiration-Timestamp where duration is in the past"):
"Put object using HTTP with attribute System-Expiration-Timestamp where duration is in the past"
):
upload_via_http_gate_curl( upload_via_http_gate_curl(
cid=user_container, cid=user_container,
filepath=file_path, filepath=file_path,
@ -177,9 +176,7 @@ class Test_http_system_header(ClusterTestBase):
error_pattern=f"{EXPIRATION_TIMESTAMP_HEADER} must be in the future", error_pattern=f"{EXPIRATION_TIMESTAMP_HEADER} must be in the future",
) )
@allure.title( @allure.title("[NEGATIVE] Put object using HTTP with attribute System-Expiration-RFC3339 where duration is in the past")
"[NEGATIVE] Put object using HTTP with attribute System-Expiration-RFC3339 where duration is in the past"
)
def test_unable_put_expired_rfc(self, user_container: str, simple_object_size: ObjectSize): def test_unable_put_expired_rfc(self, user_container: str, simple_object_size: ObjectSize):
headers = attr_into_str_header_curl({"System-Expiration-RFC3339": "2021-11-22T09:55:49Z"}) headers = attr_into_str_header_curl({"System-Expiration-RFC3339": "2021-11-22T09:55:49Z"})
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
@ -204,9 +201,7 @@ class Test_http_system_header(ClusterTestBase):
with reporter.step( with reporter.step(
f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr"
): ):
oid, head_info = self.oid_header_info_for_object( oid, head_info = self.oid_header_info_for_object(file_path=file_path, attributes=attributes, user_container=user_container)
file_path=file_path, attributes=attributes, user_container=user_container
)
self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch) self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch)
with reporter.step("Check that object becomes unavailable when epoch is expired"): with reporter.step("Check that object becomes unavailable when epoch is expired"):
for _ in range(0, epoch_count + 1): for _ in range(0, epoch_count + 1):
@ -243,9 +238,7 @@ class Test_http_system_header(ClusterTestBase):
with reporter.step( with reporter.step(
f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr"
): ):
oid, head_info = self.oid_header_info_for_object( oid, head_info = self.oid_header_info_for_object(file_path=file_path, attributes=attributes, user_container=user_container)
file_path=file_path, attributes=attributes, user_container=user_container
)
self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch) self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch)
with reporter.step("Check that object becomes unavailable when epoch is expired"): with reporter.step("Check that object becomes unavailable when epoch is expired"):
for _ in range(0, epoch_count + 1): for _ in range(0, epoch_count + 1):
@ -276,17 +269,13 @@ class Test_http_system_header(ClusterTestBase):
) )
attributes = { attributes = {
SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(epoch_duration=epoch_duration, epoch=2), SYSTEM_EXPIRATION_TIMESTAMP: self.epoch_count_into_timestamp(epoch_duration=epoch_duration, epoch=2),
SYSTEM_EXPIRATION_RFC3339: self.epoch_count_into_timestamp( SYSTEM_EXPIRATION_RFC3339: self.epoch_count_into_timestamp(epoch_duration=epoch_duration, epoch=1, rfc3339=True),
epoch_duration=epoch_duration, epoch=1, rfc3339=True
),
} }
file_path = generate_file(object_size.value) file_path = generate_file(object_size.value)
with reporter.step( with reporter.step(
f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr"
): ):
oid, head_info = self.oid_header_info_for_object( oid, head_info = self.oid_header_info_for_object(file_path=file_path, attributes=attributes, user_container=user_container)
file_path=file_path, attributes=attributes, user_container=user_container
)
self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch) self.validation_for_http_header_attr(head_info=head_info, expected_epoch=expected_epoch)
with reporter.step("Check that object becomes unavailable when epoch is expired"): with reporter.step("Check that object becomes unavailable when epoch is expired"):
for _ in range(0, epoch_count + 1): for _ in range(0, epoch_count + 1):
@ -314,20 +303,14 @@ class Test_http_system_header(ClusterTestBase):
["simple"], ["simple"],
indirect=True, indirect=True,
) )
def test_http_rfc_object_unavailable_after_expir( def test_http_rfc_object_unavailable_after_expir(self, user_container: str, object_size: ObjectSize, epoch_duration: int):
self, user_container: str, object_size: ObjectSize, epoch_duration: int
):
self.tick_epoch() self.tick_epoch()
epoch_count = 2 epoch_count = 2
expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count expected_epoch = get_epoch(self.shell, self.cluster) + epoch_count
logger.info( logger.info(
f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}" f"epoch duration={epoch_duration}, current_epoch= {get_epoch(self.shell, self.cluster)} expected_epoch {expected_epoch}"
) )
attributes = { attributes = {SYSTEM_EXPIRATION_RFC3339: self.epoch_count_into_timestamp(epoch_duration=epoch_duration, epoch=2, rfc3339=True)}
SYSTEM_EXPIRATION_RFC3339: self.epoch_count_into_timestamp(
epoch_duration=epoch_duration, epoch=2, rfc3339=True
)
}
file_path = generate_file(object_size.value) file_path = generate_file(object_size.value)
with reporter.step( with reporter.step(
f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr" f"Put objects using HTTP with attributes and head command should display {EXPIRATION_EPOCH_HEADER}: {expected_epoch} attr"

View file

@ -9,6 +9,7 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.nightly
@pytest.mark.acl @pytest.mark.acl
@pytest.mark.s3_gate @pytest.mark.s3_gate
class TestS3GateACL: class TestS3GateACL:

View file

@ -1,3 +1,4 @@
import string
from datetime import datetime, timedelta from datetime import datetime, timedelta
import allure import allure
@ -6,9 +7,17 @@ from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils import string_utils
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
VALID_SYMBOLS_WITHOUT_DOT = string.ascii_lowercase + string.digits + "-"
VALID_AND_INVALID_SYMBOLS = string.ascii_letters + string.punctuation
# TODO: The dot symbol is temporarily not supported.
VALID_SYMBOLS_WITH_DOT = VALID_SYMBOLS_WITHOUT_DOT + "."
@pytest.mark.nightly
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_bucket @pytest.mark.s3_gate_bucket
class TestS3GateBucket: class TestS3GateBucket:
@ -139,3 +148,87 @@ class TestS3GateBucket:
s3_client.delete_bucket(bucket) s3_client.delete_bucket(bucket)
with pytest.raises(Exception, match=r".*Not Found.*"): with pytest.raises(Exception, match=r".*Not Found.*"):
s3_client.head_bucket(bucket) s3_client.head_bucket(bucket)
@allure.title("Create bucket with valid name length (s3_client={s3_client}, length={length})")
@pytest.mark.parametrize("length", [3, 4, 32, 62, 63])
def test_s3_create_bucket_with_valid_length(self, s3_client: S3ClientWrapper, length: int):
bucket_name = string_utils.random_string(length, VALID_SYMBOLS_WITHOUT_DOT)
while not (bucket_name[0].isalnum() and bucket_name[-1].isalnum()):
bucket_name = string_utils.random_string(length, VALID_SYMBOLS_WITHOUT_DOT)
with reporter.step("Create bucket with valid name length"):
s3_client.create_bucket(bucket_name)
with reporter.step("Check bucket name in buckets"):
assert bucket_name in s3_client.list_buckets()
@allure.title("[NEGATIVE] Bucket with invalid name length should not be created (s3_client={s3_client}, length={length})")
@pytest.mark.parametrize("length", [2, 64, 254, 255, 256])
def test_s3_create_bucket_with_invalid_length(self, s3_client: S3ClientWrapper, length: int):
bucket_name = string_utils.random_string(length, VALID_SYMBOLS_WITHOUT_DOT)
while not (bucket_name[0].isalnum() and bucket_name[-1].isalnum()):
bucket_name = string_utils.random_string(length, VALID_SYMBOLS_WITHOUT_DOT)
with reporter.step("Create bucket with invalid name length and catch exception"):
with pytest.raises(Exception, match=".*(?:InvalidBucketName|Invalid bucket name).*"):
s3_client.create_bucket(bucket_name)
@allure.title("[NEGATIVE] Bucket with invalid name should not be created (s3_client={s3_client}, bucket_name={bucket_name})")
@pytest.mark.parametrize(
"bucket_name",
[
"BUCKET-1",
"buckeT-2",
# The following case for AWS CLI is not handled correctly
# "-bucket-3",
"bucket-4-",
".bucket-5",
"bucket-6.",
"bucket..7",
"bucket+8",
"bucket_9",
"bucket 10",
"127.10.5.11",
"xn--bucket-12",
"bucket-13-s3alias",
# The following names can be used in FrostFS but are prohibited by the AWS specification.
# "sthree-bucket-14"
# "sthree-configurator-bucket-15"
# "amzn-s3-demo-bucket-16"
# "sthree-bucket-17"
# "bucket-18--ol-s3"
# "bucket-19--x-s3"
# "bucket-20.mrap"
],
)
def test_s3_create_bucket_with_invalid_name(self, s3_client: S3ClientWrapper, bucket_name: str):
with reporter.step("Create bucket with invalid name and catch exception"):
with pytest.raises(Exception, match=".*(?:InvalidBucketName|Invalid bucket name).*"):
s3_client.create_bucket(bucket_name)
@allure.title("[NEGATIVE] Delete non-empty bucket (s3_client={s3_client})")
def test_s3_check_availability_non_empty_bucket_after_deleting(
self,
bucket: str,
simple_object_size: ObjectSize,
s3_client: S3ClientWrapper,
):
object_path = generate_file(simple_object_size.value)
object_name = s3_helper.object_key_from_file_path(object_path)
with reporter.step("Put object into bucket"):
s3_client.put_object(bucket, object_path)
with reporter.step("Check that object appears in bucket"):
objects = s3_client.list_objects(bucket)
assert objects, f"Expected bucket with object, got empty {objects}"
assert object_name in objects, f"Object {object_name} not found in bucket object list {objects}"
with reporter.step("Try to delete not empty bucket and get error"):
with pytest.raises(Exception, match=r".*The bucket you tried to delete is not empty.*"):
s3_client.delete_bucket(bucket)
with reporter.step("Check bucket availability"):
objects = s3_client.list_objects(bucket)
assert objects, f"Expected bucket with object, got empty {objects}"
assert object_name in objects, f"Object {object_name} not found in bucket object list {objects}"

View file

@ -22,14 +22,13 @@ def bucket_no_lock(s3_client: S3ClientWrapper):
return s3_client.create_bucket(object_lock_enabled_for_bucket=False) return s3_client.create_bucket(object_lock_enabled_for_bucket=False)
@pytest.mark.nightly
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_locking @pytest.mark.s3_gate_locking
@pytest.mark.parametrize("version_id", [None, "second"]) @pytest.mark.parametrize("version_id", [None, "second"])
class TestS3GateLocking: class TestS3GateLocking:
@allure.title("Retention period and legal lock on object (version_id={version_id}, s3_client={s3_client})") @allure.title("Retention period and legal lock on object (version_id={version_id}, s3_client={s3_client})")
def test_s3_object_locking( def test_s3_object_locking(self, s3_client: S3ClientWrapper, bucket_w_lock: str, version_id: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket_w_lock: str, version_id: str, simple_object_size: ObjectSize
):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 2 retention_period = 2
@ -73,9 +72,7 @@ class TestS3GateLocking:
s3_client.delete_object(bucket_w_lock, file_name, version_id) s3_client.delete_object(bucket_w_lock, file_name, version_id)
@allure.title("Impossible to change retention mode COMPLIANCE (version_id={version_id}, s3_client={s3_client})") @allure.title("Impossible to change retention mode COMPLIANCE (version_id={version_id}, s3_client={s3_client})")
def test_s3_mode_compliance( def test_s3_mode_compliance(self, s3_client: S3ClientWrapper, bucket_w_lock: str, version_id: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket_w_lock: str, version_id: str, simple_object_size: ObjectSize
):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 2 retention_period = 2
@ -105,9 +102,7 @@ class TestS3GateLocking:
s3_client.put_object_retention(bucket_w_lock, file_name, retention, version_id) s3_client.put_object_retention(bucket_w_lock, file_name, retention, version_id)
@allure.title("Change retention mode GOVERNANCE (version_id={version_id}, s3_client={s3_client})") @allure.title("Change retention mode GOVERNANCE (version_id={version_id}, s3_client={s3_client})")
def test_s3_mode_governance( def test_s3_mode_governance(self, s3_client: S3ClientWrapper, bucket_w_lock: str, version_id: str, simple_object_size: ObjectSize):
self, s3_client: S3ClientWrapper, bucket_w_lock: str, version_id: str, simple_object_size: ObjectSize
):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
retention_period = 3 retention_period = 3
@ -155,12 +150,8 @@ class TestS3GateLocking:
s3_client.put_object_retention(bucket_w_lock, file_name, retention, version_id, True) s3_client.put_object_retention(bucket_w_lock, file_name, retention, version_id, True)
s3_helper.assert_object_lock_mode(s3_client, bucket_w_lock, file_name, "GOVERNANCE", date_obj, "OFF") s3_helper.assert_object_lock_mode(s3_client, bucket_w_lock, file_name, "GOVERNANCE", date_obj, "OFF")
@allure.title( @allure.title("[NEGATIVE] Lock object in bucket with disabled locking (version_id={version_id}, s3_client={s3_client})")
"[NEGATIVE] Lock object in bucket with disabled locking (version_id={version_id}, s3_client={s3_client})" def test_s3_legal_hold(self, s3_client: S3ClientWrapper, bucket_no_lock: str, version_id: str, simple_object_size: ObjectSize):
)
def test_s3_legal_hold(
self, s3_client: S3ClientWrapper, bucket_no_lock: str, version_id: str, simple_object_size: ObjectSize
):
file_path = generate_file(simple_object_size.value) file_path = generate_file(simple_object_size.value)
file_name = s3_helper.object_key_from_file_path(file_path) file_name = s3_helper.object_key_from_file_path(file_path)
@ -174,6 +165,7 @@ class TestS3GateLocking:
s3_client.put_object_legal_hold(bucket_no_lock, file_name, "ON", version_id) s3_client.put_object_legal_hold(bucket_no_lock, file_name, "ON", version_id)
@pytest.mark.nightly
@pytest.mark.s3_gate @pytest.mark.s3_gate
class TestS3GateLockingBucket: class TestS3GateLockingBucket:
@allure.title("Bucket Lock (s3_client={s3_client})") @allure.title("Bucket Lock (s3_client={s3_client})")

View file

@ -2,7 +2,8 @@ import allure
import pytest import pytest
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.cli.container import list_objects, search_container_by_name from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.steps.cli.container import list_objects
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
@ -13,6 +14,7 @@ from frostfs_testlib.utils.file_utils import generate_file, get_file_hash, split
PART_SIZE = 5 * 1024 * 1024 PART_SIZE = 5 * 1024 * 1024
@pytest.mark.nightly
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_multipart @pytest.mark.s3_gate_multipart
class TestS3GateMultipart(ClusterTestBase): class TestS3GateMultipart(ClusterTestBase):
@ -21,7 +23,12 @@ class TestS3GateMultipart(ClusterTestBase):
@allure.title("Object Multipart API (s3_client={s3_client}, bucket versioning = {versioning_status})") @allure.title("Object Multipart API (s3_client={s3_client}, bucket versioning = {versioning_status})")
@pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED, VersioningStatus.UNDEFINED], indirect=True) @pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED, VersioningStatus.UNDEFINED], indirect=True)
def test_s3_object_multipart( def test_s3_object_multipart(
self, s3_client: S3ClientWrapper, bucket: str, default_wallet: WalletInfo, versioning_status: str self,
s3_client: S3ClientWrapper,
bucket: str,
default_wallet: WalletInfo,
versioning_status: str,
bucket_container_resolver: BucketContainerResolver,
): ):
parts_count = 5 parts_count = 5
file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part
@ -31,7 +38,7 @@ class TestS3GateMultipart(ClusterTestBase):
with reporter.step(f"Get related container_id for bucket"): with reporter.step(f"Get related container_id for bucket"):
for cluster_node in self.cluster.cluster_nodes: for cluster_node in self.cluster.cluster_nodes:
container_id = search_container_by_name(bucket, cluster_node) container_id = bucket_container_resolver.resolve(cluster_node, bucket)
if container_id: if container_id:
break break
@ -86,6 +93,7 @@ class TestS3GateMultipart(ClusterTestBase):
bucket: str, bucket: str,
simple_object_size: ObjectSize, simple_object_size: ObjectSize,
complex_object_size: ObjectSize, complex_object_size: ObjectSize,
bucket_container_resolver: BucketContainerResolver,
): ):
complex_file = generate_file(complex_object_size.value) complex_file = generate_file(complex_object_size.value)
simple_file = generate_file(simple_object_size.value) simple_file = generate_file(simple_object_size.value)
@ -95,7 +103,7 @@ class TestS3GateMultipart(ClusterTestBase):
with reporter.step("Get related container_id for bucket"): with reporter.step("Get related container_id for bucket"):
for cluster_node in self.cluster.cluster_nodes: for cluster_node in self.cluster.cluster_nodes:
container_id = search_container_by_name(bucket, cluster_node) container_id = bucket_container_resolver.resolve(cluster_node, bucket)
if container_id: if container_id:
break break

View file

@ -16,15 +16,10 @@ from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils import wallet_utils from frostfs_testlib.utils import wallet_utils
from frostfs_testlib.utils.file_utils import ( from frostfs_testlib.utils.file_utils import TestFile, concat_files, generate_file, generate_file_with_content, get_file_hash
TestFile,
concat_files,
generate_file,
generate_file_with_content,
get_file_hash,
)
@pytest.mark.nightly
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_object @pytest.mark.s3_gate_object
class TestS3GateObject: class TestS3GateObject:
@ -356,9 +351,7 @@ class TestS3GateObject:
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)
with reporter.step("Put several versions of object into bucket"): with reporter.step("Put several versions of object into bucket"):
version_id_1 = s3_client.put_object(bucket, file_name_simple) version_id_1 = s3_client.put_object(bucket, file_name_simple)
file_name_1 = generate_file_with_content( file_name_1 = generate_file_with_content(simple_object_size.value, file_path=file_name_simple, content=version_2_content)
simple_object_size.value, file_path=file_name_simple, content=version_2_content
)
version_id_2 = s3_client.put_object(bucket, file_name_1) version_id_2 = s3_client.put_object(bucket, file_name_1)
with reporter.step("Get first version of object"): with reporter.step("Get first version of object"):
@ -444,9 +437,7 @@ class TestS3GateObject:
assert get_file_hash(con_file_1) == get_file_hash(file_name_1), "Hashes must be the same" assert get_file_hash(con_file_1) == get_file_hash(file_name_1), "Hashes must be the same"
with reporter.step("Get object"): with reporter.step("Get object"):
object_3_part_1 = s3_client.get_object( object_3_part_1 = s3_client.get_object(bucket, file_name, object_range=[0, int(simple_object_size.value / 3)])
bucket, file_name, object_range=[0, int(simple_object_size.value / 3)]
)
object_3_part_2 = s3_client.get_object( object_3_part_2 = s3_client.get_object(
bucket, bucket,
file_name, file_name,
@ -560,9 +551,7 @@ class TestS3GateObject:
elif list_type == "v2": elif list_type == "v2":
list_obj = s3_client.list_objects_v2(bucket) list_obj = s3_client.list_objects_v2(bucket)
assert len(list_obj) == 2, "bucket should have 2 objects" assert len(list_obj) == 2, "bucket should have 2 objects"
assert ( assert list_obj.sort() == [file_name, file_name_2].sort(), f"bucket should have object key {file_name, file_name_2}"
list_obj.sort() == [file_name, file_name_2].sort()
), f"bucket should have object key {file_name, file_name_2}"
with reporter.step("Delete object"): with reporter.step("Delete object"):
delete_obj = s3_client.delete_object(bucket, file_name) delete_obj = s3_client.delete_object(bucket, file_name)
@ -695,13 +684,9 @@ class TestS3GateObject:
with pytest.raises(Exception, match=S3_BUCKET_DOES_NOT_ALLOW_ACL): with pytest.raises(Exception, match=S3_BUCKET_DOES_NOT_ALLOW_ACL):
s3_client.put_object(bucket, file_path, grant_full_control=f"id={second_wallet_public_key}") s3_client.put_object(bucket, file_path, grant_full_control=f"id={second_wallet_public_key}")
with reporter.step( with reporter.step("[NEGATIVE] Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"):
"[NEGATIVE] Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
):
with pytest.raises(Exception, match=S3_BUCKET_DOES_NOT_ALLOW_ACL): with pytest.raises(Exception, match=S3_BUCKET_DOES_NOT_ALLOW_ACL):
s3_client.put_object( s3_client.put_object(bucket, file_path, grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers")
bucket, file_path, grant_read="uri=http://acs.amazonaws.com/groups/global/AllUsers"
)
@allure.title("Put object with lock-mode (s3_client={s3_client})") @allure.title("Put object with lock-mode (s3_client={s3_client})")
def test_s3_put_object_lock_mode( def test_s3_put_object_lock_mode(
@ -727,9 +712,7 @@ class TestS3GateObject:
) )
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF") s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "GOVERNANCE", date_obj, "OFF")
with reporter.step( with reporter.step("Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"):
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"
):
date_obj = datetime.utcnow() + timedelta(days=2) date_obj = datetime.utcnow() + timedelta(days=2)
generate_file_with_content(simple_object_size.value, file_path=file_path_1) generate_file_with_content(simple_object_size.value, file_path=file_path_1)
s3_client.put_object( s3_client.put_object(
@ -740,9 +723,7 @@ class TestS3GateObject:
) )
s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF") s3_helper.assert_object_lock_mode(s3_client, bucket, file_name, "COMPLIANCE", date_obj, "OFF")
with reporter.step( with reporter.step("Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"):
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"
):
date_obj = datetime.utcnow() + timedelta(days=3) date_obj = datetime.utcnow() + timedelta(days=3)
generate_file_with_content(simple_object_size.value, file_path=file_path_1) generate_file_with_content(simple_object_size.value, file_path=file_path_1)
s3_client.put_object( s3_client.put_object(
@ -809,9 +790,7 @@ class TestS3GateObject:
with reporter.step(f"Check all objects put in bucket_{i} successfully"): with reporter.step(f"Check all objects put in bucket_{i} successfully"):
bucket_objects = s3_client.list_objects_v2(bucket) bucket_objects = s3_client.list_objects_v2(bucket)
assert set(put_objects) == set( assert set(put_objects) == set(bucket_objects), f"Expected all objects {put_objects} in objects list {bucket_objects}"
bucket_objects
), f"Expected all objects {put_objects} in objects list {bucket_objects}"
with reporter.step("Delete some objects from bucket_1 one by one"): with reporter.step("Delete some objects from bucket_1 one by one"):
objects_to_delete_b1 = random.sample(put_objects, k=max_delete_objects) objects_to_delete_b1 = random.sample(put_objects, k=max_delete_objects)
@ -871,9 +850,7 @@ class TestS3GateObject:
with reporter.step("Check these are the same objects"): with reporter.step("Check these are the same objects"):
for obj_key in objects: for obj_key in objects:
got_object = s3_client.get_object(bucket, obj_key) got_object = s3_client.get_object(bucket, obj_key)
assert get_file_hash(got_object) == get_file_hash( assert get_file_hash(got_object) == get_file_hash(key_to_path.get(obj_key)), "Expected hashes are the same"
key_to_path.get(obj_key)
), "Expected hashes are the same"
obj_head = s3_client.head_object(bucket, obj_key) obj_head = s3_client.head_object(bucket, obj_key)
assert obj_head.get("Metadata") == object_metadata, f"Metadata of object is {object_metadata}" assert obj_head.get("Metadata") == object_metadata, f"Metadata of object is {object_metadata}"
object_grants = s3_client.get_object_acl(bucket, obj_key) object_grants = s3_client.get_object_acl(bucket, obj_key)
@ -890,11 +867,7 @@ class TestS3GateObject:
with reporter.step("Put object"): with reporter.step("Put object"):
test_file = generate_file(simple_object_size.value) test_file = generate_file(simple_object_size.value)
obj_key = ( obj_key = "/" + "/".join(["".join(random.choices(key_characters_sample, k=5)) for _ in range(10)]) + "/test_file_1"
"/"
+ "/".join(["".join(random.choices(key_characters_sample, k=5)) for _ in range(10)])
+ "/test_file_1"
)
s3_client.put_object(bucket, test_file, obj_key) s3_client.put_object(bucket, test_file, obj_key)
with reporter.step("Check object can be downloaded"): with reporter.step("Check object can be downloaded"):

View file

@ -1,12 +1,11 @@
import json import json
import os
import allure import allure
import pytest import pytest
from botocore.exceptions import ClientError from botocore.exceptions import ClientError
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.cli.container import search_container_by_name from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.steps.storage_policy import get_simple_object_copies from frostfs_testlib.steps.storage_policy import get_simple_object_copies
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
@ -15,13 +14,20 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
from ....resources.common import S3_POLICY_FILE_LOCATION
@pytest.mark.nightly
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.parametrize("s3_policy", ["pytest_tests/resources/files/policy.json"], indirect=True) @pytest.mark.parametrize("s3_policy", [S3_POLICY_FILE_LOCATION], indirect=True)
class TestS3GatePolicy(ClusterTestBase): class TestS3GatePolicy(ClusterTestBase):
@allure.title("Bucket creation with retention policy applied (s3_client={s3_client})") @allure.title("Bucket creation with retention policy applied (s3_client={s3_client})")
def test_s3_bucket_location( def test_s3_bucket_location(
self, default_wallet: WalletInfo, s3_client: S3ClientWrapper, simple_object_size: ObjectSize self,
default_wallet: WalletInfo,
s3_client: S3ClientWrapper,
simple_object_size: ObjectSize,
bucket_container_resolver: BucketContainerResolver,
): ):
file_path_1 = generate_file(simple_object_size.value) file_path_1 = generate_file(simple_object_size.value)
file_name_1 = s3_helper.object_key_from_file_path(file_path_1) file_name_1 = s3_helper.object_key_from_file_path(file_path_1)
@ -34,9 +40,7 @@ class TestS3GatePolicy(ClusterTestBase):
bucket_2 = s3_client.create_bucket(location_constraint="rep-3") bucket_2 = s3_client.create_bucket(location_constraint="rep-3")
s3_helper.set_bucket_versioning(s3_client, bucket_2, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket_2, VersioningStatus.ENABLED)
list_buckets = s3_client.list_buckets() list_buckets = s3_client.list_buckets()
assert ( assert bucket_1 in list_buckets and bucket_2 in list_buckets, f"Expected two buckets {bucket_1, bucket_2}, got {list_buckets}"
bucket_1 in list_buckets and bucket_2 in list_buckets
), f"Expected two buckets {bucket_1, bucket_2}, got {list_buckets}"
with reporter.step("Check head buckets"): with reporter.step("Check head buckets"):
with expect_not_raises(): with expect_not_raises():
@ -57,7 +61,7 @@ class TestS3GatePolicy(ClusterTestBase):
with reporter.step("Check object policy"): with reporter.step("Check object policy"):
for cluster_node in self.cluster.cluster_nodes: for cluster_node in self.cluster.cluster_nodes:
cid_1 = search_container_by_name(name=bucket_1, node=cluster_node) cid_1 = bucket_container_resolver.resolve(cluster_node, bucket_1)
if cid_1: if cid_1:
break break
copies_1 = get_simple_object_copies( copies_1 = get_simple_object_copies(
@ -69,7 +73,7 @@ class TestS3GatePolicy(ClusterTestBase):
) )
assert copies_1 == 1 assert copies_1 == 1
for cluster_node in self.cluster.cluster_nodes: for cluster_node in self.cluster.cluster_nodes:
cid_2 = search_container_by_name(name=bucket_2, node=cluster_node) cid_2 = bucket_container_resolver.resolve(cluster_node, bucket_2)
if cid_2: if cid_2:
break break
copies_2 = get_simple_object_copies( copies_2 = get_simple_object_copies(
@ -97,7 +101,6 @@ class TestS3GatePolicy(ClusterTestBase):
s3_client.get_bucket_policy(bucket) s3_client.get_bucket_policy(bucket)
with reporter.step("Put new policy"): with reporter.step("Put new policy"):
custom_policy = f"file://{os.getcwd()}/pytest_tests/resources/files/bucket_policy.json"
custom_policy = { custom_policy = {
"Version": "2012-10-17", "Version": "2012-10-17",
"Id": "aaaa-bbbb-cccc-dddd", "Id": "aaaa-bbbb-cccc-dddd",

View file

@ -11,6 +11,7 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.nightly
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_tagging @pytest.mark.s3_gate_tagging
class TestS3GateTagging: class TestS3GateTagging:

View file

@ -9,6 +9,7 @@ from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content, get_file_content from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content, get_file_content
@pytest.mark.nightly
@pytest.mark.s3_gate @pytest.mark.s3_gate
@pytest.mark.s3_gate_versioning @pytest.mark.s3_gate_versioning
class TestS3GateVersioning: class TestS3GateVersioning:
@ -77,9 +78,7 @@ class TestS3GateVersioning:
file_name = s3_client.get_object(bucket, obj_key) file_name = s3_client.get_object(bucket, obj_key)
got_content = get_file_content(file_name) got_content = get_file_content(file_name)
assert ( assert got_content == version_2_content, f"Expected object content is\n{version_2_content}\nGot\n{got_content}"
got_content == version_2_content
), f"Expected object content is\n{version_2_content}\nGot\n{got_content}"
@allure.title("Enable and disable versioning without object_lock (s3_client={s3_client})") @allure.title("Enable and disable versioning without object_lock (s3_client={s3_client})")
def test_s3_version(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize): def test_s3_version(self, s3_client: S3ClientWrapper, simple_object_size: ObjectSize):
@ -97,9 +96,7 @@ class TestS3GateVersioning:
actual_version = [version.get("VersionId") for version in object_version if version.get("Key") == file_name] actual_version = [version.get("VersionId") for version in object_version if version.get("Key") == file_name]
assert actual_version == ["null"], f"Expected version is null in list-object-versions, got {object_version}" assert actual_version == ["null"], f"Expected version is null in list-object-versions, got {object_version}"
object_0 = s3_client.head_object(bucket, file_name) object_0 = s3_client.head_object(bucket, file_name)
assert ( assert object_0.get("VersionId") == "null", f"Expected version is null in head-object, got {object_0.get('VersionId')}"
object_0.get("VersionId") == "null"
), f"Expected version is null in head-object, got {object_0.get('VersionId')}"
s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED) s3_helper.set_bucket_versioning(s3_client, bucket, VersioningStatus.ENABLED)

View file

@ -1,10 +1,9 @@
from datetime import datetime
import pytest import pytest
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.storage.cluster import Cluster from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.utils import string_utils
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
@ -15,12 +14,12 @@ def owner_wallet(default_wallet: WalletInfo) -> WalletInfo:
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def user_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo: def user_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
with reporter.step("Create user wallet which will use objects from owner via static session"): with reporter.step("Create user wallet which will use objects from owner via static session"):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}") user = User(string_utils.unique_name("user-"))
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0]) return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def stranger_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo: def stranger_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
with reporter.step("Create stranger user wallet which should fail to obtain data"): with reporter.step("Create stranger user wallet which should fail to obtain data"):
user = User(f"user_{hex(int(datetime.now().timestamp() * 1000000))}") user = User(string_utils.unique_name("user-"))
return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0]) return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])

View file

@ -10,10 +10,10 @@ from frostfs_testlib.steps.session_token import create_session_token
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils import wallet_utils
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.nightly
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.session_token @pytest.mark.session_token
class TestDynamicObjectSession(ClusterTestBase): class TestDynamicObjectSession(ClusterTestBase):

View file

@ -3,12 +3,7 @@ import logging
import allure import allure
import pytest import pytest
from frostfs_testlib import reporter from frostfs_testlib import reporter
from frostfs_testlib.resources.error_patterns import ( from frostfs_testlib.resources.error_patterns import EXPIRED_SESSION_TOKEN, MALFORMED_REQUEST, OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
EXPIRED_SESSION_TOKEN,
MALFORMED_REQUEST,
OBJECT_ACCESS_DENIED,
OBJECT_NOT_FOUND,
)
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import create_container from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import ( from frostfs_testlib.steps.cli.object import (
@ -138,6 +133,7 @@ def static_sessions(
} }
@pytest.mark.nightly
@pytest.mark.static_session @pytest.mark.static_session
class TestObjectStaticSession(ClusterTestBase): class TestObjectStaticSession(ClusterTestBase):
@allure.title("Read operations with static session (method={method_under_test.__name__}, obj_size={object_size})") @allure.title("Read operations with static session (method={method_under_test.__name__}, obj_size={object_size})")

View file

@ -8,6 +8,7 @@ from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
@pytest.mark.nightly
@pytest.mark.static_session_container @pytest.mark.static_session_container
class TestSessionTokenContainer(ClusterTestBase): class TestSessionTokenContainer(ClusterTestBase):
@pytest.fixture(scope="module") @pytest.fixture(scope="module")

View file

@ -18,6 +18,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.nightly
@pytest.mark.shard @pytest.mark.shard
class TestControlShard(ClusterTestBase): class TestControlShard(ClusterTestBase):
@staticmethod @staticmethod

View file

@ -31,7 +31,17 @@ class TestLogs:
if not os.path.exists(logs_dir): if not os.path.exists(logs_dir):
os.makedirs(logs_dir) os.makedirs(logs_dir)
issues_regex = r"\bpanic\b|\boom\b|too many|insufficient funds|insufficient amount of gas|cannot assign requested address|\bunable to process\b" regexes = [
r"\bpanic\b",
r"\boom\b",
r"too many",
r"insufficient funds",
r"insufficient amount of gas",
r"cannot assign requested address",
r"\bunable to process\b",
r"\bmaximum number of subscriptions is reached\b",
]
issues_regex = "|".join(regexes)
exclude_filter = r"too many requests" exclude_filter = r"too many requests"
log_level_priority = "3" # will include 0-3 priority logs (0: emergency 1: alerts 2: critical 3: errors) log_level_priority = "3" # will include 0-3 priority logs (0: emergency 1: alerts 2: critical 3: errors)

View file

@ -1,8 +1,7 @@
allure-pytest==2.13.2 allure-pytest==2.13.2
allure-python-commons==2.13.2 allure-python-commons==2.13.2
base58==2.1.0 base58==2.1.0
boto3==1.16.33 boto3==1.35.30
botocore==1.19.33
configobj==5.0.6 configobj==5.0.6
neo-mamba==1.0.0 neo-mamba==1.0.0
pexpect==4.8.0 pexpect==4.8.0