From 91cd71de19fce444d69105ec44971b84536dc103 Mon Sep 17 00:00:00 2001 From: "a.berezin" Date: Sat, 16 Nov 2024 14:33:56 +0300 Subject: [PATCH] [#324] Remove basic-acl part two Signed-off-by: a.berezin --- pytest_tests/helpers/container_creation.py | 56 ++++++ pytest_tests/helpers/container_request.py | 58 ++++++ pytest_tests/helpers/container_spec.py | 48 ----- .../testsuites/access/acl/__init__.py | 0 .../testsuites/access/acl/test_acl.py | 105 ----------- .../testsuites/access/ape/test_ape.py | 21 ++- .../testsuites/access/ape/test_ape_filters.py | 9 +- .../testsuites/access/ape/test_bearer.py | 33 ++-- pytest_tests/testsuites/conftest.py | 172 ++++-------------- .../testsuites/container/test_container.py | 73 +++----- .../testsuites/container/test_policy.py | 1 - .../management/test_node_management.py | 2 +- .../metrics/test_container_metrics.py | 35 ++-- .../testsuites/object/test_object_api.py | 125 +++++++------ .../object/test_object_api_bearer.py | 14 +- .../testsuites/object/test_object_lifetime.py | 5 +- .../testsuites/object/test_object_lock.py | 54 ++---- .../object/test_object_tombstone.py | 5 +- .../object/test_object_without_user.py | 170 +++++++---------- 19 files changed, 386 insertions(+), 600 deletions(-) create mode 100644 pytest_tests/helpers/container_creation.py create mode 100644 pytest_tests/helpers/container_request.py delete mode 100644 pytest_tests/helpers/container_spec.py delete mode 100644 pytest_tests/testsuites/access/acl/__init__.py delete mode 100644 pytest_tests/testsuites/access/acl/test_acl.py diff --git a/pytest_tests/helpers/container_creation.py b/pytest_tests/helpers/container_creation.py new file mode 100644 index 00000000..a881339c --- /dev/null +++ b/pytest_tests/helpers/container_creation.py @@ -0,0 +1,56 @@ +import json +import time + +from frostfs_testlib import reporter +from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli +from frostfs_testlib.resources.common import MORPH_BLOCK_TIME +from frostfs_testlib.shell.interfaces import Shell +from frostfs_testlib.steps.cli.container import create_container, search_nodes_with_container +from frostfs_testlib.storage.cluster import Cluster +from frostfs_testlib.storage.dataclasses import ape +from frostfs_testlib.storage.dataclasses.wallet import WalletInfo +from frostfs_testlib.utils import datetime_utils + +from .container_request import ContainerRequest + + +def create_container_with_ape( + frostfs_cli: FrostfsCli, wallet: WalletInfo, shell: Shell, cluster: Cluster, endpoint: str, container_request: ContainerRequest +): + with reporter.step("Create container"): + cid = _create_container_by_spec(wallet, shell, cluster, endpoint, container_request) + + with reporter.step("Apply APE rules for container"): + if container_request.ape_rules: + _apply_ape_rules(frostfs_cli, endpoint, cid, container_request.ape_rules) + + with reporter.step("Wait for one block"): + time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) + + with reporter.step("Search nodes holding the container"): + container_holder_nodes = search_nodes_with_container(wallet, cid, shell, cluster.default_rpc_endpoint, cluster) + report_data = {node.id: node.host_ip for node in container_holder_nodes} + + reporter.attach(json.dumps(report_data, indent=2), "container_nodes.json") + + return cid + + +@reporter.step("Create container by spec {container_request}") +def _create_container_by_spec( + wallet: WalletInfo, shell: Shell, cluster: Cluster, endpoint: str, container_request: ContainerRequest +) -> str: + return create_container(wallet, shell, endpoint, container_request.parsed_rule(cluster), wait_for_creation=False) + + +def _apply_ape_rules(frostfs_cli: FrostfsCli, endpoint: str, cid: str, ape_rules: list[ape.Rule]): + for ape_rule in ape_rules: + rule_str = ape_rule.as_string() + with reporter.step(f"Apply APE rule '{rule_str}' for container {cid}"): + frostfs_cli.ape_manager.add( + endpoint, + ape_rule.chain_id, + target_name=cid, + target_type="container", + rule=rule_str, + ) diff --git a/pytest_tests/helpers/container_request.py b/pytest_tests/helpers/container_request.py new file mode 100644 index 00000000..9905112d --- /dev/null +++ b/pytest_tests/helpers/container_request.py @@ -0,0 +1,58 @@ +from dataclasses import dataclass + +from frostfs_testlib.steps.cli.container import DEFAULT_PLACEMENT_RULE +from frostfs_testlib.storage.cluster import Cluster +from frostfs_testlib.storage.dataclasses import ape + +APE_EVERYONE_ALLOW_ALL = [ape.Rule(ape.Verb.ALLOW, ape.ObjectOperations.WILDCARD_ALL)] +# In case if we need container operations +# ape.Rule(ape.Verb.ALLOW, ape.ContainerOperations.WILDCARD_ALL)] +APE_OWNER_ALLOW_ALL = [ape.Rule(ape.Verb.ALLOW, ape.ObjectOperations.WILDCARD_ALL, ape.Condition.by_role(ape.Role.OWNER))] +# In case if we need container operations +# ape.Rule(ape.Verb.ALLOW, ape.ContainerOperations.WILDCARD_ALL, ape.Condition.by_role(ape.Role.OWNER))] + + +@dataclass +class ContainerRequest: + policy: str = None + ape_rules: list[ape.Rule] = None + + short_name: str | None = None + + def __post_init__(self): + if self.ape_rules is None: + self.ape_rules = [] + + # For pytest instead of ids=[...] everywhere + self.__name__ = self.short_name + + def parsed_rule(self, cluster: Cluster): + if self.policy is None: + return None + + substitutions = {"%NODE_COUNT%": str(len(cluster.cluster_nodes))} + + parsed_rule = self.policy + for sub, replacement in substitutions.items(): + parsed_rule = parsed_rule.replace(sub, replacement) + + return parsed_rule + + def __repr__(self): + if self.short_name: + return self.short_name + + spec_info: list[str] = [] + + if self.policy: + spec_info.append(f"policy='{self.policy}'") + if self.ape_rules: + ape_rules_list = ", ".join([f"'{rule.as_string()}'" for rule in self.ape_rules]) + spec_info.append(f"ape_rules=[{ape_rules_list}]") + + return f"({', '.join(spec_info)})" + + +EVERYONE_ALLOW_ALL = ContainerRequest(policy=DEFAULT_PLACEMENT_RULE, ape_rules=APE_EVERYONE_ALLOW_ALL, short_name="Everyone_Allow_All") +OWNER_ALLOW_ALL = ContainerRequest(policy=DEFAULT_PLACEMENT_RULE, ape_rules=APE_OWNER_ALLOW_ALL, short_name="Owner_Allow_All") +PRIVATE = ContainerRequest(policy=DEFAULT_PLACEMENT_RULE, ape_rules=[], short_name="Private_No_APE") diff --git a/pytest_tests/helpers/container_spec.py b/pytest_tests/helpers/container_spec.py deleted file mode 100644 index 921f37a9..00000000 --- a/pytest_tests/helpers/container_spec.py +++ /dev/null @@ -1,48 +0,0 @@ -from dataclasses import dataclass - -from frostfs_testlib.steps.cli.container import DEFAULT_PLACEMENT_RULE -from frostfs_testlib.storage.cluster import Cluster -from frostfs_testlib.storage.dataclasses import ape - -APE_PUBLIC_READ_WRITE = [ape.Rule(ape.Verb.ALLOW, ape.ObjectOperations.WILDCARD_ALL)] - - -@dataclass -class ContainerSpec: - rule: str = DEFAULT_PLACEMENT_RULE - # TODO: Deprecated - basic_acl: str = None - # TODO: Deprecated - allow_owner_via_ape: bool = False - ape_rules: list[ape.Rule] = None - - def __post_init__(self): - if self.ape_rules is None: - self.ape_rules = [] - - def parsed_rule(self, cluster: Cluster): - if self.rule is None: - return None - - substitutions = {"%NODE_COUNT%": str(len(cluster.cluster_nodes))} - - parsed_rule = self.rule - for sub, replacement in substitutions.items(): - parsed_rule = parsed_rule.replace(sub, replacement) - - return parsed_rule - - def __repr__(self): - spec_info: list[str] = [] - - if self.rule: - spec_info.append(f"rule='{self.rule}'") - if self.ape_rules: - ape_rules_list = ", ".join([f"'{rule.as_string()}'" for rule in self.ape_rules]) - spec_info.append(f"ape_rules=[{ape_rules_list}]") - - return f"ContainerSpec({', '.join(spec_info)})" - - -class ContainerSpecs: - PublicReadWrite = ContainerSpec(ape_rules=APE_PUBLIC_READ_WRITE) diff --git a/pytest_tests/testsuites/access/acl/__init__.py b/pytest_tests/testsuites/access/acl/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pytest_tests/testsuites/access/acl/test_acl.py b/pytest_tests/testsuites/access/acl/test_acl.py deleted file mode 100644 index af64d0cf..00000000 --- a/pytest_tests/testsuites/access/acl/test_acl.py +++ /dev/null @@ -1,105 +0,0 @@ -import allure -import pytest -from frostfs_testlib import reporter -from frostfs_testlib.resources.wellknown_acl import PRIVATE_ACL_F, PUBLIC_ACL_F, READONLY_ACL_F -from frostfs_testlib.shell import Shell -from frostfs_testlib.steps.cli.object import put_object_to_random_node -from frostfs_testlib.storage.cluster import Cluster -from frostfs_testlib.storage.dataclasses.wallet import WalletInfo -from frostfs_testlib.testing.cluster_test_base import ClusterTestBase - -from ....helpers.container_access import assert_full_access_to_container, assert_no_access_to_container, assert_read_only_container -from ....helpers.container_spec import ContainerSpec - - -@pytest.mark.nightly -@pytest.mark.sanity -@pytest.mark.acl -class TestACLBasic(ClusterTestBase): - @allure.title("Operations in public container available to everyone (obj_size={object_size})") - @pytest.mark.container(ContainerSpec(basic_acl=PUBLIC_ACL_F)) - def test_basic_acl_public( - self, - default_wallet: WalletInfo, - other_wallet: WalletInfo, - client_shell: Shell, - container: str, - file_path: str, - cluster: Cluster, - ): - """ - Test access to object operations in public container. - """ - - for wallet, role in ((default_wallet, "owner"), (other_wallet, "others")): - with reporter.step("Put objects to container"): - # We create new objects for each wallet because assert_full_access_to_container - # deletes the object - owner_object_oid = put_object_to_random_node( - default_wallet, - file_path, - container, - shell=self.shell, - cluster=self.cluster, - attributes={"created": "owner"}, - ) - other_object_oid = put_object_to_random_node( - other_wallet, - file_path, - container, - shell=self.shell, - cluster=self.cluster, - attributes={"created": "other"}, - ) - - with reporter.step(f"Check {role} has full access to public container"): - assert_full_access_to_container(wallet, container, owner_object_oid, file_path, client_shell, cluster) - assert_full_access_to_container(wallet, container, other_object_oid, file_path, client_shell, cluster) - - @allure.title("Operations in private container only available to owner (obj_size={object_size})") - @pytest.mark.container(ContainerSpec(basic_acl=PRIVATE_ACL_F)) - def test_basic_acl_private( - self, - default_wallet: WalletInfo, - other_wallet: WalletInfo, - client_shell: Shell, - container: str, - file_path: str, - cluster: Cluster, - ): - """ - Test access to object operations in private container. - """ - - with reporter.step("Put object to container"): - owner_object_oid = put_object_to_random_node(default_wallet, file_path, container, client_shell, cluster) - - with reporter.step("Check no one except owner has access to operations with container"): - assert_no_access_to_container(other_wallet, container, owner_object_oid, file_path, client_shell, cluster) - - with reporter.step("Check owner has full access to private container"): - assert_full_access_to_container(default_wallet, container, owner_object_oid, file_path, self.shell, cluster) - - @allure.title("Read operations in readonly container available to others (obj_size={object_size})") - @pytest.mark.container(ContainerSpec(basic_acl=READONLY_ACL_F)) - def test_basic_acl_readonly( - self, - default_wallet: WalletInfo, - other_wallet: WalletInfo, - client_shell: Shell, - container: str, - file_path: str, - cluster: Cluster, - ): - """ - Test access to object operations in readonly container. - """ - - with reporter.step("Put object to container"): - object_oid = put_object_to_random_node(default_wallet, file_path, container, client_shell, cluster) - - with reporter.step("Check others has read-only access to operations with container"): - assert_read_only_container(other_wallet, container, object_oid, file_path, client_shell, cluster) - - with reporter.step("Check owner has full access to public container"): - assert_full_access_to_container(default_wallet, container, object_oid, file_path, client_shell, cluster) diff --git a/pytest_tests/testsuites/access/ape/test_ape.py b/pytest_tests/testsuites/access/ape/test_ape.py index f36d038e..cb0b7409 100644 --- a/pytest_tests/testsuites/access/ape/test_ape.py +++ b/pytest_tests/testsuites/access/ape/test_ape.py @@ -2,7 +2,6 @@ import allure import pytest from frostfs_testlib import reporter from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli -from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL from frostfs_testlib.steps.cli.object import put_object_to_random_node from frostfs_testlib.steps.node_management import drop_object from frostfs_testlib.storage.dataclasses import ape @@ -18,7 +17,7 @@ from ....helpers.container_access import ( assert_full_access_to_container, assert_no_access_to_container, ) -from ....helpers.container_spec import ContainerSpec +from ....helpers.container_request import APE_EVERYONE_ALLOW_ALL, OWNER_ALLOW_ALL, ContainerRequest @pytest.fixture @@ -113,7 +112,11 @@ class TestApeContainer(ClusterTestBase): assert_full_access_to_container(other_wallet_2, container, objects.pop(), file_path, self.shell, self.cluster) @allure.title("Replication works with APE deny rules on OWNER and OTHERS (obj_size={object_size})") - @pytest.mark.container(ContainerSpec(f"REP %NODE_COUNT% IN X CBF 1 SELECT %NODE_COUNT% FROM * AS X", PUBLIC_ACL)) + @pytest.mark.parametrize( + "container_request", + [ContainerRequest(f"REP %NODE_COUNT% IN X CBF 1 SELECT %NODE_COUNT% FROM * AS X", APE_EVERYONE_ALLOW_ALL, "custom")], + indirect=True, + ) def test_replication_works_with_deny_rules( self, default_wallet: WalletInfo, @@ -149,6 +152,7 @@ class TestApeContainer(ClusterTestBase): wait_object_replication(container, oid, len(self.cluster.storage_nodes), self.shell, self.cluster.storage_nodes) @allure.title("Deny operations via APE by role (role=ir, obj_size={object_size})") + @pytest.mark.parametrize("container_request", [OWNER_ALLOW_ALL], indirect=True) def test_deny_operations_via_ape_by_role_ir( self, frostfs_cli: FrostfsCli, ir_wallet: WalletInfo, container: str, objects: list[str], rpc_endpoint: str, file_path: TestFile ): @@ -156,7 +160,7 @@ class TestApeContainer(ClusterTestBase): ape.ObjectOperations.PUT: False, ape.ObjectOperations.GET: True, ape.ObjectOperations.HEAD: True, - ape.ObjectOperations.GET_RANGE: False, + ape.ObjectOperations.GET_RANGE: True, ape.ObjectOperations.GET_RANGE_HASH: True, ape.ObjectOperations.SEARCH: True, ape.ObjectOperations.DELETE: False, @@ -185,6 +189,7 @@ class TestApeContainer(ClusterTestBase): assert_access_to_container(default_ir_access, ir_wallet, container, objects[0], file_path, self.shell, self.cluster) @allure.title("Deny operations via APE by role (role=container, obj_size={object_size})") + @pytest.mark.parametrize("container_request", [OWNER_ALLOW_ALL], indirect=True) def test_deny_operations_via_ape_by_role_container( self, frostfs_cli: FrostfsCli, @@ -198,10 +203,10 @@ class TestApeContainer(ClusterTestBase): ape.ObjectOperations.PUT: True, ape.ObjectOperations.GET: True, ape.ObjectOperations.HEAD: True, - ape.ObjectOperations.GET_RANGE: False, + ape.ObjectOperations.GET_RANGE: True, ape.ObjectOperations.GET_RANGE_HASH: True, ape.ObjectOperations.SEARCH: True, - ape.ObjectOperations.DELETE: False, + ape.ObjectOperations.DELETE: True, } with reporter.step("Assert CONTAINER wallet access in default state"): @@ -216,7 +221,7 @@ class TestApeContainer(ClusterTestBase): self.wait_for_blocks() with reporter.step("Assert CONTAINER wallet ignores APE rule"): - assert_access_to_container(access_matrix, container_node_wallet, container, objects[0], file_path, self.shell, self.cluster) + assert_access_to_container(access_matrix, container_node_wallet, container, objects[1], file_path, self.shell, self.cluster) with reporter.step("Remove APE rule"): frostfs_cli.ape_manager.remove(rpc_endpoint, rule.chain_id, target_name=container, target_type="container") @@ -225,4 +230,4 @@ class TestApeContainer(ClusterTestBase): self.wait_for_blocks() with reporter.step("Assert CONTAINER wallet access after rule was removed"): - assert_access_to_container(access_matrix, container_node_wallet, container, objects[0], file_path, self.shell, self.cluster) + assert_access_to_container(access_matrix, container_node_wallet, container, objects[2], file_path, self.shell, self.cluster) diff --git a/pytest_tests/testsuites/access/ape/test_ape_filters.py b/pytest_tests/testsuites/access/ape/test_ape_filters.py index 561ed81a..4ac79cfa 100644 --- a/pytest_tests/testsuites/access/ape/test_ape_filters.py +++ b/pytest_tests/testsuites/access/ape/test_ape_filters.py @@ -18,7 +18,7 @@ from ....helpers.container_access import ( assert_full_access_to_container, assert_no_access_to_container, ) -from ....helpers.container_spec import ContainerSpec +from ....helpers.container_request import OWNER_ALLOW_ALL from ....helpers.object_access import OBJECT_ACCESS_DENIED @@ -241,8 +241,7 @@ class TestApeFilters(ClusterTestBase): @allure.title("Operations with allow APE rule with resource filters (match_type={match_type}, obj_size={object_size})") @pytest.mark.parametrize("match_type", [ape.MatchType.EQUAL, ape.MatchType.NOT_EQUAL]) - @pytest.mark.parametrize("object_size", ["simple"], indirect=True) - @pytest.mark.container(ContainerSpec(basic_acl="0", allow_owner_via_ape=True)) + @pytest.mark.parametrize("object_size, container_request", [("simple", OWNER_ALLOW_ALL)], indirect=True) def test_ape_allow_filters_object( self, frostfs_cli: FrostfsCli, @@ -342,7 +341,7 @@ class TestApeFilters(ClusterTestBase): put_object_to_random_node(other_wallet, file_path, container, self.shell, self.cluster, bearer, attributes=allow_attribute) @allure.title("PUT and GET object using bearer with objectID in filter (obj_size={object_size}, match_type=NOT_EQUAL)") - @pytest.mark.container(ContainerSpec(basic_acl="0", allow_owner_via_ape=True)) + @pytest.mark.parametrize("container_request", [OWNER_ALLOW_ALL], indirect=True) def test_ape_filter_object_id_not_equals( self, frostfs_cli: FrostfsCli, @@ -370,7 +369,7 @@ class TestApeFilters(ClusterTestBase): get_object_from_random_node(other_wallet, container, oid, self.shell, self.cluster, bearer) @allure.title("PUT and GET object using bearer with objectID in filter (obj_size={object_size}, match_type=EQUAL)") - @pytest.mark.container(ContainerSpec(basic_acl="0", allow_owner_via_ape=True)) + @pytest.mark.parametrize("container_request", [OWNER_ALLOW_ALL], indirect=True) def test_ape_filter_object_id_equals( self, frostfs_cli: FrostfsCli, diff --git a/pytest_tests/testsuites/access/ape/test_bearer.py b/pytest_tests/testsuites/access/ape/test_bearer.py index 64a89ffd..68280810 100644 --- a/pytest_tests/testsuites/access/ape/test_bearer.py +++ b/pytest_tests/testsuites/access/ape/test_bearer.py @@ -75,7 +75,7 @@ class TestApeBearer(ClusterTestBase): temp_directory: str, default_wallet: WalletInfo, other_wallet: WalletInfo, - container: tuple[str, list[str], str], + container: str, objects: list[str], rpc_endpoint: str, file_path: TestFile, @@ -113,25 +113,24 @@ class TestApeBearer(ClusterTestBase): bt_access_map = { ape.Role.OWNER: { - ape.ObjectOperations.PUT: True, - ape.ObjectOperations.GET: True, - ape.ObjectOperations.HEAD: True, - ape.ObjectOperations.GET_RANGE: True, - ape.ObjectOperations.GET_RANGE_HASH: True, - ape.ObjectOperations.SEARCH: True, - ape.ObjectOperations.DELETE: True, - }, - ape.Role.OTHERS: { ape.ObjectOperations.PUT: True, ape.ObjectOperations.GET: False, ape.ObjectOperations.HEAD: True, + ape.ObjectOperations.GET_RANGE: True, + ape.ObjectOperations.GET_RANGE_HASH: False, + ape.ObjectOperations.SEARCH: False, + ape.ObjectOperations.DELETE: True, + }, + # Bearer Token COMPLETLY overrides chains set for the specific target. + # Thus, any restictions or permissions should be explicitly defined in BT. + ape.Role.OTHERS: { + ape.ObjectOperations.PUT: False, + ape.ObjectOperations.GET: False, + ape.ObjectOperations.HEAD: False, ape.ObjectOperations.GET_RANGE: False, ape.ObjectOperations.GET_RANGE_HASH: False, - # Although SEARCH is denied by the APE chain defined in Policy contract, - # Bearer Token COMPLETLY overrides chains set for the specific target. - # Thus, any restictions or permissions should be explicitly defined in BT. - ape.ObjectOperations.SEARCH: True, - ape.ObjectOperations.DELETE: True, + ape.ObjectOperations.SEARCH: False, + ape.ObjectOperations.DELETE: False, }, } @@ -148,9 +147,11 @@ class TestApeBearer(ClusterTestBase): # Operations that we will allow for each role with bearer token bearer_map = { ape.Role.OWNER: [ - ape.ObjectOperations.DELETE, ape.ObjectOperations.PUT, + ape.ObjectOperations.HEAD, ape.ObjectOperations.GET_RANGE, + # Delete also requires PUT (to make tobstone) and HEAD (to get simple objects header) + ape.ObjectOperations.DELETE, ], ape.Role.OTHERS: [ ape.ObjectOperations.GET, diff --git a/pytest_tests/testsuites/conftest.py b/pytest_tests/testsuites/conftest.py index 957dfaae..ba6ac7a4 100644 --- a/pytest_tests/testsuites/conftest.py +++ b/pytest_tests/testsuites/conftest.py @@ -1,7 +1,5 @@ -import json import logging import random -import time from datetime import datetime, timedelta, timezone from typing import Optional @@ -14,23 +12,16 @@ from frostfs_testlib.credentials.interfaces import CredentialsProvider, User from frostfs_testlib.healthcheck.interfaces import Healthcheck from frostfs_testlib.hosting import Hosting from frostfs_testlib.resources import optionals -from frostfs_testlib.resources.common import COMPLEX_OBJECT_CHUNKS_COUNT, COMPLEX_OBJECT_TAIL_SIZE, MORPH_BLOCK_TIME, SIMPLE_OBJECT_SIZE +from frostfs_testlib.resources.common import COMPLEX_OBJECT_CHUNKS_COUNT, COMPLEX_OBJECT_TAIL_SIZE, SIMPLE_OBJECT_SIZE from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus from frostfs_testlib.s3.interfaces import BucketContainerResolver from frostfs_testlib.shell import LocalShell, Shell -from frostfs_testlib.steps.cli.container import ( - DEFAULT_EC_PLACEMENT_RULE, - DEFAULT_PLACEMENT_RULE, - FROSTFS_CLI_EXEC, - create_container, - search_nodes_with_container, -) +from frostfs_testlib.steps.cli.container import DEFAULT_EC_PLACEMENT_RULE, DEFAULT_PLACEMENT_RULE, FROSTFS_CLI_EXEC from frostfs_testlib.steps.cli.object import get_netmap_netinfo from frostfs_testlib.steps.epoch import ensure_fresh_epoch from frostfs_testlib.steps.s3 import s3_helper from frostfs_testlib.storage.cluster import Cluster, ClusterNode from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController -from frostfs_testlib.storage.dataclasses import ape from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy @@ -40,10 +31,11 @@ from frostfs_testlib.storage.grpc_operations.interfaces import GrpcClientWrapper from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.parallel import parallel from frostfs_testlib.testing.test_control import cached_fixture, run_optionally, wait_for_success -from frostfs_testlib.utils import datetime_utils, env_utils, string_utils, version_utils +from frostfs_testlib.utils import env_utils, string_utils, version_utils from frostfs_testlib.utils.file_utils import TestFile, generate_file -from ..helpers.container_spec import ContainerSpec, ContainerSpecs +from ..helpers.container_creation import create_container_with_ape +from ..helpers.container_request import EVERYONE_ALLOW_ALL, ContainerRequest from ..resources.common import TEST_CYCLES_COUNT logger = logging.getLogger("NeoLogger") @@ -191,6 +183,11 @@ def test_file(object_size: ObjectSize) -> TestFile: return generate_file(object_size.value) +@pytest.fixture(scope="module") +def test_file_module(object_size: ObjectSize) -> TestFile: + return generate_file(object_size.value) + + # Deprecated. Please migrate all to test_file @pytest.fixture() def file_path(test_file: TestFile) -> TestFile: @@ -227,8 +224,10 @@ def placement_policy( ) -> PlacementPolicy: if request.param == "rep": return rep_placement_policy + elif request.param == "ec": + return ec_placement_policy - return ec_placement_policy + return request.param @pytest.fixture(scope="session") @@ -451,6 +450,7 @@ def default_wallet(default_user: User) -> WalletInfo: @pytest.fixture(scope="session") +@cached_fixture(optionals.OPTIONAL_CACHE_FIXTURES) def wallets_pool(credentials_provider: CredentialsProvider, cluster: Cluster) -> list[WalletInfo]: users = [User(string_utils.unique_name("user-")) for _ in range(WALLTETS_IN_POOL)] parallel(credentials_provider.GRPC.provide, users, cluster_node=cluster.cluster_nodes[0]) @@ -488,133 +488,39 @@ def bucket_container_resolver(node_under_test: ClusterNode) -> BucketContainerRe return resolver +@pytest.fixture(scope="session", params=[pytest.param(EVERYONE_ALLOW_ALL)]) +def container_request(request: pytest.FixtureRequest) -> ContainerRequest: + if "param" in request.__dict__: + return request.param + + container_marker = request.node.get_closest_marker("container") + # let default container to be public at the moment + container_request = EVERYONE_ALLOW_ALL + + if container_marker: + if len(container_marker.args) != 1: + raise RuntimeError(f"Something wrong with container marker: {container_marker}") + container_request = container_marker.args[0] + + if not container_request: + raise RuntimeError( + f"""Container specification is empty. + Add @pytest.mark.parametrize("container_request", [ContainerRequest(...)], indirect=True) decorator.""" + ) + + return container_request + + @pytest.fixture def container( default_wallet: WalletInfo, frostfs_cli: FrostfsCli, client_shell: Shell, cluster: Cluster, - request: pytest.FixtureRequest, rpc_endpoint: str, + container_request: ContainerRequest, ) -> str: - with reporter.step("Get container specification for test"): - container_spec = _get_container_spec(request) - - with reporter.step("Create container"): - cid = _create_container_by_spec(default_wallet, client_shell, cluster, rpc_endpoint, container_spec) - # TODO: deprecate this. Use generic ContainerSpec.ape_rule param - if container_spec.allow_owner_via_ape: - with reporter.step("Allow owner via APE on container"): - _allow_owner_via_ape(frostfs_cli, cluster, cid) - - with reporter.step("Apply APE rules for container"): - if container_spec.ape_rules: - _apply_ape_rules(frostfs_cli, cluster, cid, container_spec.ape_rules) - - # Add marker if we want to run all tests with container - request.node.add_marker("requires_container") - - return cid - - -@pytest.fixture(scope="module") -def container_module_scope( - default_wallet: WalletInfo, - frostfs_cli: FrostfsCli, - client_shell: Shell, - cluster: Cluster, - request: pytest.FixtureRequest, - rpc_endpoint: str, -) -> str: - with reporter.step("Get container specification for test"): - container_spec = _get_container_spec(request) - - with reporter.step("Create container"): - cid = _create_container_by_spec(default_wallet, client_shell, cluster, rpc_endpoint, container_spec) - # TODO: deprecate this. Use generic ContainerSpec.ape_rule param - if container_spec.allow_owner_via_ape: - with reporter.step("Allow owner via APE on container"): - _allow_owner_via_ape(frostfs_cli, cluster, cid) - - with reporter.step("Apply APE rules for container"): - if container_spec.ape_rules: - _apply_ape_rules(frostfs_cli, cluster, cid, container_spec.ape_rules) - - # Add marker if we want to run all tests with container - request.node.add_marker("requires_container") - - return cid - - -def _apply_ape_rules(frostfs_cli: FrostfsCli, cluster: Cluster, container: str, ape_rules: list[ape.Rule]): - for ape_rule in ape_rules: - rule_str = ape_rule.as_string() - with reporter.step(f"Apply APE rule '{rule_str}' for container {container}"): - frostfs_cli.ape_manager.add( - cluster.default_rpc_endpoint, - ape_rule.chain_id, - target_name=container, - target_type="container", - rule=rule_str, - ) - - with reporter.step("Wait for one block"): - time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) - - -def _create_container_by_spec( - default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, rpc_endpoint: str, container_spec: ContainerSpec -) -> str: - with reporter.step(f"Create container by spec {container_spec}"): - cid = create_container(default_wallet, client_shell, rpc_endpoint, container_spec.parsed_rule(cluster)) - - with reporter.step("Search nodes holding the container"): - container_holder_nodes = search_nodes_with_container(default_wallet, cid, client_shell, cluster.default_rpc_endpoint, cluster) - report_data = {node.id: node.host_ip for node in container_holder_nodes} - - reporter.attach(json.dumps(report_data, indent=2), "container_nodes.json") - - return cid - - -def _get_container_spec(request: pytest.FixtureRequest) -> ContainerSpec: - container_marker = request.node.get_closest_marker("container") - # let default container to be public at the moment - container_spec = ContainerSpecs.PublicReadWrite - - if container_marker: - if len(container_marker.args) != 1: - raise RuntimeError(f"Something wrong with container marker: {container_marker}") - container_spec = container_marker.args[0] - - if "param" in request.__dict__: - container_spec = request.param - - if not container_spec: - raise RuntimeError( - f"""Container specification is empty. - Either add @pytest.mark.container(ContainerSpec(...)) or - @pytest.mark.parametrize(\"container\", [ContainerSpec(...)], indirect=True) decorator""" - ) - - return container_spec - - -def _allow_owner_via_ape(frostfs_cli: FrostfsCli, cluster: Cluster, container: str): - with reporter.step("Create allow APE rule for container owner"): - role_condition = ape.Condition.by_role(ape.Role.OWNER) - ape_rule = ape.Rule(ape.Verb.ALLOW, ape.ObjectOperations.WILDCARD_ALL, role_condition) - - frostfs_cli.ape_manager.add( - cluster.default_rpc_endpoint, - ape_rule.chain_id, - target_name=container, - target_type="container", - rule=ape_rule.as_string(), - ) - - with reporter.step("Wait for one block"): - time.sleep(datetime_utils.parse_time(MORPH_BLOCK_TIME)) + return create_container_with_ape(frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint, container_request) @pytest.fixture() diff --git a/pytest_tests/testsuites/container/test_container.py b/pytest_tests/testsuites/container/test_container.py index ce1f760d..0790e043 100644 --- a/pytest_tests/testsuites/container/test_container.py +++ b/pytest_tests/testsuites/container/test_container.py @@ -20,31 +20,20 @@ from ...helpers.utility import placement_policy_from_container @pytest.mark.sanity @pytest.mark.container class TestContainer(ClusterTestBase): + PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X" + @allure.title("Create container (name={name})") @pytest.mark.parametrize("name", ["", "test-container"], ids=["No name", "Set particular name"]) @pytest.mark.smoke - def test_container_creation(self, default_wallet: WalletInfo, name: str): + def test_container_creation(self, default_wallet: WalletInfo, name: str, rpc_endpoint: str): wallet = default_wallet - placement_rule = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X" - cid = create_container( - wallet, - rule=placement_rule, - name=name, - shell=self.shell, - endpoint=self.cluster.default_rpc_endpoint, - ) + cid = create_container(wallet, self.shell, rpc_endpoint, self.PLACEMENT_RULE, name=name) - containers = list_containers(wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint) + containers = list_containers(wallet, self.shell, rpc_endpoint) assert cid in containers, f"Expected container {cid} in containers: {containers}" - container_info: str = get_container( - wallet, - cid, - json_mode=False, - shell=self.shell, - endpoint=self.cluster.default_rpc_endpoint, - ) + container_info: str = get_container(wallet, cid, self.shell, rpc_endpoint, False) container_info = container_info.casefold() # To ignore case when comparing with expected values info_to_check = { @@ -56,7 +45,7 @@ class TestContainer(ClusterTestBase): info_to_check.add(f"Name={name}") with reporter.step("Check container has correct information"): - expected_policy = placement_rule.casefold() + expected_policy = self.PLACEMENT_RULE.casefold() actual_policy = placement_policy_from_container(container_info) assert actual_policy == expected_policy, f"Expected policy\n{expected_policy} but got policy\n{actual_policy}" @@ -65,50 +54,42 @@ class TestContainer(ClusterTestBase): assert expected_info in container_info, f"Expected {expected_info} in container info:\n{container_info}" with reporter.step("Delete container and check it was deleted"): - delete_container( - wallet, - cid, - shell=self.shell, - endpoint=self.cluster.default_rpc_endpoint, - await_mode=True, - ) + # Force to skip frostfs-cli verifictions before delete. + # Because no APE rules assigned to container, those verifications will fail due to APE requests denial. + delete_container(wallet, cid, self.shell, rpc_endpoint, force=True, await_mode=True) self.tick_epoch() - wait_for_container_deletion(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint) + wait_for_container_deletion(wallet, cid, self.shell, rpc_endpoint) + + @allure.title("Delete container without force (name={name})") + @pytest.mark.smoke + def test_container_deletion_no_force(self, container: str, default_wallet: WalletInfo, rpc_endpoint: str): + with reporter.step("Delete container and check it was deleted"): + delete_container(default_wallet, container, self.shell, rpc_endpoint, await_mode=True) + self.tick_epoch() + wait_for_container_deletion(default_wallet, container, self.shell, rpc_endpoint) @allure.title("Parallel container creation and deletion") - def test_container_creation_deletion_parallel(self, default_wallet: WalletInfo): + def test_container_creation_deletion_parallel(self, default_wallet: WalletInfo, rpc_endpoint: str): containers_count = 3 wallet = default_wallet - placement_rule = "REP 2 IN X CBF 1 SELECT 2 FROM * AS X" iteration_count = 10 - for iteration in range(iteration_count): + for _ in range(iteration_count): cids: list[str] = [] with reporter.step(f"Create {containers_count} containers"): for _ in range(containers_count): cids.append( - create_container( - wallet, - rule=placement_rule, - await_mode=False, - shell=self.shell, - endpoint=self.cluster.default_rpc_endpoint, - wait_for_creation=False, - ) + create_container(wallet, self.shell, rpc_endpoint, self.PLACEMENT_RULE, await_mode=False, wait_for_creation=False) ) with reporter.step("Wait for containers occur in container list"): for cid in cids: - wait_for_container_creation( - wallet, - cid, - sleep_interval=containers_count, - shell=self.shell, - endpoint=self.cluster.default_rpc_endpoint, - ) + wait_for_container_creation(wallet, cid, self.shell, rpc_endpoint, sleep_interval=containers_count) with reporter.step("Delete containers and check they were deleted"): for cid in cids: - delete_container(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, await_mode=True) - containers_list = list_containers(wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint) + # Force to skip frostfs-cli verifictions before delete. + # Because no APE rules assigned to container, those verifications will fail due to APE requests denial. + delete_container(wallet, cid, self.shell, rpc_endpoint, force=True, await_mode=True) + containers_list = list_containers(wallet, self.shell, rpc_endpoint) assert cid not in containers_list, "Container not deleted" diff --git a/pytest_tests/testsuites/container/test_policy.py b/pytest_tests/testsuites/container/test_policy.py index f0b9dd28..68479048 100644 --- a/pytest_tests/testsuites/container/test_policy.py +++ b/pytest_tests/testsuites/container/test_policy.py @@ -21,7 +21,6 @@ from ...resources.policy_error_patterns import NOT_ENOUGH_TO_SELECT, NOT_FOUND_F @pytest.mark.nightly -@pytest.mark.container @pytest.mark.policy class TestPolicy(ClusterTestBase): @wait_for_success(1200, 60, title="Wait for full field price on node", expected_result=True) diff --git a/pytest_tests/testsuites/management/test_node_management.py b/pytest_tests/testsuites/management/test_node_management.py index 0948f609..cae2f44a 100644 --- a/pytest_tests/testsuites/management/test_node_management.py +++ b/pytest_tests/testsuites/management/test_node_management.py @@ -364,7 +364,7 @@ class TestMaintenanceMode(ClusterTestBase): cluster_state_controller: ClusterStateController, restore_node_status: list[ClusterNode], ): - with reporter.step("Create container and create\put object"): + with reporter.step("Create container and put object"): cid = create_container( wallet=default_wallet, shell=self.shell, diff --git a/pytest_tests/testsuites/metrics/test_container_metrics.py b/pytest_tests/testsuites/metrics/test_container_metrics.py index 86f5ba25..ed740879 100644 --- a/pytest_tests/testsuites/metrics/test_container_metrics.py +++ b/pytest_tests/testsuites/metrics/test_container_metrics.py @@ -1,7 +1,6 @@ import math import allure -from frostfs_testlib.testing.parallel import parallel import pytest from frostfs_testlib import reporter from frostfs_testlib.steps.cli.container import create_container, delete_container, search_nodes_with_container, wait_for_container_deletion @@ -12,19 +11,20 @@ from frostfs_testlib.storage.cluster import Cluster, ClusterNode from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase +from frostfs_testlib.testing.parallel import parallel from frostfs_testlib.utils.file_utils import generate_file from ...helpers.utility import are_numbers_similar @pytest.mark.nightly -@pytest.mark.container +@pytest.mark.metrics class TestContainerMetrics(ClusterTestBase): @reporter.step("Put object to container: {cid}") def put_object_parallel(self, file_path: str, wallet: WalletInfo, cid: str): oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster) return oid - + @reporter.step("Get metrics value from node") def get_metrics_search_by_greps_parallel(self, node: ClusterNode, **greps): try: @@ -139,12 +139,7 @@ class TestContainerMetrics(ClusterTestBase): @allure.title("Container size metrics put {objects_count} objects (obj_size={object_size})") @pytest.mark.parametrize("objects_count", [5, 10, 20]) - def test_container_size_metrics_more_objects( - self, - object_size: ObjectSize, - default_wallet: WalletInfo, - objects_count: int - ): + def test_container_size_metrics_more_objects(self, object_size: ObjectSize, default_wallet: WalletInfo, objects_count: int): with reporter.step(f"Create container"): cid = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint) @@ -154,10 +149,14 @@ class TestContainerMetrics(ClusterTestBase): oids = [future.result() for future in futures] with reporter.step("Check metric appears in all nodes"): - metric_values = [get_metrics_value(node, command="frostfs_node_engine_container_size_bytes", cid=cid) for node in self.cluster.cluster_nodes] - actual_value = sum(metric_values) // 2 # for policy REP 2, value divide by 2 + metric_values = [ + get_metrics_value(node, command="frostfs_node_engine_container_size_bytes", cid=cid) for node in self.cluster.cluster_nodes + ] + actual_value = sum(metric_values) // 2 # for policy REP 2, value divide by 2 expected_value = object_size.value * objects_count - assert are_numbers_similar(actual_value, expected_value, tolerance_percentage=2), "metric container size bytes value not correct" + assert are_numbers_similar( + actual_value, expected_value, tolerance_percentage=2 + ), "metric container size bytes value not correct" with reporter.step("Delete file, wait until gc remove object"): tombstones_size = 0 @@ -173,16 +172,10 @@ class TestContainerMetrics(ClusterTestBase): assert act_metric >= 0, "Metrics value is negative" assert sum(metrics_value_nodes) // len(self.cluster.cluster_nodes) == tombstones_size, "tomstone size of objects not correct" - @allure.title("Container metrics (policy={policy})") @pytest.mark.parametrize("placement_policy, policy", [("REP 2 IN X CBF 2 SELECT 2 FROM * AS X", "REP"), ("EC 1.1 CBF 1", "EC")]) def test_container_metrics_delete_complex_objects( - self, - complex_object_size: ObjectSize, - default_wallet: WalletInfo, - cluster: Cluster, - placement_policy: str, - policy: str + self, complex_object_size: ObjectSize, default_wallet: WalletInfo, cluster: Cluster, placement_policy: str, policy: str ): copies = 2 if policy == "REP" else 1 objects_count = 2 @@ -201,9 +194,9 @@ class TestContainerMetrics(ClusterTestBase): with reporter.step("Delete objects and container"): for oid in oids: delete_object(default_wallet, cid, oid, self.shell, cluster.default_rpc_endpoint) - + delete_container(default_wallet, cid, self.shell, cluster.default_rpc_endpoint) - + with reporter.step("Tick epoch and check container was deleted"): self.tick_epoch() wait_for_container_deletion(default_wallet, cid, shell=self.shell, endpoint=cluster.default_rpc_endpoint) diff --git a/pytest_tests/testsuites/object/test_object_api.py b/pytest_tests/testsuites/object/test_object_api.py index e438e730..1023e68e 100755 --- a/pytest_tests/testsuites/object/test_object_api.py +++ b/pytest_tests/testsuites/object/test_object_api.py @@ -5,6 +5,7 @@ import sys import allure import pytest from frostfs_testlib import reporter +from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli from frostfs_testlib.resources.error_patterns import ( INVALID_LENGTH_SPECIFIER, INVALID_OFFSET_SPECIFIER, @@ -14,7 +15,7 @@ from frostfs_testlib.resources.error_patterns import ( OUT_OF_RANGE, ) from frostfs_testlib.shell import Shell -from frostfs_testlib.steps.cli.container import create_container, search_nodes_with_container +from frostfs_testlib.steps.cli.container import DEFAULT_EC_PLACEMENT_RULE, DEFAULT_PLACEMENT_RULE, search_nodes_with_container from frostfs_testlib.steps.cli.object import ( get_object_from_random_node, get_range, @@ -33,12 +34,16 @@ from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase -from frostfs_testlib.utils.file_utils import generate_file, get_file_content, get_file_hash +from frostfs_testlib.utils.file_utils import TestFile, get_file_content, get_file_hash + +from ...helpers.container_creation import create_container_with_ape +from ...helpers.container_request import APE_EVERYONE_ALLOW_ALL, ContainerRequest logger = logging.getLogger("NeoLogger") CLEANUP_TIMEOUT = 10 COMMON_ATTRIBUTE = {"common_key": "common_value"} +COMMON_CONTAINER_RULE = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" # Will upload object for each attribute set OBJECT_ATTRIBUTES = [ None, @@ -80,7 +85,7 @@ def generate_ranges(storage_object: StorageObjectInfo, max_object_size: int, she for offset, length in file_ranges: range_length = random.randint(RANGE_MIN_LEN, RANGE_MAX_LEN) - range_start = random.randint(offset, offset + length) + range_start = random.randint(offset, offset + length - 1) file_ranges_to_test.append((range_start, min(range_length, storage_object.size - range_start))) @@ -90,12 +95,15 @@ def generate_ranges(storage_object: StorageObjectInfo, max_object_size: int, she @pytest.fixture(scope="module") -def common_container(default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster) -> str: - rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" - with reporter.step(f"Create container with {rule} and put object"): - cid = create_container(default_wallet, client_shell, cluster.default_rpc_endpoint, rule) - - return cid +def common_container( + frostfs_cli: FrostfsCli, + default_wallet: WalletInfo, + client_shell: Shell, + cluster: Cluster, + rpc_endpoint: str, +) -> str: + container_request = ContainerRequest(COMMON_CONTAINER_RULE, APE_EVERYONE_ALLOW_ALL) + return create_container_with_ape(frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint, container_request) @pytest.fixture(scope="module") @@ -117,33 +125,35 @@ def storage_objects( client_shell: Shell, cluster: Cluster, object_size: ObjectSize, + test_file_module: TestFile, + frostfs_cli: FrostfsCli, placement_policy: PlacementPolicy, + rpc_endpoint: str, ) -> list[StorageObjectInfo]: - wallet = default_wallet - # Separate containers for complex/simple objects to avoid side-effects - cid = create_container(wallet, shell=client_shell, rule=placement_policy.value, endpoint=cluster.default_rpc_endpoint) - - file_path = generate_file(object_size.value) - file_hash = get_file_hash(file_path) + cid = create_container_with_ape( + frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint, ContainerRequest(placement_policy.value, APE_EVERYONE_ALLOW_ALL) + ) + with reporter.step("Generate file"): + file_hash = get_file_hash(test_file_module.path) storage_objects = [] with reporter.step("Put objects"): # We need to upload objects multiple times with different attributes for attributes in OBJECT_ATTRIBUTES: storage_object_id = put_object_to_random_node( - wallet=wallet, - path=file_path, - cid=cid, - shell=client_shell, - cluster=cluster, + default_wallet, + test_file_module.path, + cid, + client_shell, + cluster, attributes=attributes, ) storage_object = StorageObjectInfo(cid, storage_object_id) storage_object.size = object_size.value - storage_object.wallet = wallet - storage_object.file_path = file_path + storage_object.wallet = default_wallet + storage_object.file_path = test_file_module.path storage_object.file_hash = file_hash storage_object.attributes = attributes @@ -241,21 +251,26 @@ class TestObjectApi(ClusterTestBase): ) self.check_header_is_presented(head_info, storage_object_2.attributes) - @allure.title("Head deleted object with --raw arg (obj_size={object_size}, policy={placement_policy})") - def test_object_head_raw(self, default_wallet: str, object_size: ObjectSize, placement_policy: PlacementPolicy): - with reporter.step("Create container"): - cid = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint, placement_policy.value) - + @allure.title("Head deleted object with --raw arg (obj_size={object_size}, policy={container_request})") + @pytest.mark.parametrize( + "container_request", + [ + ContainerRequest(DEFAULT_PLACEMENT_RULE, APE_EVERYONE_ALLOW_ALL, "rep"), + ContainerRequest(DEFAULT_EC_PLACEMENT_RULE, APE_EVERYONE_ALLOW_ALL, "ec"), + ], + indirect=True, + ids=["rep", "ec"], + ) + def test_object_head_raw(self, default_wallet: str, container: str, test_file: TestFile): with reporter.step("Upload object"): - file_path = generate_file(object_size.value) - oid = put_object_to_random_node(default_wallet, file_path, cid, self.shell, self.cluster) + oid = put_object_to_random_node(default_wallet, test_file.path, container, self.shell, self.cluster) with reporter.step("Delete object"): - delete_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint) + delete_object(default_wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint) with reporter.step("Call object head --raw and expect error"): with pytest.raises(Exception, match=OBJECT_ALREADY_REMOVED): - head_object(default_wallet, cid, oid, self.shell, self.cluster.default_rpc_endpoint, is_raw=True) + head_object(default_wallet, container, oid, self.shell, self.cluster.default_rpc_endpoint, is_raw=True) @allure.title("Search objects by native API (obj_size={object_size}, policy={placement_policy})") def test_search_object_api(self, storage_objects: list[StorageObjectInfo]): @@ -299,54 +314,50 @@ class TestObjectApi(ClusterTestBase): assert sorted(expected_oids) == sorted(result) @allure.title("Search objects with removed items (obj_size={object_size})") - def test_object_search_should_return_tombstone_items(self, default_wallet: WalletInfo, object_size: ObjectSize): + def test_object_search_should_return_tombstone_items( + self, + default_wallet: WalletInfo, + container: str, + object_size: ObjectSize, + rpc_endpoint: str, + test_file: TestFile, + ): """ Validate object search with removed items """ - wallet = default_wallet - cid = create_container(wallet, self.shell, self.cluster.default_rpc_endpoint) - - with reporter.step("Upload file"): - file_path = generate_file(object_size.value) - file_hash = get_file_hash(file_path) - + with reporter.step("Put object to container"): storage_object = StorageObjectInfo( - cid=cid, - oid=put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster), + cid=container, + oid=put_object_to_random_node(default_wallet, test_file.path, container, self.shell, self.cluster), size=object_size.value, - wallet=wallet, - file_path=file_path, - file_hash=file_hash, + wallet=default_wallet, + file_path=test_file.path, + file_hash=get_file_hash(test_file.path), ) with reporter.step("Search object"): # Root Search object should return root object oid - result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True) - assert result == [storage_object.oid] + result = search_object(default_wallet, container, self.shell, rpc_endpoint, root=True) + objects_before_deletion = len(result) + assert storage_object.oid in result - with reporter.step("Delete file"): + with reporter.step("Delete object"): delete_objects([storage_object], self.shell, self.cluster) with reporter.step("Search deleted object with --root"): # Root Search object should return nothing - result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True) + result = search_object(default_wallet, container, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True) assert len(result) == 0 with reporter.step("Search deleted object with --phy should return only tombstones"): # Physical Search object should return only tombstones - result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, phy=True) + result = search_object(default_wallet, container, self.shell, rpc_endpoint, phy=True) assert storage_object.tombstone in result, "Search result should contain tombstone of removed object" assert storage_object.oid not in result, "Search result should not contain ObjectId of removed object" for tombstone_oid in result: - header = head_object( - wallet, - cid, - tombstone_oid, - shell=self.shell, - endpoint=self.cluster.default_rpc_endpoint, - )["header"] - object_type = header["objectType"] + head_info = head_object(default_wallet, container, tombstone_oid, self.shell, rpc_endpoint) + object_type = head_info["header"]["objectType"] assert object_type == "TOMBSTONE", f"Object wasn't deleted properly. Found object {tombstone_oid} with type {object_type}" @allure.title("Get range hash by native API (obj_size={object_size}, policy={placement_policy})") diff --git a/pytest_tests/testsuites/object/test_object_api_bearer.py b/pytest_tests/testsuites/object/test_object_api_bearer.py index f6f9d404..ba031326 100644 --- a/pytest_tests/testsuites/object/test_object_api_bearer.py +++ b/pytest_tests/testsuites/object/test_object_api_bearer.py @@ -2,14 +2,12 @@ import allure import pytest from frostfs_testlib import reporter from frostfs_testlib.cli import FrostfsCli -from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL from frostfs_testlib.shell import Shell from frostfs_testlib.steps.cli.container import ( REP_2_FOR_3_NODES_PLACEMENT_RULE, SINGLE_PLACEMENT_RULE, StorageContainer, StorageContainerInfo, - create_container, ) from frostfs_testlib.steps.cli.object import delete_object, get_object from frostfs_testlib.steps.storage_object import StorageObjectInfo @@ -23,13 +21,18 @@ from pytest import FixtureRequest from ...helpers.bearer_token import create_bearer_token from ...helpers.container_access import assert_full_access_to_container +from ...helpers.container_creation import create_container_with_ape +from ...helpers.container_request import ContainerRequest @pytest.fixture(scope="session") @allure.title("Create user container for bearer token usage") -def user_container(default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, request: FixtureRequest) -> StorageContainer: - rule = request.param if "param" in request.__dict__ else SINGLE_PLACEMENT_RULE - container_id = create_container(default_wallet, client_shell, cluster.default_rpc_endpoint, rule, PUBLIC_ACL) +def user_container( + frostfs_cli: FrostfsCli, default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, rpc_endpoint: str, request: FixtureRequest +) -> StorageContainer: + policy = request.param if "param" in request.__dict__ else SINGLE_PLACEMENT_RULE + container_request = ContainerRequest(policy) + container_id = create_container_with_ape(frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint, container_request) # Deliberately using s3gate wallet here to test bearer token s3_gate_wallet = WalletInfo.from_node(cluster.s3_gates[0]) @@ -65,6 +68,7 @@ def storage_objects( @pytest.mark.nightly @pytest.mark.bearer @pytest.mark.ape +@pytest.mark.grpc_api class TestObjectApiWithBearerToken(ClusterTestBase): @allure.title("Object can be deleted from any node using s3gate wallet with bearer token (obj_size={object_size})") @pytest.mark.parametrize( diff --git a/pytest_tests/testsuites/object/test_object_lifetime.py b/pytest_tests/testsuites/object/test_object_lifetime.py index 65245fe0..b0926f10 100644 --- a/pytest_tests/testsuites/object/test_object_lifetime.py +++ b/pytest_tests/testsuites/object/test_object_lifetime.py @@ -6,13 +6,11 @@ from frostfs_testlib import reporter from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND from frostfs_testlib.steps.cli.object import get_object_from_random_node, head_object, put_object_to_random_node from frostfs_testlib.steps.epoch import get_epoch -from frostfs_testlib.storage.dataclasses.object_size import ObjectSize from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.test_control import expect_not_raises from frostfs_testlib.utils.file_utils import TestFile -from ...helpers.container_spec import ContainerSpecs from ...helpers.utility import wait_for_gc_pass_on_storage_nodes logger = logging.getLogger("NeoLogger") @@ -22,9 +20,8 @@ logger = logging.getLogger("NeoLogger") @pytest.mark.sanity @pytest.mark.grpc_api class TestObjectApiLifetime(ClusterTestBase): - @pytest.mark.container(ContainerSpecs.PublicReadWrite) @allure.title("Object is removed when lifetime expired (obj_size={object_size})") - def test_object_api_lifetime(self, container: str, test_file: TestFile, default_wallet: WalletInfo, object_size: ObjectSize): + def test_object_api_lifetime(self, container: str, test_file: TestFile, default_wallet: WalletInfo): """ Test object deleted after expiration epoch. """ diff --git a/pytest_tests/testsuites/object/test_object_lock.py b/pytest_tests/testsuites/object/test_object_lock.py index f190bfc0..0b9fe488 100755 --- a/pytest_tests/testsuites/object/test_object_lock.py +++ b/pytest_tests/testsuites/object/test_object_lock.py @@ -3,7 +3,7 @@ import logging import allure import pytest from frostfs_testlib import reporter -from frostfs_testlib.credentials.interfaces import CredentialsProvider, User +from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli from frostfs_testlib.resources.common import STORAGE_GC_TIME from frostfs_testlib.resources.error_patterns import ( LIFETIME_REQUIRED, @@ -27,9 +27,10 @@ from frostfs_testlib.storage.dataclasses.storage_object_info import LockObjectIn from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.test_control import expect_not_raises, wait_for_success -from frostfs_testlib.utils import datetime_utils, string_utils +from frostfs_testlib.utils import datetime_utils -from ...helpers.container_spec import ContainerSpecs +from ...helpers.container_creation import create_container_with_ape +from ...helpers.container_request import EVERYONE_ALLOW_ALL from ...helpers.utility import wait_for_gc_pass_on_storage_nodes logger = logging.getLogger("NeoLogger") @@ -37,34 +38,22 @@ logger = logging.getLogger("NeoLogger") FIXTURE_LOCK_LIFETIME = 5 FIXTURE_OBJECT_LIFETIME = 10 -pytestmark = pytest.mark.container(ContainerSpecs.PublicReadWrite) + +@pytest.fixture(scope="module") +def user_container( + frostfs_cli: FrostfsCli, default_wallet: WalletInfo, client_shell: Shell, cluster: Cluster, rpc_endpoint: str +) -> StorageContainer: + cid = create_container_with_ape(frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint, EVERYONE_ALLOW_ALL) + return StorageContainer(StorageContainerInfo(cid, default_wallet), client_shell, cluster) @pytest.fixture(scope="module") -def user_wallet(credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo: - with reporter.step("Create user wallet with container"): - user = User(string_utils.unique_name("user-")) - return credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0]) - - -@pytest.fixture(scope="module") -def user_container(container_module_scope: str, user_wallet: WalletInfo, client_shell: Shell, cluster: Cluster): - return StorageContainer(StorageContainerInfo(container_module_scope, user_wallet), client_shell, cluster) - - -@pytest.fixture(scope="module") -def locked_storage_object( - new_epoch_module_scope: int, - user_container: StorageContainer, - client_shell: Shell, - cluster: Cluster, - object_size: ObjectSize, -): +def locked_storage_object(user_container: StorageContainer, client_shell: Shell, cluster: Cluster, object_size: ObjectSize): """ Intention of this fixture is to provide storage object which is NOT expected to be deleted during test act phase """ with reporter.step("Creating locked object"): - current_epoch = new_epoch_module_scope + current_epoch = ensure_fresh_epoch(client_shell, cluster) expiration_epoch = current_epoch + FIXTURE_LOCK_LIFETIME storage_object = user_container.generate_object(object_size.value, expire_at=current_epoch + FIXTURE_OBJECT_LIFETIME) @@ -84,27 +73,16 @@ def locked_storage_object( @wait_for_success(datetime_utils.parse_time(STORAGE_GC_TIME)) def check_object_not_found(wallet: WalletInfo, cid: str, oid: str, shell: Shell, rpc_endpoint: str): with pytest.raises(Exception, match=OBJECT_NOT_FOUND): - head_object( - wallet, - cid, - oid, - shell, - rpc_endpoint, - ) + head_object(wallet, cid, oid, shell, rpc_endpoint) def verify_object_available(wallet: WalletInfo, cid: str, oid: str, shell: Shell, rpc_endpoint: str): with expect_not_raises(): - head_object( - wallet, - cid, - oid, - shell, - rpc_endpoint, - ) + head_object(wallet, cid, oid, shell, rpc_endpoint) @pytest.mark.nightly +@pytest.mark.grpc_api @pytest.mark.grpc_object_lock class TestObjectLockWithGrpc(ClusterTestBase): @pytest.fixture() diff --git a/pytest_tests/testsuites/object/test_object_tombstone.py b/pytest_tests/testsuites/object/test_object_tombstone.py index 941fb9bf..24135cd7 100644 --- a/pytest_tests/testsuites/object/test_object_tombstone.py +++ b/pytest_tests/testsuites/object/test_object_tombstone.py @@ -10,9 +10,9 @@ from frostfs_testlib.storage.grpc_operations.interfaces import GrpcClientWrapper from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.utils.file_utils import TestFile -from ....pytest_tests.helpers.container_spec import APE_PUBLIC_READ_WRITE, ContainerSpec - +@pytest.mark.nightly +@pytest.mark.grpc_api class TestObjectTombstone(ClusterTestBase): @pytest.fixture() @allure.title("Change tombstone lifetime") @@ -24,7 +24,6 @@ class TestObjectTombstone(ClusterTestBase): config_manager.revert_all(True) - @pytest.mark.container(ContainerSpec(ape_rules=APE_PUBLIC_READ_WRITE)) @pytest.mark.parametrize("object_size, tombstone_lifetime", [("simple", 2)], indirect=True) @allure.title("Tombstone object should be removed after expiration") def test_tombstone_lifetime( diff --git a/pytest_tests/testsuites/object/test_object_without_user.py b/pytest_tests/testsuites/object/test_object_without_user.py index 98bf39f4..9ee9b245 100644 --- a/pytest_tests/testsuites/object/test_object_without_user.py +++ b/pytest_tests/testsuites/object/test_object_without_user.py @@ -7,9 +7,7 @@ from frostfs_testlib import reporter from frostfs_testlib.cli import FrostfsCli from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_CLI_EXEC from frostfs_testlib.resources.error_patterns import OBJECT_IS_LOCKED -from frostfs_testlib.resources.wellknown_acl import PUBLIC_ACL_F from frostfs_testlib.shell import Shell -from frostfs_testlib.steps.cli.container import create_container from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing.cluster_test_base import ClusterTestBase from frostfs_testlib.testing.test_control import expect_not_raises @@ -19,6 +17,7 @@ logger = logging.getLogger("NeoLogger") @pytest.mark.nightly +@pytest.mark.grpc_api @pytest.mark.grpc_without_user class TestObjectApiWithoutUser(ClusterTestBase): def _parse_oid(self, stdout: str) -> str: @@ -31,96 +30,72 @@ class TestObjectApiWithoutUser(ClusterTestBase): tombstone = id_str.split(":")[1] return tombstone.strip() - @pytest.fixture(scope="function") - def public_container(self, default_wallet: WalletInfo) -> str: - with reporter.step("Create public container"): - cid_public = create_container( - default_wallet, - self.shell, - self.cluster.default_rpc_endpoint, - basic_acl=PUBLIC_ACL_F, - ) - - return cid_public - @pytest.fixture(scope="class") - def frostfs_cli(self, client_shell: Shell) -> FrostfsCli: + def cli_without_wallet(self, client_shell: Shell) -> FrostfsCli: return FrostfsCli(client_shell, FROSTFS_CLI_EXEC) @allure.title("Get public container by native API with generate private key") - def test_get_container_with_generated_key(self, frostfs_cli: FrostfsCli, public_container: str): + def test_get_container_with_generated_key(self, cli_without_wallet: FrostfsCli, container: str, rpc_endpoint: str): """ Validate `container get` native API with flag `--generate-key`. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint - with reporter.step("Get container with generate key"): with expect_not_raises(): - frostfs_cli.container.get(rpc_endpoint, cid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) + cli_without_wallet.container.get(rpc_endpoint, container, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) @allure.title("Get list containers by native API with generate private key") - def test_list_containers_with_generated_key(self, frostfs_cli: FrostfsCli, default_wallet: WalletInfo, public_container: str): + def test_list_containers_with_generated_key( + self, cli_without_wallet: FrostfsCli, default_wallet: WalletInfo, container: str, rpc_endpoint: str + ): """ Validate `container list` native API with flag `--generate-key`. """ - - rpc_endpoint = self.cluster.default_rpc_endpoint owner = default_wallet.get_address_from_json(0) with reporter.step("List containers with generate key"): with expect_not_raises(): - result = frostfs_cli.container.list(rpc_endpoint, owner=owner, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) + result = cli_without_wallet.container.list(rpc_endpoint, owner=owner, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) with reporter.step("Expect container in received containers list"): containers = result.stdout.split() - assert public_container in containers + assert container in containers @allure.title("Get list of public container objects by native API with generate private key") - def test_list_objects_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str): + def test_list_objects_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, rpc_endpoint: str): """ Validate `container list_objects` native API with flag `--generate-key`. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint - with reporter.step("List objects with generate key"): with expect_not_raises(): - result = frostfs_cli.container.list_objects(rpc_endpoint, cid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) + result = cli_without_wallet.container.list_objects(rpc_endpoint, container, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) with reporter.step("Expect empty objects list"): objects = result.stdout.split() assert len(objects) == 0, objects @allure.title("Search public container nodes by native API with generate private key") - def test_search_nodes_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str): + def test_search_nodes_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, rpc_endpoint: str): """ Validate `container search_node` native API with flag `--generate-key`. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint - with reporter.step("Search nodes with generate key"): with expect_not_raises(): - frostfs_cli.container.search_node(rpc_endpoint, cid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) + cli_without_wallet.container.search_node(rpc_endpoint, container, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) @allure.title("Put object into public container by native API with generate private key (obj_size={object_size})") - def test_put_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): + def test_put_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str): """ Validate `object put` into container with public ACL and flag `--generate-key`. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint - with reporter.step("Put object with generate key"): with expect_not_raises(): - result = frostfs_cli.object.put( + result = cli_without_wallet.object.put( rpc_endpoint, - cid, + container, file_path, generate_key=True, no_progress=True, @@ -130,26 +105,24 @@ class TestObjectApiWithoutUser(ClusterTestBase): oid = self._parse_oid(result.stdout) with reporter.step("List objects with generate key"): - result = frostfs_cli.container.list_objects(rpc_endpoint, cid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) + result = cli_without_wallet.container.list_objects(rpc_endpoint, container, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) with reporter.step("Expect object in received objects list"): objects = result.stdout.split() assert oid in objects, objects @allure.title("Get public container object by native API with generate private key (obj_size={object_size})") - def test_get_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): + def test_get_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str): """ Validate `object get` for container with public ACL and flag `--generate-key`. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint expected_hash = get_file_hash(file_path) with reporter.step("Put object with generate key"): - result = frostfs_cli.object.put( + result = cli_without_wallet.object.put( rpc_endpoint, - cid, + container, file_path, generate_key=True, no_progress=True, @@ -160,9 +133,9 @@ class TestObjectApiWithoutUser(ClusterTestBase): with reporter.step("Get object with generate key"): with expect_not_raises(): - frostfs_cli.object.get( + cli_without_wallet.object.get( rpc_endpoint, - cid, + container, oid, file=file_path, generate_key=True, @@ -176,18 +149,15 @@ class TestObjectApiWithoutUser(ClusterTestBase): assert expected_hash == downloaded_hash @allure.title("Head public container object by native API with generate private key (obj_size={object_size})") - def test_head_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): + def test_head_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str): """ Validate `object head` for container with public ACL and flag `--generate-key`. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint - with reporter.step("Put object with generate key"): - result = frostfs_cli.object.put( + result = cli_without_wallet.object.put( rpc_endpoint, - cid, + container, file_path, generate_key=True, no_progress=True, @@ -198,21 +168,18 @@ class TestObjectApiWithoutUser(ClusterTestBase): with reporter.step("Head object with generate key"): with expect_not_raises(): - frostfs_cli.object.head(rpc_endpoint, cid, oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) + cli_without_wallet.object.head(rpc_endpoint, container, oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) @allure.title("Delete public container object by native API with generate private key (obj_size={object_size})") - def test_delete_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): + def test_delete_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str): """ Validate `object delete` for container with public ACL and flag `--generate key`. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint - with reporter.step("Put object with generate key"): - result = frostfs_cli.object.put( + result = cli_without_wallet.object.put( rpc_endpoint, - cid, + container, file_path, generate_key=True, no_progress=True, @@ -223,14 +190,14 @@ class TestObjectApiWithoutUser(ClusterTestBase): with reporter.step("Delete object with generate key"): with expect_not_raises(): - result = frostfs_cli.object.delete(rpc_endpoint, cid, oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) + result = cli_without_wallet.object.delete(rpc_endpoint, container, oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) oid = self._parse_tombstone_oid(result.stdout) with reporter.step("Head object with generate key"): - result = frostfs_cli.object.head( + result = cli_without_wallet.object.head( rpc_endpoint, - cid, + container, oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT, @@ -241,19 +208,16 @@ class TestObjectApiWithoutUser(ClusterTestBase): assert object_type == "TOMBSTONE", object_type @allure.title("Lock public container object by native API with generate private key (obj_size={object_size})") - def test_lock_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): + def test_lock_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str): """ Validate `object lock` for container with public ACL and flag `--generate-key`. Attempt to delete the locked object. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint - with reporter.step("Put object with generate key"): - result = frostfs_cli.object.put( + result = cli_without_wallet.object.put( rpc_endpoint, - cid, + container, file_path, generate_key=True, no_progress=True, @@ -264,9 +228,9 @@ class TestObjectApiWithoutUser(ClusterTestBase): with reporter.step("Lock object with generate key"): with expect_not_raises(): - frostfs_cli.object.lock( + cli_without_wallet.object.lock( rpc_endpoint, - cid, + container, oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT, @@ -275,27 +239,24 @@ class TestObjectApiWithoutUser(ClusterTestBase): with reporter.step("Delete locked object with generate key and expect error"): with pytest.raises(Exception, match=OBJECT_IS_LOCKED): - frostfs_cli.object.delete( + cli_without_wallet.object.delete( rpc_endpoint, - cid, + container, oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT, ) @allure.title("Search public container objects by native API with generate private key (obj_size={object_size})") - def test_search_object_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): + def test_search_object_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str): """ Validate `object search` for container with public ACL and flag `--generate-key`. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint - with reporter.step("Put object with generate key"): - result = frostfs_cli.object.put( + result = cli_without_wallet.object.put( rpc_endpoint, - cid, + container, file_path, generate_key=True, no_progress=True, @@ -306,25 +267,22 @@ class TestObjectApiWithoutUser(ClusterTestBase): with reporter.step("Object search with generate key"): with expect_not_raises(): - result = frostfs_cli.object.search(rpc_endpoint, cid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) + result = cli_without_wallet.object.search(rpc_endpoint, container, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT) with reporter.step("Expect object in received objects list of container"): object_ids = re.findall(r"(\w{43,44})", result.stdout) assert oid in object_ids @allure.title("Get range of public container object by native API with generate private key (obj_size={object_size})") - def test_range_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): + def test_range_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str): """ Validate `object range` for container with public ACL and `--generate-key`. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint - with reporter.step("Put object with generate key"): - result = frostfs_cli.object.put( + result = cli_without_wallet.object.put( rpc_endpoint, - cid, + container, file_path, generate_key=True, no_progress=True, @@ -335,9 +293,9 @@ class TestObjectApiWithoutUser(ClusterTestBase): with reporter.step("Get range of object with generate key"): with expect_not_raises(): - frostfs_cli.object.range( + cli_without_wallet.object.range( rpc_endpoint, - cid, + container, oid, "0:10", file=file_path, @@ -346,18 +304,15 @@ class TestObjectApiWithoutUser(ClusterTestBase): ) @allure.title("Get hash of public container object by native API with generate private key (obj_size={object_size})") - def test_hash_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): + def test_hash_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str): """ Validate `object hash` for container with public ACL and `--generate-key`. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint - with reporter.step("Put object with generate key"): - result = frostfs_cli.object.put( + result = cli_without_wallet.object.put( rpc_endpoint, - cid, + container, file_path, generate_key=True, no_progress=True, @@ -368,9 +323,9 @@ class TestObjectApiWithoutUser(ClusterTestBase): with reporter.step("Get range hash of object with generate key"): with expect_not_raises(): - frostfs_cli.object.hash( + cli_without_wallet.object.hash( rpc_endpoint, - cid, + container, oid, range="0:10", generate_key=True, @@ -378,18 +333,15 @@ class TestObjectApiWithoutUser(ClusterTestBase): ) @allure.title("Get public container object nodes by native API with generate private key (obj_size={object_size})") - def test_nodes_with_generate_key(self, frostfs_cli: FrostfsCli, public_container: str, file_path: TestFile): + def test_nodes_with_generate_key(self, cli_without_wallet: FrostfsCli, container: str, file_path: TestFile, rpc_endpoint: str): """ Validate `object nodes` for container with public ACL and `--generate-key`. """ - cid = public_container - rpc_endpoint = self.cluster.default_rpc_endpoint - with reporter.step("Put object with generate key"): - result = frostfs_cli.object.put( + result = cli_without_wallet.object.put( rpc_endpoint, - cid, + container, file_path, no_progress=True, generate_key=True, @@ -401,14 +353,14 @@ class TestObjectApiWithoutUser(ClusterTestBase): with reporter.step("Configure frostfs-cli for alive remote node"): alive_node = self.cluster.cluster_nodes[0] node_shell = alive_node.host.get_shell() - rpc_endpoint = alive_node.storage_node.get_rpc_endpoint() - node_frostfs_cli = FrostfsCli(node_shell, FROSTFS_CLI_EXEC) + alive_endpoint = alive_node.storage_node.get_rpc_endpoint() + node_frostfs_cli_wo_wallet = FrostfsCli(node_shell, FROSTFS_CLI_EXEC) with reporter.step("Get object nodes with generate key"): with expect_not_raises(): - node_frostfs_cli.object.nodes( - rpc_endpoint, - cid, + node_frostfs_cli_wo_wallet.object.nodes( + alive_endpoint, + container, oid=oid, generate_key=True, timeout=CLI_DEFAULT_TIMEOUT,