Merge branch 'master' into fix/add-missing-titles

pytest_tests/helpers/object_access.py

@@ -2,7 +2,7 @@ from typing import Optional
 
 from frostfs_testlib import reporter
 from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
-from frostfs_testlib.resources.error_patterns import OBJECT_ACCESS_DENIED
+from frostfs_testlib.resources.error_patterns import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
 from frostfs_testlib.shell import Shell
 from frostfs_testlib.steps.cli.object import (
     delete_object,
@@ -20,6 +20,10 @@ from frostfs_testlib.utils.file_utils import get_file_hash
 
 OPERATION_ERROR_TYPE = RuntimeError
 
+# TODO: Revert to just OBJECT_ACCESS_DENIED when the issue is fixed
+# https://git.frostfs.info/TrueCloudLab/frostfs-node/issues/1297
+OBJECT_NO_ACCESS = rf"(?:{OBJECT_NOT_FOUND}|{OBJECT_ACCESS_DENIED})"
+
 
 def can_get_object(
     wallet: WalletInfo,
@@ -43,9 +47,7 @@ def can_get_object(
                 cluster=cluster,
             )
         except OPERATION_ERROR_TYPE as err:
-            assert string_utils.is_str_match_pattern(
-                err, OBJECT_ACCESS_DENIED
-            ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
+            assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}"
             return False
         assert get_file_hash(file_name) == get_file_hash(got_file_path)
     return True
@@ -74,9 +76,7 @@ def can_put_object(
                 cluster=cluster,
             )
         except OPERATION_ERROR_TYPE as err:
-            assert string_utils.is_str_match_pattern(
-                err, OBJECT_ACCESS_DENIED
-            ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
+            assert string_utils.is_str_match_pattern(err, OBJECT_ACCESS_DENIED), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
             return False
     return True
 
@@ -102,9 +102,7 @@ def can_delete_object(
                 endpoint=endpoint,
             )
         except OPERATION_ERROR_TYPE as err:
-            assert string_utils.is_str_match_pattern(
-                err, OBJECT_ACCESS_DENIED
-            ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
+            assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}"
             return False
     return True
 
@@ -132,9 +130,7 @@ def can_get_head_object(
                 timeout=timeout,
             )
         except OPERATION_ERROR_TYPE as err:
-            assert string_utils.is_str_match_pattern(
-                err, OBJECT_ACCESS_DENIED
-            ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
+            assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}"
             return False
     return True
 
@@ -163,9 +159,7 @@ def can_get_range_of_object(
                 timeout=timeout,
             )
         except OPERATION_ERROR_TYPE as err:
-            assert string_utils.is_str_match_pattern(
-                err, OBJECT_ACCESS_DENIED
-            ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
+            assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}"
             return False
     return True
 
@@ -194,9 +188,7 @@ def can_get_range_hash_of_object(
                 timeout=timeout,
             )
         except OPERATION_ERROR_TYPE as err:
-            assert string_utils.is_str_match_pattern(
-                err, OBJECT_ACCESS_DENIED
-            ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
+            assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}"
             return False
     return True
 
@@ -223,9 +215,7 @@ def can_search_object(
                 timeout=timeout,
             )
         except OPERATION_ERROR_TYPE as err:
-            assert string_utils.is_str_match_pattern(
-                err, OBJECT_ACCESS_DENIED
-            ), f"Expected {err} to match {OBJECT_ACCESS_DENIED}"
+            assert string_utils.is_str_match_pattern(err, OBJECT_NO_ACCESS), f"Expected {err} to match {OBJECT_NO_ACCESS}"
             return False
         if oid:
             return oid in oids

pytest_tests/testsuites/access/ape/test_ape_filters.py

@@ -21,8 +21,7 @@ from pytest_tests.helpers.container_access import (
     assert_full_access_to_container,
     assert_no_access_to_container,
 )
-
-OBJECT_NO_ACCESS = f"(?:{OBJECT_NOT_FOUND}|{OBJECT_ACCESS_DENIED})"
+from pytest_tests.helpers.object_access import OBJECT_NO_ACCESS
 
 
 @pytest.mark.ape
@@ -58,7 +57,7 @@ class TestApeFilters(ClusterTestBase):
         return cid, objects_with_header, objects_with_other_header, objects_without_header, file_path
 
     @pytest.fixture(scope="function")
-    def container_with_objects(self, default_wallet: WalletInfo, file_path: TestFile, frostfs_cli: FrostfsCli, cluster: Cluster):
+    def private_container(self, default_wallet: WalletInfo, frostfs_cli: FrostfsCli, cluster: Cluster):
         with reporter.step("Create private container"):
             cid = create_container(default_wallet, self.shell, self.cluster.default_rpc_endpoint, basic_acl="0")
 
@@ -77,9 +76,14 @@ class TestApeFilters(ClusterTestBase):
         with reporter.step("Wait for one block"):
             self.wait_for_blocks()
 
-        objects_with_header, objects_with_other_header, objects_without_header = self._fill_container(default_wallet, file_path, cid)
+        return cid
 
-        return cid, objects_with_header, objects_with_other_header, objects_without_header, file_path
+    @pytest.fixture(scope="function")
+    def container_with_objects(self, private_container: str, default_wallet: WalletInfo, file_path: TestFile):
+        objects_with_header, objects_with_other_header, objects_without_header = self._fill_container(
+            default_wallet, file_path, private_container
+        )
+        return private_container, objects_with_header, objects_with_other_header, objects_without_header, file_path
 
     @reporter.step("Add objects to container")
     def _fill_container(self, wallet: WalletInfo, test_file: TestFile, cid: str):
@@ -120,7 +124,7 @@ class TestApeFilters(ClusterTestBase):
         endpoint = self.cluster.default_rpc_endpoint
 
         with reporter.step("Deny all operations for others via APE with request condition"):
-            request_condition = ape.Condition('"check_key"', '"check_value"', ape.ConditionType.REQUEST, match_type)
+            request_condition = ape.Condition('"frostfs:xheader/check_key"', '"check_value"', ape.ConditionType.REQUEST, match_type)
             role_condition = ape.Condition.by_role(ape.Role.OTHERS)
             deny_rule = ape.Rule(ape.Verb.DENY, ALL_OBJECT_OPERATIONS, [request_condition, role_condition])
 
@@ -168,15 +172,15 @@ class TestApeFilters(ClusterTestBase):
         # TODO: Refactor this to be fixtures, not test logic
         (
             cid,
-            objects_with_header,
-            objects_with_other_header,
-            objs_without_header,
+            objects_with_attributes,
+            objects_with_other_attributes,
+            objs_without_attributes,
             file_path,
         ) = public_container_with_objects
         endpoint = self.cluster.default_rpc_endpoint
 
-        allow_objects = objects_with_other_header if match_type == ape.MatchType.EQUAL else objects_with_header
-        deny_objects = objects_with_header if match_type == ape.MatchType.EQUAL else objects_with_other_header
+        allow_objects = objects_with_other_attributes if match_type == ape.MatchType.EQUAL else objects_with_attributes
+        deny_objects = objects_with_attributes if match_type == ape.MatchType.EQUAL else objects_with_other_attributes
 
         # When there is no attribute on the object, it's the same as "", and "" is not equal to "<some_value>"
         # So it's the same as deny_objects
@@ -192,10 +196,23 @@ class TestApeFilters(ClusterTestBase):
                 ape.ObjectOperations.DELETE: False,  # Denied by restricted PUT
             },
         }
+        allowed_access = {
+            ape.MatchType.EQUAL: FULL_ACCESS,
+            ape.MatchType.NOT_EQUAL: {
+                ape.ObjectOperations.PUT: False,  # because currently we are put without attributes
+                ape.ObjectOperations.GET: True,
+                ape.ObjectOperations.HEAD: True,
+                ape.ObjectOperations.GET_RANGE: True,
+                ape.ObjectOperations.GET_RANGE_HASH: True,
+                ape.ObjectOperations.SEARCH: True,
+                ape.ObjectOperations.DELETE: False,  # Because delete needs to put a tombstone without attributes
+            },
+        }
         # End of refactor
 
         with reporter.step("Deny operations for others via APE with resource condition"):
             resource_condition = ape.Condition('"check_key"', '"check_value"', ape.ConditionType.RESOURCE, match_type)
+            not_a_tombstone_condition = ape.Condition.by_object_type("TOMBSTONE", ape.ConditionType.RESOURCE, ape.MatchType.NOT_EQUAL)
             role_condition = ape.Condition.by_role(ape.Role.OTHERS)
             deny_rule = ape.Rule(ape.Verb.DENY, self.RESOURCE_OPERATIONS, [resource_condition, role_condition])
 
@@ -222,7 +239,7 @@ class TestApeFilters(ClusterTestBase):
                     no_attributes_access[match_type],
                     other_wallet,
                     cid,
-                    objs_without_header.pop(),
+                    objs_without_attributes.pop(),
                     file_path,
                     self.shell,
                     self.cluster,
@@ -230,7 +247,9 @@ class TestApeFilters(ClusterTestBase):
                 )
 
             with reporter.step("Check others have full access to objects without deny attribute"):
-                assert_full_access_to_container(other_wallet, cid, allow_objects.pop(), file_path, self.shell, self.cluster, xhdr=xhdr)
+                assert_access_to_container(
+                    allowed_access[match_type], other_wallet, cid, allow_objects.pop(), file_path, self.shell, self.cluster, xhdr=xhdr
+                )
 
             with reporter.step("Check others have no access to objects with deny attribute"):
                 with pytest.raises(Exception, match=OBJECT_NO_ACCESS):
@@ -268,23 +287,25 @@ class TestApeFilters(ClusterTestBase):
         # TODO: Refactor this to be fixtures, not test logic!
         (
             cid,
-            objects_with_header,
-            objects_with_other_header,
-            objects_without_header,
+            objects_with_attributes,
+            objects_with_other_attributes,
+            objects_without_attributes,
             file_path,
         ) = container_with_objects
         endpoint = self.cluster.default_rpc_endpoint
 
         if match_type == ape.MatchType.EQUAL:
-            allow_objects = objects_with_header
-            deny_objects = objects_with_other_header
+            allow_objects = objects_with_attributes
+            deny_objects = objects_with_other_attributes
             allow_attribute = self.HEADER
             deny_attribute = self.OTHER_HEADER
+            no_attributes_match_context = pytest.raises(Exception, match=OBJECT_NO_ACCESS)
         else:
-            allow_objects = objects_with_other_header
-            deny_objects = objects_with_header
+            allow_objects = objects_with_other_attributes
+            deny_objects = objects_with_attributes
             allow_attribute = self.OTHER_HEADER
             deny_attribute = self.HEADER
+            no_attributes_match_context = expect_not_raises()
         # End of refactor block
 
         with reporter.step("Allow operations for others except few operations by resource condition via APE"):
@@ -297,15 +318,15 @@ class TestApeFilters(ClusterTestBase):
         with reporter.step("Wait for one block"):
             self.wait_for_blocks()
 
-        with reporter.step("Check others cannot get and put objects without attributes"):
-            oid = objects_without_header.pop()
-            with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
+        with reporter.step("Check GET, PUT and HEAD operations with objects without attributes for OTHERS role"):
+            oid = objects_without_attributes.pop()
+            with no_attributes_match_context:
                 assert head_object(other_wallet, cid, oid, self.shell, endpoint)
 
-            with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
+            with no_attributes_match_context:
                 assert get_object_from_random_node(other_wallet, cid, oid, self.shell, self.cluster)
 
-            with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
+            with no_attributes_match_context:
                 assert put_object_to_random_node(other_wallet, file_path, cid, self.shell, self.cluster)
 
         with reporter.step("Create bearer token with everything allowed for others role"):
@@ -314,7 +335,7 @@ class TestApeFilters(ClusterTestBase):
             bearer = create_bearer_token(frostfs_cli, temp_directory, cid, rule, endpoint)
 
         with reporter.step("Check others can get and put objects without attributes and using bearer token"):
-            oid = objects_without_header[0]
+            oid = objects_without_attributes[0]
             with expect_not_raises():
                 head_object(other_wallet, cid, oid, self.shell, endpoint, bearer)
 
@@ -337,13 +358,13 @@ class TestApeFilters(ClusterTestBase):
 
         with reporter.step("Check others cannot get and put objects without attributes matching the filter"):
             oid = deny_objects[0]
-            with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
+            with pytest.raises(Exception, match=OBJECT_NO_ACCESS):
                 head_object(other_wallet, cid, oid, self.shell, endpoint)
 
-            with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
-                assert get_object_from_random_node(other_wallet, cid, oid, file_path, self.shell, self.cluster)
+            with pytest.raises(Exception, match=OBJECT_NO_ACCESS):
+                assert get_object_from_random_node(other_wallet, cid, oid, self.shell, self.cluster)
 
-            with pytest.raises(Exception, match=OBJECT_ACCESS_DENIED):
+            with pytest.raises(Exception, match=OBJECT_NO_ACCESS):
                 assert put_object_to_random_node(other_wallet, file_path, cid, self.shell, self.cluster, attributes=deny_attribute)
 
         with reporter.step("Check others can get and put objects without attributes matching the filter with bearer token"):
@@ -356,3 +377,57 @@ class TestApeFilters(ClusterTestBase):
 
             with expect_not_raises():
                 put_object_to_random_node(other_wallet, file_path, cid, self.shell, self.cluster, bearer, attributes=allow_attribute)
+
+    @allure.title("PUT and GET object using bearer with objectID in filter (obj_size={object_size}, match_type=NOT_EQUAL)")
+    def test_ape_filter_object_id_not_equals(
+        self,
+        frostfs_cli: FrostfsCli,
+        default_wallet: WalletInfo,
+        other_wallet: WalletInfo,
+        private_container: str,
+        temp_directory: str,
+        file_path: TestFile,
+    ):
+        with reporter.step("Put object to container"):
+            oid = put_object_to_random_node(default_wallet, file_path, private_container, self.shell, self.cluster)
+
+        with reporter.step("Create bearer token with objectID filter"):
+            role_condition = ape.Condition.by_role(ape.Role.OTHERS)
+            object_condition = ape.Condition.by_object_id(oid, ape.ConditionType.RESOURCE, ape.MatchType.NOT_EQUAL)
+            rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, [role_condition, object_condition])
+            bearer = create_bearer_token(frostfs_cli, temp_directory, private_container, rule, self.cluster.default_rpc_endpoint)
+
+        with reporter.step("Others should be able to put object using bearer token"):
+            with expect_not_raises():
+                put_object_to_random_node(other_wallet, file_path, private_container, self.shell, self.cluster, bearer)
+
+        with reporter.step("Others should not be able to get object matching the filter"):
+            with pytest.raises(Exception, match=OBJECT_NO_ACCESS):
+                get_object_from_random_node(other_wallet, private_container, oid, self.shell, self.cluster, bearer)
+
+    @allure.title("PUT and GET object using bearer with objectID in filter (obj_size={object_size}, match_type=EQUAL)")
+    def test_ape_filter_object_id_equals(
+        self,
+        frostfs_cli: FrostfsCli,
+        default_wallet: WalletInfo,
+        other_wallet: WalletInfo,
+        private_container: str,
+        temp_directory: str,
+        file_path: TestFile,
+    ):
+        with reporter.step("Put object to container"):
+            oid = put_object_to_random_node(default_wallet, file_path, private_container, self.shell, self.cluster)
+
+        with reporter.step("Create bearer token with objectID filter"):
+            role_condition = ape.Condition.by_role(ape.Role.OTHERS)
+            object_condition = ape.Condition.by_object_id(oid, ape.ConditionType.RESOURCE, ape.MatchType.EQUAL)
+            rule = ape.Rule(ape.Verb.ALLOW, ALL_OBJECT_OPERATIONS, [role_condition, object_condition])
+            bearer = create_bearer_token(frostfs_cli, temp_directory, private_container, rule, self.cluster.default_rpc_endpoint)
+
+        with reporter.step("Others should not be able to put object using bearer token"):
+            with pytest.raises(Exception, match=OBJECT_NO_ACCESS):
+                put_object_to_random_node(other_wallet, file_path, private_container, self.shell, self.cluster, bearer)
+
+        with reporter.step("Others should be able to get object matching the filter"):
+            with expect_not_raises():
+                get_object_from_random_node(other_wallet, private_container, oid, self.shell, self.cluster, bearer)

pytest_tests/testsuites/conftest.py

@@ -13,6 +13,7 @@ from frostfs_testlib.cli import FrostfsCli
 from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
 from frostfs_testlib.healthcheck.interfaces import Healthcheck
 from frostfs_testlib.hosting import Hosting
+from frostfs_testlib.resources import optionals
 from frostfs_testlib.resources.common import ASSETS_DIR, COMPLEX_OBJECT_CHUNKS_COUNT, COMPLEX_OBJECT_TAIL_SIZE, SIMPLE_OBJECT_SIZE
 from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
 from frostfs_testlib.shell import LocalShell, Shell
@@ -27,7 +28,7 @@ from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy
 from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
 from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
 from frostfs_testlib.testing.parallel import parallel
-from frostfs_testlib.testing.test_control import wait_for_success
+from frostfs_testlib.testing.test_control import run_optionally, wait_for_success
 from frostfs_testlib.utils import env_utils, string_utils, version_utils
 from frostfs_testlib.utils.file_utils import TestFile, generate_file
 
@@ -347,6 +348,7 @@ def two_buckets(buckets_pool: list[str], s3_client: S3ClientWrapper) -> list[str
 
 @allure.title("[Autouse/Session] Collect binary versions")
 @pytest.fixture(scope="session", autouse=True)
+@run_optionally(optionals.OPTIONAL_AUTOUSE_FIXTURES_ENABLED)
 def collect_binary_versions(hosting: Hosting, client_shell: Shell, request: pytest.FixtureRequest):
     environment_dir = request.config.getoption("--alluredir")
     if not environment_dir:
@@ -392,6 +394,7 @@ def session_start_time(configure_testlib):
 
 @allure.title("[Autouse/Session] After deploy healthcheck")
 @pytest.fixture(scope="session", autouse=True)
+@run_optionally(optionals.OPTIONAL_AUTOUSE_FIXTURES_ENABLED)
 def after_deploy_healthcheck(cluster: Cluster):
     with reporter.step("Wait for cluster readiness after deploy"):
         parallel(readiness_on_node, cluster.cluster_nodes)

pytest_tests/testsuites/failovers/test_failover_server.py

@@ -123,7 +123,7 @@ class TestFailoverServer(ClusterTestBase):
     @reporter.step("Verify objects")
     def verify_objects(self, nodes: list[StorageNode], storage_objects: list[StorageObjectInfo]) -> None:
         workers_count = os.environ.get("PARALLEL_CUSTOM_LIMIT", 50)
-        with parallel_workers_limit(workers_count):
+        with parallel_workers_limit(int(workers_count)):
             parallel(self._verify_object, storage_objects * len(nodes), node=itertools.cycle(nodes))
 
     @allure.title("Full shutdown node")