From 05924784abec5f1285241e34e803ebc1327ca15f Mon Sep 17 00:00:00 2001 From: Vladimir Avdeev Date: Wed, 7 Dec 2022 15:38:56 +0300 Subject: [PATCH] Remove SIMPLE_OBJ_SIZE and COMPLEX_OBJ_SIZE from env Signed-off-by: Vladimir Avdeev --- pytest_tests/helpers/file_helper.py | 7 +- pytest_tests/testsuites/acl/conftest.py | 4 +- .../acl/storage_group/test_storagegroup.py | 26 +++- pytest_tests/testsuites/conftest.py | 27 ++++ .../failovers/test_failover_network.py | 6 +- .../failovers/test_failover_storage.py | 14 +- .../network/test_node_management.py | 42 ++++-- .../testsuites/object/test_object_api.py | 35 ++--- .../object/test_object_api_bearer.py | 11 +- .../testsuites/object/test_object_lifetime.py | 7 +- .../testsuites/object/test_object_lock.py | 34 +++-- .../services/s3_gate/test_s3_ACL.py | 4 +- .../services/s3_gate/test_s3_bucket.py | 10 +- .../services/s3_gate/test_s3_gate.py | 52 ++++--- .../services/s3_gate/test_s3_locking.py | 22 +-- .../services/s3_gate/test_s3_object.py | 133 ++++++++++-------- .../services/s3_gate/test_s3_policy.py | 20 +-- .../services/s3_gate/test_s3_tagging.py | 4 +- .../services/s3_gate/test_s3_versioning.py | 6 +- .../testsuites/services/test_http_gate.py | 39 ++--- .../test_object_session_token.py | 4 +- .../test_static_object_session_token.py | 48 +++---- .../test_static_session_token_container.py | 3 +- requirements.txt | 1 + .../lib/python_keywords/storage_group.py | 10 +- robot/variables/common.py | 6 +- 26 files changed, 322 insertions(+), 253 deletions(-) diff --git a/pytest_tests/helpers/file_helper.py b/pytest_tests/helpers/file_helper.py index c036b7f4..0a503895 100644 --- a/pytest_tests/helpers/file_helper.py +++ b/pytest_tests/helpers/file_helper.py @@ -5,12 +5,12 @@ import uuid from typing import Any, Optional import allure -from common import ASSETS_DIR, SIMPLE_OBJ_SIZE +from common import ASSETS_DIR logger = logging.getLogger("NeoLogger") -def generate_file(size: int = SIMPLE_OBJ_SIZE) -> str: +def generate_file(size: int) -> str: """Generates a binary file with the specified size in bytes. Args: @@ -28,6 +28,7 @@ def generate_file(size: int = SIMPLE_OBJ_SIZE) -> str: def generate_file_with_content( + size: int, file_path: Optional[str] = None, content: Optional[str] = None, ) -> str: @@ -44,7 +45,7 @@ def generate_file_with_content( """ mode = "w+" if content is None: - content = os.urandom(SIMPLE_OBJ_SIZE) + content = os.urandom(size) mode = "wb" if not file_path: diff --git a/pytest_tests/testsuites/acl/conftest.py b/pytest_tests/testsuites/acl/conftest.py index 1670ccb5..80a9c605 100644 --- a/pytest_tests/testsuites/acl/conftest.py +++ b/pytest_tests/testsuites/acl/conftest.py @@ -68,8 +68,8 @@ def wallets(default_wallet, temp_directory, cluster: Cluster) -> Wallets: @pytest.fixture(scope="module") -def file_path(): - yield generate_file() +def file_path(simple_object_size): + yield generate_file(simple_object_size) @pytest.fixture(scope="function") diff --git a/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py b/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py index 7f2a9a14..497bd812 100644 --- a/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py +++ b/pytest_tests/testsuites/acl/storage_group/test_storagegroup.py @@ -6,7 +6,7 @@ from typing import Optional import allure import pytest from cluster_test_base import ClusterTestBase -from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, FREE_STORAGE, SIMPLE_OBJ_SIZE, WALLET_PASS +from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS from file_helper import generate_file from grpc_responses import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND from neofs_testlib.utils.wallet import init_wallet @@ -37,7 +37,7 @@ deposit = 30 @pytest.mark.parametrize( "object_size", - [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], ids=["simple object", "complex object"], ) @pytest.mark.sanity @@ -68,7 +68,7 @@ class TestStorageGroup(ClusterTestBase): ) @allure.title("Test Storage Group in Private Container") - def test_storagegroup_basic_private_container(self, object_size): + def test_storagegroup_basic_private_container(self, object_size, max_object_size): cid = create_container( self.main_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint ) @@ -88,6 +88,7 @@ class TestStorageGroup(ClusterTestBase): cid=cid, obj_list=objects, object_size=object_size, + max_object_size=max_object_size, ) self.expect_failure_for_storagegroup_operations( wallet=self.other_wallet, @@ -103,7 +104,7 @@ class TestStorageGroup(ClusterTestBase): ) @allure.title("Test Storage Group in Public Container") - def test_storagegroup_basic_public_container(self, object_size): + def test_storagegroup_basic_public_container(self, object_size, max_object_size): cid = create_container( self.main_wallet, basic_acl="public-read-write", @@ -120,12 +121,14 @@ class TestStorageGroup(ClusterTestBase): cid=cid, obj_list=objects, object_size=object_size, + max_object_size=max_object_size, ) self.expect_success_for_storagegroup_operations( wallet=self.other_wallet, cid=cid, obj_list=objects, object_size=object_size, + max_object_size=max_object_size, ) self.storagegroup_operations_by_system_ro_container( wallet=self.main_wallet, @@ -135,7 +138,7 @@ class TestStorageGroup(ClusterTestBase): ) @allure.title("Test Storage Group in Read-Only Container") - def test_storagegroup_basic_ro_container(self, object_size): + def test_storagegroup_basic_ro_container(self, object_size, max_object_size): cid = create_container( self.main_wallet, basic_acl="public-read", @@ -152,6 +155,7 @@ class TestStorageGroup(ClusterTestBase): cid=cid, obj_list=objects, object_size=object_size, + max_object_size=max_object_size, ) self.storagegroup_operations_by_other_ro_container( owner_wallet=self.main_wallet, @@ -168,7 +172,7 @@ class TestStorageGroup(ClusterTestBase): ) @allure.title("Test Storage Group with Bearer Allow") - def test_storagegroup_bearer_allow(self, object_size): + def test_storagegroup_bearer_allow(self, object_size, max_object_size): cid = create_container( self.main_wallet, basic_acl="eacl-public-read-write", @@ -185,6 +189,7 @@ class TestStorageGroup(ClusterTestBase): cid=cid, obj_list=objects, object_size=object_size, + max_object_size=max_object_size, ) storage_group = put_storagegroup( self.shell, self.cluster.default_rpc_endpoint, self.main_wallet, cid, objects @@ -219,6 +224,7 @@ class TestStorageGroup(ClusterTestBase): cid=cid, obj_list=objects, object_size=object_size, + max_object_size=max_object_size, bearer=bearer_file, ) @@ -259,6 +265,7 @@ class TestStorageGroup(ClusterTestBase): cid: str, obj_list: list, object_size: int, + max_object_size: int, bearer: Optional[str] = None, ): """ @@ -285,6 +292,7 @@ class TestStorageGroup(ClusterTestBase): gid=storage_group, obj_list=obj_list, object_size=object_size, + max_object_size=max_object_size, bearer=bearer, ) delete_storagegroup( @@ -381,7 +389,11 @@ class TestStorageGroup(ClusterTestBase): @allure.step("Run Storage Group Operations On Systems's Behalf In RO Container") def storagegroup_operations_by_system_ro_container( - self, wallet: str, cid: str, obj_list: list, object_size: int + self, + wallet: str, + cid: str, + obj_list: list, + object_size: int, ): """ In this func we create a Storage Group on Inner Ring's key behalf diff --git a/pytest_tests/testsuites/conftest.py b/pytest_tests/testsuites/conftest.py index e55538a8..7aec546d 100644 --- a/pytest_tests/testsuites/conftest.py +++ b/pytest_tests/testsuites/conftest.py @@ -16,11 +16,14 @@ from common import ( BACKGROUND_OBJ_SIZE, BACKGROUND_READERS_COUNT, BACKGROUND_WRITERS_COUNT, + COMPLEX_OBJECT_CHUNKS_COUNT, + COMPLEX_OBJECT_TAIL_SIZE, FREE_STORAGE, HOSTING_CONFIG_FILE, LOAD_NODE_SSH_PRIVATE_KEY_PATH, LOAD_NODE_SSH_USER, LOAD_NODES, + SIMPLE_OBJECT_SIZE, STORAGE_NODE_SERVICE_NAME_REGEX, WALLET_PASS, ) @@ -32,6 +35,8 @@ from neofs_testlib.reporter import AllureHandler, get_reporter from neofs_testlib.shell import LocalShell, Shell from neofs_testlib.utils.wallet import init_wallet from payment_neogo import deposit_gas, transfer_gas +from pytest import FixtureRequest +from python_keywords.neofs_verbs import get_netmap_netinfo from python_keywords.node_management import storage_node_healthcheck from helpers.wallet import WalletFactory @@ -81,6 +86,28 @@ def require_multiple_hosts(hosting: Hosting): yield +@pytest.fixture(scope="session") +def max_object_size(cluster: Cluster, client_shell: Shell) -> int: + storage_node = cluster.storage_nodes[0] + net_info = get_netmap_netinfo( + wallet=storage_node.get_wallet_path(), + wallet_config=storage_node.get_wallet_config_path(), + endpoint=storage_node.get_rpc_endpoint(), + shell=client_shell, + ) + yield net_info["maximum_object_size"] + + +@pytest.fixture(scope="session") +def simple_object_size(max_object_size: int) -> int: + yield int(SIMPLE_OBJECT_SIZE) if int(SIMPLE_OBJECT_SIZE) < max_object_size else max_object_size + + +@pytest.fixture(scope="session") +def complex_object_size(max_object_size: int) -> int: + return max_object_size * int(COMPLEX_OBJECT_CHUNKS_COUNT) + int(COMPLEX_OBJECT_TAIL_SIZE) + + @pytest.fixture(scope="session") def wallet_factory(temp_directory: str, client_shell: Shell, cluster: Cluster) -> WalletFactory: return WalletFactory(temp_directory, client_shell, cluster) diff --git a/pytest_tests/testsuites/failovers/test_failover_network.py b/pytest_tests/testsuites/failovers/test_failover_network.py index 1ee52b86..eee214bc 100644 --- a/pytest_tests/testsuites/failovers/test_failover_network.py +++ b/pytest_tests/testsuites/failovers/test_failover_network.py @@ -38,7 +38,9 @@ class TestFailoverNetwork(ClusterTestBase): wait_all_storage_nodes_returned(self.cluster) @allure.title("Block Storage node traffic") - def test_block_storage_node_traffic(self, default_wallet, require_multiple_hosts): + def test_block_storage_node_traffic( + self, default_wallet, require_multiple_hosts, simple_object_size + ): """ Block storage nodes traffic using iptables and wait for replication for objects. """ @@ -47,7 +49,7 @@ class TestFailoverNetwork(ClusterTestBase): wakeup_node_timeout = 10 # timeout to let nodes detect that traffic has blocked nodes_to_block_count = 2 - source_file_path = generate_file() + source_file_path = generate_file(simple_object_size) cid = create_container( wallet, shell=self.shell, diff --git a/pytest_tests/testsuites/failovers/test_failover_storage.py b/pytest_tests/testsuites/failovers/test_failover_storage.py index 0ec153f7..f0dd952e 100644 --- a/pytest_tests/testsuites/failovers/test_failover_storage.py +++ b/pytest_tests/testsuites/failovers/test_failover_storage.py @@ -47,14 +47,11 @@ class TestFailoverStorage(ClusterTestBase): @pytest.mark.parametrize("hard_reboot", [True, False]) @pytest.mark.failover_reboot def test_lose_storage_node_host( - self, - default_wallet, - hard_reboot: bool, - require_multiple_hosts, + self, default_wallet, hard_reboot: bool, require_multiple_hosts, simple_object_size ): wallet = default_wallet placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" - source_file_path = generate_file() + source_file_path = generate_file(simple_object_size) cid = create_container( wallet, shell=self.shell, @@ -106,14 +103,11 @@ class TestFailoverStorage(ClusterTestBase): @pytest.mark.parametrize("sequence", [True, False]) @pytest.mark.failover_panic def test_panic_storage_node_host( - self, - default_wallet, - require_multiple_hosts, - sequence: bool, + self, default_wallet, require_multiple_hosts, sequence: bool, simple_object_size ): wallet = default_wallet placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" - source_file_path = generate_file() + source_file_path = generate_file(simple_object_size) cid = create_container( wallet, shell=self.shell, diff --git a/pytest_tests/testsuites/network/test_node_management.py b/pytest_tests/testsuites/network/test_node_management.py index e2647150..2710d0c5 100644 --- a/pytest_tests/testsuites/network/test_node_management.py +++ b/pytest_tests/testsuites/network/test_node_management.py @@ -7,7 +7,7 @@ import allure import pytest from cluster import StorageNode from cluster_test_base import ClusterTestBase -from common import COMPLEX_OBJ_SIZE, MORPH_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT +from common import MORPH_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT from epoch import tick_epoch from file_helper import generate_file from grpc_responses import OBJECT_NOT_FOUND, error_matches_status @@ -49,9 +49,10 @@ check_nodes: list[StorageNode] = [] class TestNodeManagement(ClusterTestBase): @pytest.fixture @allure.title("Create container and pick the node with data") - def create_container_and_pick_node(self, default_wallet: str) -> Tuple[str, StorageNode]: - default_wallet - file_path = generate_file() + def create_container_and_pick_node( + self, default_wallet: str, simple_object_size + ) -> Tuple[str, StorageNode]: + file_path = generate_file(simple_object_size) placement_rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" endpoint = self.cluster.default_rpc_endpoint @@ -126,6 +127,7 @@ class TestNodeManagement(ClusterTestBase): self, default_wallet, return_nodes_after_test_run, + simple_object_size, ): """ This test remove one node from cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status). @@ -133,7 +135,7 @@ class TestNodeManagement(ClusterTestBase): wallet = default_wallet placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X" placement_rule_4 = "REP 4 IN X CBF 1 SELECT 4 FROM * AS X" - source_file_path = generate_file() + source_file_path = generate_file(simple_object_size) storage_nodes = self.cluster.storage_nodes random_node = random.choice(storage_nodes[1:]) @@ -219,12 +221,14 @@ class TestNodeManagement(ClusterTestBase): ) @pytest.mark.node_mgmt @allure.title("Test object copies based on placement policy") - def test_placement_policy(self, default_wallet, placement_rule, expected_copies): + def test_placement_policy( + self, default_wallet, placement_rule, expected_copies, simple_object_size + ): """ This test checks object's copies based on container's placement policy. """ wallet = default_wallet - file_path = generate_file() + file_path = generate_file(simple_object_size) self.validate_object_copies(wallet, placement_rule, file_path, expected_copies) @pytest.mark.parametrize( @@ -279,14 +283,19 @@ class TestNodeManagement(ClusterTestBase): @pytest.mark.node_mgmt @allure.title("Test object copies and storage nodes based on placement policy") def test_placement_policy_with_nodes( - self, default_wallet, placement_rule, expected_copies, expected_nodes_id: set[int] + self, + default_wallet, + placement_rule, + expected_copies, + expected_nodes_id: set[int], + simple_object_size, ): """ Based on container's placement policy check that storage nodes are piked correctly and object has correct copies amount. """ wallet = default_wallet - file_path = generate_file() + file_path = generate_file(simple_object_size) cid, oid, found_nodes = self.validate_object_copies( wallet, placement_rule, file_path, expected_copies ) @@ -303,24 +312,28 @@ class TestNodeManagement(ClusterTestBase): ) @pytest.mark.node_mgmt @allure.title("Negative cases for placement policy") - def test_placement_policy_negative(self, default_wallet, placement_rule, expected_copies): + def test_placement_policy_negative( + self, default_wallet, placement_rule, expected_copies, simple_object_size + ): """ Negative test for placement policy. """ wallet = default_wallet - file_path = generate_file() + file_path = generate_file(simple_object_size) with pytest.raises(RuntimeError, match=".*not enough nodes to SELECT from.*"): self.validate_object_copies(wallet, placement_rule, file_path, expected_copies) @pytest.mark.node_mgmt @allure.title("NeoFS object could be dropped using control command") - def test_drop_object(self, default_wallet): + def test_drop_object(self, default_wallet, complex_object_size, simple_object_size): """ Test checks object could be dropped using `neofs-cli control drop-objects` command. """ wallet = default_wallet endpoint = self.cluster.default_rpc_endpoint - file_path_simple, file_path_complex = generate_file(), generate_file(COMPLEX_OBJ_SIZE) + file_path_simple, file_path_complex = generate_file(simple_object_size), generate_file( + complex_object_size + ) locode = get_locode_from_random_node(self.cluster) rule = f"REP 1 CBF 1 SELECT 1 FROM * FILTER 'UN-LOCODE' EQ '{locode}' AS LOC" @@ -358,9 +371,10 @@ class TestNodeManagement(ClusterTestBase): self, default_wallet, create_container_and_pick_node, + simple_object_size, ): wallet = default_wallet - file_path = generate_file() + file_path = generate_file(simple_object_size) cid, node = create_container_and_pick_node original_oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster) diff --git a/pytest_tests/testsuites/object/test_object_api.py b/pytest_tests/testsuites/object/test_object_api.py index a22247cd..02f2d14b 100755 --- a/pytest_tests/testsuites/object/test_object_api.py +++ b/pytest_tests/testsuites/object/test_object_api.py @@ -5,14 +5,12 @@ import sys import allure import pytest from cluster import Cluster -from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE -from container import create_container from file_helper import generate_file, get_file_content, get_file_hash from grpc_responses import OUT_OF_RANGE from neofs_testlib.shell import Shell from pytest import FixtureRequest +from python_keywords.container import create_container from python_keywords.neofs_verbs import ( - get_netmap_netinfo, get_object_from_random_node, get_range, get_range_hash, @@ -42,10 +40,7 @@ RANGES_COUNT = 4 # by quarters RANGE_MIN_LEN = 10 RANGE_MAX_LEN = 500 # Used for static ranges found with issues -STATIC_RANGES = { - SIMPLE_OBJ_SIZE: [], - COMPLEX_OBJ_SIZE: [], -} +STATIC_RANGES = {} def generate_ranges(file_size: int, max_object_size: int) -> list[(int, int)]: @@ -58,10 +53,10 @@ def generate_ranges(file_size: int, max_object_size: int) -> list[(int, int)]: file_ranges.append((int(file_range_step * i), int(file_range_step * (i + 1)))) # For simple object we can read all file ranges without too much time for testing - if file_size == SIMPLE_OBJ_SIZE: + if file_size < max_object_size: file_ranges_to_test.extend(file_ranges) # For complex object we need to fetch multiple child objects from different nodes. - if file_size == COMPLEX_OBJ_SIZE: + else: assert ( file_size >= RANGE_MAX_LEN + max_object_size ), f"Complex object size should be at least {max_object_size + RANGE_MAX_LEN}. Current: {file_size}" @@ -83,7 +78,7 @@ def generate_ranges(file_size: int, max_object_size: int) -> list[(int, int)]: @pytest.fixture( - params=[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + params=[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], ids=["simple object", "complex object"], # Scope session to upload/delete each files set only once scope="module", @@ -132,7 +127,7 @@ def storage_objects( class TestObjectApi(ClusterTestBase): @allure.title("Validate object storage policy by native API") def test_object_storage_policies( - self, request: FixtureRequest, storage_objects: list[StorageObjectInfo] + self, request: FixtureRequest, storage_objects: list[StorageObjectInfo], simple_object_size ): """ Validate object storage policy @@ -143,7 +138,7 @@ class TestObjectApi(ClusterTestBase): with allure.step("Validate storage policy for objects"): for storage_object in storage_objects: - if storage_object.size == SIMPLE_OBJ_SIZE: + if storage_object.size == simple_object_size: copies = get_simple_object_copies( storage_object.wallet_file_path, storage_object.cid, @@ -257,7 +252,9 @@ class TestObjectApi(ClusterTestBase): @allure.title("Validate object search with removed items") @pytest.mark.parametrize( - "object_size", [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], ids=["simple object", "complex object"] + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], ) def test_object_search_should_return_tombstone_items( self, default_wallet: str, request: FixtureRequest, object_size: int @@ -330,7 +327,7 @@ class TestObjectApi(ClusterTestBase): @pytest.mark.sanity @pytest.mark.grpc_api def test_object_get_range_hash( - self, request: FixtureRequest, storage_objects: list[StorageObjectInfo] + self, request: FixtureRequest, storage_objects: list[StorageObjectInfo], max_object_size ): """ Validate get_range_hash for object by common gRPC API @@ -343,10 +340,6 @@ class TestObjectApi(ClusterTestBase): cid = storage_objects[0].cid oids = [storage_object.oid for storage_object in storage_objects[:2]] file_path = storage_objects[0].file_path - net_info = get_netmap_netinfo( - wallet, self.shell, endpoint=self.cluster.default_rpc_endpoint - ) - max_object_size = net_info["maximum_object_size"] file_ranges_to_test = generate_ranges(storage_objects[0].size, max_object_size) logging.info(f"Ranges used in test {file_ranges_to_test}") @@ -372,7 +365,7 @@ class TestObjectApi(ClusterTestBase): @pytest.mark.sanity @pytest.mark.grpc_api def test_object_get_range( - self, request: FixtureRequest, storage_objects: list[StorageObjectInfo] + self, request: FixtureRequest, storage_objects: list[StorageObjectInfo], max_object_size ): """ Validate get_range for object by common gRPC API @@ -383,10 +376,6 @@ class TestObjectApi(ClusterTestBase): cid = storage_objects[0].cid oids = [storage_object.oid for storage_object in storage_objects[:2]] file_path = storage_objects[0].file_path - net_info = get_netmap_netinfo( - wallet, self.shell, endpoint=self.cluster.default_rpc_endpoint - ) - max_object_size = net_info["maximum_object_size"] file_ranges_to_test = generate_ranges(storage_objects[0].size, max_object_size) logging.info(f"Ranges used in test {file_ranges_to_test}") diff --git a/pytest_tests/testsuites/object/test_object_api_bearer.py b/pytest_tests/testsuites/object/test_object_api_bearer.py index 63688b47..153fc152 100644 --- a/pytest_tests/testsuites/object/test_object_api_bearer.py +++ b/pytest_tests/testsuites/object/test_object_api_bearer.py @@ -1,7 +1,6 @@ import allure import pytest from cluster import Cluster -from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE from container import REP_2_FOR_3_NODES_PLACEMENT_RULE, SINGLE_PLACEMENT_RULE, create_container from epoch import get_epoch from neofs_testlib.shell import Shell @@ -88,7 +87,7 @@ class TestObjectApiWithBearerToken(ClusterTestBase): ) @pytest.mark.parametrize( "storage_objects", - [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], ids=["simple object", "complex object"], indirect=True, ) @@ -99,7 +98,8 @@ class TestObjectApiWithBearerToken(ClusterTestBase): request: FixtureRequest, ): allure.dynamic.title( - f"Check that objects can be deleted from any node using s3gate wallet with bearer token for {request.node.callspec.id}" + f"Check that objects can be deleted from any node using s3gate wallet with bearer " + f"token for {request.node.callspec.id}" ) s3_gate_wallet = self.cluster.s3gates[0] @@ -124,7 +124,7 @@ class TestObjectApiWithBearerToken(ClusterTestBase): ) @pytest.mark.parametrize( "file_size", - [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], ids=["simple object", "complex object"], ) def test_get_object_with_s3_wallet_bearer_from_all_nodes( @@ -135,7 +135,8 @@ class TestObjectApiWithBearerToken(ClusterTestBase): request: FixtureRequest, ): allure.dynamic.title( - f"Check that objects can be deleted from any node using s3gate wallet with bearer token for {request.node.callspec.id}" + "Check that objects can be deleted from any node using s3gate wallet with bearer " + f"token for {request.node.callspec.id}" ) s3_gate_wallet = self.cluster.s3gates[0] diff --git a/pytest_tests/testsuites/object/test_object_lifetime.py b/pytest_tests/testsuites/object/test_object_lifetime.py index 81475c61..65fab1ce 100644 --- a/pytest_tests/testsuites/object/test_object_lifetime.py +++ b/pytest_tests/testsuites/object/test_object_lifetime.py @@ -2,12 +2,11 @@ import logging import allure import pytest -from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE -from container import create_container from epoch import get_epoch, tick_epoch from file_helper import generate_file, get_file_hash from grpc_responses import OBJECT_NOT_FOUND from pytest import FixtureRequest +from python_keywords.container import create_container from python_keywords.neofs_verbs import get_object_from_random_node, put_object_to_random_node from utility import wait_for_gc_pass_on_storage_nodes @@ -21,7 +20,9 @@ logger = logging.getLogger("NeoLogger") class ObjectApiLifetimeTest(ClusterTestBase): @allure.title("Test object life time") @pytest.mark.parametrize( - "object_size", [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], ids=["simple object", "complex object"] + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], ) def test_object_api_lifetime( self, default_wallet: str, request: FixtureRequest, object_size: int diff --git a/pytest_tests/testsuites/object/test_object_lock.py b/pytest_tests/testsuites/object/test_object_lock.py index b4379be1..c6b689b1 100755 --- a/pytest_tests/testsuites/object/test_object_lock.py +++ b/pytest_tests/testsuites/object/test_object_lock.py @@ -5,9 +5,8 @@ import allure import pytest from cluster import Cluster from cluster_test_base import ClusterTestBase -from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE, STORAGE_GC_TIME +from common import STORAGE_GC_TIME from complex_object_actions import get_link_object -from container import create_container from epoch import ensure_fresh_epoch, get_epoch, tick_epoch from grpc_responses import ( LIFETIME_REQUIRED, @@ -20,6 +19,7 @@ from grpc_responses import ( ) from neofs_testlib.shell import Shell from pytest import FixtureRequest +from python_keywords.container import create_container from python_keywords.neofs_verbs import delete_object, head_object, lock_object from test_control import expect_not_raises, wait_for_success from utility import parse_time, wait_for_gc_pass_on_storage_nodes @@ -143,7 +143,7 @@ class TestObjectLockWithGrpc(ClusterTestBase): @allure.title("Locked object should be protected from deletion") @pytest.mark.parametrize( "locked_storage_object", - [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], ids=["simple object", "complex object"], indirect=True, ) @@ -170,7 +170,9 @@ class TestObjectLockWithGrpc(ClusterTestBase): @allure.title("Lock object itself should be protected from deletion") # We operate with only lock object here so no complex object needed in this test - @pytest.mark.parametrize("locked_storage_object", [SIMPLE_OBJ_SIZE], indirect=True) + @pytest.mark.parametrize( + "locked_storage_object", [pytest.lazy_fixture("simple_object_size")], indirect=True + ) def test_lock_object_itself_cannot_be_deleted( self, locked_storage_object: StorageObjectInfo, @@ -193,7 +195,9 @@ class TestObjectLockWithGrpc(ClusterTestBase): @allure.title("Lock object itself cannot be locked") # We operate with only lock object here so no complex object needed in this test - @pytest.mark.parametrize("locked_storage_object", [SIMPLE_OBJ_SIZE], indirect=True) + @pytest.mark.parametrize( + "locked_storage_object", [pytest.lazy_fixture("simple_object_size")], indirect=True + ) def test_lock_object_cannot_be_locked( self, locked_storage_object: StorageObjectInfo, @@ -217,7 +221,9 @@ class TestObjectLockWithGrpc(ClusterTestBase): @allure.title("Cannot lock object without lifetime and expire_at fields") # We operate with only lock object here so no complex object needed in this test - @pytest.mark.parametrize("locked_storage_object", [SIMPLE_OBJ_SIZE], indirect=True) + @pytest.mark.parametrize( + "locked_storage_object", [pytest.lazy_fixture("simple_object_size")], indirect=True + ) @pytest.mark.parametrize( "wrong_lifetime,wrong_expire_at,expected_error", [ @@ -259,7 +265,9 @@ class TestObjectLockWithGrpc(ClusterTestBase): @allure.title("Expired object should be deleted after locks are expired") @pytest.mark.parametrize( - "object_size", [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], ids=["simple object", "complex object"] + "object_size", + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], + ids=["simple object", "complex object"], ) def test_expired_object_should_be_deleted_after_locks_are_expired( self, @@ -327,7 +335,7 @@ class TestObjectLockWithGrpc(ClusterTestBase): @allure.title("Should be possible to lock multiple objects at once") @pytest.mark.parametrize( "object_size", - [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], ids=["simple object", "complex object"], ) def test_should_be_possible_to_lock_multiple_objects_at_once( @@ -382,7 +390,7 @@ class TestObjectLockWithGrpc(ClusterTestBase): @allure.title("Already outdated lock should not be applied") @pytest.mark.parametrize( "object_size", - [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], ids=["simple object", "complex object"], ) def test_already_outdated_lock_should_not_be_applied( @@ -421,7 +429,7 @@ class TestObjectLockWithGrpc(ClusterTestBase): @allure.title("After lock expiration with lifetime user should be able to delete object") @pytest.mark.parametrize( "object_size", - [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], ids=["simple object", "complex object"], ) @expect_not_raises() @@ -463,7 +471,7 @@ class TestObjectLockWithGrpc(ClusterTestBase): @allure.title("After lock expiration with expire_at user should be able to delete object") @pytest.mark.parametrize( "object_size", - [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], ids=["simple object", "complex object"], ) @expect_not_raises() @@ -508,7 +516,7 @@ class TestObjectLockWithGrpc(ClusterTestBase): @pytest.mark.parametrize( # Only complex objects are required for this test "locked_storage_object", - [COMPLEX_OBJ_SIZE], + [pytest.lazy_fixture("complex_object_size")], indirect=True, ) def test_complex_object_chunks_should_also_be_protected_from_deletion( @@ -535,7 +543,7 @@ class TestObjectLockWithGrpc(ClusterTestBase): @pytest.mark.parametrize( # Only complex objects are required for this test "locked_storage_object", - [COMPLEX_OBJ_SIZE], + [pytest.lazy_fixture("complex_object_size")], indirect=True, ) def test_link_object_of_complex_object_should_also_be_protected_from_deletion( diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py b/pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py index f3887779..188f7b4b 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py @@ -17,8 +17,8 @@ def pytest_generate_tests(metafunc): @pytest.mark.s3_gate class TestS3GateACL(TestS3GateBase): @allure.title("Test S3: Object ACL") - def test_s3_object_ACL(self, bucket): - file_path = generate_file() + def test_s3_object_ACL(self, bucket, simple_object_size): + file_path = generate_file(simple_object_size) file_name = object_key_from_file_path(file_path) with allure.step("Put object into bucket, Check ACL is empty"): diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py b/pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py index e4836213..2cca5214 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py @@ -103,8 +103,8 @@ class TestS3GateBucket(TestS3GateBase): ], "Permission for CanonicalUser is FULL_CONTROL" @allure.title("Test S3: create bucket with object lock") - def test_s3_bucket_object_lock(self): - file_path = generate_file() + def test_s3_bucket_object_lock(self, simple_object_size): + file_path = generate_file(simple_object_size) file_name = object_key_from_file_path(file_path) with allure.step("Create bucket with --no-object-lock-enabled-for-bucket"): @@ -138,10 +138,10 @@ class TestS3GateBucket(TestS3GateBase): ) @allure.title("Test S3: delete bucket") - def test_s3_delete_bucket(self): - file_path_1 = generate_file() + def test_s3_delete_bucket(self, simple_object_size): + file_path_1 = generate_file(simple_object_size) file_name_1 = object_key_from_file_path(file_path_1) - file_path_2 = generate_file() + file_path_2 = generate_file(simple_object_size) file_name_2 = object_key_from_file_path(file_path_2) bucket = s3_gate_bucket.create_bucket_s3(self.s3_client) diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_gate.py b/pytest_tests/testsuites/services/s3_gate/test_s3_gate.py index 61dd2640..fe7960fe 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_gate.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_gate.py @@ -5,7 +5,7 @@ from random import choice, choices import allure import pytest from aws_cli_client import AwsCliClient -from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE +from common import ASSETS_DIR from epoch import tick_epoch from file_helper import ( generate_file, @@ -39,12 +39,12 @@ def pytest_generate_tests(metafunc): @pytest.mark.s3_gate_base class TestS3Gate(TestS3GateBase): @allure.title("Test S3 Bucket API") - def test_s3_buckets(self): + def test_s3_buckets(self, simple_object_size): """ Test base S3 Bucket API (Create/List/Head/Delete). """ - file_path = generate_file() + file_path = generate_file(simple_object_size) file_name = self.object_key_from_file_path(file_path) with allure.step("Create buckets"): @@ -109,11 +109,13 @@ class TestS3Gate(TestS3GateBase): @pytest.mark.parametrize( "file_type", ["simple", "large"], ids=["Simple object", "Large object"] ) - def test_s3_api_object(self, file_type, two_buckets): + def test_s3_api_object(self, file_type, two_buckets, simple_object_size, complex_object_size): """ Test base S3 Object API (Put/Head/List) for simple and large objects. """ - file_path = generate_file(SIMPLE_OBJ_SIZE if file_type == "simple" else COMPLEX_OBJ_SIZE) + file_path = generate_file( + simple_object_size if file_type == "simple" else complex_object_size + ) file_name = self.object_key_from_file_path(file_path) bucket_1, bucket_2 = two_buckets @@ -136,7 +138,7 @@ class TestS3Gate(TestS3GateBase): s3_gate_object.get_object_attributes(self.s3_client, bucket, file_name, *attrs) @allure.title("Test S3 Sync directory") - def test_s3_sync_dir(self, bucket): + def test_s3_sync_dir(self, bucket, simple_object_size): """ Test checks sync directory with AWS CLI utility. """ @@ -147,8 +149,8 @@ class TestS3Gate(TestS3GateBase): if not isinstance(self.s3_client, AwsCliClient): pytest.skip("This test is not supported with boto3 client") - generate_file_with_content(file_path=file_path_1) - generate_file_with_content(file_path=file_path_2) + generate_file_with_content(simple_object_size, file_path=file_path_1) + generate_file_with_content(simple_object_size, file_path=file_path_2) self.s3_client.sync(bucket_name=bucket, dir_path=os.path.dirname(file_path_1)) @@ -166,19 +168,21 @@ class TestS3Gate(TestS3GateBase): ), "Expected hashes are the same" @allure.title("Test S3 Object versioning") - def test_s3_api_versioning(self, bucket): + def test_s3_api_versioning(self, bucket, simple_object_size): """ Test checks basic versioning functionality for S3 bucket. """ version_1_content = "Version 1" version_2_content = "Version 2" - file_name_simple = generate_file_with_content(content=version_1_content) + file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content) obj_key = os.path.basename(file_name_simple) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) with allure.step("Put several versions of object into bucket"): version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) - generate_file_with_content(file_path=file_name_simple, content=version_2_content) + generate_file_with_content( + simple_object_size, file_path=file_name_simple, content=version_2_content + ) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) with allure.step("Check bucket shows all versions"): @@ -246,13 +250,15 @@ class TestS3Gate(TestS3GateBase): @pytest.mark.s3_gate_multipart @allure.title("Test S3 Object Multipart API") - def test_s3_api_multipart(self, bucket): + def test_s3_api_multipart(self, bucket, simple_object_size): """ Test checks S3 Multipart API (Create multipart upload/Abort multipart upload/List multipart upload/ Upload part/List parts/Complete multipart upload). """ parts_count = 3 - file_name_large = generate_file(SIMPLE_OBJ_SIZE * 1024 * 6 * parts_count) # 5Mb - min part + file_name_large = generate_file( + simple_object_size * 1024 * 6 * parts_count + ) # 5Mb - min part object_key = self.object_key_from_file_path(file_name_large) part_files = split_file(file_name_large, parts_count) parts = [] @@ -320,7 +326,7 @@ class TestS3Gate(TestS3GateBase): check_tags_by_bucket(self.s3_client, bucket, []) @allure.title("Test S3 Object tagging API") - def test_s3_api_object_tagging(self, bucket): + def test_s3_api_object_tagging(self, bucket, simple_object_size): """ Test checks S3 Object tagging API (Put tag/Get tag/Update tag). """ @@ -330,7 +336,7 @@ class TestS3Gate(TestS3GateBase): ("some-key--obj2", "some-value--obj2"), ] key_value_pair_obj_new = [("some-key-obj-new", "some-value-obj-new")] - file_name_simple = generate_file(SIMPLE_OBJ_SIZE) + file_name_simple = generate_file(simple_object_size) obj_key = self.object_key_from_file_path(file_name_simple) s3_gate_bucket.put_bucket_tagging(self.s3_client, bucket, key_value_pair_bucket) @@ -350,7 +356,7 @@ class TestS3Gate(TestS3GateBase): check_tags_by_object(self.s3_client, bucket, obj_key, []) @allure.title("Test S3: Delete object & delete objects S3 API") - def test_s3_api_delete(self, two_buckets): + def test_s3_api_delete(self, two_buckets, simple_object_size, complex_object_size): """ Check delete_object and delete_objects S3 API operation. From first bucket some objects deleted one by one. From second bucket some objects deleted all at once. @@ -359,7 +365,7 @@ class TestS3Gate(TestS3GateBase): max_delete_objects = 17 put_objects = [] file_paths = [] - obj_sizes = [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE] + obj_sizes = [simple_object_size, complex_object_size] bucket_1, bucket_2 = two_buckets @@ -406,12 +412,14 @@ class TestS3Gate(TestS3GateBase): try_to_get_objects_and_expect_error(self.s3_client, bucket_2, objects_to_delete_b2) @allure.title("Test S3: Copy object to the same bucket") - def test_s3_copy_same_bucket(self, bucket): + def test_s3_copy_same_bucket(self, bucket, complex_object_size, simple_object_size): """ Test object can be copied to the same bucket. #TODO: delete after test_s3_copy_object will be merge """ - file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) file_name_simple = self.object_key_from_file_path(file_path_simple) file_name_large = self.object_key_from_file_path(file_path_large) bucket_objects = [file_name_simple, file_name_large] @@ -448,12 +456,14 @@ class TestS3Gate(TestS3GateBase): ) @allure.title("Test S3: Copy object to another bucket") - def test_s3_copy_to_another_bucket(self, two_buckets): + def test_s3_copy_to_another_bucket(self, two_buckets, complex_object_size, simple_object_size): """ Test object can be copied to another bucket. #TODO: delete after test_s3_copy_object will be merge """ - file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) file_name_simple = self.object_key_from_file_path(file_path_simple) file_name_large = self.object_key_from_file_path(file_path_large) bucket_1_objects = [file_name_simple, file_name_large] diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_locking.py b/pytest_tests/testsuites/services/s3_gate/test_s3_locking.py index bafb2ac4..758e77c7 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_locking.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_locking.py @@ -21,8 +21,8 @@ def pytest_generate_tests(metafunc): @pytest.mark.parametrize("version_id", [None, "second"]) class TestS3GateLocking(TestS3GateBase): @allure.title("Test S3: Checking the operation of retention period & legal lock on the object") - def test_s3_object_locking(self, version_id): - file_path = generate_file() + def test_s3_object_locking(self, version_id, simple_object_size): + file_path = generate_file(simple_object_size) file_name = object_key_from_file_path(file_path) retention_period = 2 @@ -30,7 +30,7 @@ class TestS3GateLocking(TestS3GateBase): with allure.step("Put several versions of object into bucket"): s3_gate_object.put_object_s3(self.s3_client, bucket, file_path) - file_name_1 = generate_file_with_content(file_path=file_path) + file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) check_objects_in_bucket(self.s3_client, bucket, [file_name]) if version_id: @@ -74,8 +74,8 @@ class TestS3GateLocking(TestS3GateBase): s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name, version_id) @allure.title("Test S3: Checking the impossibility to change the retention mode COMPLIANCE") - def test_s3_mode_compliance(self, version_id): - file_path = generate_file() + def test_s3_mode_compliance(self, version_id, simple_object_size): + file_path = generate_file(simple_object_size) file_name = object_key_from_file_path(file_path) retention_period = 2 retention_period_1 = 1 @@ -115,8 +115,8 @@ class TestS3GateLocking(TestS3GateBase): ) @allure.title("Test S3: Checking the ability to change retention mode GOVERNANCE") - def test_s3_mode_governance(self, version_id): - file_path = generate_file() + def test_s3_mode_governance(self, version_id, simple_object_size): + file_path = generate_file(simple_object_size) file_name = object_key_from_file_path(file_path) retention_period = 3 retention_period_1 = 2 @@ -183,8 +183,8 @@ class TestS3GateLocking(TestS3GateBase): ) @allure.title("Test S3: Checking if an Object Cannot Be Locked") - def test_s3_legal_hold(self, version_id): - file_path = generate_file() + def test_s3_legal_hold(self, version_id, simple_object_size): + file_path = generate_file(simple_object_size) file_name = object_key_from_file_path(file_path) bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, False) @@ -205,8 +205,8 @@ class TestS3GateLocking(TestS3GateBase): @pytest.mark.s3_gate class TestS3GateLockingBucket(TestS3GateBase): @allure.title("Test S3: Bucket Lock") - def test_s3_bucket_lock(self): - file_path = generate_file() + def test_s3_bucket_lock(self, simple_object_size): + file_path = generate_file(simple_object_size) file_name = object_key_from_file_path(file_path) configuration = {"Rule": {"DefaultRetention": {"Mode": "COMPLIANCE", "Days": 1}}} diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_object.py b/pytest_tests/testsuites/services/s3_gate/test_s3_object.py index 5d265d6b..ca19ae73 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_object.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_object.py @@ -7,7 +7,7 @@ from random import choices, sample import allure import pytest from aws_cli_client import AwsCliClient -from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, FREE_STORAGE, SIMPLE_OBJ_SIZE, WALLET_PASS +from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS from data_formatters import get_wallet_public_key from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash from neofs_testlib.utils.wallet import init_wallet @@ -32,8 +32,8 @@ class TestS3GateObject(TestS3GateBase): return os.path.basename(full_path) @allure.title("Test S3: Copy object") - def test_s3_copy_object(self, two_buckets): - file_path = generate_file() + def test_s3_copy_object(self, two_buckets, simple_object_size): + file_path = generate_file(simple_object_size) file_name = self.object_key_from_file_path(file_path) bucket_1_objects = [file_name] @@ -81,7 +81,7 @@ class TestS3GateObject(TestS3GateBase): @allure.title("Test S3: Copy version of object") def test_s3_copy_version_object(self, two_buckets): version_1_content = "Version 1" - file_name_simple = generate_file_with_content(content=version_1_content) + file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content) obj_key = os.path.basename(file_name_simple) bucket_1, bucket_2 = two_buckets @@ -115,9 +115,9 @@ class TestS3GateObject(TestS3GateBase): s3_gate_object.copy_object_s3(self.s3_client, bucket_1, obj_key) @allure.title("Test S3: Checking copy with acl") - def test_s3_copy_acl(self, bucket): + def test_s3_copy_acl(self, bucket, simple_object_size): version_1_content = "Version 1" - file_name_simple = generate_file_with_content(content=version_1_content) + file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content) obj_key = os.path.basename(file_name_simple) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) @@ -137,9 +137,9 @@ class TestS3GateObject(TestS3GateBase): ), "Permission for all groups is FULL_CONTROL" @allure.title("Test S3: Copy object with metadata") - def test_s3_copy_metadate(self, bucket): + def test_s3_copy_metadate(self, bucket, simple_object_size): object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} - file_path = generate_file() + file_path = generate_file(simple_object_size) file_name = self.object_key_from_file_path(file_path) bucket_1_objects = [file_name] @@ -187,9 +187,9 @@ class TestS3GateObject(TestS3GateBase): ), f"Metadata must be {object_metadata_1}" @allure.title("Test S3: Copy object with tagging") - def test_s3_copy_tagging(self, bucket): + def test_s3_copy_tagging(self, bucket, simple_object_size): object_tagging = [(f"{uuid.uuid4()}", f"{uuid.uuid4()}")] - file_path = generate_file() + file_path = generate_file(simple_object_size) file_name_simple = self.object_key_from_file_path(file_path) bucket_1_objects = [file_name_simple] @@ -239,10 +239,10 @@ class TestS3GateObject(TestS3GateBase): assert tag in got_tags, f"Expected tag {tag} in {got_tags}" @allure.title("Test S3: Delete version of object") - def test_s3_delete_versioning(self, bucket): + def test_s3_delete_versioning(self, bucket, complex_object_size, simple_object_size): version_1_content = "Version 1" version_2_content = "Version 2" - file_name_simple = generate_file_with_content(content=version_1_content) + file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content) obj_key = os.path.basename(file_name_simple) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) @@ -250,7 +250,7 @@ class TestS3GateObject(TestS3GateBase): with allure.step("Put several versions of object into bucket"): version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) file_name_1 = generate_file_with_content( - file_path=file_name_simple, content=version_2_content + simple_object_size, file_path=file_name_simple, content=version_2_content ) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) @@ -287,7 +287,7 @@ class TestS3GateObject(TestS3GateBase): assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found" with allure.step("Put new object into bucket"): - file_name_simple = generate_file(COMPLEX_OBJ_SIZE) + file_name_simple = generate_file(complex_object_size) obj_key = os.path.basename(file_name_simple) version_id = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) @@ -298,12 +298,12 @@ class TestS3GateObject(TestS3GateBase): assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker" @allure.title("Test S3: bulk delete version of object") - def test_s3_bulk_delete_versioning(self, bucket): + def test_s3_bulk_delete_versioning(self, bucket, simple_object_size): version_1_content = "Version 1" version_2_content = "Version 2" version_3_content = "Version 3" version_4_content = "Version 4" - file_name_1 = generate_file_with_content(content=version_1_content) + file_name_1 = generate_file_with_content(simple_object_size, content=version_1_content) obj_key = os.path.basename(file_name_1) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) @@ -311,15 +311,15 @@ class TestS3GateObject(TestS3GateBase): with allure.step("Put several versions of object into bucket"): version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) file_name_2 = generate_file_with_content( - file_path=file_name_1, content=version_2_content + simple_object_size, file_path=file_name_1, content=version_2_content ) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_2) file_name_3 = generate_file_with_content( - file_path=file_name_1, content=version_3_content + simple_object_size, file_path=file_name_1, content=version_3_content ) version_id_3 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_3) file_name_4 = generate_file_with_content( - file_path=file_name_1, content=version_4_content + simple_object_size, file_path=file_name_1, content=version_4_content ) version_id_4 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_4) version_ids = {version_id_1, version_id_2, version_id_3, version_id_4} @@ -352,14 +352,14 @@ class TestS3GateObject(TestS3GateBase): def test_s3_get_versioning(self, bucket): version_1_content = "Version 1" version_2_content = "Version 2" - file_name_simple = generate_file_with_content(content=version_1_content) + file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content) obj_key = os.path.basename(file_name_simple) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) with allure.step("Put several versions of object into bucket"): version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) file_name_1 = generate_file_with_content( - file_path=file_name_simple, content=version_2_content + simple_object_size, file_path=file_name_simple, content=version_2_content ) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) @@ -388,14 +388,14 @@ class TestS3GateObject(TestS3GateBase): ), f"Get object with version {version_id_2}" @allure.title("Test S3: Get range") - def test_s3_get_range(self, bucket): - file_path = generate_file(COMPLEX_OBJ_SIZE) + def test_s3_get_range(self, bucket, complex_object_size: int, simple_object_size: int): + file_path = generate_file(complex_object_size) file_name = self.object_key_from_file_path(file_path) file_hash = get_file_hash(file_path) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) with allure.step("Put several versions of object into bucket"): version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path) - file_name_1 = generate_file_with_content(file_path=file_path) + file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) with allure.step("Get first version of object"): @@ -404,42 +404,46 @@ class TestS3GateObject(TestS3GateBase): bucket, file_name, version_id_1, - range=[0, int(COMPLEX_OBJ_SIZE / 3)], + range=[0, int(complex_object_size / 3)], ) object_1_part_2 = s3_gate_object.get_object_s3( self.s3_client, bucket, file_name, version_id_1, - range=[int(COMPLEX_OBJ_SIZE / 3) + 1, 2 * int(COMPLEX_OBJ_SIZE / 3)], + range=[int(complex_object_size / 3) + 1, 2 * int(complex_object_size / 3)], ) object_1_part_3 = s3_gate_object.get_object_s3( self.s3_client, bucket, file_name, version_id_1, - range=[2 * int(COMPLEX_OBJ_SIZE / 3) + 1, COMPLEX_OBJ_SIZE], + range=[2 * int(complex_object_size / 3) + 1, complex_object_size], ) con_file = concat_files([object_1_part_1, object_1_part_2, object_1_part_3]) assert get_file_hash(con_file) == file_hash, "Hashes must be the same" with allure.step("Get second version of object"): object_2_part_1 = s3_gate_object.get_object_s3( - self.s3_client, bucket, file_name, version_id_2, range=[0, int(SIMPLE_OBJ_SIZE / 3)] + self.s3_client, + bucket, + file_name, + version_id_2, + range=[0, int(complex_object_size / 3)], ) object_2_part_2 = s3_gate_object.get_object_s3( self.s3_client, bucket, file_name, version_id_2, - range=[int(SIMPLE_OBJ_SIZE / 3) + 1, 2 * int(SIMPLE_OBJ_SIZE / 3)], + range=[int(simple_object_size / 3) + 1, 2 * int(simple_object_size / 3)], ) object_2_part_3 = s3_gate_object.get_object_s3( self.s3_client, bucket, file_name, version_id_2, - range=[2 * int(SIMPLE_OBJ_SIZE / 3) + 1, COMPLEX_OBJ_SIZE], + range=[2 * int(simple_object_size / 3) + 1, complex_object_size], ) con_file_1 = concat_files([object_2_part_1, object_2_part_2, object_2_part_3]) assert get_file_hash(con_file_1) == get_file_hash( @@ -448,28 +452,28 @@ class TestS3GateObject(TestS3GateBase): with allure.step("Get object"): object_3_part_1 = s3_gate_object.get_object_s3( - self.s3_client, bucket, file_name, range=[0, int(SIMPLE_OBJ_SIZE / 3)] + self.s3_client, bucket, file_name, range=[0, int(simple_object_size / 3)] ) object_3_part_2 = s3_gate_object.get_object_s3( self.s3_client, bucket, file_name, - range=[int(SIMPLE_OBJ_SIZE / 3) + 1, 2 * int(SIMPLE_OBJ_SIZE / 3)], + range=[int(simple_object_size / 3) + 1, 2 * int(simple_object_size / 3)], ) object_3_part_3 = s3_gate_object.get_object_s3( self.s3_client, bucket, file_name, - range=[2 * int(SIMPLE_OBJ_SIZE / 3) + 1, COMPLEX_OBJ_SIZE], + range=[2 * int(simple_object_size / 3) + 1, complex_object_size], ) con_file = concat_files([object_3_part_1, object_3_part_2, object_3_part_3]) assert get_file_hash(con_file) == get_file_hash(file_name_1), "Hashes must be the same" @allure.title("Test S3: Copy object with metadata") @pytest.mark.smoke - def test_s3_head_object(self, bucket): + def test_s3_head_object(self, bucket, complex_object_size, simple_object_size): object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} - file_path = generate_file(COMPLEX_OBJ_SIZE) + file_path = generate_file(complex_object_size) file_name = self.object_key_from_file_path(file_path) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) @@ -477,7 +481,7 @@ class TestS3GateObject(TestS3GateBase): version_id_1 = s3_gate_object.put_object_s3( self.s3_client, bucket, file_path, Metadata=object_metadata ) - file_name_1 = generate_file_with_content(file_path=file_path) + file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) with allure.step("Get head of first version of object"): @@ -506,10 +510,10 @@ class TestS3GateObject(TestS3GateBase): @allure.title("Test S3: list of object with versions") @pytest.mark.parametrize("list_type", ["v1", "v2"]) - def test_s3_list_object(self, list_type: str, bucket): - file_path_1 = generate_file(COMPLEX_OBJ_SIZE) + def test_s3_list_object(self, list_type: str, bucket, complex_object_size): + file_path_1 = generate_file(complex_object_size) file_name = self.object_key_from_file_path(file_path_1) - file_path_2 = generate_file(COMPLEX_OBJ_SIZE) + file_path_2 = generate_file(complex_object_size) file_name_2 = self.object_key_from_file_path(file_path_2) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) @@ -543,8 +547,8 @@ class TestS3GateObject(TestS3GateBase): assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker" @allure.title("Test S3: put object") - def test_s3_put_object(self, bucket): - file_path_1 = generate_file(COMPLEX_OBJ_SIZE) + def test_s3_put_object(self, bucket, complex_object_size, simple_object_size): + file_path_1 = generate_file(complex_object_size) file_name = self.object_key_from_file_path(file_path_1) object_1_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} tag_key_1 = "tag1" @@ -569,7 +573,7 @@ class TestS3GateObject(TestS3GateBase): ], "Tags must be the same" with allure.step("Rewrite file into bucket"): - file_path_2 = generate_file_with_content(file_path=file_path_1) + file_path_2 = generate_file_with_content(simple_object_size, file_path=file_path_1) s3_gate_object.put_object_s3( self.s3_client, bucket, file_path_2, Metadata=object_2_metadata, Tagging=tag_2 ) @@ -583,7 +587,7 @@ class TestS3GateObject(TestS3GateBase): set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) - file_path_3 = generate_file(COMPLEX_OBJ_SIZE) + file_path_3 = generate_file(complex_object_size) file_hash = get_file_hash(file_path_3) file_name_3 = self.object_key_from_file_path(file_path_3) object_3_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} @@ -604,7 +608,7 @@ class TestS3GateObject(TestS3GateBase): ], "Tags must be the same" with allure.step("Put new version of file into bucket"): - file_path_4 = generate_file_with_content(file_path=file_path_3) + file_path_4 = generate_file_with_content(simple_object_size, file_path=file_path_3) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_4) versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket) obj_versions = { @@ -680,8 +684,15 @@ class TestS3GateObject(TestS3GateBase): @allure.title("Test S3: put object with ACL") @pytest.mark.parametrize("bucket_versioning", ["ENABLED", "SUSPENDED"]) - def test_s3_put_object_acl(self, prepare_two_wallets, bucket_versioning, bucket): - file_path_1 = generate_file(COMPLEX_OBJ_SIZE) + def test_s3_put_object_acl( + self, + prepare_two_wallets, + bucket_versioning, + bucket, + complex_object_size, + simple_object_size, + ): + file_path_1 = generate_file(complex_object_size) file_name = self.object_key_from_file_path(file_path_1) if bucket_versioning == "ENABLED": status = s3_gate_bucket.VersioningStatus.ENABLED @@ -698,7 +709,7 @@ class TestS3GateObject(TestS3GateBase): assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same" with allure.step("Put object with acl public-read"): - file_path_2 = generate_file_with_content(file_path=file_path_1) + file_path_2 = generate_file_with_content(simple_object_size, file_path=file_path_1) s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_2, ACL="public-read") obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) obj_permission = [permission.get("Permission") for permission in obj_acl] @@ -710,7 +721,7 @@ class TestS3GateObject(TestS3GateBase): assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same" with allure.step("Put object with acl public-read-write"): - file_path_3 = generate_file_with_content(file_path=file_path_1) + file_path_3 = generate_file_with_content(simple_object_size, file_path=file_path_1) s3_gate_object.put_object_s3( self.s3_client, bucket, file_path_3, ACL="public-read-write" ) @@ -724,7 +735,7 @@ class TestS3GateObject(TestS3GateBase): assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same" with allure.step("Put object with acl authenticated-read"): - file_path_4 = generate_file_with_content(file_path=file_path_1) + file_path_4 = generate_file_with_content(simple_object_size, file_path=file_path_1) s3_gate_object.put_object_s3( self.s3_client, bucket, file_path_4, ACL="authenticated-read" ) @@ -737,11 +748,11 @@ class TestS3GateObject(TestS3GateBase): object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name) assert get_file_hash(file_path_4) == get_file_hash(object_4), "Hashes must be the same" - file_path_5 = generate_file(COMPLEX_OBJ_SIZE) + file_path_5 = generate_file(complex_object_size) file_name_5 = self.object_key_from_file_path(file_path_5) with allure.step("Put object with --grant-full-control id=mycanonicaluserid"): - file_path_6 = generate_file_with_content(file_path=file_path_5) + file_path_6 = generate_file_with_content(simple_object_size, file_path=file_path_5) s3_gate_object.put_object_s3( self.s3_client, bucket, @@ -760,7 +771,7 @@ class TestS3GateObject(TestS3GateBase): with allure.step( "Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers" ): - file_path_7 = generate_file_with_content(file_path=file_path_5) + file_path_7 = generate_file_with_content(simple_object_size, file_path=file_path_5) s3_gate_object.put_object_s3( self.s3_client, bucket, @@ -777,9 +788,9 @@ class TestS3GateObject(TestS3GateBase): assert get_file_hash(file_path_7) == get_file_hash(object_7), "Hashes must be the same" @allure.title("Test S3: put object with lock-mode") - def test_s3_put_object_lock_mode(self, bucket): + def test_s3_put_object_lock_mode(self, bucket, complex_object_size, simple_object_size): - file_path_1 = generate_file(COMPLEX_OBJ_SIZE) + file_path_1 = generate_file(complex_object_size) file_name = self.object_key_from_file_path(file_path_1) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) @@ -803,7 +814,7 @@ class TestS3GateObject(TestS3GateBase): "Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]" ): date_obj = datetime.utcnow() + timedelta(days=2) - file_name_1 = generate_file_with_content(file_path=file_path_1) + file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1) s3_gate_object.put_object_s3( self.s3_client, bucket, @@ -819,7 +830,7 @@ class TestS3GateObject(TestS3GateBase): "Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]" ): date_obj = datetime.utcnow() + timedelta(days=3) - file_name_1 = generate_file_with_content(file_path=file_path_1) + file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1) s3_gate_object.put_object_s3( self.s3_client, bucket, @@ -857,7 +868,7 @@ class TestS3GateObject(TestS3GateBase): @allure.title("Test S3 Sync directory") @pytest.mark.parametrize("sync_type", ["sync", "cp"]) - def test_s3_sync_dir(self, sync_type, bucket): + def test_s3_sync_dir(self, sync_type, bucket, simple_object_size): file_path_1 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_1") file_path_2 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_2") object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} @@ -866,8 +877,8 @@ class TestS3GateObject(TestS3GateBase): if not isinstance(self.s3_client, AwsCliClient): pytest.skip("This test is not supported with boto3 client") - generate_file_with_content(file_path=file_path_1) - generate_file_with_content(file_path=file_path_2) + generate_file_with_content(simple_object_size, file_path=file_path_1) + generate_file_with_content(simple_object_size, file_path=file_path_2) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) # TODO: return ACL, when https://github.com/nspcc-dev/neofs-s3-gw/issues/685 will be closed if sync_type == "sync": @@ -909,10 +920,10 @@ class TestS3GateObject(TestS3GateBase): # ], "Permission for all groups is FULL_CONTROL" @allure.title("Test S3 Put 10 nested level object") - def test_s3_put_10_folder(self, bucket, temp_directory): + def test_s3_put_10_folder(self, bucket, temp_directory, simple_object_size): path = "/".join(["".join(choices(string.ascii_letters, k=3)) for _ in range(10)]) file_path_1 = os.path.join(temp_directory, path, "test_file_1") - generate_file_with_content(file_path=file_path_1) + generate_file_with_content(simple_object_size, file_path=file_path_1) file_name = self.object_key_from_file_path(file_path_1) objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket) assert not objects_list, f"Expected empty bucket, got {objects_list}" diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py b/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py index a70f9474..3933bfeb 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_policy.py @@ -1,21 +1,11 @@ import os -import time -from datetime import datetime, timedelta -from random import choice -from string import ascii_letters -from typing import Tuple import allure import pytest -from file_helper import generate_file, generate_file_with_content +from file_helper import generate_file from python_keywords.container import search_container_by_name from python_keywords.storage_policy import get_simple_object_copies -from s3_helper import ( - assert_object_lock_mode, - check_objects_in_bucket, - object_key_from_file_path, - set_bucket_versioning, -) +from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning from steps import s3_gate_bucket, s3_gate_object from steps.s3_gate_base import TestS3GateBase @@ -35,10 +25,10 @@ def pytest_generate_tests(metafunc): @pytest.mark.s3_gate class TestS3GatePolicy(TestS3GateBase): @allure.title("Test S3: Verify bucket creation with retention policy applied") - def test_s3_bucket_location(self): - file_path_1 = generate_file() + def test_s3_bucket_location(self, simple_object_size): + file_path_1 = generate_file(simple_object_size) file_name_1 = object_key_from_file_path(file_path_1) - file_path_2 = generate_file() + file_path_2 = generate_file(simple_object_size) file_name_2 = object_key_from_file_path(file_path_2) with allure.step("Create two buckets with different bucket configuration"): diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_tagging.py b/pytest_tests/testsuites/services/s3_gate/test_s3_tagging.py index d1f6a912..2379a28a 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_tagging.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_tagging.py @@ -32,8 +32,8 @@ class TestS3GateTagging(TestS3GateBase): return tags @allure.title("Test S3: Object tagging") - def test_s3_object_tagging(self, bucket): - file_path = generate_file() + def test_s3_object_tagging(self, bucket, simple_object_size): + file_path = generate_file(simple_object_size) file_name = object_key_from_file_path(file_path) with allure.step("Put with 3 tags object into bucket"): diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_versioning.py b/pytest_tests/testsuites/services/s3_gate/test_s3_versioning.py index e0cd1584..a9ac5b85 100644 --- a/pytest_tests/testsuites/services/s3_gate/test_s3_versioning.py +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_versioning.py @@ -30,8 +30,8 @@ class TestS3GateVersioning(TestS3GateBase): set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.SUSPENDED) @allure.title("Test S3: Enable and disable versioning") - def test_s3_version(self): - file_path = generate_file() + def test_s3_version(self, simple_object_size): + file_path = generate_file(simple_object_size) file_name = self.object_key_from_file_path(file_path) bucket_objects = [file_name] bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, False) @@ -61,7 +61,7 @@ class TestS3GateVersioning(TestS3GateBase): with allure.step("Put several versions of object into bucket"): version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path) - file_name_1 = generate_file_with_content(file_path=file_path) + file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) with allure.step("Check bucket shows all versions"): diff --git a/pytest_tests/testsuites/services/test_http_gate.py b/pytest_tests/testsuites/services/test_http_gate.py index 0b3b9f03..4efcc645 100644 --- a/pytest_tests/testsuites/services/test_http_gate.py +++ b/pytest_tests/testsuites/services/test_http_gate.py @@ -5,10 +5,9 @@ from time import sleep import allure import pytest -from common import COMPLEX_OBJ_SIZE -from container import create_container from epoch import get_epoch, tick_epoch from file_helper import generate_file, get_file_hash +from python_keywords.container import create_container from python_keywords.http_gate import ( get_via_http_curl, get_via_http_gate, @@ -50,7 +49,7 @@ class TestHttpGate(ClusterTestBase): TestHttpGate.wallet = default_wallet @allure.title("Test Put over gRPC, Get over HTTP") - def test_put_grpc_get_http(self): + def test_put_grpc_get_http(self, complex_object_size, simple_object_size): """ Test that object can be put using gRPC interface and get using HTTP. @@ -72,7 +71,9 @@ class TestHttpGate(ClusterTestBase): rule=self.PLACEMENT_RULE_1, basic_acl=PUBLIC_ACL, ) - file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) with allure.step("Put objects using gRPC"): oid_simple = put_object_to_random_node( @@ -97,7 +98,7 @@ class TestHttpGate(ClusterTestBase): @allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading") @allure.title("Test Put over HTTP, Get over HTTP") @pytest.mark.smoke - def test_put_http_get_http(self): + def test_put_http_get_http(self, complex_object_size, simple_object_size): """ Test that object can be put and get using HTTP interface. @@ -117,7 +118,9 @@ class TestHttpGate(ClusterTestBase): rule=self.PLACEMENT_RULE_2, basic_acl=PUBLIC_ACL, ) - file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) with allure.step("Put objects using HTTP"): oid_simple = upload_via_http_gate( @@ -143,7 +146,7 @@ class TestHttpGate(ClusterTestBase): ], ids=["simple", "hyphen", "percent"], ) - def test_put_http_get_http_with_headers(self, attributes: dict): + def test_put_http_get_http_with_headers(self, attributes: dict, simple_object_size): """ Test that object can be downloaded using different attributes in HTTP header. @@ -163,7 +166,7 @@ class TestHttpGate(ClusterTestBase): rule=self.PLACEMENT_RULE_2, basic_acl=PUBLIC_ACL, ) - file_path = generate_file() + file_path = generate_file(simple_object_size) with allure.step("Put objects using HTTP with attribute"): headers = self._attr_into_header(attributes) @@ -179,7 +182,7 @@ class TestHttpGate(ClusterTestBase): self.get_object_by_attr_and_verify_hashes(oid, file_path, cid, attributes) @allure.title("Test Expiration-Epoch in HTTP header") - def test_expiration_epoch_in_http(self): + def test_expiration_epoch_in_http(self, simple_object_size): endpoint = self.cluster.default_rpc_endpoint http_endpoint = self.cluster.default_http_gate_endpoint @@ -190,7 +193,7 @@ class TestHttpGate(ClusterTestBase): rule=self.PLACEMENT_RULE_2, basic_acl=PUBLIC_ACL, ) - file_path = generate_file() + file_path = generate_file(simple_object_size) oids = [] curr_epoch = get_epoch(self.shell, self.cluster) @@ -228,7 +231,7 @@ class TestHttpGate(ClusterTestBase): get_via_http_gate(cid=cid, oid=oid, endpoint=http_endpoint) @allure.title("Test Zip in HTTP header") - def test_zip_in_http(self): + def test_zip_in_http(self, complex_object_size, simple_object_size): cid = create_container( self.wallet, shell=self.shell, @@ -236,7 +239,9 @@ class TestHttpGate(ClusterTestBase): rule=self.PLACEMENT_RULE_2, basic_acl=PUBLIC_ACL, ) - file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) common_prefix = "my_files" headers1 = {"X-Attribute-FilePath": f"{common_prefix}/file1"} @@ -267,7 +272,7 @@ class TestHttpGate(ClusterTestBase): @pytest.mark.long @allure.title("Test Put over HTTP/Curl, Get over HTTP/Curl for large object") - def test_put_http_get_http_large_file(self): + def test_put_http_get_http_large_file(self, complex_object_size): """ This test checks upload and download using curl with 'large' object. Large is object with size up to 20Mb. @@ -280,7 +285,7 @@ class TestHttpGate(ClusterTestBase): basic_acl=PUBLIC_ACL, ) - obj_size = int(os.getenv("BIG_OBJ_SIZE", COMPLEX_OBJ_SIZE)) + obj_size = int(os.getenv("BIG_OBJ_SIZE", complex_object_size)) file_path = generate_file(obj_size) with allure.step("Put objects using HTTP"): @@ -304,7 +309,7 @@ class TestHttpGate(ClusterTestBase): ) @allure.title("Test Put/Get over HTTP using Curl utility") - def test_put_http_get_http_curl(self): + def test_put_http_get_http_curl(self, complex_object_size, simple_object_size): """ Test checks upload and download over HTTP using curl utility. """ @@ -315,7 +320,9 @@ class TestHttpGate(ClusterTestBase): rule=self.PLACEMENT_RULE_2, basic_acl=PUBLIC_ACL, ) - file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) + file_path_simple, file_path_large = generate_file(simple_object_size), generate_file( + complex_object_size + ) with allure.step("Put objects using curl utility"): oid_simple = upload_via_http_gate_curl( diff --git a/pytest_tests/testsuites/session_token/test_object_session_token.py b/pytest_tests/testsuites/session_token/test_object_session_token.py index 13f91d41..a8799892 100644 --- a/pytest_tests/testsuites/session_token/test_object_session_token.py +++ b/pytest_tests/testsuites/session_token/test_object_session_token.py @@ -3,7 +3,7 @@ import random import allure import pytest from cluster_test_base import ClusterTestBase -from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE, WALLET_PASS +from common import WALLET_PASS from file_helper import generate_file from grpc_responses import SESSION_NOT_FOUND from neofs_testlib.utils.wallet import get_last_address_from_wallet @@ -19,7 +19,7 @@ class TestDynamicObjectSession(ClusterTestBase): @allure.title("Test Object Operations with Session Token") @pytest.mark.parametrize( "object_size", - [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], ids=["simple object", "complex object"], ) def test_object_session_token(self, default_wallet, object_size): diff --git a/pytest_tests/testsuites/session_token/test_static_object_session_token.py b/pytest_tests/testsuites/session_token/test_static_object_session_token.py index d7c145bf..eaeb432e 100644 --- a/pytest_tests/testsuites/session_token/test_static_object_session_token.py +++ b/pytest_tests/testsuites/session_token/test_static_object_session_token.py @@ -4,7 +4,6 @@ import allure import pytest from cluster import Cluster from cluster_test_base import ClusterTestBase -from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE from epoch import ensure_fresh_epoch from file_helper import generate_file from grpc_responses import MALFORMED_REQUEST, OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND @@ -13,7 +12,6 @@ from pytest import FixtureRequest from python_keywords.container import create_container from python_keywords.neofs_verbs import ( delete_object, - get_netmap_netinfo, get_object, get_object_from_random_node, get_range, @@ -58,7 +56,7 @@ def storage_containers( @pytest.fixture( - params=[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], + params=[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")], ids=["simple object", "complex object"], # Scope module to upload/delete each files set only once scope="module", @@ -98,16 +96,15 @@ def storage_objects( @allure.step("Get ranges for test") -def get_ranges(storage_object: StorageObjectInfo, shell: Shell, endpoint: str) -> list[str]: +def get_ranges( + storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, endpoint: str +) -> list[str]: """ Returns ranges to test range/hash methods via static session """ object_size = storage_object.size - if object_size == COMPLEX_OBJ_SIZE: - net_info = get_netmap_netinfo(storage_object.wallet_file_path, shell, endpoint) - max_object_size = net_info["maximum_object_size"] - # make sure to test multiple parts of complex object + if object_size > max_object_size: assert object_size >= max_object_size + RANGE_OFFSET_FOR_COMPLEX_OBJECT return [ "0:10", @@ -160,9 +157,9 @@ class TestObjectStaticSession(ClusterTestBase): self, user_wallet: WalletFile, storage_objects: list[StorageObjectInfo], - static_sessions: list[str], + static_sessions: dict[ObjectVerb, str], method_under_test, - verb: str, + verb: ObjectVerb, request: FixtureRequest, ): """ @@ -175,9 +172,9 @@ class TestObjectStaticSession(ClusterTestBase): for node in self.cluster.storage_nodes: for storage_object in storage_objects[0:2]: method_under_test( - user_wallet.path, - storage_object.cid, - storage_object.oid, + wallet=user_wallet.path, + cid=storage_object.cid, + oid=storage_object.oid, shell=self.shell, endpoint=node.get_rpc_endpoint(), session=static_sessions[verb], @@ -193,10 +190,11 @@ class TestObjectStaticSession(ClusterTestBase): self, user_wallet: WalletFile, storage_objects: list[StorageObjectInfo], - static_sessions: list[str], + static_sessions: dict[ObjectVerb, str], method_under_test, - verb: str, + verb: ObjectVerb, request: FixtureRequest, + max_object_size, ): """ Validate static session with range operations @@ -205,7 +203,9 @@ class TestObjectStaticSession(ClusterTestBase): f"Validate static session with range operations for {request.node.callspec.id}" ) storage_object = storage_objects[0] - ranges_to_test = get_ranges(storage_object, self.shell, self.cluster.default_rpc_endpoint) + ranges_to_test = get_ranges( + storage_object, max_object_size, self.shell, self.cluster.default_rpc_endpoint + ) for range_to_test in ranges_to_test: with allure.step(f"Check range {range_to_test}"): @@ -227,7 +227,7 @@ class TestObjectStaticSession(ClusterTestBase): self, user_wallet: WalletFile, storage_objects: list[StorageObjectInfo], - static_sessions: list[str], + static_sessions: dict[ObjectVerb, str], request: FixtureRequest, ): """ @@ -253,7 +253,7 @@ class TestObjectStaticSession(ClusterTestBase): self, user_wallet: WalletFile, storage_objects: list[StorageObjectInfo], - static_sessions: list[str], + static_sessions: dict[ObjectVerb, str], request: FixtureRequest, ): """ @@ -278,7 +278,7 @@ class TestObjectStaticSession(ClusterTestBase): self, stranger_wallet: WalletFile, storage_objects: list[StorageObjectInfo], - static_sessions: list[str], + static_sessions: dict[ObjectVerb, str], request: FixtureRequest, ): """ @@ -305,7 +305,7 @@ class TestObjectStaticSession(ClusterTestBase): self, user_wallet: WalletFile, storage_objects: list[StorageObjectInfo], - static_sessions: list[str], + static_sessions: dict[ObjectVerb, str], request: FixtureRequest, ): """ @@ -333,7 +333,7 @@ class TestObjectStaticSession(ClusterTestBase): user_wallet: WalletFile, storage_objects: list[StorageObjectInfo], storage_containers: list[str], - static_sessions: list[str], + static_sessions: dict[ObjectVerb, str], request: FixtureRequest, ): """ @@ -361,7 +361,7 @@ class TestObjectStaticSession(ClusterTestBase): owner_wallet: WalletFile, user_wallet: WalletFile, stranger_wallet: WalletFile, - storage_containers: list[int], + storage_containers: list[str], storage_objects: list[StorageObjectInfo], temp_directory: str, request: FixtureRequest, @@ -638,7 +638,7 @@ class TestObjectStaticSession(ClusterTestBase): self, user_wallet: WalletFile, storage_objects: list[StorageObjectInfo], - static_sessions: list[str], + static_sessions: dict[ObjectVerb, str], request: FixtureRequest, ): """ @@ -663,7 +663,7 @@ class TestObjectStaticSession(ClusterTestBase): self, user_wallet: WalletFile, storage_objects: list[StorageObjectInfo], - static_sessions: list[str], + static_sessions: dict[ObjectVerb, str], request: FixtureRequest, ): """ diff --git a/pytest_tests/testsuites/session_token/test_static_session_token_container.py b/pytest_tests/testsuites/session_token/test_static_session_token_container.py index d1ba8799..6dd2f388 100644 --- a/pytest_tests/testsuites/session_token/test_static_session_token_container.py +++ b/pytest_tests/testsuites/session_token/test_static_session_token_container.py @@ -148,6 +148,7 @@ class TestSessionTokenContainer(ClusterTestBase): user_wallet: WalletFile, stranger_wallet: WalletFile, static_sessions: dict[ContainerVerb, str], + simple_object_size, ): """ Validate static session with set eacl operation @@ -159,7 +160,7 @@ class TestSessionTokenContainer(ClusterTestBase): shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, ) - file_path = generate_file() + file_path = generate_file(simple_object_size) assert can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster) with allure.step(f"Deny all operations for other via eACL"): diff --git a/requirements.txt b/requirements.txt index 7aeb0d60..5c9ce036 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,6 +55,7 @@ pyflakes==2.4.0 pyparsing==3.0.9 pyrsistent==0.18.1 pytest==7.1.2 +pytest-lazy-fixture==0.6.3 python-dateutil==2.8.2 pyyaml==6.0 requests==2.28.0 diff --git a/robot/resources/lib/python_keywords/storage_group.py b/robot/resources/lib/python_keywords/storage_group.py index 8a00bb7b..0309fecd 100644 --- a/robot/resources/lib/python_keywords/storage_group.py +++ b/robot/resources/lib/python_keywords/storage_group.py @@ -7,7 +7,7 @@ from typing import Optional import allure from cluster import Cluster -from common import COMPLEX_OBJ_SIZE, NEOFS_CLI_EXEC, SIMPLE_OBJ_SIZE, WALLET_CONFIG +from common import NEOFS_CLI_EXEC, WALLET_CONFIG from complex_object_actions import get_link_object from neofs_testlib.cli import NeofsCli from neofs_testlib.shell import Shell @@ -201,12 +201,13 @@ def verify_get_storage_group( gid: str, obj_list: list, object_size: int, + max_object_size: int, bearer: str = None, wallet_config: str = WALLET_CONFIG, ): obj_parts = [] endpoint = cluster.default_rpc_endpoint - if object_size == COMPLEX_OBJ_SIZE: + if object_size > max_object_size: for obj in obj_list: link_oid = get_link_object( wallet, @@ -239,11 +240,10 @@ def verify_get_storage_group( bearer=bearer, wallet_config=wallet_config, ) - if object_size == SIMPLE_OBJ_SIZE: - exp_size = SIMPLE_OBJ_SIZE * obj_num + exp_size = object_size * obj_num + if object_size < max_object_size: assert int(storagegroup_data["Group size"]) == exp_size assert storagegroup_data["Members"] == obj_list else: - exp_size = COMPLEX_OBJ_SIZE * obj_num assert int(storagegroup_data["Group size"]) == exp_size assert storagegroup_data["Members"] == obj_parts diff --git a/robot/variables/common.py b/robot/variables/common.py index a4a8b41d..ce6e3840 100644 --- a/robot/variables/common.py +++ b/robot/variables/common.py @@ -4,9 +4,9 @@ import yaml CONTAINER_WAIT_INTERVAL = "1m" -# TODO: Get object size data from a node config -SIMPLE_OBJ_SIZE = int(os.getenv("SIMPLE_OBJ_SIZE", "1000")) -COMPLEX_OBJ_SIZE = int(os.getenv("COMPLEX_OBJ_SIZE", "2000")) +SIMPLE_OBJECT_SIZE = os.getenv("SIMPLE_OBJECT_SIZE", "1000") +COMPLEX_OBJECT_CHUNKS_COUNT = os.getenv("COMPLEX_OBJECT_CHUNKS_COUNT", "3") +COMPLEX_OBJECT_TAIL_SIZE = os.getenv("COMPLEX_OBJECT_TAIL_SIZE", "1000") MAINNET_BLOCK_TIME = os.getenv("MAINNET_BLOCK_TIME", "1s") MAINNET_TIMEOUT = os.getenv("MAINNET_TIMEOUT", "1min")