TrueCloudLab/frostfs-testcases
24
0
Fork 19
Code Issues 2 Pull requests 1 Releases Activity Actions

Remove SIMPLE_OBJ_SIZE and COMPLEX_OBJ_SIZE from env

Browse source 
Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
This commit is contained in:
Vladimir Avdeev 2022-12-07 15:38:56 +03:00 committed by Vladimir Avdeev
parent 76c5d40e63
commit 05924784ab
26 changed files with 322 additions and 253 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
pytest_tests/helpers/file_helper.py
View file

@ -5,12 +5,12 @@ import uuid
from typing import Any, Optional from typing import Any, Optional
import allure import allure
from common import ASSETS_DIR, SIMPLE_OBJ_SIZE from common import ASSETS_DIR
logger = logging.getLogger("NeoLogger") logger = logging.getLogger("NeoLogger")
def generate_file(size: int = SIMPLE_OBJ_SIZE) -> str: def generate_file(size: int) -> str:
"""Generates a binary file with the specified size in bytes. """Generates a binary file with the specified size in bytes.
Args: Args:
@ -28,6 +28,7 @@ def generate_file(size: int = SIMPLE_OBJ_SIZE) -> str:
def generate_file_with_content( def generate_file_with_content(
size: int,
file_path: Optional[str] = None, file_path: Optional[str] = None,
content: Optional[str] = None, content: Optional[str] = None,
) -> str: ) -> str:
@ -44,7 +45,7 @@ def generate_file_with_content(
""" """
mode = "w+" mode = "w+"
if content is None: if content is None:
content = os.urandom(SIMPLE_OBJ_SIZE) content = os.urandom(size)
mode = "wb" mode = "wb"
if not file_path: if not file_path:

4
pytest_tests/testsuites/acl/conftest.py
View file

@ -68,8 +68,8 @@ def wallets(default_wallet, temp_directory, cluster: Cluster) -> Wallets:
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def file_path(): def file_path(simple_object_size):
yield generate_file() yield generate_file(simple_object_size)
@pytest.fixture(scope="function") @pytest.fixture(scope="function")

26
pytest_tests/testsuites/acl/storage_group/test_storagegroup.py
View file

@ -6,7 +6,7 @@ from typing import Optional
import allure import allure
import pytest import pytest
from cluster_test_base import ClusterTestBase from cluster_test_base import ClusterTestBase
from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, FREE_STORAGE, SIMPLE_OBJ_SIZE, WALLET_PASS from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from file_helper import generate_file from file_helper import generate_file
from grpc_responses import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND from grpc_responses import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
from neofs_testlib.utils.wallet import init_wallet from neofs_testlib.utils.wallet import init_wallet
@ -37,7 +37,7 @@ deposit = 30
@pytest.mark.parametrize( @pytest.mark.parametrize(
"object_size", "object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"], ids=["simple object", "complex object"],
) )
@pytest.mark.sanity @pytest.mark.sanity
@ -68,7 +68,7 @@ class TestStorageGroup(ClusterTestBase):
) )
@allure.title("Test Storage Group in Private Container") @allure.title("Test Storage Group in Private Container")
def test_storagegroup_basic_private_container(self, object_size): def test_storagegroup_basic_private_container(self, object_size, max_object_size):
cid = create_container( cid = create_container(
self.main_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint self.main_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
) )
@ -88,6 +88,7 @@ class TestStorageGroup(ClusterTestBase):
cid=cid, cid=cid,
obj_list=objects, obj_list=objects,
object_size=object_size, object_size=object_size,
max_object_size=max_object_size,
) )
self.expect_failure_for_storagegroup_operations( self.expect_failure_for_storagegroup_operations(
wallet=self.other_wallet, wallet=self.other_wallet,
@ -103,7 +104,7 @@ class TestStorageGroup(ClusterTestBase):
) )
@allure.title("Test Storage Group in Public Container") @allure.title("Test Storage Group in Public Container")
def test_storagegroup_basic_public_container(self, object_size): def test_storagegroup_basic_public_container(self, object_size, max_object_size):
cid = create_container( cid = create_container(
self.main_wallet, self.main_wallet,
basic_acl="public-read-write", basic_acl="public-read-write",
@ -120,12 +121,14 @@ class TestStorageGroup(ClusterTestBase):
cid=cid, cid=cid,
obj_list=objects, obj_list=objects,
object_size=object_size, object_size=object_size,
max_object_size=max_object_size,
) )
self.expect_success_for_storagegroup_operations( self.expect_success_for_storagegroup_operations(
wallet=self.other_wallet, wallet=self.other_wallet,
cid=cid, cid=cid,
obj_list=objects, obj_list=objects,
object_size=object_size, object_size=object_size,
max_object_size=max_object_size,
) )
self.storagegroup_operations_by_system_ro_container( self.storagegroup_operations_by_system_ro_container(
wallet=self.main_wallet, wallet=self.main_wallet,
@ -135,7 +138,7 @@ class TestStorageGroup(ClusterTestBase):
) )
@allure.title("Test Storage Group in Read-Only Container") @allure.title("Test Storage Group in Read-Only Container")
def test_storagegroup_basic_ro_container(self, object_size): def test_storagegroup_basic_ro_container(self, object_size, max_object_size):
cid = create_container( cid = create_container(
self.main_wallet, self.main_wallet,
basic_acl="public-read", basic_acl="public-read",
@ -152,6 +155,7 @@ class TestStorageGroup(ClusterTestBase):
cid=cid, cid=cid,
obj_list=objects, obj_list=objects,
object_size=object_size, object_size=object_size,
max_object_size=max_object_size,
) )
self.storagegroup_operations_by_other_ro_container( self.storagegroup_operations_by_other_ro_container(
owner_wallet=self.main_wallet, owner_wallet=self.main_wallet,
@ -168,7 +172,7 @@ class TestStorageGroup(ClusterTestBase):
) )
@allure.title("Test Storage Group with Bearer Allow") @allure.title("Test Storage Group with Bearer Allow")
def test_storagegroup_bearer_allow(self, object_size): def test_storagegroup_bearer_allow(self, object_size, max_object_size):
cid = create_container( cid = create_container(
self.main_wallet, self.main_wallet,
basic_acl="eacl-public-read-write", basic_acl="eacl-public-read-write",
@ -185,6 +189,7 @@ class TestStorageGroup(ClusterTestBase):
cid=cid, cid=cid,
obj_list=objects, obj_list=objects,
object_size=object_size, object_size=object_size,
max_object_size=max_object_size,
) )
storage_group = put_storagegroup( storage_group = put_storagegroup(
self.shell, self.cluster.default_rpc_endpoint, self.main_wallet, cid, objects self.shell, self.cluster.default_rpc_endpoint, self.main_wallet, cid, objects
@ -219,6 +224,7 @@ class TestStorageGroup(ClusterTestBase):
cid=cid, cid=cid,
obj_list=objects, obj_list=objects,
object_size=object_size, object_size=object_size,
max_object_size=max_object_size,
bearer=bearer_file, bearer=bearer_file,
) )
@ -259,6 +265,7 @@ class TestStorageGroup(ClusterTestBase):
cid: str, cid: str,
obj_list: list, obj_list: list,
object_size: int, object_size: int,
max_object_size: int,
bearer: Optional[str] = None, bearer: Optional[str] = None,
): ):
""" """
@ -285,6 +292,7 @@ class TestStorageGroup(ClusterTestBase):
gid=storage_group, gid=storage_group,
obj_list=obj_list, obj_list=obj_list,
object_size=object_size, object_size=object_size,
max_object_size=max_object_size,
bearer=bearer, bearer=bearer,
) )
delete_storagegroup( delete_storagegroup(
@ -381,7 +389,11 @@ class TestStorageGroup(ClusterTestBase):
@allure.step("Run Storage Group Operations On Systems's Behalf In RO Container") @allure.step("Run Storage Group Operations On Systems's Behalf In RO Container")
def storagegroup_operations_by_system_ro_container( def storagegroup_operations_by_system_ro_container(
self, wallet: str, cid: str, obj_list: list, object_size: int self,
wallet: str,
cid: str,
obj_list: list,
object_size: int,
): ):
""" """
In this func we create a Storage Group on Inner Ring's key behalf In this func we create a Storage Group on Inner Ring's key behalf

27
pytest_tests/testsuites/conftest.py
View file

@ -16,11 +16,14 @@ from common import (
BACKGROUND_OBJ_SIZE, BACKGROUND_OBJ_SIZE,
BACKGROUND_READERS_COUNT, BACKGROUND_READERS_COUNT,
BACKGROUND_WRITERS_COUNT, BACKGROUND_WRITERS_COUNT,
COMPLEX_OBJECT_CHUNKS_COUNT,
COMPLEX_OBJECT_TAIL_SIZE,
FREE_STORAGE, FREE_STORAGE,
HOSTING_CONFIG_FILE, HOSTING_CONFIG_FILE,
LOAD_NODE_SSH_PRIVATE_KEY_PATH, LOAD_NODE_SSH_PRIVATE_KEY_PATH,
LOAD_NODE_SSH_USER, LOAD_NODE_SSH_USER,
LOAD_NODES, LOAD_NODES,
SIMPLE_OBJECT_SIZE,
STORAGE_NODE_SERVICE_NAME_REGEX, STORAGE_NODE_SERVICE_NAME_REGEX,
WALLET_PASS, WALLET_PASS,
) )
@ -32,6 +35,8 @@ from neofs_testlib.reporter import AllureHandler, get_reporter
from neofs_testlib.shell import LocalShell, Shell from neofs_testlib.shell import LocalShell, Shell
from neofs_testlib.utils.wallet import init_wallet from neofs_testlib.utils.wallet import init_wallet
from payment_neogo import deposit_gas, transfer_gas from payment_neogo import deposit_gas, transfer_gas
from pytest import FixtureRequest
from python_keywords.neofs_verbs import get_netmap_netinfo
from python_keywords.node_management import storage_node_healthcheck from python_keywords.node_management import storage_node_healthcheck
from helpers.wallet import WalletFactory from helpers.wallet import WalletFactory
@ -81,6 +86,28 @@ def require_multiple_hosts(hosting: Hosting):
yield yield
@pytest.fixture(scope="session")
def max_object_size(cluster: Cluster, client_shell: Shell) -> int:
storage_node = cluster.storage_nodes[0]
net_info = get_netmap_netinfo(
wallet=storage_node.get_wallet_path(),
wallet_config=storage_node.get_wallet_config_path(),
endpoint=storage_node.get_rpc_endpoint(),
shell=client_shell,
)
yield net_info["maximum_object_size"]
@pytest.fixture(scope="session")
def simple_object_size(max_object_size: int) -> int:
yield int(SIMPLE_OBJECT_SIZE) if int(SIMPLE_OBJECT_SIZE) < max_object_size else max_object_size
@pytest.fixture(scope="session")
def complex_object_size(max_object_size: int) -> int:
return max_object_size * int(COMPLEX_OBJECT_CHUNKS_COUNT) + int(COMPLEX_OBJECT_TAIL_SIZE)
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def wallet_factory(temp_directory: str, client_shell: Shell, cluster: Cluster) -> WalletFactory: def wallet_factory(temp_directory: str, client_shell: Shell, cluster: Cluster) -> WalletFactory:
return WalletFactory(temp_directory, client_shell, cluster) return WalletFactory(temp_directory, client_shell, cluster)

6
pytest_tests/testsuites/failovers/test_failover_network.py
View file

@ -38,7 +38,9 @@ class TestFailoverNetwork(ClusterTestBase):
wait_all_storage_nodes_returned(self.cluster) wait_all_storage_nodes_returned(self.cluster)
@allure.title("Block Storage node traffic") @allure.title("Block Storage node traffic")
def test_block_storage_node_traffic(self, default_wallet, require_multiple_hosts): def test_block_storage_node_traffic(
self, default_wallet, require_multiple_hosts, simple_object_size
):
""" """
Block storage nodes traffic using iptables and wait for replication for objects. Block storage nodes traffic using iptables and wait for replication for objects.
""" """
@ -47,7 +49,7 @@ class TestFailoverNetwork(ClusterTestBase):
wakeup_node_timeout = 10 # timeout to let nodes detect that traffic has blocked wakeup_node_timeout = 10 # timeout to let nodes detect that traffic has blocked
nodes_to_block_count = 2 nodes_to_block_count = 2
source_file_path = generate_file() source_file_path = generate_file(simple_object_size)
cid = create_container( cid = create_container(
wallet, wallet,
shell=self.shell, shell=self.shell,

14
pytest_tests/testsuites/failovers/test_failover_storage.py
View file

@ -47,14 +47,11 @@ class TestFailoverStorage(ClusterTestBase):
@pytest.mark.parametrize("hard_reboot", [True, False]) @pytest.mark.parametrize("hard_reboot", [True, False])
@pytest.mark.failover_reboot @pytest.mark.failover_reboot
def test_lose_storage_node_host( def test_lose_storage_node_host(
self, self, default_wallet, hard_reboot: bool, require_multiple_hosts, simple_object_size
default_wallet,
hard_reboot: bool,
require_multiple_hosts,
): ):
wallet = default_wallet wallet = default_wallet
placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
source_file_path = generate_file() source_file_path = generate_file(simple_object_size)
cid = create_container( cid = create_container(
wallet, wallet,
shell=self.shell, shell=self.shell,
@ -106,14 +103,11 @@ class TestFailoverStorage(ClusterTestBase):
@pytest.mark.parametrize("sequence", [True, False]) @pytest.mark.parametrize("sequence", [True, False])
@pytest.mark.failover_panic @pytest.mark.failover_panic
def test_panic_storage_node_host( def test_panic_storage_node_host(
self, self, default_wallet, require_multiple_hosts, sequence: bool, simple_object_size
default_wallet,
require_multiple_hosts,
sequence: bool,
): ):
wallet = default_wallet wallet = default_wallet
placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X" placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
source_file_path = generate_file() source_file_path = generate_file(simple_object_size)
cid = create_container( cid = create_container(
wallet, wallet,
shell=self.shell, shell=self.shell,

42
pytest_tests/testsuites/network/test_node_management.py
View file

@ -7,7 +7,7 @@ import allure
import pytest import pytest
from cluster import StorageNode from cluster import StorageNode
from cluster_test_base import ClusterTestBase from cluster_test_base import ClusterTestBase
from common import COMPLEX_OBJ_SIZE, MORPH_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT from common import MORPH_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT
from epoch import tick_epoch from epoch import tick_epoch
from file_helper import generate_file from file_helper import generate_file
from grpc_responses import OBJECT_NOT_FOUND, error_matches_status from grpc_responses import OBJECT_NOT_FOUND, error_matches_status
@ -49,9 +49,10 @@ check_nodes: list[StorageNode] = []
class TestNodeManagement(ClusterTestBase): class TestNodeManagement(ClusterTestBase):
@pytest.fixture @pytest.fixture
@allure.title("Create container and pick the node with data") @allure.title("Create container and pick the node with data")
def create_container_and_pick_node(self, default_wallet: str) -> Tuple[str, StorageNode]: def create_container_and_pick_node(
default_wallet self, default_wallet: str, simple_object_size
file_path = generate_file() ) -> Tuple[str, StorageNode]:
file_path = generate_file(simple_object_size)
placement_rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X" placement_rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
@ -126,6 +127,7 @@ class TestNodeManagement(ClusterTestBase):
self, self,
default_wallet, default_wallet,
return_nodes_after_test_run, return_nodes_after_test_run,
simple_object_size,
): ):
""" """
This test remove one node from cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status). This test remove one node from cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
@ -133,7 +135,7 @@ class TestNodeManagement(ClusterTestBase):
wallet = default_wallet wallet = default_wallet
placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X" placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X"
placement_rule_4 = "REP 4 IN X CBF 1 SELECT 4 FROM * AS X" placement_rule_4 = "REP 4 IN X CBF 1 SELECT 4 FROM * AS X"
source_file_path = generate_file() source_file_path = generate_file(simple_object_size)
storage_nodes = self.cluster.storage_nodes storage_nodes = self.cluster.storage_nodes
random_node = random.choice(storage_nodes[1:]) random_node = random.choice(storage_nodes[1:])
@ -219,12 +221,14 @@ class TestNodeManagement(ClusterTestBase):
) )
@pytest.mark.node_mgmt @pytest.mark.node_mgmt
@allure.title("Test object copies based on placement policy") @allure.title("Test object copies based on placement policy")
def test_placement_policy(self, default_wallet, placement_rule, expected_copies): def test_placement_policy(
self, default_wallet, placement_rule, expected_copies, simple_object_size
):
""" """
This test checks object's copies based on container's placement policy. This test checks object's copies based on container's placement policy.
""" """
wallet = default_wallet wallet = default_wallet
file_path = generate_file() file_path = generate_file(simple_object_size)
self.validate_object_copies(wallet, placement_rule, file_path, expected_copies) self.validate_object_copies(wallet, placement_rule, file_path, expected_copies)
@pytest.mark.parametrize( @pytest.mark.parametrize(
@ -279,14 +283,19 @@ class TestNodeManagement(ClusterTestBase):
@pytest.mark.node_mgmt @pytest.mark.node_mgmt
@allure.title("Test object copies and storage nodes based on placement policy") @allure.title("Test object copies and storage nodes based on placement policy")
def test_placement_policy_with_nodes( def test_placement_policy_with_nodes(
self, default_wallet, placement_rule, expected_copies, expected_nodes_id: set[int] self,
default_wallet,
placement_rule,
expected_copies,
expected_nodes_id: set[int],
simple_object_size,
): ):
""" """
Based on container's placement policy check that storage nodes are piked correctly and object has Based on container's placement policy check that storage nodes are piked correctly and object has
correct copies amount. correct copies amount.
""" """
wallet = default_wallet wallet = default_wallet
file_path = generate_file() file_path = generate_file(simple_object_size)
cid, oid, found_nodes = self.validate_object_copies( cid, oid, found_nodes = self.validate_object_copies(
wallet, placement_rule, file_path, expected_copies wallet, placement_rule, file_path, expected_copies
) )
@ -303,24 +312,28 @@ class TestNodeManagement(ClusterTestBase):
) )
@pytest.mark.node_mgmt @pytest.mark.node_mgmt
@allure.title("Negative cases for placement policy") @allure.title("Negative cases for placement policy")
def test_placement_policy_negative(self, default_wallet, placement_rule, expected_copies): def test_placement_policy_negative(
self, default_wallet, placement_rule, expected_copies, simple_object_size
):
""" """
Negative test for placement policy. Negative test for placement policy.
""" """
wallet = default_wallet wallet = default_wallet
file_path = generate_file() file_path = generate_file(simple_object_size)
with pytest.raises(RuntimeError, match=".*not enough nodes to SELECT from.*"): with pytest.raises(RuntimeError, match=".*not enough nodes to SELECT from.*"):
self.validate_object_copies(wallet, placement_rule, file_path, expected_copies) self.validate_object_copies(wallet, placement_rule, file_path, expected_copies)
@pytest.mark.node_mgmt @pytest.mark.node_mgmt
@allure.title("NeoFS object could be dropped using control command") @allure.title("NeoFS object could be dropped using control command")
def test_drop_object(self, default_wallet): def test_drop_object(self, default_wallet, complex_object_size, simple_object_size):
""" """
Test checks object could be dropped using `neofs-cli control drop-objects` command. Test checks object could be dropped using `neofs-cli control drop-objects` command.
""" """
wallet = default_wallet wallet = default_wallet
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
file_path_simple, file_path_complex = generate_file(), generate_file(COMPLEX_OBJ_SIZE) file_path_simple, file_path_complex = generate_file(simple_object_size), generate_file(
complex_object_size
)
locode = get_locode_from_random_node(self.cluster) locode = get_locode_from_random_node(self.cluster)
rule = f"REP 1 CBF 1 SELECT 1 FROM * FILTER 'UN-LOCODE' EQ '{locode}' AS LOC" rule = f"REP 1 CBF 1 SELECT 1 FROM * FILTER 'UN-LOCODE' EQ '{locode}' AS LOC"
@ -358,9 +371,10 @@ class TestNodeManagement(ClusterTestBase):
self, self,
default_wallet, default_wallet,
create_container_and_pick_node, create_container_and_pick_node,
simple_object_size,
): ):
wallet = default_wallet wallet = default_wallet
file_path = generate_file() file_path = generate_file(simple_object_size)
cid, node = create_container_and_pick_node cid, node = create_container_and_pick_node
original_oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster) original_oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)

35
pytest_tests/testsuites/object/test_object_api.py
View file

@ -5,14 +5,12 @@ import sys
import allure import allure
import pytest import pytest
from cluster import Cluster from cluster import Cluster
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
from container import create_container
from file_helper import generate_file, get_file_content, get_file_hash from file_helper import generate_file, get_file_content, get_file_hash
from grpc_responses import OUT_OF_RANGE from grpc_responses import OUT_OF_RANGE
from neofs_testlib.shell import Shell from neofs_testlib.shell import Shell
from pytest import FixtureRequest from pytest import FixtureRequest
from python_keywords.container import create_container
from python_keywords.neofs_verbs import ( from python_keywords.neofs_verbs import (
get_netmap_netinfo,
get_object_from_random_node, get_object_from_random_node,
get_range, get_range,
get_range_hash, get_range_hash,
@ -42,10 +40,7 @@ RANGES_COUNT = 4 # by quarters
RANGE_MIN_LEN = 10 RANGE_MIN_LEN = 10
RANGE_MAX_LEN = 500 RANGE_MAX_LEN = 500
# Used for static ranges found with issues # Used for static ranges found with issues
STATIC_RANGES = { STATIC_RANGES = {}
SIMPLE_OBJ_SIZE: [],
COMPLEX_OBJ_SIZE: [],
}
def generate_ranges(file_size: int, max_object_size: int) -> list[(int, int)]: def generate_ranges(file_size: int, max_object_size: int) -> list[(int, int)]:
@ -58,10 +53,10 @@ def generate_ranges(file_size: int, max_object_size: int) -> list[(int, int)]:
file_ranges.append((int(file_range_step * i), int(file_range_step * (i + 1)))) file_ranges.append((int(file_range_step * i), int(file_range_step * (i + 1))))
# For simple object we can read all file ranges without too much time for testing # For simple object we can read all file ranges without too much time for testing
if file_size == SIMPLE_OBJ_SIZE: if file_size < max_object_size:
file_ranges_to_test.extend(file_ranges) file_ranges_to_test.extend(file_ranges)
# For complex object we need to fetch multiple child objects from different nodes. # For complex object we need to fetch multiple child objects from different nodes.
if file_size == COMPLEX_OBJ_SIZE: else:
assert ( assert (
file_size >= RANGE_MAX_LEN + max_object_size file_size >= RANGE_MAX_LEN + max_object_size
), f"Complex object size should be at least {max_object_size + RANGE_MAX_LEN}. Current: {file_size}" ), f"Complex object size should be at least {max_object_size + RANGE_MAX_LEN}. Current: {file_size}"
@ -83,7 +78,7 @@ def generate_ranges(file_size: int, max_object_size: int) -> list[(int, int)]:
@pytest.fixture( @pytest.fixture(
params=[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], params=[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"], ids=["simple object", "complex object"],
# Scope session to upload/delete each files set only once # Scope session to upload/delete each files set only once
scope="module", scope="module",
@ -132,7 +127,7 @@ def storage_objects(
class TestObjectApi(ClusterTestBase): class TestObjectApi(ClusterTestBase):
@allure.title("Validate object storage policy by native API") @allure.title("Validate object storage policy by native API")
def test_object_storage_policies( def test_object_storage_policies(
self, request: FixtureRequest, storage_objects: list[StorageObjectInfo] self, request: FixtureRequest, storage_objects: list[StorageObjectInfo], simple_object_size
): ):
""" """
Validate object storage policy Validate object storage policy
@ -143,7 +138,7 @@ class TestObjectApi(ClusterTestBase):
with allure.step("Validate storage policy for objects"): with allure.step("Validate storage policy for objects"):
for storage_object in storage_objects: for storage_object in storage_objects:
if storage_object.size == SIMPLE_OBJ_SIZE: if storage_object.size == simple_object_size:
copies = get_simple_object_copies( copies = get_simple_object_copies(
storage_object.wallet_file_path, storage_object.wallet_file_path,
storage_object.cid, storage_object.cid,
@ -257,7 +252,9 @@ class TestObjectApi(ClusterTestBase):
@allure.title("Validate object search with removed items") @allure.title("Validate object search with removed items")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"object_size", [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], ids=["simple object", "complex object"] "object_size",
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
) )
def test_object_search_should_return_tombstone_items( def test_object_search_should_return_tombstone_items(
self, default_wallet: str, request: FixtureRequest, object_size: int self, default_wallet: str, request: FixtureRequest, object_size: int
@ -330,7 +327,7 @@ class TestObjectApi(ClusterTestBase):
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.grpc_api @pytest.mark.grpc_api
def test_object_get_range_hash( def test_object_get_range_hash(
self, request: FixtureRequest, storage_objects: list[StorageObjectInfo] self, request: FixtureRequest, storage_objects: list[StorageObjectInfo], max_object_size
): ):
""" """
Validate get_range_hash for object by common gRPC API Validate get_range_hash for object by common gRPC API
@ -343,10 +340,6 @@ class TestObjectApi(ClusterTestBase):
cid = storage_objects[0].cid cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]] oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path file_path = storage_objects[0].file_path
net_info = get_netmap_netinfo(
wallet, self.shell, endpoint=self.cluster.default_rpc_endpoint
)
max_object_size = net_info["maximum_object_size"]
file_ranges_to_test = generate_ranges(storage_objects[0].size, max_object_size) file_ranges_to_test = generate_ranges(storage_objects[0].size, max_object_size)
logging.info(f"Ranges used in test {file_ranges_to_test}") logging.info(f"Ranges used in test {file_ranges_to_test}")
@ -372,7 +365,7 @@ class TestObjectApi(ClusterTestBase):
@pytest.mark.sanity @pytest.mark.sanity
@pytest.mark.grpc_api @pytest.mark.grpc_api
def test_object_get_range( def test_object_get_range(
self, request: FixtureRequest, storage_objects: list[StorageObjectInfo] self, request: FixtureRequest, storage_objects: list[StorageObjectInfo], max_object_size
): ):
""" """
Validate get_range for object by common gRPC API Validate get_range for object by common gRPC API
@ -383,10 +376,6 @@ class TestObjectApi(ClusterTestBase):
cid = storage_objects[0].cid cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]] oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path file_path = storage_objects[0].file_path
net_info = get_netmap_netinfo(
wallet, self.shell, endpoint=self.cluster.default_rpc_endpoint
)
max_object_size = net_info["maximum_object_size"]
file_ranges_to_test = generate_ranges(storage_objects[0].size, max_object_size) file_ranges_to_test = generate_ranges(storage_objects[0].size, max_object_size)
logging.info(f"Ranges used in test {file_ranges_to_test}") logging.info(f"Ranges used in test {file_ranges_to_test}")

11
pytest_tests/testsuites/object/test_object_api_bearer.py
View file

@ -1,7 +1,6 @@
import allure import allure
import pytest import pytest
from cluster import Cluster from cluster import Cluster
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
from container import REP_2_FOR_3_NODES_PLACEMENT_RULE, SINGLE_PLACEMENT_RULE, create_container from container import REP_2_FOR_3_NODES_PLACEMENT_RULE, SINGLE_PLACEMENT_RULE, create_container
from epoch import get_epoch from epoch import get_epoch
from neofs_testlib.shell import Shell from neofs_testlib.shell import Shell
@ -88,7 +87,7 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
) )
@pytest.mark.parametrize( @pytest.mark.parametrize(
"storage_objects", "storage_objects",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"], ids=["simple object", "complex object"],
indirect=True, indirect=True,
) )
@ -99,7 +98,8 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
request: FixtureRequest, request: FixtureRequest,
): ):
allure.dynamic.title( allure.dynamic.title(
f"Check that objects can be deleted from any node using s3gate wallet with bearer token for {request.node.callspec.id}" f"Check that objects can be deleted from any node using s3gate wallet with bearer "
f"token for {request.node.callspec.id}"
) )
s3_gate_wallet = self.cluster.s3gates[0] s3_gate_wallet = self.cluster.s3gates[0]
@ -124,7 +124,7 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
) )
@pytest.mark.parametrize( @pytest.mark.parametrize(
"file_size", "file_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"], ids=["simple object", "complex object"],
) )
def test_get_object_with_s3_wallet_bearer_from_all_nodes( def test_get_object_with_s3_wallet_bearer_from_all_nodes(
@ -135,7 +135,8 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
request: FixtureRequest, request: FixtureRequest,
): ):
allure.dynamic.title( allure.dynamic.title(
f"Check that objects can be deleted from any node using s3gate wallet with bearer token for {request.node.callspec.id}" "Check that objects can be deleted from any node using s3gate wallet with bearer "
f"token for {request.node.callspec.id}"
) )
s3_gate_wallet = self.cluster.s3gates[0] s3_gate_wallet = self.cluster.s3gates[0]

7
pytest_tests/testsuites/object/test_object_lifetime.py
View file

@ -2,12 +2,11 @@ import logging
import allure import allure
import pytest import pytest
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
from container import create_container
from epoch import get_epoch, tick_epoch from epoch import get_epoch, tick_epoch
from file_helper import generate_file, get_file_hash from file_helper import generate_file, get_file_hash
from grpc_responses import OBJECT_NOT_FOUND from grpc_responses import OBJECT_NOT_FOUND
from pytest import FixtureRequest from pytest import FixtureRequest
from python_keywords.container import create_container
from python_keywords.neofs_verbs import get_object_from_random_node, put_object_to_random_node from python_keywords.neofs_verbs import get_object_from_random_node, put_object_to_random_node
from utility import wait_for_gc_pass_on_storage_nodes from utility import wait_for_gc_pass_on_storage_nodes
@ -21,7 +20,9 @@ logger = logging.getLogger("NeoLogger")
class ObjectApiLifetimeTest(ClusterTestBase): class ObjectApiLifetimeTest(ClusterTestBase):
@allure.title("Test object life time") @allure.title("Test object life time")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"object_size", [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], ids=["simple object", "complex object"] "object_size",
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
) )
def test_object_api_lifetime( def test_object_api_lifetime(
self, default_wallet: str, request: FixtureRequest, object_size: int self, default_wallet: str, request: FixtureRequest, object_size: int

34
pytest_tests/testsuites/object/test_object_lock.py
View file

@ -5,9 +5,8 @@ import allure
import pytest import pytest
from cluster import Cluster from cluster import Cluster
from cluster_test_base import ClusterTestBase from cluster_test_base import ClusterTestBase
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE, STORAGE_GC_TIME from common import STORAGE_GC_TIME
from complex_object_actions import get_link_object from complex_object_actions import get_link_object
from container import create_container
from epoch import ensure_fresh_epoch, get_epoch, tick_epoch from epoch import ensure_fresh_epoch, get_epoch, tick_epoch
from grpc_responses import ( from grpc_responses import (
LIFETIME_REQUIRED, LIFETIME_REQUIRED,
@ -20,6 +19,7 @@ from grpc_responses import (
) )
from neofs_testlib.shell import Shell from neofs_testlib.shell import Shell
from pytest import FixtureRequest from pytest import FixtureRequest
from python_keywords.container import create_container
from python_keywords.neofs_verbs import delete_object, head_object, lock_object from python_keywords.neofs_verbs import delete_object, head_object, lock_object
from test_control import expect_not_raises, wait_for_success from test_control import expect_not_raises, wait_for_success
from utility import parse_time, wait_for_gc_pass_on_storage_nodes from utility import parse_time, wait_for_gc_pass_on_storage_nodes
@ -143,7 +143,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Locked object should be protected from deletion") @allure.title("Locked object should be protected from deletion")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"locked_storage_object", "locked_storage_object",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"], ids=["simple object", "complex object"],
indirect=True, indirect=True,
) )
@ -170,7 +170,9 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Lock object itself should be protected from deletion") @allure.title("Lock object itself should be protected from deletion")
# We operate with only lock object here so no complex object needed in this test # We operate with only lock object here so no complex object needed in this test
@pytest.mark.parametrize("locked_storage_object", [SIMPLE_OBJ_SIZE], indirect=True) @pytest.mark.parametrize(
"locked_storage_object", [pytest.lazy_fixture("simple_object_size")], indirect=True
)
def test_lock_object_itself_cannot_be_deleted( def test_lock_object_itself_cannot_be_deleted(
self, self,
locked_storage_object: StorageObjectInfo, locked_storage_object: StorageObjectInfo,
@ -193,7 +195,9 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Lock object itself cannot be locked") @allure.title("Lock object itself cannot be locked")
# We operate with only lock object here so no complex object needed in this test # We operate with only lock object here so no complex object needed in this test
@pytest.mark.parametrize("locked_storage_object", [SIMPLE_OBJ_SIZE], indirect=True) @pytest.mark.parametrize(
"locked_storage_object", [pytest.lazy_fixture("simple_object_size")], indirect=True
)
def test_lock_object_cannot_be_locked( def test_lock_object_cannot_be_locked(
self, self,
locked_storage_object: StorageObjectInfo, locked_storage_object: StorageObjectInfo,
@ -217,7 +221,9 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Cannot lock object without lifetime and expire_at fields") @allure.title("Cannot lock object without lifetime and expire_at fields")
# We operate with only lock object here so no complex object needed in this test # We operate with only lock object here so no complex object needed in this test
@pytest.mark.parametrize("locked_storage_object", [SIMPLE_OBJ_SIZE], indirect=True) @pytest.mark.parametrize(
"locked_storage_object", [pytest.lazy_fixture("simple_object_size")], indirect=True
)
@pytest.mark.parametrize( @pytest.mark.parametrize(
"wrong_lifetime,wrong_expire_at,expected_error", "wrong_lifetime,wrong_expire_at,expected_error",
[ [
@ -259,7 +265,9 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Expired object should be deleted after locks are expired") @allure.title("Expired object should be deleted after locks are expired")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"object_size", [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], ids=["simple object", "complex object"] "object_size",
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
) )
def test_expired_object_should_be_deleted_after_locks_are_expired( def test_expired_object_should_be_deleted_after_locks_are_expired(
self, self,
@ -327,7 +335,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Should be possible to lock multiple objects at once") @allure.title("Should be possible to lock multiple objects at once")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"object_size", "object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"], ids=["simple object", "complex object"],
) )
def test_should_be_possible_to_lock_multiple_objects_at_once( def test_should_be_possible_to_lock_multiple_objects_at_once(
@ -382,7 +390,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Already outdated lock should not be applied") @allure.title("Already outdated lock should not be applied")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"object_size", "object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"], ids=["simple object", "complex object"],
) )
def test_already_outdated_lock_should_not_be_applied( def test_already_outdated_lock_should_not_be_applied(
@ -421,7 +429,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("After lock expiration with lifetime user should be able to delete object") @allure.title("After lock expiration with lifetime user should be able to delete object")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"object_size", "object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"], ids=["simple object", "complex object"],
) )
@expect_not_raises() @expect_not_raises()
@ -463,7 +471,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("After lock expiration with expire_at user should be able to delete object") @allure.title("After lock expiration with expire_at user should be able to delete object")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"object_size", "object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"], ids=["simple object", "complex object"],
) )
@expect_not_raises() @expect_not_raises()
@ -508,7 +516,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@pytest.mark.parametrize( @pytest.mark.parametrize(
# Only complex objects are required for this test # Only complex objects are required for this test
"locked_storage_object", "locked_storage_object",
[COMPLEX_OBJ_SIZE], [pytest.lazy_fixture("complex_object_size")],
indirect=True, indirect=True,
) )
def test_complex_object_chunks_should_also_be_protected_from_deletion( def test_complex_object_chunks_should_also_be_protected_from_deletion(
@ -535,7 +543,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@pytest.mark.parametrize( @pytest.mark.parametrize(
# Only complex objects are required for this test # Only complex objects are required for this test
"locked_storage_object", "locked_storage_object",
[COMPLEX_OBJ_SIZE], [pytest.lazy_fixture("complex_object_size")],
indirect=True, indirect=True,
) )
def test_link_object_of_complex_object_should_also_be_protected_from_deletion( def test_link_object_of_complex_object_should_also_be_protected_from_deletion(

4
pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py
View file

@ -17,8 +17,8 @@ def pytest_generate_tests(metafunc):
@pytest.mark.s3_gate @pytest.mark.s3_gate
class TestS3GateACL(TestS3GateBase): class TestS3GateACL(TestS3GateBase):
@allure.title("Test S3: Object ACL") @allure.title("Test S3: Object ACL")
def test_s3_object_ACL(self, bucket): def test_s3_object_ACL(self, bucket, simple_object_size):
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path) file_name = object_key_from_file_path(file_path)
with allure.step("Put object into bucket, Check ACL is empty"): with allure.step("Put object into bucket, Check ACL is empty"):

10
pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py
View file

@ -103,8 +103,8 @@ class TestS3GateBucket(TestS3GateBase):
], "Permission for CanonicalUser is FULL_CONTROL" ], "Permission for CanonicalUser is FULL_CONTROL"
@allure.title("Test S3: create bucket with object lock") @allure.title("Test S3: create bucket with object lock")
def test_s3_bucket_object_lock(self): def test_s3_bucket_object_lock(self, simple_object_size):
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path) file_name = object_key_from_file_path(file_path)
with allure.step("Create bucket with --no-object-lock-enabled-for-bucket"): with allure.step("Create bucket with --no-object-lock-enabled-for-bucket"):
@ -138,10 +138,10 @@ class TestS3GateBucket(TestS3GateBase):
) )
@allure.title("Test S3: delete bucket") @allure.title("Test S3: delete bucket")
def test_s3_delete_bucket(self): def test_s3_delete_bucket(self, simple_object_size):
file_path_1 = generate_file() file_path_1 = generate_file(simple_object_size)
file_name_1 = object_key_from_file_path(file_path_1) file_name_1 = object_key_from_file_path(file_path_1)
file_path_2 = generate_file() file_path_2 = generate_file(simple_object_size)
file_name_2 = object_key_from_file_path(file_path_2) file_name_2 = object_key_from_file_path(file_path_2)
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client) bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)

52
pytest_tests/testsuites/services/s3_gate/test_s3_gate.py
View file

@ -5,7 +5,7 @@ from random import choice, choices
import allure import allure
import pytest import pytest
from aws_cli_client import AwsCliClient from aws_cli_client import AwsCliClient
from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE from common import ASSETS_DIR
from epoch import tick_epoch from epoch import tick_epoch
from file_helper import ( from file_helper import (
generate_file, generate_file,
@ -39,12 +39,12 @@ def pytest_generate_tests(metafunc):
@pytest.mark.s3_gate_base @pytest.mark.s3_gate_base
class TestS3Gate(TestS3GateBase): class TestS3Gate(TestS3GateBase):
@allure.title("Test S3 Bucket API") @allure.title("Test S3 Bucket API")
def test_s3_buckets(self): def test_s3_buckets(self, simple_object_size):
""" """
Test base S3 Bucket API (Create/List/Head/Delete). Test base S3 Bucket API (Create/List/Head/Delete).
""" """
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = self.object_key_from_file_path(file_path) file_name = self.object_key_from_file_path(file_path)
with allure.step("Create buckets"): with allure.step("Create buckets"):
@ -109,11 +109,13 @@ class TestS3Gate(TestS3GateBase):
@pytest.mark.parametrize( @pytest.mark.parametrize(
"file_type", ["simple", "large"], ids=["Simple object", "Large object"] "file_type", ["simple", "large"], ids=["Simple object", "Large object"]
) )
def test_s3_api_object(self, file_type, two_buckets): def test_s3_api_object(self, file_type, two_buckets, simple_object_size, complex_object_size):
""" """
Test base S3 Object API (Put/Head/List) for simple and large objects. Test base S3 Object API (Put/Head/List) for simple and large objects.
""" """
file_path = generate_file(SIMPLE_OBJ_SIZE if file_type == "simple" else COMPLEX_OBJ_SIZE) file_path = generate_file(
simple_object_size if file_type == "simple" else complex_object_size
)
file_name = self.object_key_from_file_path(file_path) file_name = self.object_key_from_file_path(file_path)
bucket_1, bucket_2 = two_buckets bucket_1, bucket_2 = two_buckets
@ -136,7 +138,7 @@ class TestS3Gate(TestS3GateBase):
s3_gate_object.get_object_attributes(self.s3_client, bucket, file_name, *attrs) s3_gate_object.get_object_attributes(self.s3_client, bucket, file_name, *attrs)
@allure.title("Test S3 Sync directory") @allure.title("Test S3 Sync directory")
def test_s3_sync_dir(self, bucket): def test_s3_sync_dir(self, bucket, simple_object_size):
""" """
Test checks sync directory with AWS CLI utility. Test checks sync directory with AWS CLI utility.
""" """
@ -147,8 +149,8 @@ class TestS3Gate(TestS3GateBase):
if not isinstance(self.s3_client, AwsCliClient): if not isinstance(self.s3_client, AwsCliClient):
pytest.skip("This test is not supported with boto3 client") pytest.skip("This test is not supported with boto3 client")
generate_file_with_content(file_path=file_path_1) generate_file_with_content(simple_object_size, file_path=file_path_1)
generate_file_with_content(file_path=file_path_2) generate_file_with_content(simple_object_size, file_path=file_path_2)
self.s3_client.sync(bucket_name=bucket, dir_path=os.path.dirname(file_path_1)) self.s3_client.sync(bucket_name=bucket, dir_path=os.path.dirname(file_path_1))
@ -166,19 +168,21 @@ class TestS3Gate(TestS3GateBase):
), "Expected hashes are the same" ), "Expected hashes are the same"
@allure.title("Test S3 Object versioning") @allure.title("Test S3 Object versioning")
def test_s3_api_versioning(self, bucket): def test_s3_api_versioning(self, bucket, simple_object_size):
""" """
Test checks basic versioning functionality for S3 bucket. Test checks basic versioning functionality for S3 bucket.
""" """
version_1_content = "Version 1" version_1_content = "Version 1"
version_2_content = "Version 2" version_2_content = "Version 2"
file_name_simple = generate_file_with_content(content=version_1_content) file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
with allure.step("Put several versions of object into bucket"): with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
generate_file_with_content(file_path=file_name_simple, content=version_2_content) generate_file_with_content(
simple_object_size, file_path=file_name_simple, content=version_2_content
)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
with allure.step("Check bucket shows all versions"): with allure.step("Check bucket shows all versions"):
@ -246,13 +250,15 @@ class TestS3Gate(TestS3GateBase):
@pytest.mark.s3_gate_multipart @pytest.mark.s3_gate_multipart
@allure.title("Test S3 Object Multipart API") @allure.title("Test S3 Object Multipart API")
def test_s3_api_multipart(self, bucket): def test_s3_api_multipart(self, bucket, simple_object_size):
""" """
Test checks S3 Multipart API (Create multipart upload/Abort multipart upload/List multipart upload/ Test checks S3 Multipart API (Create multipart upload/Abort multipart upload/List multipart upload/
Upload part/List parts/Complete multipart upload). Upload part/List parts/Complete multipart upload).
""" """
parts_count = 3 parts_count = 3
file_name_large = generate_file(SIMPLE_OBJ_SIZE * 1024 * 6 * parts_count) # 5Mb - min part file_name_large = generate_file(
simple_object_size * 1024 * 6 * parts_count
) # 5Mb - min part
object_key = self.object_key_from_file_path(file_name_large) object_key = self.object_key_from_file_path(file_name_large)
part_files = split_file(file_name_large, parts_count) part_files = split_file(file_name_large, parts_count)
parts = [] parts = []
@ -320,7 +326,7 @@ class TestS3Gate(TestS3GateBase):
check_tags_by_bucket(self.s3_client, bucket, []) check_tags_by_bucket(self.s3_client, bucket, [])
@allure.title("Test S3 Object tagging API") @allure.title("Test S3 Object tagging API")
def test_s3_api_object_tagging(self, bucket): def test_s3_api_object_tagging(self, bucket, simple_object_size):
""" """
Test checks S3 Object tagging API (Put tag/Get tag/Update tag). Test checks S3 Object tagging API (Put tag/Get tag/Update tag).
""" """
@ -330,7 +336,7 @@ class TestS3Gate(TestS3GateBase):
("some-key--obj2", "some-value--obj2"), ("some-key--obj2", "some-value--obj2"),
] ]
key_value_pair_obj_new = [("some-key-obj-new", "some-value-obj-new")] key_value_pair_obj_new = [("some-key-obj-new", "some-value-obj-new")]
file_name_simple = generate_file(SIMPLE_OBJ_SIZE) file_name_simple = generate_file(simple_object_size)
obj_key = self.object_key_from_file_path(file_name_simple) obj_key = self.object_key_from_file_path(file_name_simple)
s3_gate_bucket.put_bucket_tagging(self.s3_client, bucket, key_value_pair_bucket) s3_gate_bucket.put_bucket_tagging(self.s3_client, bucket, key_value_pair_bucket)
@ -350,7 +356,7 @@ class TestS3Gate(TestS3GateBase):
check_tags_by_object(self.s3_client, bucket, obj_key, []) check_tags_by_object(self.s3_client, bucket, obj_key, [])
@allure.title("Test S3: Delete object & delete objects S3 API") @allure.title("Test S3: Delete object & delete objects S3 API")
def test_s3_api_delete(self, two_buckets): def test_s3_api_delete(self, two_buckets, simple_object_size, complex_object_size):
""" """
Check delete_object and delete_objects S3 API operation. From first bucket some objects deleted one by one. Check delete_object and delete_objects S3 API operation. From first bucket some objects deleted one by one.
From second bucket some objects deleted all at once. From second bucket some objects deleted all at once.
@ -359,7 +365,7 @@ class TestS3Gate(TestS3GateBase):
max_delete_objects = 17 max_delete_objects = 17
put_objects = [] put_objects = []
file_paths = [] file_paths = []
obj_sizes = [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE] obj_sizes = [simple_object_size, complex_object_size]
bucket_1, bucket_2 = two_buckets bucket_1, bucket_2 = two_buckets
@ -406,12 +412,14 @@ class TestS3Gate(TestS3GateBase):
try_to_get_objects_and_expect_error(self.s3_client, bucket_2, objects_to_delete_b2) try_to_get_objects_and_expect_error(self.s3_client, bucket_2, objects_to_delete_b2)
@allure.title("Test S3: Copy object to the same bucket") @allure.title("Test S3: Copy object to the same bucket")
def test_s3_copy_same_bucket(self, bucket): def test_s3_copy_same_bucket(self, bucket, complex_object_size, simple_object_size):
""" """
Test object can be copied to the same bucket. Test object can be copied to the same bucket.
#TODO: delete after test_s3_copy_object will be merge #TODO: delete after test_s3_copy_object will be merge
""" """
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
file_name_simple = self.object_key_from_file_path(file_path_simple) file_name_simple = self.object_key_from_file_path(file_path_simple)
file_name_large = self.object_key_from_file_path(file_path_large) file_name_large = self.object_key_from_file_path(file_path_large)
bucket_objects = [file_name_simple, file_name_large] bucket_objects = [file_name_simple, file_name_large]
@ -448,12 +456,14 @@ class TestS3Gate(TestS3GateBase):
) )
@allure.title("Test S3: Copy object to another bucket") @allure.title("Test S3: Copy object to another bucket")
def test_s3_copy_to_another_bucket(self, two_buckets): def test_s3_copy_to_another_bucket(self, two_buckets, complex_object_size, simple_object_size):
""" """
Test object can be copied to another bucket. Test object can be copied to another bucket.
#TODO: delete after test_s3_copy_object will be merge #TODO: delete after test_s3_copy_object will be merge
""" """
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
file_name_simple = self.object_key_from_file_path(file_path_simple) file_name_simple = self.object_key_from_file_path(file_path_simple)
file_name_large = self.object_key_from_file_path(file_path_large) file_name_large = self.object_key_from_file_path(file_path_large)
bucket_1_objects = [file_name_simple, file_name_large] bucket_1_objects = [file_name_simple, file_name_large]

22
pytest_tests/testsuites/services/s3_gate/test_s3_locking.py
View file

@ -21,8 +21,8 @@ def pytest_generate_tests(metafunc):
@pytest.mark.parametrize("version_id", [None, "second"]) @pytest.mark.parametrize("version_id", [None, "second"])
class TestS3GateLocking(TestS3GateBase): class TestS3GateLocking(TestS3GateBase):
@allure.title("Test S3: Checking the operation of retention period & legal lock on the object") @allure.title("Test S3: Checking the operation of retention period & legal lock on the object")
def test_s3_object_locking(self, version_id): def test_s3_object_locking(self, version_id, simple_object_size):
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path) file_name = object_key_from_file_path(file_path)
retention_period = 2 retention_period = 2
@ -30,7 +30,7 @@ class TestS3GateLocking(TestS3GateBase):
with allure.step("Put several versions of object into bucket"): with allure.step("Put several versions of object into bucket"):
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path) s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
file_name_1 = generate_file_with_content(file_path=file_path) file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
check_objects_in_bucket(self.s3_client, bucket, [file_name]) check_objects_in_bucket(self.s3_client, bucket, [file_name])
if version_id: if version_id:
@ -74,8 +74,8 @@ class TestS3GateLocking(TestS3GateBase):
s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name, version_id) s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name, version_id)
@allure.title("Test S3: Checking the impossibility to change the retention mode COMPLIANCE") @allure.title("Test S3: Checking the impossibility to change the retention mode COMPLIANCE")
def test_s3_mode_compliance(self, version_id): def test_s3_mode_compliance(self, version_id, simple_object_size):
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path) file_name = object_key_from_file_path(file_path)
retention_period = 2 retention_period = 2
retention_period_1 = 1 retention_period_1 = 1
@ -115,8 +115,8 @@ class TestS3GateLocking(TestS3GateBase):
) )
@allure.title("Test S3: Checking the ability to change retention mode GOVERNANCE") @allure.title("Test S3: Checking the ability to change retention mode GOVERNANCE")
def test_s3_mode_governance(self, version_id): def test_s3_mode_governance(self, version_id, simple_object_size):
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path) file_name = object_key_from_file_path(file_path)
retention_period = 3 retention_period = 3
retention_period_1 = 2 retention_period_1 = 2
@ -183,8 +183,8 @@ class TestS3GateLocking(TestS3GateBase):
) )
@allure.title("Test S3: Checking if an Object Cannot Be Locked") @allure.title("Test S3: Checking if an Object Cannot Be Locked")
def test_s3_legal_hold(self, version_id): def test_s3_legal_hold(self, version_id, simple_object_size):
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path) file_name = object_key_from_file_path(file_path)
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, False) bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, False)
@ -205,8 +205,8 @@ class TestS3GateLocking(TestS3GateBase):
@pytest.mark.s3_gate @pytest.mark.s3_gate
class TestS3GateLockingBucket(TestS3GateBase): class TestS3GateLockingBucket(TestS3GateBase):
@allure.title("Test S3: Bucket Lock") @allure.title("Test S3: Bucket Lock")
def test_s3_bucket_lock(self): def test_s3_bucket_lock(self, simple_object_size):
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path) file_name = object_key_from_file_path(file_path)
configuration = {"Rule": {"DefaultRetention": {"Mode": "COMPLIANCE", "Days": 1}}} configuration = {"Rule": {"DefaultRetention": {"Mode": "COMPLIANCE", "Days": 1}}}

133
pytest_tests/testsuites/services/s3_gate/test_s3_object.py
View file

@ -7,7 +7,7 @@ from random import choices, sample
import allure import allure
import pytest import pytest
from aws_cli_client import AwsCliClient from aws_cli_client import AwsCliClient
from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, FREE_STORAGE, SIMPLE_OBJ_SIZE, WALLET_PASS from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from data_formatters import get_wallet_public_key from data_formatters import get_wallet_public_key
from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash
from neofs_testlib.utils.wallet import init_wallet from neofs_testlib.utils.wallet import init_wallet
@ -32,8 +32,8 @@ class TestS3GateObject(TestS3GateBase):
return os.path.basename(full_path) return os.path.basename(full_path)
@allure.title("Test S3: Copy object") @allure.title("Test S3: Copy object")
def test_s3_copy_object(self, two_buckets): def test_s3_copy_object(self, two_buckets, simple_object_size):
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = self.object_key_from_file_path(file_path) file_name = self.object_key_from_file_path(file_path)
bucket_1_objects = [file_name] bucket_1_objects = [file_name]
@ -81,7 +81,7 @@ class TestS3GateObject(TestS3GateBase):
@allure.title("Test S3: Copy version of object") @allure.title("Test S3: Copy version of object")
def test_s3_copy_version_object(self, two_buckets): def test_s3_copy_version_object(self, two_buckets):
version_1_content = "Version 1" version_1_content = "Version 1"
file_name_simple = generate_file_with_content(content=version_1_content) file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
bucket_1, bucket_2 = two_buckets bucket_1, bucket_2 = two_buckets
@ -115,9 +115,9 @@ class TestS3GateObject(TestS3GateBase):
s3_gate_object.copy_object_s3(self.s3_client, bucket_1, obj_key) s3_gate_object.copy_object_s3(self.s3_client, bucket_1, obj_key)
@allure.title("Test S3: Checking copy with acl") @allure.title("Test S3: Checking copy with acl")
def test_s3_copy_acl(self, bucket): def test_s3_copy_acl(self, bucket, simple_object_size):
version_1_content = "Version 1" version_1_content = "Version 1"
file_name_simple = generate_file_with_content(content=version_1_content) file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -137,9 +137,9 @@ class TestS3GateObject(TestS3GateBase):
), "Permission for all groups is FULL_CONTROL" ), "Permission for all groups is FULL_CONTROL"
@allure.title("Test S3: Copy object with metadata") @allure.title("Test S3: Copy object with metadata")
def test_s3_copy_metadate(self, bucket): def test_s3_copy_metadate(self, bucket, simple_object_size):
object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = self.object_key_from_file_path(file_path) file_name = self.object_key_from_file_path(file_path)
bucket_1_objects = [file_name] bucket_1_objects = [file_name]
@ -187,9 +187,9 @@ class TestS3GateObject(TestS3GateBase):
), f"Metadata must be {object_metadata_1}" ), f"Metadata must be {object_metadata_1}"
@allure.title("Test S3: Copy object with tagging") @allure.title("Test S3: Copy object with tagging")
def test_s3_copy_tagging(self, bucket): def test_s3_copy_tagging(self, bucket, simple_object_size):
object_tagging = [(f"{uuid.uuid4()}", f"{uuid.uuid4()}")] object_tagging = [(f"{uuid.uuid4()}", f"{uuid.uuid4()}")]
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name_simple = self.object_key_from_file_path(file_path) file_name_simple = self.object_key_from_file_path(file_path)
bucket_1_objects = [file_name_simple] bucket_1_objects = [file_name_simple]
@ -239,10 +239,10 @@ class TestS3GateObject(TestS3GateBase):
assert tag in got_tags, f"Expected tag {tag} in {got_tags}" assert tag in got_tags, f"Expected tag {tag} in {got_tags}"
@allure.title("Test S3: Delete version of object") @allure.title("Test S3: Delete version of object")
def test_s3_delete_versioning(self, bucket): def test_s3_delete_versioning(self, bucket, complex_object_size, simple_object_size):
version_1_content = "Version 1" version_1_content = "Version 1"
version_2_content = "Version 2" version_2_content = "Version 2"
file_name_simple = generate_file_with_content(content=version_1_content) file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -250,7 +250,7 @@ class TestS3GateObject(TestS3GateBase):
with allure.step("Put several versions of object into bucket"): with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
file_name_1 = generate_file_with_content( file_name_1 = generate_file_with_content(
file_path=file_name_simple, content=version_2_content simple_object_size, file_path=file_name_simple, content=version_2_content
) )
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
@ -287,7 +287,7 @@ class TestS3GateObject(TestS3GateBase):
assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found" assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found"
with allure.step("Put new object into bucket"): with allure.step("Put new object into bucket"):
file_name_simple = generate_file(COMPLEX_OBJ_SIZE) file_name_simple = generate_file(complex_object_size)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
version_id = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) version_id = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
@ -298,12 +298,12 @@ class TestS3GateObject(TestS3GateBase):
assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker" assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker"
@allure.title("Test S3: bulk delete version of object") @allure.title("Test S3: bulk delete version of object")
def test_s3_bulk_delete_versioning(self, bucket): def test_s3_bulk_delete_versioning(self, bucket, simple_object_size):
version_1_content = "Version 1" version_1_content = "Version 1"
version_2_content = "Version 2" version_2_content = "Version 2"
version_3_content = "Version 3" version_3_content = "Version 3"
version_4_content = "Version 4" version_4_content = "Version 4"
file_name_1 = generate_file_with_content(content=version_1_content) file_name_1 = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_1) obj_key = os.path.basename(file_name_1)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -311,15 +311,15 @@ class TestS3GateObject(TestS3GateBase):
with allure.step("Put several versions of object into bucket"): with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
file_name_2 = generate_file_with_content( file_name_2 = generate_file_with_content(
file_path=file_name_1, content=version_2_content simple_object_size, file_path=file_name_1, content=version_2_content
) )
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_2) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_2)
file_name_3 = generate_file_with_content( file_name_3 = generate_file_with_content(
file_path=file_name_1, content=version_3_content simple_object_size, file_path=file_name_1, content=version_3_content
) )
version_id_3 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_3) version_id_3 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_3)
file_name_4 = generate_file_with_content( file_name_4 = generate_file_with_content(
file_path=file_name_1, content=version_4_content simple_object_size, file_path=file_name_1, content=version_4_content
) )
version_id_4 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_4) version_id_4 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_4)
version_ids = {version_id_1, version_id_2, version_id_3, version_id_4} version_ids = {version_id_1, version_id_2, version_id_3, version_id_4}
@ -352,14 +352,14 @@ class TestS3GateObject(TestS3GateBase):
def test_s3_get_versioning(self, bucket): def test_s3_get_versioning(self, bucket):
version_1_content = "Version 1" version_1_content = "Version 1"
version_2_content = "Version 2" version_2_content = "Version 2"
file_name_simple = generate_file_with_content(content=version_1_content) file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_simple) obj_key = os.path.basename(file_name_simple)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
with allure.step("Put several versions of object into bucket"): with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple) version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
file_name_1 = generate_file_with_content( file_name_1 = generate_file_with_content(
file_path=file_name_simple, content=version_2_content simple_object_size, file_path=file_name_simple, content=version_2_content
) )
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
@ -388,14 +388,14 @@ class TestS3GateObject(TestS3GateBase):
), f"Get object with version {version_id_2}" ), f"Get object with version {version_id_2}"
@allure.title("Test S3: Get range") @allure.title("Test S3: Get range")
def test_s3_get_range(self, bucket): def test_s3_get_range(self, bucket, complex_object_size: int, simple_object_size: int):
file_path = generate_file(COMPLEX_OBJ_SIZE) file_path = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path) file_name = self.object_key_from_file_path(file_path)
file_hash = get_file_hash(file_path) file_hash = get_file_hash(file_path)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
with allure.step("Put several versions of object into bucket"): with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path) version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
file_name_1 = generate_file_with_content(file_path=file_path) file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
with allure.step("Get first version of object"): with allure.step("Get first version of object"):
@ -404,42 +404,46 @@ class TestS3GateObject(TestS3GateBase):
bucket, bucket,
file_name, file_name,
version_id_1, version_id_1,
range=[0, int(COMPLEX_OBJ_SIZE / 3)], range=[0, int(complex_object_size / 3)],
) )
object_1_part_2 = s3_gate_object.get_object_s3( object_1_part_2 = s3_gate_object.get_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
file_name, file_name,
version_id_1, version_id_1,
range=[int(COMPLEX_OBJ_SIZE / 3) + 1, 2 * int(COMPLEX_OBJ_SIZE / 3)], range=[int(complex_object_size / 3) + 1, 2 * int(complex_object_size / 3)],
) )
object_1_part_3 = s3_gate_object.get_object_s3( object_1_part_3 = s3_gate_object.get_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
file_name, file_name,
version_id_1, version_id_1,
range=[2 * int(COMPLEX_OBJ_SIZE / 3) + 1, COMPLEX_OBJ_SIZE], range=[2 * int(complex_object_size / 3) + 1, complex_object_size],
) )
con_file = concat_files([object_1_part_1, object_1_part_2, object_1_part_3]) con_file = concat_files([object_1_part_1, object_1_part_2, object_1_part_3])
assert get_file_hash(con_file) == file_hash, "Hashes must be the same" assert get_file_hash(con_file) == file_hash, "Hashes must be the same"
with allure.step("Get second version of object"): with allure.step("Get second version of object"):
object_2_part_1 = s3_gate_object.get_object_s3( object_2_part_1 = s3_gate_object.get_object_s3(
self.s3_client, bucket, file_name, version_id_2, range=[0, int(SIMPLE_OBJ_SIZE / 3)] self.s3_client,
bucket,
file_name,
version_id_2,
range=[0, int(complex_object_size / 3)],
) )
object_2_part_2 = s3_gate_object.get_object_s3( object_2_part_2 = s3_gate_object.get_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
file_name, file_name,
version_id_2, version_id_2,
range=[int(SIMPLE_OBJ_SIZE / 3) + 1, 2 * int(SIMPLE_OBJ_SIZE / 3)], range=[int(simple_object_size / 3) + 1, 2 * int(simple_object_size / 3)],
) )
object_2_part_3 = s3_gate_object.get_object_s3( object_2_part_3 = s3_gate_object.get_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
file_name, file_name,
version_id_2, version_id_2,
range=[2 * int(SIMPLE_OBJ_SIZE / 3) + 1, COMPLEX_OBJ_SIZE], range=[2 * int(simple_object_size / 3) + 1, complex_object_size],
) )
con_file_1 = concat_files([object_2_part_1, object_2_part_2, object_2_part_3]) con_file_1 = concat_files([object_2_part_1, object_2_part_2, object_2_part_3])
assert get_file_hash(con_file_1) == get_file_hash( assert get_file_hash(con_file_1) == get_file_hash(
@ -448,28 +452,28 @@ class TestS3GateObject(TestS3GateBase):
with allure.step("Get object"): with allure.step("Get object"):
object_3_part_1 = s3_gate_object.get_object_s3( object_3_part_1 = s3_gate_object.get_object_s3(
self.s3_client, bucket, file_name, range=[0, int(SIMPLE_OBJ_SIZE / 3)] self.s3_client, bucket, file_name, range=[0, int(simple_object_size / 3)]
) )
object_3_part_2 = s3_gate_object.get_object_s3( object_3_part_2 = s3_gate_object.get_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
file_name, file_name,
range=[int(SIMPLE_OBJ_SIZE / 3) + 1, 2 * int(SIMPLE_OBJ_SIZE / 3)], range=[int(simple_object_size / 3) + 1, 2 * int(simple_object_size / 3)],
) )
object_3_part_3 = s3_gate_object.get_object_s3( object_3_part_3 = s3_gate_object.get_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
file_name, file_name,
range=[2 * int(SIMPLE_OBJ_SIZE / 3) + 1, COMPLEX_OBJ_SIZE], range=[2 * int(simple_object_size / 3) + 1, complex_object_size],
) )
con_file = concat_files([object_3_part_1, object_3_part_2, object_3_part_3]) con_file = concat_files([object_3_part_1, object_3_part_2, object_3_part_3])
assert get_file_hash(con_file) == get_file_hash(file_name_1), "Hashes must be the same" assert get_file_hash(con_file) == get_file_hash(file_name_1), "Hashes must be the same"
@allure.title("Test S3: Copy object with metadata") @allure.title("Test S3: Copy object with metadata")
@pytest.mark.smoke @pytest.mark.smoke
def test_s3_head_object(self, bucket): def test_s3_head_object(self, bucket, complex_object_size, simple_object_size):
object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
file_path = generate_file(COMPLEX_OBJ_SIZE) file_path = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path) file_name = self.object_key_from_file_path(file_path)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -477,7 +481,7 @@ class TestS3GateObject(TestS3GateBase):
version_id_1 = s3_gate_object.put_object_s3( version_id_1 = s3_gate_object.put_object_s3(
self.s3_client, bucket, file_path, Metadata=object_metadata self.s3_client, bucket, file_path, Metadata=object_metadata
) )
file_name_1 = generate_file_with_content(file_path=file_path) file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
with allure.step("Get head of first version of object"): with allure.step("Get head of first version of object"):
@ -506,10 +510,10 @@ class TestS3GateObject(TestS3GateBase):
@allure.title("Test S3: list of object with versions") @allure.title("Test S3: list of object with versions")
@pytest.mark.parametrize("list_type", ["v1", "v2"]) @pytest.mark.parametrize("list_type", ["v1", "v2"])
def test_s3_list_object(self, list_type: str, bucket): def test_s3_list_object(self, list_type: str, bucket, complex_object_size):
file_path_1 = generate_file(COMPLEX_OBJ_SIZE) file_path_1 = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path_1) file_name = self.object_key_from_file_path(file_path_1)
file_path_2 = generate_file(COMPLEX_OBJ_SIZE) file_path_2 = generate_file(complex_object_size)
file_name_2 = self.object_key_from_file_path(file_path_2) file_name_2 = self.object_key_from_file_path(file_path_2)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -543,8 +547,8 @@ class TestS3GateObject(TestS3GateBase):
assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker" assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker"
@allure.title("Test S3: put object") @allure.title("Test S3: put object")
def test_s3_put_object(self, bucket): def test_s3_put_object(self, bucket, complex_object_size, simple_object_size):
file_path_1 = generate_file(COMPLEX_OBJ_SIZE) file_path_1 = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path_1) file_name = self.object_key_from_file_path(file_path_1)
object_1_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} object_1_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
tag_key_1 = "tag1" tag_key_1 = "tag1"
@ -569,7 +573,7 @@ class TestS3GateObject(TestS3GateBase):
], "Tags must be the same" ], "Tags must be the same"
with allure.step("Rewrite file into bucket"): with allure.step("Rewrite file into bucket"):
file_path_2 = generate_file_with_content(file_path=file_path_1) file_path_2 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3( s3_gate_object.put_object_s3(
self.s3_client, bucket, file_path_2, Metadata=object_2_metadata, Tagging=tag_2 self.s3_client, bucket, file_path_2, Metadata=object_2_metadata, Tagging=tag_2
) )
@ -583,7 +587,7 @@ class TestS3GateObject(TestS3GateBase):
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
file_path_3 = generate_file(COMPLEX_OBJ_SIZE) file_path_3 = generate_file(complex_object_size)
file_hash = get_file_hash(file_path_3) file_hash = get_file_hash(file_path_3)
file_name_3 = self.object_key_from_file_path(file_path_3) file_name_3 = self.object_key_from_file_path(file_path_3)
object_3_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} object_3_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
@ -604,7 +608,7 @@ class TestS3GateObject(TestS3GateBase):
], "Tags must be the same" ], "Tags must be the same"
with allure.step("Put new version of file into bucket"): with allure.step("Put new version of file into bucket"):
file_path_4 = generate_file_with_content(file_path=file_path_3) file_path_4 = generate_file_with_content(simple_object_size, file_path=file_path_3)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_4) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_4)
versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket) versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket)
obj_versions = { obj_versions = {
@ -680,8 +684,15 @@ class TestS3GateObject(TestS3GateBase):
@allure.title("Test S3: put object with ACL") @allure.title("Test S3: put object with ACL")
@pytest.mark.parametrize("bucket_versioning", ["ENABLED", "SUSPENDED"]) @pytest.mark.parametrize("bucket_versioning", ["ENABLED", "SUSPENDED"])
def test_s3_put_object_acl(self, prepare_two_wallets, bucket_versioning, bucket): def test_s3_put_object_acl(
file_path_1 = generate_file(COMPLEX_OBJ_SIZE) self,
prepare_two_wallets,
bucket_versioning,
bucket,
complex_object_size,
simple_object_size,
):
file_path_1 = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path_1) file_name = self.object_key_from_file_path(file_path_1)
if bucket_versioning == "ENABLED": if bucket_versioning == "ENABLED":
status = s3_gate_bucket.VersioningStatus.ENABLED status = s3_gate_bucket.VersioningStatus.ENABLED
@ -698,7 +709,7 @@ class TestS3GateObject(TestS3GateBase):
assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same" assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same"
with allure.step("Put object with acl public-read"): with allure.step("Put object with acl public-read"):
file_path_2 = generate_file_with_content(file_path=file_path_1) file_path_2 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_2, ACL="public-read") s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_2, ACL="public-read")
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name) obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
obj_permission = [permission.get("Permission") for permission in obj_acl] obj_permission = [permission.get("Permission") for permission in obj_acl]
@ -710,7 +721,7 @@ class TestS3GateObject(TestS3GateBase):
assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same" assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same"
with allure.step("Put object with acl public-read-write"): with allure.step("Put object with acl public-read-write"):
file_path_3 = generate_file_with_content(file_path=file_path_1) file_path_3 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3( s3_gate_object.put_object_s3(
self.s3_client, bucket, file_path_3, ACL="public-read-write" self.s3_client, bucket, file_path_3, ACL="public-read-write"
) )
@ -724,7 +735,7 @@ class TestS3GateObject(TestS3GateBase):
assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same" assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same"
with allure.step("Put object with acl authenticated-read"): with allure.step("Put object with acl authenticated-read"):
file_path_4 = generate_file_with_content(file_path=file_path_1) file_path_4 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3( s3_gate_object.put_object_s3(
self.s3_client, bucket, file_path_4, ACL="authenticated-read" self.s3_client, bucket, file_path_4, ACL="authenticated-read"
) )
@ -737,11 +748,11 @@ class TestS3GateObject(TestS3GateBase):
object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name) object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name)
assert get_file_hash(file_path_4) == get_file_hash(object_4), "Hashes must be the same" assert get_file_hash(file_path_4) == get_file_hash(object_4), "Hashes must be the same"
file_path_5 = generate_file(COMPLEX_OBJ_SIZE) file_path_5 = generate_file(complex_object_size)
file_name_5 = self.object_key_from_file_path(file_path_5) file_name_5 = self.object_key_from_file_path(file_path_5)
with allure.step("Put object with --grant-full-control id=mycanonicaluserid"): with allure.step("Put object with --grant-full-control id=mycanonicaluserid"):
file_path_6 = generate_file_with_content(file_path=file_path_5) file_path_6 = generate_file_with_content(simple_object_size, file_path=file_path_5)
s3_gate_object.put_object_s3( s3_gate_object.put_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
@ -760,7 +771,7 @@ class TestS3GateObject(TestS3GateBase):
with allure.step( with allure.step(
"Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers" "Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
): ):
file_path_7 = generate_file_with_content(file_path=file_path_5) file_path_7 = generate_file_with_content(simple_object_size, file_path=file_path_5)
s3_gate_object.put_object_s3( s3_gate_object.put_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
@ -777,9 +788,9 @@ class TestS3GateObject(TestS3GateBase):
assert get_file_hash(file_path_7) == get_file_hash(object_7), "Hashes must be the same" assert get_file_hash(file_path_7) == get_file_hash(object_7), "Hashes must be the same"
@allure.title("Test S3: put object with lock-mode") @allure.title("Test S3: put object with lock-mode")
def test_s3_put_object_lock_mode(self, bucket): def test_s3_put_object_lock_mode(self, bucket, complex_object_size, simple_object_size):
file_path_1 = generate_file(COMPLEX_OBJ_SIZE) file_path_1 = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path_1) file_name = self.object_key_from_file_path(file_path_1)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -803,7 +814,7 @@ class TestS3GateObject(TestS3GateBase):
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]" "Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"
): ):
date_obj = datetime.utcnow() + timedelta(days=2) date_obj = datetime.utcnow() + timedelta(days=2)
file_name_1 = generate_file_with_content(file_path=file_path_1) file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3( s3_gate_object.put_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
@ -819,7 +830,7 @@ class TestS3GateObject(TestS3GateBase):
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]" "Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"
): ):
date_obj = datetime.utcnow() + timedelta(days=3) date_obj = datetime.utcnow() + timedelta(days=3)
file_name_1 = generate_file_with_content(file_path=file_path_1) file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3( s3_gate_object.put_object_s3(
self.s3_client, self.s3_client,
bucket, bucket,
@ -857,7 +868,7 @@ class TestS3GateObject(TestS3GateBase):
@allure.title("Test S3 Sync directory") @allure.title("Test S3 Sync directory")
@pytest.mark.parametrize("sync_type", ["sync", "cp"]) @pytest.mark.parametrize("sync_type", ["sync", "cp"])
def test_s3_sync_dir(self, sync_type, bucket): def test_s3_sync_dir(self, sync_type, bucket, simple_object_size):
file_path_1 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_1") file_path_1 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_1")
file_path_2 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_2") file_path_2 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_2")
object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"} object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
@ -866,8 +877,8 @@ class TestS3GateObject(TestS3GateBase):
if not isinstance(self.s3_client, AwsCliClient): if not isinstance(self.s3_client, AwsCliClient):
pytest.skip("This test is not supported with boto3 client") pytest.skip("This test is not supported with boto3 client")
generate_file_with_content(file_path=file_path_1) generate_file_with_content(simple_object_size, file_path=file_path_1)
generate_file_with_content(file_path=file_path_2) generate_file_with_content(simple_object_size, file_path=file_path_2)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
# TODO: return ACL, when https://github.com/nspcc-dev/neofs-s3-gw/issues/685 will be closed # TODO: return ACL, when https://github.com/nspcc-dev/neofs-s3-gw/issues/685 will be closed
if sync_type == "sync": if sync_type == "sync":
@ -909,10 +920,10 @@ class TestS3GateObject(TestS3GateBase):
# ], "Permission for all groups is FULL_CONTROL" # ], "Permission for all groups is FULL_CONTROL"
@allure.title("Test S3 Put 10 nested level object") @allure.title("Test S3 Put 10 nested level object")
def test_s3_put_10_folder(self, bucket, temp_directory): def test_s3_put_10_folder(self, bucket, temp_directory, simple_object_size):
path = "/".join(["".join(choices(string.ascii_letters, k=3)) for _ in range(10)]) path = "/".join(["".join(choices(string.ascii_letters, k=3)) for _ in range(10)])
file_path_1 = os.path.join(temp_directory, path, "test_file_1") file_path_1 = os.path.join(temp_directory, path, "test_file_1")
generate_file_with_content(file_path=file_path_1) generate_file_with_content(simple_object_size, file_path=file_path_1)
file_name = self.object_key_from_file_path(file_path_1) file_name = self.object_key_from_file_path(file_path_1)
objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket) objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}" assert not objects_list, f"Expected empty bucket, got {objects_list}"

20
pytest_tests/testsuites/services/s3_gate/test_s3_policy.py
View file

@ -1,21 +1,11 @@
import os import os
import time
from datetime import datetime, timedelta
from random import choice
from string import ascii_letters
from typing import Tuple
import allure import allure
import pytest import pytest
from file_helper import generate_file, generate_file_with_content from file_helper import generate_file
from python_keywords.container import search_container_by_name from python_keywords.container import search_container_by_name
from python_keywords.storage_policy import get_simple_object_copies from python_keywords.storage_policy import get_simple_object_copies
from s3_helper import ( from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning
assert_object_lock_mode,
check_objects_in_bucket,
object_key_from_file_path,
set_bucket_versioning,
)
from steps import s3_gate_bucket, s3_gate_object from steps import s3_gate_bucket, s3_gate_object
from steps.s3_gate_base import TestS3GateBase from steps.s3_gate_base import TestS3GateBase
@ -35,10 +25,10 @@ def pytest_generate_tests(metafunc):
@pytest.mark.s3_gate @pytest.mark.s3_gate
class TestS3GatePolicy(TestS3GateBase): class TestS3GatePolicy(TestS3GateBase):
@allure.title("Test S3: Verify bucket creation with retention policy applied") @allure.title("Test S3: Verify bucket creation with retention policy applied")
def test_s3_bucket_location(self): def test_s3_bucket_location(self, simple_object_size):
file_path_1 = generate_file() file_path_1 = generate_file(simple_object_size)
file_name_1 = object_key_from_file_path(file_path_1) file_name_1 = object_key_from_file_path(file_path_1)
file_path_2 = generate_file() file_path_2 = generate_file(simple_object_size)
file_name_2 = object_key_from_file_path(file_path_2) file_name_2 = object_key_from_file_path(file_path_2)
with allure.step("Create two buckets with different bucket configuration"): with allure.step("Create two buckets with different bucket configuration"):

4
pytest_tests/testsuites/services/s3_gate/test_s3_tagging.py
View file

@ -32,8 +32,8 @@ class TestS3GateTagging(TestS3GateBase):
return tags return tags
@allure.title("Test S3: Object tagging") @allure.title("Test S3: Object tagging")
def test_s3_object_tagging(self, bucket): def test_s3_object_tagging(self, bucket, simple_object_size):
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path) file_name = object_key_from_file_path(file_path)
with allure.step("Put with 3 tags object into bucket"): with allure.step("Put with 3 tags object into bucket"):

6
pytest_tests/testsuites/services/s3_gate/test_s3_versioning.py
View file

@ -30,8 +30,8 @@ class TestS3GateVersioning(TestS3GateBase):
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.SUSPENDED) set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.SUSPENDED)
@allure.title("Test S3: Enable and disable versioning") @allure.title("Test S3: Enable and disable versioning")
def test_s3_version(self): def test_s3_version(self, simple_object_size):
file_path = generate_file() file_path = generate_file(simple_object_size)
file_name = self.object_key_from_file_path(file_path) file_name = self.object_key_from_file_path(file_path)
bucket_objects = [file_name] bucket_objects = [file_name]
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, False) bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, False)
@ -61,7 +61,7 @@ class TestS3GateVersioning(TestS3GateBase):
with allure.step("Put several versions of object into bucket"): with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path) version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
file_name_1 = generate_file_with_content(file_path=file_path) file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1) version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
with allure.step("Check bucket shows all versions"): with allure.step("Check bucket shows all versions"):

39
pytest_tests/testsuites/services/test_http_gate.py
View file

@ -5,10 +5,9 @@ from time import sleep
import allure import allure
import pytest import pytest
from common import COMPLEX_OBJ_SIZE
from container import create_container
from epoch import get_epoch, tick_epoch from epoch import get_epoch, tick_epoch
from file_helper import generate_file, get_file_hash from file_helper import generate_file, get_file_hash
from python_keywords.container import create_container
from python_keywords.http_gate import ( from python_keywords.http_gate import (
get_via_http_curl, get_via_http_curl,
get_via_http_gate, get_via_http_gate,
@ -50,7 +49,7 @@ class TestHttpGate(ClusterTestBase):
TestHttpGate.wallet = default_wallet TestHttpGate.wallet = default_wallet
@allure.title("Test Put over gRPC, Get over HTTP") @allure.title("Test Put over gRPC, Get over HTTP")
def test_put_grpc_get_http(self): def test_put_grpc_get_http(self, complex_object_size, simple_object_size):
""" """
Test that object can be put using gRPC interface and get using HTTP. Test that object can be put using gRPC interface and get using HTTP.
@ -72,7 +71,9 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_1, rule=self.PLACEMENT_RULE_1,
basic_acl=PUBLIC_ACL, basic_acl=PUBLIC_ACL,
) )
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
with allure.step("Put objects using gRPC"): with allure.step("Put objects using gRPC"):
oid_simple = put_object_to_random_node( oid_simple = put_object_to_random_node(
@ -97,7 +98,7 @@ class TestHttpGate(ClusterTestBase):
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading") @allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading")
@allure.title("Test Put over HTTP, Get over HTTP") @allure.title("Test Put over HTTP, Get over HTTP")
@pytest.mark.smoke @pytest.mark.smoke
def test_put_http_get_http(self): def test_put_http_get_http(self, complex_object_size, simple_object_size):
""" """
Test that object can be put and get using HTTP interface. Test that object can be put and get using HTTP interface.
@ -117,7 +118,9 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_2, rule=self.PLACEMENT_RULE_2,
basic_acl=PUBLIC_ACL, basic_acl=PUBLIC_ACL,
) )
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
with allure.step("Put objects using HTTP"): with allure.step("Put objects using HTTP"):
oid_simple = upload_via_http_gate( oid_simple = upload_via_http_gate(
@ -143,7 +146,7 @@ class TestHttpGate(ClusterTestBase):
], ],
ids=["simple", "hyphen", "percent"], ids=["simple", "hyphen", "percent"],
) )
def test_put_http_get_http_with_headers(self, attributes: dict): def test_put_http_get_http_with_headers(self, attributes: dict, simple_object_size):
""" """
Test that object can be downloaded using different attributes in HTTP header. Test that object can be downloaded using different attributes in HTTP header.
@ -163,7 +166,7 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_2, rule=self.PLACEMENT_RULE_2,
basic_acl=PUBLIC_ACL, basic_acl=PUBLIC_ACL,
) )
file_path = generate_file() file_path = generate_file(simple_object_size)
with allure.step("Put objects using HTTP with attribute"): with allure.step("Put objects using HTTP with attribute"):
headers = self._attr_into_header(attributes) headers = self._attr_into_header(attributes)
@ -179,7 +182,7 @@ class TestHttpGate(ClusterTestBase):
self.get_object_by_attr_and_verify_hashes(oid, file_path, cid, attributes) self.get_object_by_attr_and_verify_hashes(oid, file_path, cid, attributes)
@allure.title("Test Expiration-Epoch in HTTP header") @allure.title("Test Expiration-Epoch in HTTP header")
def test_expiration_epoch_in_http(self): def test_expiration_epoch_in_http(self, simple_object_size):
endpoint = self.cluster.default_rpc_endpoint endpoint = self.cluster.default_rpc_endpoint
http_endpoint = self.cluster.default_http_gate_endpoint http_endpoint = self.cluster.default_http_gate_endpoint
@ -190,7 +193,7 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_2, rule=self.PLACEMENT_RULE_2,
basic_acl=PUBLIC_ACL, basic_acl=PUBLIC_ACL,
) )
file_path = generate_file() file_path = generate_file(simple_object_size)
oids = [] oids = []
curr_epoch = get_epoch(self.shell, self.cluster) curr_epoch = get_epoch(self.shell, self.cluster)
@ -228,7 +231,7 @@ class TestHttpGate(ClusterTestBase):
get_via_http_gate(cid=cid, oid=oid, endpoint=http_endpoint) get_via_http_gate(cid=cid, oid=oid, endpoint=http_endpoint)
@allure.title("Test Zip in HTTP header") @allure.title("Test Zip in HTTP header")
def test_zip_in_http(self): def test_zip_in_http(self, complex_object_size, simple_object_size):
cid = create_container( cid = create_container(
self.wallet, self.wallet,
shell=self.shell, shell=self.shell,
@ -236,7 +239,9 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_2, rule=self.PLACEMENT_RULE_2,
basic_acl=PUBLIC_ACL, basic_acl=PUBLIC_ACL,
) )
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
common_prefix = "my_files" common_prefix = "my_files"
headers1 = {"X-Attribute-FilePath": f"{common_prefix}/file1"} headers1 = {"X-Attribute-FilePath": f"{common_prefix}/file1"}
@ -267,7 +272,7 @@ class TestHttpGate(ClusterTestBase):
@pytest.mark.long @pytest.mark.long
@allure.title("Test Put over HTTP/Curl, Get over HTTP/Curl for large object") @allure.title("Test Put over HTTP/Curl, Get over HTTP/Curl for large object")
def test_put_http_get_http_large_file(self): def test_put_http_get_http_large_file(self, complex_object_size):
""" """
This test checks upload and download using curl with 'large' object. This test checks upload and download using curl with 'large' object.
Large is object with size up to 20Mb. Large is object with size up to 20Mb.
@ -280,7 +285,7 @@ class TestHttpGate(ClusterTestBase):
basic_acl=PUBLIC_ACL, basic_acl=PUBLIC_ACL,
) )
obj_size = int(os.getenv("BIG_OBJ_SIZE", COMPLEX_OBJ_SIZE)) obj_size = int(os.getenv("BIG_OBJ_SIZE", complex_object_size))
file_path = generate_file(obj_size) file_path = generate_file(obj_size)
with allure.step("Put objects using HTTP"): with allure.step("Put objects using HTTP"):
@ -304,7 +309,7 @@ class TestHttpGate(ClusterTestBase):
) )
@allure.title("Test Put/Get over HTTP using Curl utility") @allure.title("Test Put/Get over HTTP using Curl utility")
def test_put_http_get_http_curl(self): def test_put_http_get_http_curl(self, complex_object_size, simple_object_size):
""" """
Test checks upload and download over HTTP using curl utility. Test checks upload and download over HTTP using curl utility.
""" """
@ -315,7 +320,9 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_2, rule=self.PLACEMENT_RULE_2,
basic_acl=PUBLIC_ACL, basic_acl=PUBLIC_ACL,
) )
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE) file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
with allure.step("Put objects using curl utility"): with allure.step("Put objects using curl utility"):
oid_simple = upload_via_http_gate_curl( oid_simple = upload_via_http_gate_curl(

4
pytest_tests/testsuites/session_token/test_object_session_token.py
View file

@ -3,7 +3,7 @@ import random
import allure import allure
import pytest import pytest
from cluster_test_base import ClusterTestBase from cluster_test_base import ClusterTestBase
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE, WALLET_PASS from common import WALLET_PASS
from file_helper import generate_file from file_helper import generate_file
from grpc_responses import SESSION_NOT_FOUND from grpc_responses import SESSION_NOT_FOUND
from neofs_testlib.utils.wallet import get_last_address_from_wallet from neofs_testlib.utils.wallet import get_last_address_from_wallet
@ -19,7 +19,7 @@ class TestDynamicObjectSession(ClusterTestBase):
@allure.title("Test Object Operations with Session Token") @allure.title("Test Object Operations with Session Token")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"object_size", "object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], [pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"], ids=["simple object", "complex object"],
) )
def test_object_session_token(self, default_wallet, object_size): def test_object_session_token(self, default_wallet, object_size):

48
pytest_tests/testsuites/session_token/test_static_object_session_token.py
View file

@ -4,7 +4,6 @@ import allure
import pytest import pytest
from cluster import Cluster from cluster import Cluster
from cluster_test_base import ClusterTestBase from cluster_test_base import ClusterTestBase
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
from epoch import ensure_fresh_epoch from epoch import ensure_fresh_epoch
from file_helper import generate_file from file_helper import generate_file
from grpc_responses import MALFORMED_REQUEST, OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND from grpc_responses import MALFORMED_REQUEST, OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
@ -13,7 +12,6 @@ from pytest import FixtureRequest
from python_keywords.container import create_container from python_keywords.container import create_container
from python_keywords.neofs_verbs import ( from python_keywords.neofs_verbs import (
delete_object, delete_object,
get_netmap_netinfo,
get_object, get_object,
get_object_from_random_node, get_object_from_random_node,
get_range, get_range,
@ -58,7 +56,7 @@ def storage_containers(
@pytest.fixture( @pytest.fixture(
params=[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], params=[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"], ids=["simple object", "complex object"],
# Scope module to upload/delete each files set only once # Scope module to upload/delete each files set only once
scope="module", scope="module",
@ -98,16 +96,15 @@ def storage_objects(
@allure.step("Get ranges for test") @allure.step("Get ranges for test")
def get_ranges(storage_object: StorageObjectInfo, shell: Shell, endpoint: str) -> list[str]: def get_ranges(
storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, endpoint: str
) -> list[str]:
""" """
Returns ranges to test range/hash methods via static session Returns ranges to test range/hash methods via static session
""" """
object_size = storage_object.size object_size = storage_object.size
if object_size == COMPLEX_OBJ_SIZE: if object_size > max_object_size:
net_info = get_netmap_netinfo(storage_object.wallet_file_path, shell, endpoint)
max_object_size = net_info["maximum_object_size"]
# make sure to test multiple parts of complex object
assert object_size >= max_object_size + RANGE_OFFSET_FOR_COMPLEX_OBJECT assert object_size >= max_object_size + RANGE_OFFSET_FOR_COMPLEX_OBJECT
return [ return [
"0:10", "0:10",
@ -160,9 +157,9 @@ class TestObjectStaticSession(ClusterTestBase):
self, self,
user_wallet: WalletFile, user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo], storage_objects: list[StorageObjectInfo],
static_sessions: list[str], static_sessions: dict[ObjectVerb, str],
method_under_test, method_under_test,
verb: str, verb: ObjectVerb,
request: FixtureRequest, request: FixtureRequest,
): ):
""" """
@ -175,9 +172,9 @@ class TestObjectStaticSession(ClusterTestBase):
for node in self.cluster.storage_nodes: for node in self.cluster.storage_nodes:
for storage_object in storage_objects[0:2]: for storage_object in storage_objects[0:2]:
method_under_test( method_under_test(
user_wallet.path, wallet=user_wallet.path,
storage_object.cid, cid=storage_object.cid,
storage_object.oid, oid=storage_object.oid,
shell=self.shell, shell=self.shell,
endpoint=node.get_rpc_endpoint(), endpoint=node.get_rpc_endpoint(),
session=static_sessions[verb], session=static_sessions[verb],
@ -193,10 +190,11 @@ class TestObjectStaticSession(ClusterTestBase):
self, self,
user_wallet: WalletFile, user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo], storage_objects: list[StorageObjectInfo],
static_sessions: list[str], static_sessions: dict[ObjectVerb, str],
method_under_test, method_under_test,
verb: str, verb: ObjectVerb,
request: FixtureRequest, request: FixtureRequest,
max_object_size,
): ):
""" """
Validate static session with range operations Validate static session with range operations
@ -205,7 +203,9 @@ class TestObjectStaticSession(ClusterTestBase):
f"Validate static session with range operations for {request.node.callspec.id}" f"Validate static session with range operations for {request.node.callspec.id}"
) )
storage_object = storage_objects[0] storage_object = storage_objects[0]
ranges_to_test = get_ranges(storage_object, self.shell, self.cluster.default_rpc_endpoint) ranges_to_test = get_ranges(
storage_object, max_object_size, self.shell, self.cluster.default_rpc_endpoint
)
for range_to_test in ranges_to_test: for range_to_test in ranges_to_test:
with allure.step(f"Check range {range_to_test}"): with allure.step(f"Check range {range_to_test}"):
@ -227,7 +227,7 @@ class TestObjectStaticSession(ClusterTestBase):
self, self,
user_wallet: WalletFile, user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo], storage_objects: list[StorageObjectInfo],
static_sessions: list[str], static_sessions: dict[ObjectVerb, str],
request: FixtureRequest, request: FixtureRequest,
): ):
""" """
@ -253,7 +253,7 @@ class TestObjectStaticSession(ClusterTestBase):
self, self,
user_wallet: WalletFile, user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo], storage_objects: list[StorageObjectInfo],
static_sessions: list[str], static_sessions: dict[ObjectVerb, str],
request: FixtureRequest, request: FixtureRequest,
): ):
""" """
@ -278,7 +278,7 @@ class TestObjectStaticSession(ClusterTestBase):
self, self,
stranger_wallet: WalletFile, stranger_wallet: WalletFile,
storage_objects: list[StorageObjectInfo], storage_objects: list[StorageObjectInfo],
static_sessions: list[str], static_sessions: dict[ObjectVerb, str],
request: FixtureRequest, request: FixtureRequest,
): ):
""" """
@ -305,7 +305,7 @@ class TestObjectStaticSession(ClusterTestBase):
self, self,
user_wallet: WalletFile, user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo], storage_objects: list[StorageObjectInfo],
static_sessions: list[str], static_sessions: dict[ObjectVerb, str],
request: FixtureRequest, request: FixtureRequest,
): ):
""" """
@ -333,7 +333,7 @@ class TestObjectStaticSession(ClusterTestBase):
user_wallet: WalletFile, user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo], storage_objects: list[StorageObjectInfo],
storage_containers: list[str], storage_containers: list[str],
static_sessions: list[str], static_sessions: dict[ObjectVerb, str],
request: FixtureRequest, request: FixtureRequest,
): ):
""" """
@ -361,7 +361,7 @@ class TestObjectStaticSession(ClusterTestBase):
owner_wallet: WalletFile, owner_wallet: WalletFile,
user_wallet: WalletFile, user_wallet: WalletFile,
stranger_wallet: WalletFile, stranger_wallet: WalletFile,
storage_containers: list[int], storage_containers: list[str],
storage_objects: list[StorageObjectInfo], storage_objects: list[StorageObjectInfo],
temp_directory: str, temp_directory: str,
request: FixtureRequest, request: FixtureRequest,
@ -638,7 +638,7 @@ class TestObjectStaticSession(ClusterTestBase):
self, self,
user_wallet: WalletFile, user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo], storage_objects: list[StorageObjectInfo],
static_sessions: list[str], static_sessions: dict[ObjectVerb, str],
request: FixtureRequest, request: FixtureRequest,
): ):
""" """
@ -663,7 +663,7 @@ class TestObjectStaticSession(ClusterTestBase):
self, self,
user_wallet: WalletFile, user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo], storage_objects: list[StorageObjectInfo],
static_sessions: list[str], static_sessions: dict[ObjectVerb, str],
request: FixtureRequest, request: FixtureRequest,
): ):
""" """

3
pytest_tests/testsuites/session_token/test_static_session_token_container.py
View file

@ -148,6 +148,7 @@ class TestSessionTokenContainer(ClusterTestBase):
user_wallet: WalletFile, user_wallet: WalletFile,
stranger_wallet: WalletFile, stranger_wallet: WalletFile,
static_sessions: dict[ContainerVerb, str], static_sessions: dict[ContainerVerb, str],
simple_object_size,
): ):
""" """
Validate static session with set eacl operation Validate static session with set eacl operation
@ -159,7 +160,7 @@ class TestSessionTokenContainer(ClusterTestBase):
shell=self.shell, shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint, endpoint=self.cluster.default_rpc_endpoint,
) )
file_path = generate_file() file_path = generate_file(simple_object_size)
assert can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster) assert can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster)
with allure.step(f"Deny all operations for other via eACL"): with allure.step(f"Deny all operations for other via eACL"):

1
requirements.txt
View file

@ -55,6 +55,7 @@ pyflakes==2.4.0
pyparsing==3.0.9 pyparsing==3.0.9
pyrsistent==0.18.1 pyrsistent==0.18.1
pytest==7.1.2 pytest==7.1.2
pytest-lazy-fixture==0.6.3
python-dateutil==2.8.2 python-dateutil==2.8.2
pyyaml==6.0 pyyaml==6.0
requests==2.28.0 requests==2.28.0

10
robot/resources/lib/python_keywords/storage_group.py
View file

@ -7,7 +7,7 @@ from typing import Optional
import allure import allure
from cluster import Cluster from cluster import Cluster
from common import COMPLEX_OBJ_SIZE, NEOFS_CLI_EXEC, SIMPLE_OBJ_SIZE, WALLET_CONFIG from common import NEOFS_CLI_EXEC, WALLET_CONFIG
from complex_object_actions import get_link_object from complex_object_actions import get_link_object
from neofs_testlib.cli import NeofsCli from neofs_testlib.cli import NeofsCli
from neofs_testlib.shell import Shell from neofs_testlib.shell import Shell
@ -201,12 +201,13 @@ def verify_get_storage_group(
gid: str, gid: str,
obj_list: list, obj_list: list,
object_size: int, object_size: int,
max_object_size: int,
bearer: str = None, bearer: str = None,
wallet_config: str = WALLET_CONFIG, wallet_config: str = WALLET_CONFIG,
): ):
obj_parts = [] obj_parts = []
endpoint = cluster.default_rpc_endpoint endpoint = cluster.default_rpc_endpoint
if object_size == COMPLEX_OBJ_SIZE: if object_size > max_object_size:
for obj in obj_list: for obj in obj_list:
link_oid = get_link_object( link_oid = get_link_object(
wallet, wallet,
@ -239,11 +240,10 @@ def verify_get_storage_group(
bearer=bearer, bearer=bearer,
wallet_config=wallet_config, wallet_config=wallet_config,
) )
if object_size == SIMPLE_OBJ_SIZE: exp_size = object_size * obj_num
exp_size = SIMPLE_OBJ_SIZE * obj_num if object_size < max_object_size:
assert int(storagegroup_data["Group size"]) == exp_size assert int(storagegroup_data["Group size"]) == exp_size
assert storagegroup_data["Members"] == obj_list assert storagegroup_data["Members"] == obj_list
else: else:
exp_size = COMPLEX_OBJ_SIZE * obj_num
assert int(storagegroup_data["Group size"]) == exp_size assert int(storagegroup_data["Group size"]) == exp_size
assert storagegroup_data["Members"] == obj_parts assert storagegroup_data["Members"] == obj_parts

6
robot/variables/common.py
View file

@ -4,9 +4,9 @@ import yaml
CONTAINER_WAIT_INTERVAL = "1m" CONTAINER_WAIT_INTERVAL = "1m"
# TODO: Get object size data from a node config SIMPLE_OBJECT_SIZE = os.getenv("SIMPLE_OBJECT_SIZE", "1000")
SIMPLE_OBJ_SIZE = int(os.getenv("SIMPLE_OBJ_SIZE", "1000")) COMPLEX_OBJECT_CHUNKS_COUNT = os.getenv("COMPLEX_OBJECT_CHUNKS_COUNT", "3")
COMPLEX_OBJ_SIZE = int(os.getenv("COMPLEX_OBJ_SIZE", "2000")) COMPLEX_OBJECT_TAIL_SIZE = os.getenv("COMPLEX_OBJECT_TAIL_SIZE", "1000")
MAINNET_BLOCK_TIME = os.getenv("MAINNET_BLOCK_TIME", "1s") MAINNET_BLOCK_TIME = os.getenv("MAINNET_BLOCK_TIME", "1s")
MAINNET_TIMEOUT = os.getenv("MAINNET_TIMEOUT", "1min") MAINNET_TIMEOUT = os.getenv("MAINNET_TIMEOUT", "1min")