TrueCloudLab
/
frostfs-testcases
22
0
Fork 18
Code Issues 2 Pull Requests 3 Actions 1 Releases Activity

Remove SIMPLE_OBJ_SIZE and COMPLEX_OBJ_SIZE from env 

Signed-off-by: Vladimir Avdeev <v.avdeev@yadro.com>
pull/1/head
Vladimir Avdeev 2022-12-07 15:38:56 +03:00 committed by Vladimir Avdeev
parent 76c5d40e63
commit 05924784ab
26 changed files with 322 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pytest_tests/helpers/file_helper.py
View File

@ -5,12 +5,12 @@ import uuid
from typing import Any, Optional
import allure
from common import ASSETS_DIR, SIMPLE_OBJ_SIZE
from common import ASSETS_DIR
logger = logging.getLogger("NeoLogger")
def generate_file(size: int = SIMPLE_OBJ_SIZE) -> str:
def generate_file(size: int) -> str:
"""Generates a binary file with the specified size in bytes.
Args:
@ -28,6 +28,7 @@ def generate_file(size: int = SIMPLE_OBJ_SIZE) -> str:
def generate_file_with_content(
size: int,
file_path: Optional[str] = None,
content: Optional[str] = None,
) -> str:
@ -44,7 +45,7 @@ def generate_file_with_content(
"""
mode = "w+"
if content is None:
content = os.urandom(SIMPLE_OBJ_SIZE)
content = os.urandom(size)
mode = "wb"
if not file_path:

4
pytest_tests/testsuites/acl/conftest.py
View File

@ -68,8 +68,8 @@ def wallets(default_wallet, temp_directory, cluster: Cluster) -> Wallets:
@pytest.fixture(scope="module")
def file_path():
yield generate_file()
def file_path(simple_object_size):
yield generate_file(simple_object_size)
@pytest.fixture(scope="function")

26
pytest_tests/testsuites/acl/storage_group/test_storagegroup.py
View File

@ -6,7 +6,7 @@ from typing import Optional
import allure
import pytest
from cluster_test_base import ClusterTestBase
from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, FREE_STORAGE, SIMPLE_OBJ_SIZE, WALLET_PASS
from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from file_helper import generate_file
from grpc_responses import OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
from neofs_testlib.utils.wallet import init_wallet
@ -37,7 +37,7 @@ deposit = 30
@pytest.mark.parametrize(
"object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE],
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
)
@pytest.mark.sanity
@ -68,7 +68,7 @@ class TestStorageGroup(ClusterTestBase):
)
@allure.title("Test Storage Group in Private Container")
def test_storagegroup_basic_private_container(self, object_size):
def test_storagegroup_basic_private_container(self, object_size, max_object_size):
cid = create_container(
self.main_wallet, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint
)
@ -88,6 +88,7 @@ class TestStorageGroup(ClusterTestBase):
cid=cid,
obj_list=objects,
object_size=object_size,
max_object_size=max_object_size,
)
self.expect_failure_for_storagegroup_operations(
wallet=self.other_wallet,
@ -103,7 +104,7 @@ class TestStorageGroup(ClusterTestBase):
)
@allure.title("Test Storage Group in Public Container")
def test_storagegroup_basic_public_container(self, object_size):
def test_storagegroup_basic_public_container(self, object_size, max_object_size):
cid = create_container(
self.main_wallet,
basic_acl="public-read-write",
@ -120,12 +121,14 @@ class TestStorageGroup(ClusterTestBase):
cid=cid,
obj_list=objects,
object_size=object_size,
max_object_size=max_object_size,
)
self.expect_success_for_storagegroup_operations(
wallet=self.other_wallet,
cid=cid,
obj_list=objects,
object_size=object_size,
max_object_size=max_object_size,
)
self.storagegroup_operations_by_system_ro_container(
wallet=self.main_wallet,
@ -135,7 +138,7 @@ class TestStorageGroup(ClusterTestBase):
)
@allure.title("Test Storage Group in Read-Only Container")
def test_storagegroup_basic_ro_container(self, object_size):
def test_storagegroup_basic_ro_container(self, object_size, max_object_size):
cid = create_container(
self.main_wallet,
basic_acl="public-read",
@ -152,6 +155,7 @@ class TestStorageGroup(ClusterTestBase):
cid=cid,
obj_list=objects,
object_size=object_size,
max_object_size=max_object_size,
)
self.storagegroup_operations_by_other_ro_container(
owner_wallet=self.main_wallet,
@ -168,7 +172,7 @@ class TestStorageGroup(ClusterTestBase):
)
@allure.title("Test Storage Group with Bearer Allow")
def test_storagegroup_bearer_allow(self, object_size):
def test_storagegroup_bearer_allow(self, object_size, max_object_size):
cid = create_container(
self.main_wallet,
basic_acl="eacl-public-read-write",
@ -185,6 +189,7 @@ class TestStorageGroup(ClusterTestBase):
cid=cid,
obj_list=objects,
object_size=object_size,
max_object_size=max_object_size,
)
storage_group = put_storagegroup(
self.shell, self.cluster.default_rpc_endpoint, self.main_wallet, cid, objects
@ -219,6 +224,7 @@ class TestStorageGroup(ClusterTestBase):
cid=cid,
obj_list=objects,
object_size=object_size,
max_object_size=max_object_size,
bearer=bearer_file,
)
@ -259,6 +265,7 @@ class TestStorageGroup(ClusterTestBase):
cid: str,
obj_list: list,
object_size: int,
max_object_size: int,
bearer: Optional[str] = None,
):
"""
@ -285,6 +292,7 @@ class TestStorageGroup(ClusterTestBase):
gid=storage_group,
obj_list=obj_list,
object_size=object_size,
max_object_size=max_object_size,
bearer=bearer,
)
delete_storagegroup(
@ -381,7 +389,11 @@ class TestStorageGroup(ClusterTestBase):
@allure.step("Run Storage Group Operations On Systems's Behalf In RO Container")
def storagegroup_operations_by_system_ro_container(
self, wallet: str, cid: str, obj_list: list, object_size: int
self,
wallet: str,
cid: str,
obj_list: list,
object_size: int,
):
"""
In this func we create a Storage Group on Inner Ring's key behalf

27
pytest_tests/testsuites/conftest.py
View File

@ -16,11 +16,14 @@ from common import (
BACKGROUND_OBJ_SIZE,
BACKGROUND_READERS_COUNT,
BACKGROUND_WRITERS_COUNT,
COMPLEX_OBJECT_CHUNKS_COUNT,
COMPLEX_OBJECT_TAIL_SIZE,
FREE_STORAGE,
HOSTING_CONFIG_FILE,
LOAD_NODE_SSH_PRIVATE_KEY_PATH,
LOAD_NODE_SSH_USER,
LOAD_NODES,
SIMPLE_OBJECT_SIZE,
STORAGE_NODE_SERVICE_NAME_REGEX,
WALLET_PASS,
)
@ -32,6 +35,8 @@ from neofs_testlib.reporter import AllureHandler, get_reporter
from neofs_testlib.shell import LocalShell, Shell
from neofs_testlib.utils.wallet import init_wallet
from payment_neogo import deposit_gas, transfer_gas
from pytest import FixtureRequest
from python_keywords.neofs_verbs import get_netmap_netinfo
from python_keywords.node_management import storage_node_healthcheck
from helpers.wallet import WalletFactory
@ -81,6 +86,28 @@ def require_multiple_hosts(hosting: Hosting):
yield
@pytest.fixture(scope="session")
def max_object_size(cluster: Cluster, client_shell: Shell) -> int:
storage_node = cluster.storage_nodes[0]
net_info = get_netmap_netinfo(
wallet=storage_node.get_wallet_path(),
wallet_config=storage_node.get_wallet_config_path(),
endpoint=storage_node.get_rpc_endpoint(),
shell=client_shell,
)
yield net_info["maximum_object_size"]
@pytest.fixture(scope="session")
def simple_object_size(max_object_size: int) -> int:
yield int(SIMPLE_OBJECT_SIZE) if int(SIMPLE_OBJECT_SIZE) < max_object_size else max_object_size
@pytest.fixture(scope="session")
def complex_object_size(max_object_size: int) -> int:
return max_object_size * int(COMPLEX_OBJECT_CHUNKS_COUNT) + int(COMPLEX_OBJECT_TAIL_SIZE)
@pytest.fixture(scope="session")
def wallet_factory(temp_directory: str, client_shell: Shell, cluster: Cluster) -> WalletFactory:
return WalletFactory(temp_directory, client_shell, cluster)

6
pytest_tests/testsuites/failovers/test_failover_network.py
View File

@ -38,7 +38,9 @@ class TestFailoverNetwork(ClusterTestBase):
wait_all_storage_nodes_returned(self.cluster)
@allure.title("Block Storage node traffic")
def test_block_storage_node_traffic(self, default_wallet, require_multiple_hosts):
def test_block_storage_node_traffic(
self, default_wallet, require_multiple_hosts, simple_object_size
):
"""
Block storage nodes traffic using iptables and wait for replication for objects.
"""
@ -47,7 +49,7 @@ class TestFailoverNetwork(ClusterTestBase):
wakeup_node_timeout = 10 # timeout to let nodes detect that traffic has blocked
nodes_to_block_count = 2
source_file_path = generate_file()
source_file_path = generate_file(simple_object_size)
cid = create_container(
wallet,
shell=self.shell,

14
pytest_tests/testsuites/failovers/test_failover_storage.py
View File

@ -47,14 +47,11 @@ class TestFailoverStorage(ClusterTestBase):
@pytest.mark.parametrize("hard_reboot", [True, False])
@pytest.mark.failover_reboot
def test_lose_storage_node_host(
self,
default_wallet,
hard_reboot: bool,
require_multiple_hosts,
self, default_wallet, hard_reboot: bool, require_multiple_hosts, simple_object_size
):
wallet = default_wallet
placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
source_file_path = generate_file()
source_file_path = generate_file(simple_object_size)
cid = create_container(
wallet,
shell=self.shell,
@ -106,14 +103,11 @@ class TestFailoverStorage(ClusterTestBase):
@pytest.mark.parametrize("sequence", [True, False])
@pytest.mark.failover_panic
def test_panic_storage_node_host(
self,
default_wallet,
require_multiple_hosts,
sequence: bool,
self, default_wallet, require_multiple_hosts, sequence: bool, simple_object_size
):
wallet = default_wallet
placement_rule = "REP 2 IN X CBF 2 SELECT 2 FROM * AS X"
source_file_path = generate_file()
source_file_path = generate_file(simple_object_size)
cid = create_container(
wallet,
shell=self.shell,

42
pytest_tests/testsuites/network/test_node_management.py
View File

@ -7,7 +7,7 @@ import allure
import pytest
from cluster import StorageNode
from cluster_test_base import ClusterTestBase
from common import COMPLEX_OBJ_SIZE, MORPH_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT
from common import MORPH_BLOCK_TIME, NEOFS_CONTRACT_CACHE_TIMEOUT
from epoch import tick_epoch
from file_helper import generate_file
from grpc_responses import OBJECT_NOT_FOUND, error_matches_status
@ -49,9 +49,10 @@ check_nodes: list[StorageNode] = []
class TestNodeManagement(ClusterTestBase):
@pytest.fixture
@allure.title("Create container and pick the node with data")
def create_container_and_pick_node(self, default_wallet: str) -> Tuple[str, StorageNode]:
default_wallet
file_path = generate_file()
def create_container_and_pick_node(
self, default_wallet: str, simple_object_size
) -> Tuple[str, StorageNode]:
file_path = generate_file(simple_object_size)
placement_rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
endpoint = self.cluster.default_rpc_endpoint
@ -126,6 +127,7 @@ class TestNodeManagement(ClusterTestBase):
self,
default_wallet,
return_nodes_after_test_run,
simple_object_size,
):
"""
This test remove one node from cluster then add it back. Test uses base control operations with storage nodes (healthcheck, netmap-snapshot, set-status).
@ -133,7 +135,7 @@ class TestNodeManagement(ClusterTestBase):
wallet = default_wallet
placement_rule_3 = "REP 3 IN X CBF 1 SELECT 3 FROM * AS X"
placement_rule_4 = "REP 4 IN X CBF 1 SELECT 4 FROM * AS X"
source_file_path = generate_file()
source_file_path = generate_file(simple_object_size)
storage_nodes = self.cluster.storage_nodes
random_node = random.choice(storage_nodes[1:])
@ -219,12 +221,14 @@ class TestNodeManagement(ClusterTestBase):
)
@pytest.mark.node_mgmt
@allure.title("Test object copies based on placement policy")
def test_placement_policy(self, default_wallet, placement_rule, expected_copies):
def test_placement_policy(
self, default_wallet, placement_rule, expected_copies, simple_object_size
):
"""
This test checks object's copies based on container's placement policy.
"""
wallet = default_wallet
file_path = generate_file()
file_path = generate_file(simple_object_size)
self.validate_object_copies(wallet, placement_rule, file_path, expected_copies)
@pytest.mark.parametrize(
@ -279,14 +283,19 @@ class TestNodeManagement(ClusterTestBase):
@pytest.mark.node_mgmt
@allure.title("Test object copies and storage nodes based on placement policy")
def test_placement_policy_with_nodes(
self, default_wallet, placement_rule, expected_copies, expected_nodes_id: set[int]
self,
default_wallet,
placement_rule,
expected_copies,
expected_nodes_id: set[int],
simple_object_size,
):
"""
Based on container's placement policy check that storage nodes are piked correctly and object has
correct copies amount.
"""
wallet = default_wallet
file_path = generate_file()
file_path = generate_file(simple_object_size)
cid, oid, found_nodes = self.validate_object_copies(
wallet, placement_rule, file_path, expected_copies
)
@ -303,24 +312,28 @@ class TestNodeManagement(ClusterTestBase):
)
@pytest.mark.node_mgmt
@allure.title("Negative cases for placement policy")
def test_placement_policy_negative(self, default_wallet, placement_rule, expected_copies):
def test_placement_policy_negative(
self, default_wallet, placement_rule, expected_copies, simple_object_size
):
"""
Negative test for placement policy.
"""
wallet = default_wallet
file_path = generate_file()
file_path = generate_file(simple_object_size)
with pytest.raises(RuntimeError, match=".*not enough nodes to SELECT from.*"):
self.validate_object_copies(wallet, placement_rule, file_path, expected_copies)
@pytest.mark.node_mgmt
@allure.title("NeoFS object could be dropped using control command")
def test_drop_object(self, default_wallet):
def test_drop_object(self, default_wallet, complex_object_size, simple_object_size):
"""
Test checks object could be dropped using `neofs-cli control drop-objects` command.
"""
wallet = default_wallet
endpoint = self.cluster.default_rpc_endpoint
file_path_simple, file_path_complex = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
file_path_simple, file_path_complex = generate_file(simple_object_size), generate_file(
complex_object_size
)
locode = get_locode_from_random_node(self.cluster)
rule = f"REP 1 CBF 1 SELECT 1 FROM * FILTER 'UN-LOCODE' EQ '{locode}' AS LOC"
@ -358,9 +371,10 @@ class TestNodeManagement(ClusterTestBase):
self,
default_wallet,
create_container_and_pick_node,
simple_object_size,
):
wallet = default_wallet
file_path = generate_file()
file_path = generate_file(simple_object_size)
cid, node = create_container_and_pick_node
original_oid = put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster)

35
pytest_tests/testsuites/object/test_object_api.py
View File

@ -5,14 +5,12 @@ import sys
import allure
import pytest
from cluster import Cluster
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
from container import create_container
from file_helper import generate_file, get_file_content, get_file_hash
from grpc_responses import OUT_OF_RANGE
from neofs_testlib.shell import Shell
from pytest import FixtureRequest
from python_keywords.container import create_container
from python_keywords.neofs_verbs import (
get_netmap_netinfo,
get_object_from_random_node,
get_range,
get_range_hash,
@ -42,10 +40,7 @@ RANGES_COUNT = 4 # by quarters
RANGE_MIN_LEN = 10
RANGE_MAX_LEN = 500
# Used for static ranges found with issues
STATIC_RANGES = {
SIMPLE_OBJ_SIZE: [],
COMPLEX_OBJ_SIZE: [],
}
STATIC_RANGES = {}
def generate_ranges(file_size: int, max_object_size: int) -> list[(int, int)]:
@ -58,10 +53,10 @@ def generate_ranges(file_size: int, max_object_size: int) -> list[(int, int)]:
file_ranges.append((int(file_range_step * i), int(file_range_step * (i + 1))))
# For simple object we can read all file ranges without too much time for testing
if file_size == SIMPLE_OBJ_SIZE:
if file_size < max_object_size:
file_ranges_to_test.extend(file_ranges)
# For complex object we need to fetch multiple child objects from different nodes.
if file_size == COMPLEX_OBJ_SIZE:
else:
assert (
file_size >= RANGE_MAX_LEN + max_object_size
), f"Complex object size should be at least {max_object_size + RANGE_MAX_LEN}. Current: {file_size}"
@ -83,7 +78,7 @@ def generate_ranges(file_size: int, max_object_size: int) -> list[(int, int)]:
@pytest.fixture(
params=[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE],
params=[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
# Scope session to upload/delete each files set only once
scope="module",
@ -132,7 +127,7 @@ def storage_objects(
class TestObjectApi(ClusterTestBase):
@allure.title("Validate object storage policy by native API")
def test_object_storage_policies(
self, request: FixtureRequest, storage_objects: list[StorageObjectInfo]
self, request: FixtureRequest, storage_objects: list[StorageObjectInfo], simple_object_size
):
"""
Validate object storage policy
@ -143,7 +138,7 @@ class TestObjectApi(ClusterTestBase):
with allure.step("Validate storage policy for objects"):
for storage_object in storage_objects:
if storage_object.size == SIMPLE_OBJ_SIZE:
if storage_object.size == simple_object_size:
copies = get_simple_object_copies(
storage_object.wallet_file_path,
storage_object.cid,
@ -257,7 +252,9 @@ class TestObjectApi(ClusterTestBase):
@allure.title("Validate object search with removed items")
@pytest.mark.parametrize(
"object_size", [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], ids=["simple object", "complex object"]
"object_size",
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
)
def test_object_search_should_return_tombstone_items(
self, default_wallet: str, request: FixtureRequest, object_size: int
@ -330,7 +327,7 @@ class TestObjectApi(ClusterTestBase):
@pytest.mark.sanity
@pytest.mark.grpc_api
def test_object_get_range_hash(
self, request: FixtureRequest, storage_objects: list[StorageObjectInfo]
self, request: FixtureRequest, storage_objects: list[StorageObjectInfo], max_object_size
):
"""
Validate get_range_hash for object by common gRPC API
@ -343,10 +340,6 @@ class TestObjectApi(ClusterTestBase):
cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path
net_info = get_netmap_netinfo(
wallet, self.shell, endpoint=self.cluster.default_rpc_endpoint
)
max_object_size = net_info["maximum_object_size"]
file_ranges_to_test = generate_ranges(storage_objects[0].size, max_object_size)
logging.info(f"Ranges used in test {file_ranges_to_test}")
@ -372,7 +365,7 @@ class TestObjectApi(ClusterTestBase):
@pytest.mark.sanity
@pytest.mark.grpc_api
def test_object_get_range(
self, request: FixtureRequest, storage_objects: list[StorageObjectInfo]
self, request: FixtureRequest, storage_objects: list[StorageObjectInfo], max_object_size
):
"""
Validate get_range for object by common gRPC API
@ -383,10 +376,6 @@ class TestObjectApi(ClusterTestBase):
cid = storage_objects[0].cid
oids = [storage_object.oid for storage_object in storage_objects[:2]]
file_path = storage_objects[0].file_path
net_info = get_netmap_netinfo(
wallet, self.shell, endpoint=self.cluster.default_rpc_endpoint
)
max_object_size = net_info["maximum_object_size"]
file_ranges_to_test = generate_ranges(storage_objects[0].size, max_object_size)
logging.info(f"Ranges used in test {file_ranges_to_test}")

11
pytest_tests/testsuites/object/test_object_api_bearer.py
View File

@ -1,7 +1,6 @@
import allure
import pytest
from cluster import Cluster
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
from container import REP_2_FOR_3_NODES_PLACEMENT_RULE, SINGLE_PLACEMENT_RULE, create_container
from epoch import get_epoch
from neofs_testlib.shell import Shell
@ -88,7 +87,7 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
)
@pytest.mark.parametrize(
"storage_objects",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE],
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
indirect=True,
)
@ -99,7 +98,8 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
request: FixtureRequest,
):
allure.dynamic.title(
f"Check that objects can be deleted from any node using s3gate wallet with bearer token for {request.node.callspec.id}"
f"Check that objects can be deleted from any node using s3gate wallet with bearer "
f"token for {request.node.callspec.id}"
)
s3_gate_wallet = self.cluster.s3gates[0]
@ -124,7 +124,7 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
)
@pytest.mark.parametrize(
"file_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE],
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
)
def test_get_object_with_s3_wallet_bearer_from_all_nodes(
@ -135,7 +135,8 @@ class TestObjectApiWithBearerToken(ClusterTestBase):
request: FixtureRequest,
):
allure.dynamic.title(
f"Check that objects can be deleted from any node using s3gate wallet with bearer token for {request.node.callspec.id}"
"Check that objects can be deleted from any node using s3gate wallet with bearer "
f"token for {request.node.callspec.id}"
)
s3_gate_wallet = self.cluster.s3gates[0]

7
pytest_tests/testsuites/object/test_object_lifetime.py
View File

@ -2,12 +2,11 @@ import logging
import allure
import pytest
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
from container import create_container
from epoch import get_epoch, tick_epoch
from file_helper import generate_file, get_file_hash
from grpc_responses import OBJECT_NOT_FOUND
from pytest import FixtureRequest
from python_keywords.container import create_container
from python_keywords.neofs_verbs import get_object_from_random_node, put_object_to_random_node
from utility import wait_for_gc_pass_on_storage_nodes
@ -21,7 +20,9 @@ logger = logging.getLogger("NeoLogger")
class ObjectApiLifetimeTest(ClusterTestBase):
@allure.title("Test object life time")
@pytest.mark.parametrize(
"object_size", [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], ids=["simple object", "complex object"]
"object_size",
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
)
def test_object_api_lifetime(
self, default_wallet: str, request: FixtureRequest, object_size: int

34
pytest_tests/testsuites/object/test_object_lock.py
View File

@ -5,9 +5,8 @@ import allure
import pytest
from cluster import Cluster
from cluster_test_base import ClusterTestBase
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE, STORAGE_GC_TIME
from common import STORAGE_GC_TIME
from complex_object_actions import get_link_object
from container import create_container
from epoch import ensure_fresh_epoch, get_epoch, tick_epoch
from grpc_responses import (
LIFETIME_REQUIRED,
@ -20,6 +19,7 @@ from grpc_responses import (
)
from neofs_testlib.shell import Shell
from pytest import FixtureRequest
from python_keywords.container import create_container
from python_keywords.neofs_verbs import delete_object, head_object, lock_object
from test_control import expect_not_raises, wait_for_success
from utility import parse_time, wait_for_gc_pass_on_storage_nodes
@ -143,7 +143,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Locked object should be protected from deletion")
@pytest.mark.parametrize(
"locked_storage_object",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE],
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
indirect=True,
)
@ -170,7 +170,9 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Lock object itself should be protected from deletion")
# We operate with only lock object here so no complex object needed in this test
@pytest.mark.parametrize("locked_storage_object", [SIMPLE_OBJ_SIZE], indirect=True)
@pytest.mark.parametrize(
"locked_storage_object", [pytest.lazy_fixture("simple_object_size")], indirect=True
)
def test_lock_object_itself_cannot_be_deleted(
self,
locked_storage_object: StorageObjectInfo,
@ -193,7 +195,9 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Lock object itself cannot be locked")
# We operate with only lock object here so no complex object needed in this test
@pytest.mark.parametrize("locked_storage_object", [SIMPLE_OBJ_SIZE], indirect=True)
@pytest.mark.parametrize(
"locked_storage_object", [pytest.lazy_fixture("simple_object_size")], indirect=True
)
def test_lock_object_cannot_be_locked(
self,
locked_storage_object: StorageObjectInfo,
@ -217,7 +221,9 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Cannot lock object without lifetime and expire_at fields")
# We operate with only lock object here so no complex object needed in this test
@pytest.mark.parametrize("locked_storage_object", [SIMPLE_OBJ_SIZE], indirect=True)
@pytest.mark.parametrize(
"locked_storage_object", [pytest.lazy_fixture("simple_object_size")], indirect=True
)
@pytest.mark.parametrize(
"wrong_lifetime,wrong_expire_at,expected_error",
[
@ -259,7 +265,9 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Expired object should be deleted after locks are expired")
@pytest.mark.parametrize(
"object_size", [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE], ids=["simple object", "complex object"]
"object_size",
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
)
def test_expired_object_should_be_deleted_after_locks_are_expired(
self,
@ -327,7 +335,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Should be possible to lock multiple objects at once")
@pytest.mark.parametrize(
"object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE],
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
)
def test_should_be_possible_to_lock_multiple_objects_at_once(
@ -382,7 +390,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("Already outdated lock should not be applied")
@pytest.mark.parametrize(
"object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE],
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
)
def test_already_outdated_lock_should_not_be_applied(
@ -421,7 +429,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("After lock expiration with lifetime user should be able to delete object")
@pytest.mark.parametrize(
"object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE],
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
)
@expect_not_raises()
@ -463,7 +471,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@allure.title("After lock expiration with expire_at user should be able to delete object")
@pytest.mark.parametrize(
"object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE],
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
)
@expect_not_raises()
@ -508,7 +516,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@pytest.mark.parametrize(
# Only complex objects are required for this test
"locked_storage_object",
[COMPLEX_OBJ_SIZE],
[pytest.lazy_fixture("complex_object_size")],
indirect=True,
)
def test_complex_object_chunks_should_also_be_protected_from_deletion(
@ -535,7 +543,7 @@ class TestObjectLockWithGrpc(ClusterTestBase):
@pytest.mark.parametrize(
# Only complex objects are required for this test
"locked_storage_object",
[COMPLEX_OBJ_SIZE],
[pytest.lazy_fixture("complex_object_size")],
indirect=True,
)
def test_link_object_of_complex_object_should_also_be_protected_from_deletion(

4
pytest_tests/testsuites/services/s3_gate/test_s3_ACL.py
View File

@ -17,8 +17,8 @@ def pytest_generate_tests(metafunc):
@pytest.mark.s3_gate
class TestS3GateACL(TestS3GateBase):
@allure.title("Test S3: Object ACL")
def test_s3_object_ACL(self, bucket):
file_path = generate_file()
def test_s3_object_ACL(self, bucket, simple_object_size):
file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path)
with allure.step("Put object into bucket, Check ACL is empty"):

10
pytest_tests/testsuites/services/s3_gate/test_s3_bucket.py
View File

@ -103,8 +103,8 @@ class TestS3GateBucket(TestS3GateBase):
], "Permission for CanonicalUser is FULL_CONTROL"
@allure.title("Test S3: create bucket with object lock")
def test_s3_bucket_object_lock(self):
file_path = generate_file()
def test_s3_bucket_object_lock(self, simple_object_size):
file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path)
with allure.step("Create bucket with --no-object-lock-enabled-for-bucket"):
@ -138,10 +138,10 @@ class TestS3GateBucket(TestS3GateBase):
)
@allure.title("Test S3: delete bucket")
def test_s3_delete_bucket(self):
file_path_1 = generate_file()
def test_s3_delete_bucket(self, simple_object_size):
file_path_1 = generate_file(simple_object_size)
file_name_1 = object_key_from_file_path(file_path_1)
file_path_2 = generate_file()
file_path_2 = generate_file(simple_object_size)
file_name_2 = object_key_from_file_path(file_path_2)
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)

52
pytest_tests/testsuites/services/s3_gate/test_s3_gate.py
View File

@ -5,7 +5,7 @@ from random import choice, choices
import allure
import pytest
from aws_cli_client import AwsCliClient
from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
from common import ASSETS_DIR
from epoch import tick_epoch
from file_helper import (
generate_file,
@ -39,12 +39,12 @@ def pytest_generate_tests(metafunc):
@pytest.mark.s3_gate_base
class TestS3Gate(TestS3GateBase):
@allure.title("Test S3 Bucket API")
def test_s3_buckets(self):
def test_s3_buckets(self, simple_object_size):
"""
Test base S3 Bucket API (Create/List/Head/Delete).
"""
file_path = generate_file()
file_path = generate_file(simple_object_size)
file_name = self.object_key_from_file_path(file_path)
with allure.step("Create buckets"):
@ -109,11 +109,13 @@ class TestS3Gate(TestS3GateBase):
@pytest.mark.parametrize(
"file_type", ["simple", "large"], ids=["Simple object", "Large object"]
)
def test_s3_api_object(self, file_type, two_buckets):
def test_s3_api_object(self, file_type, two_buckets, simple_object_size, complex_object_size):
"""
Test base S3 Object API (Put/Head/List) for simple and large objects.
"""
file_path = generate_file(SIMPLE_OBJ_SIZE if file_type == "simple" else COMPLEX_OBJ_SIZE)
file_path = generate_file(
simple_object_size if file_type == "simple" else complex_object_size
)
file_name = self.object_key_from_file_path(file_path)
bucket_1, bucket_2 = two_buckets
@ -136,7 +138,7 @@ class TestS3Gate(TestS3GateBase):
s3_gate_object.get_object_attributes(self.s3_client, bucket, file_name, *attrs)
@allure.title("Test S3 Sync directory")
def test_s3_sync_dir(self, bucket):
def test_s3_sync_dir(self, bucket, simple_object_size):
"""
Test checks sync directory with AWS CLI utility.
"""
@ -147,8 +149,8 @@ class TestS3Gate(TestS3GateBase):
if not isinstance(self.s3_client, AwsCliClient):
pytest.skip("This test is not supported with boto3 client")
generate_file_with_content(file_path=file_path_1)
generate_file_with_content(file_path=file_path_2)
generate_file_with_content(simple_object_size, file_path=file_path_1)
generate_file_with_content(simple_object_size, file_path=file_path_2)
self.s3_client.sync(bucket_name=bucket, dir_path=os.path.dirname(file_path_1))
@ -166,19 +168,21 @@ class TestS3Gate(TestS3GateBase):
), "Expected hashes are the same"
@allure.title("Test S3 Object versioning")
def test_s3_api_versioning(self, bucket):
def test_s3_api_versioning(self, bucket, simple_object_size):
"""
Test checks basic versioning functionality for S3 bucket.
"""
version_1_content = "Version 1"
version_2_content = "Version 2"
file_name_simple = generate_file_with_content(content=version_1_content)
file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
generate_file_with_content(file_path=file_name_simple, content=version_2_content)
generate_file_with_content(
simple_object_size, file_path=file_name_simple, content=version_2_content
)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
with allure.step("Check bucket shows all versions"):
@ -246,13 +250,15 @@ class TestS3Gate(TestS3GateBase):
@pytest.mark.s3_gate_multipart
@allure.title("Test S3 Object Multipart API")
def test_s3_api_multipart(self, bucket):
def test_s3_api_multipart(self, bucket, simple_object_size):
"""
Test checks S3 Multipart API (Create multipart upload/Abort multipart upload/List multipart upload/
Upload part/List parts/Complete multipart upload).
"""
parts_count = 3
file_name_large = generate_file(SIMPLE_OBJ_SIZE * 1024 * 6 * parts_count) # 5Mb - min part
file_name_large = generate_file(
simple_object_size * 1024 * 6 * parts_count
) # 5Mb - min part
object_key = self.object_key_from_file_path(file_name_large)
part_files = split_file(file_name_large, parts_count)
parts = []
@ -320,7 +326,7 @@ class TestS3Gate(TestS3GateBase):
check_tags_by_bucket(self.s3_client, bucket, [])
@allure.title("Test S3 Object tagging API")
def test_s3_api_object_tagging(self, bucket):
def test_s3_api_object_tagging(self, bucket, simple_object_size):
"""
Test checks S3 Object tagging API (Put tag/Get tag/Update tag).
"""
@ -330,7 +336,7 @@ class TestS3Gate(TestS3GateBase):
("some-key--obj2", "some-value--obj2"),
]
key_value_pair_obj_new = [("some-key-obj-new", "some-value-obj-new")]
file_name_simple = generate_file(SIMPLE_OBJ_SIZE)
file_name_simple = generate_file(simple_object_size)
obj_key = self.object_key_from_file_path(file_name_simple)
s3_gate_bucket.put_bucket_tagging(self.s3_client, bucket, key_value_pair_bucket)
@ -350,7 +356,7 @@ class TestS3Gate(TestS3GateBase):
check_tags_by_object(self.s3_client, bucket, obj_key, [])
@allure.title("Test S3: Delete object & delete objects S3 API")
def test_s3_api_delete(self, two_buckets):
def test_s3_api_delete(self, two_buckets, simple_object_size, complex_object_size):
"""
Check delete_object and delete_objects S3 API operation. From first bucket some objects deleted one by one.
From second bucket some objects deleted all at once.
@ -359,7 +365,7 @@ class TestS3Gate(TestS3GateBase):
max_delete_objects = 17
put_objects = []
file_paths = []
obj_sizes = [SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE]
obj_sizes = [simple_object_size, complex_object_size]
bucket_1, bucket_2 = two_buckets
@ -406,12 +412,14 @@ class TestS3Gate(TestS3GateBase):
try_to_get_objects_and_expect_error(self.s3_client, bucket_2, objects_to_delete_b2)
@allure.title("Test S3: Copy object to the same bucket")
def test_s3_copy_same_bucket(self, bucket):
def test_s3_copy_same_bucket(self, bucket, complex_object_size, simple_object_size):
"""
Test object can be copied to the same bucket.
#TODO: delete after test_s3_copy_object will be merge
"""
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
file_name_simple = self.object_key_from_file_path(file_path_simple)
file_name_large = self.object_key_from_file_path(file_path_large)
bucket_objects = [file_name_simple, file_name_large]
@ -448,12 +456,14 @@ class TestS3Gate(TestS3GateBase):
)
@allure.title("Test S3: Copy object to another bucket")
def test_s3_copy_to_another_bucket(self, two_buckets):
def test_s3_copy_to_another_bucket(self, two_buckets, complex_object_size, simple_object_size):
"""
Test object can be copied to another bucket.
#TODO: delete after test_s3_copy_object will be merge
"""
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
file_name_simple = self.object_key_from_file_path(file_path_simple)
file_name_large = self.object_key_from_file_path(file_path_large)
bucket_1_objects = [file_name_simple, file_name_large]

22
pytest_tests/testsuites/services/s3_gate/test_s3_locking.py
View File

@ -21,8 +21,8 @@ def pytest_generate_tests(metafunc):
@pytest.mark.parametrize("version_id", [None, "second"])
class TestS3GateLocking(TestS3GateBase):
@allure.title("Test S3: Checking the operation of retention period & legal lock on the object")
def test_s3_object_locking(self, version_id):
file_path = generate_file()
def test_s3_object_locking(self, version_id, simple_object_size):
file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path)
retention_period = 2
@ -30,7 +30,7 @@ class TestS3GateLocking(TestS3GateBase):
with allure.step("Put several versions of object into bucket"):
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
file_name_1 = generate_file_with_content(file_path=file_path)
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
check_objects_in_bucket(self.s3_client, bucket, [file_name])
if version_id:
@ -74,8 +74,8 @@ class TestS3GateLocking(TestS3GateBase):
s3_gate_object.delete_object_s3(self.s3_client, bucket, file_name, version_id)
@allure.title("Test S3: Checking the impossibility to change the retention mode COMPLIANCE")
def test_s3_mode_compliance(self, version_id):
file_path = generate_file()
def test_s3_mode_compliance(self, version_id, simple_object_size):
file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path)
retention_period = 2
retention_period_1 = 1
@ -115,8 +115,8 @@ class TestS3GateLocking(TestS3GateBase):
)
@allure.title("Test S3: Checking the ability to change retention mode GOVERNANCE")
def test_s3_mode_governance(self, version_id):
file_path = generate_file()
def test_s3_mode_governance(self, version_id, simple_object_size):
file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path)
retention_period = 3
retention_period_1 = 2
@ -183,8 +183,8 @@ class TestS3GateLocking(TestS3GateBase):
)
@allure.title("Test S3: Checking if an Object Cannot Be Locked")
def test_s3_legal_hold(self, version_id):
file_path = generate_file()
def test_s3_legal_hold(self, version_id, simple_object_size):
file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path)
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, False)
@ -205,8 +205,8 @@ class TestS3GateLocking(TestS3GateBase):
@pytest.mark.s3_gate
class TestS3GateLockingBucket(TestS3GateBase):
@allure.title("Test S3: Bucket Lock")
def test_s3_bucket_lock(self):
file_path = generate_file()
def test_s3_bucket_lock(self, simple_object_size):
file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path)
configuration = {"Rule": {"DefaultRetention": {"Mode": "COMPLIANCE", "Days": 1}}}

133
pytest_tests/testsuites/services/s3_gate/test_s3_object.py
View File

@ -7,7 +7,7 @@ from random import choices, sample
import allure
import pytest
from aws_cli_client import AwsCliClient
from common import ASSETS_DIR, COMPLEX_OBJ_SIZE, FREE_STORAGE, SIMPLE_OBJ_SIZE, WALLET_PASS
from common import ASSETS_DIR, FREE_STORAGE, WALLET_PASS
from data_formatters import get_wallet_public_key
from file_helper import concat_files, generate_file, generate_file_with_content, get_file_hash
from neofs_testlib.utils.wallet import init_wallet
@ -32,8 +32,8 @@ class TestS3GateObject(TestS3GateBase):
return os.path.basename(full_path)
@allure.title("Test S3: Copy object")
def test_s3_copy_object(self, two_buckets):
file_path = generate_file()
def test_s3_copy_object(self, two_buckets, simple_object_size):
file_path = generate_file(simple_object_size)
file_name = self.object_key_from_file_path(file_path)
bucket_1_objects = [file_name]
@ -81,7 +81,7 @@ class TestS3GateObject(TestS3GateBase):
@allure.title("Test S3: Copy version of object")
def test_s3_copy_version_object(self, two_buckets):
version_1_content = "Version 1"
file_name_simple = generate_file_with_content(content=version_1_content)
file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
bucket_1, bucket_2 = two_buckets
@ -115,9 +115,9 @@ class TestS3GateObject(TestS3GateBase):
s3_gate_object.copy_object_s3(self.s3_client, bucket_1, obj_key)
@allure.title("Test S3: Checking copy with acl")
def test_s3_copy_acl(self, bucket):
def test_s3_copy_acl(self, bucket, simple_object_size):
version_1_content = "Version 1"
file_name_simple = generate_file_with_content(content=version_1_content)
file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -137,9 +137,9 @@ class TestS3GateObject(TestS3GateBase):
), "Permission for all groups is FULL_CONTROL"
@allure.title("Test S3: Copy object with metadata")
def test_s3_copy_metadate(self, bucket):
def test_s3_copy_metadate(self, bucket, simple_object_size):
object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
file_path = generate_file()
file_path = generate_file(simple_object_size)
file_name = self.object_key_from_file_path(file_path)
bucket_1_objects = [file_name]
@ -187,9 +187,9 @@ class TestS3GateObject(TestS3GateBase):
), f"Metadata must be {object_metadata_1}"
@allure.title("Test S3: Copy object with tagging")
def test_s3_copy_tagging(self, bucket):
def test_s3_copy_tagging(self, bucket, simple_object_size):
object_tagging = [(f"{uuid.uuid4()}", f"{uuid.uuid4()}")]
file_path = generate_file()
file_path = generate_file(simple_object_size)
file_name_simple = self.object_key_from_file_path(file_path)
bucket_1_objects = [file_name_simple]
@ -239,10 +239,10 @@ class TestS3GateObject(TestS3GateBase):
assert tag in got_tags, f"Expected tag {tag} in {got_tags}"
@allure.title("Test S3: Delete version of object")
def test_s3_delete_versioning(self, bucket):
def test_s3_delete_versioning(self, bucket, complex_object_size, simple_object_size):
version_1_content = "Version 1"
version_2_content = "Version 2"
file_name_simple = generate_file_with_content(content=version_1_content)
file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -250,7 +250,7 @@ class TestS3GateObject(TestS3GateBase):
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
file_name_1 = generate_file_with_content(
file_path=file_name_simple, content=version_2_content
simple_object_size, file_path=file_name_simple, content=version_2_content
)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
@ -287,7 +287,7 @@ class TestS3GateObject(TestS3GateBase):
assert not "DeleteMarkers" in delete_obj.keys(), "Delete markes not found"
with allure.step("Put new object into bucket"):
file_name_simple = generate_file(COMPLEX_OBJ_SIZE)
file_name_simple = generate_file(complex_object_size)
obj_key = os.path.basename(file_name_simple)
version_id = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
@ -298,12 +298,12 @@ class TestS3GateObject(TestS3GateBase):
assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker"
@allure.title("Test S3: bulk delete version of object")
def test_s3_bulk_delete_versioning(self, bucket):
def test_s3_bulk_delete_versioning(self, bucket, simple_object_size):
version_1_content = "Version 1"
version_2_content = "Version 2"
version_3_content = "Version 3"
version_4_content = "Version 4"
file_name_1 = generate_file_with_content(content=version_1_content)
file_name_1 = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_1)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -311,15 +311,15 @@ class TestS3GateObject(TestS3GateBase):
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
file_name_2 = generate_file_with_content(
file_path=file_name_1, content=version_2_content
simple_object_size, file_path=file_name_1, content=version_2_content
)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_2)
file_name_3 = generate_file_with_content(
file_path=file_name_1, content=version_3_content
simple_object_size, file_path=file_name_1, content=version_3_content
)
version_id_3 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_3)
file_name_4 = generate_file_with_content(
file_path=file_name_1, content=version_4_content
simple_object_size, file_path=file_name_1, content=version_4_content
)
version_id_4 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_4)
version_ids = {version_id_1, version_id_2, version_id_3, version_id_4}
@ -352,14 +352,14 @@ class TestS3GateObject(TestS3GateBase):
def test_s3_get_versioning(self, bucket):
version_1_content = "Version 1"
version_2_content = "Version 2"
file_name_simple = generate_file_with_content(content=version_1_content)
file_name_simple = generate_file_with_content(simple_object_size, content=version_1_content)
obj_key = os.path.basename(file_name_simple)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_simple)
file_name_1 = generate_file_with_content(
file_path=file_name_simple, content=version_2_content
simple_object_size, file_path=file_name_simple, content=version_2_content
)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
@ -388,14 +388,14 @@ class TestS3GateObject(TestS3GateBase):
), f"Get object with version {version_id_2}"
@allure.title("Test S3: Get range")
def test_s3_get_range(self, bucket):
file_path = generate_file(COMPLEX_OBJ_SIZE)
def test_s3_get_range(self, bucket, complex_object_size: int, simple_object_size: int):
file_path = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path)
file_hash = get_file_hash(file_path)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
file_name_1 = generate_file_with_content(file_path=file_path)
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
with allure.step("Get first version of object"):
@ -404,42 +404,46 @@ class TestS3GateObject(TestS3GateBase):
bucket,
file_name,
version_id_1,
range=[0, int(COMPLEX_OBJ_SIZE / 3)],
range=[0, int(complex_object_size / 3)],
)
object_1_part_2 = s3_gate_object.get_object_s3(
self.s3_client,
bucket,
file_name,
version_id_1,
range=[int(COMPLEX_OBJ_SIZE / 3) + 1, 2 * int(COMPLEX_OBJ_SIZE / 3)],
range=[int(complex_object_size / 3) + 1, 2 * int(complex_object_size / 3)],
)
object_1_part_3 = s3_gate_object.get_object_s3(
self.s3_client,
bucket,
file_name,
version_id_1,
range=[2 * int(COMPLEX_OBJ_SIZE / 3) + 1, COMPLEX_OBJ_SIZE],
range=[2 * int(complex_object_size / 3) + 1, complex_object_size],
)
con_file = concat_files([object_1_part_1, object_1_part_2, object_1_part_3])
assert get_file_hash(con_file) == file_hash, "Hashes must be the same"
with allure.step("Get second version of object"):
object_2_part_1 = s3_gate_object.get_object_s3(
self.s3_client, bucket, file_name, version_id_2, range=[0, int(SIMPLE_OBJ_SIZE / 3)]
self.s3_client,
bucket,
file_name,
version_id_2,
range=[0, int(complex_object_size / 3)],
)
object_2_part_2 = s3_gate_object.get_object_s3(
self.s3_client,
bucket,
file_name,
version_id_2,
range=[int(SIMPLE_OBJ_SIZE / 3) + 1, 2 * int(SIMPLE_OBJ_SIZE / 3)],
range=[int(simple_object_size / 3) + 1, 2 * int(simple_object_size / 3)],
)
object_2_part_3 = s3_gate_object.get_object_s3(
self.s3_client,
bucket,
file_name,
version_id_2,
range=[2 * int(SIMPLE_OBJ_SIZE / 3) + 1, COMPLEX_OBJ_SIZE],
range=[2 * int(simple_object_size / 3) + 1, complex_object_size],
)
con_file_1 = concat_files([object_2_part_1, object_2_part_2, object_2_part_3])
assert get_file_hash(con_file_1) == get_file_hash(
@ -448,28 +452,28 @@ class TestS3GateObject(TestS3GateBase):
with allure.step("Get object"):
object_3_part_1 = s3_gate_object.get_object_s3(
self.s3_client, bucket, file_name, range=[0, int(SIMPLE_OBJ_SIZE / 3)]
self.s3_client, bucket, file_name, range=[0, int(simple_object_size / 3)]
)
object_3_part_2 = s3_gate_object.get_object_s3(
self.s3_client,
bucket,
file_name,
range=[int(SIMPLE_OBJ_SIZE / 3) + 1, 2 * int(SIMPLE_OBJ_SIZE / 3)],
range=[int(simple_object_size / 3) + 1, 2 * int(simple_object_size / 3)],
)
object_3_part_3 = s3_gate_object.get_object_s3(
self.s3_client,
bucket,
file_name,
range=[2 * int(SIMPLE_OBJ_SIZE / 3) + 1, COMPLEX_OBJ_SIZE],
range=[2 * int(simple_object_size / 3) + 1, complex_object_size],
)
con_file = concat_files([object_3_part_1, object_3_part_2, object_3_part_3])
assert get_file_hash(con_file) == get_file_hash(file_name_1), "Hashes must be the same"
@allure.title("Test S3: Copy object with metadata")
@pytest.mark.smoke
def test_s3_head_object(self, bucket):
def test_s3_head_object(self, bucket, complex_object_size, simple_object_size):
object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
file_path = generate_file(COMPLEX_OBJ_SIZE)
file_path = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -477,7 +481,7 @@ class TestS3GateObject(TestS3GateBase):
version_id_1 = s3_gate_object.put_object_s3(
self.s3_client, bucket, file_path, Metadata=object_metadata
)
file_name_1 = generate_file_with_content(file_path=file_path)
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
with allure.step("Get head of first version of object"):
@ -506,10 +510,10 @@ class TestS3GateObject(TestS3GateBase):
@allure.title("Test S3: list of object with versions")
@pytest.mark.parametrize("list_type", ["v1", "v2"])
def test_s3_list_object(self, list_type: str, bucket):
file_path_1 = generate_file(COMPLEX_OBJ_SIZE)
def test_s3_list_object(self, list_type: str, bucket, complex_object_size):
file_path_1 = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path_1)
file_path_2 = generate_file(COMPLEX_OBJ_SIZE)
file_path_2 = generate_file(complex_object_size)
file_name_2 = self.object_key_from_file_path(file_path_2)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -543,8 +547,8 @@ class TestS3GateObject(TestS3GateBase):
assert "DeleteMarker" in delete_obj.keys(), f"Expected delete Marker"
@allure.title("Test S3: put object")
def test_s3_put_object(self, bucket):
file_path_1 = generate_file(COMPLEX_OBJ_SIZE)
def test_s3_put_object(self, bucket, complex_object_size, simple_object_size):
file_path_1 = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path_1)
object_1_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
tag_key_1 = "tag1"
@ -569,7 +573,7 @@ class TestS3GateObject(TestS3GateBase):
], "Tags must be the same"
with allure.step("Rewrite file into bucket"):
file_path_2 = generate_file_with_content(file_path=file_path_1)
file_path_2 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3(
self.s3_client, bucket, file_path_2, Metadata=object_2_metadata, Tagging=tag_2
)
@ -583,7 +587,7 @@ class TestS3GateObject(TestS3GateBase):
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
file_path_3 = generate_file(COMPLEX_OBJ_SIZE)
file_path_3 = generate_file(complex_object_size)
file_hash = get_file_hash(file_path_3)
file_name_3 = self.object_key_from_file_path(file_path_3)
object_3_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
@ -604,7 +608,7 @@ class TestS3GateObject(TestS3GateBase):
], "Tags must be the same"
with allure.step("Put new version of file into bucket"):
file_path_4 = generate_file_with_content(file_path=file_path_3)
file_path_4 = generate_file_with_content(simple_object_size, file_path=file_path_3)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_4)
versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket)
obj_versions = {
@ -680,8 +684,15 @@ class TestS3GateObject(TestS3GateBase):
@allure.title("Test S3: put object with ACL")
@pytest.mark.parametrize("bucket_versioning", ["ENABLED", "SUSPENDED"])
def test_s3_put_object_acl(self, prepare_two_wallets, bucket_versioning, bucket):
file_path_1 = generate_file(COMPLEX_OBJ_SIZE)
def test_s3_put_object_acl(
self,
prepare_two_wallets,
bucket_versioning,
bucket,
complex_object_size,
simple_object_size,
):
file_path_1 = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path_1)
if bucket_versioning == "ENABLED":
status = s3_gate_bucket.VersioningStatus.ENABLED
@ -698,7 +709,7 @@ class TestS3GateObject(TestS3GateBase):
assert get_file_hash(file_path_1) == get_file_hash(object_1), "Hashes must be the same"
with allure.step("Put object with acl public-read"):
file_path_2 = generate_file_with_content(file_path=file_path_1)
file_path_2 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path_2, ACL="public-read")
obj_acl = s3_gate_object.get_object_acl_s3(self.s3_client, bucket, file_name)
obj_permission = [permission.get("Permission") for permission in obj_acl]
@ -710,7 +721,7 @@ class TestS3GateObject(TestS3GateBase):
assert get_file_hash(file_path_2) == get_file_hash(object_2), "Hashes must be the same"
with allure.step("Put object with acl public-read-write"):
file_path_3 = generate_file_with_content(file_path=file_path_1)
file_path_3 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3(
self.s3_client, bucket, file_path_3, ACL="public-read-write"
)
@ -724,7 +735,7 @@ class TestS3GateObject(TestS3GateBase):
assert get_file_hash(file_path_3) == get_file_hash(object_3), "Hashes must be the same"
with allure.step("Put object with acl authenticated-read"):
file_path_4 = generate_file_with_content(file_path=file_path_1)
file_path_4 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3(
self.s3_client, bucket, file_path_4, ACL="authenticated-read"
)
@ -737,11 +748,11 @@ class TestS3GateObject(TestS3GateBase):
object_4 = s3_gate_object.get_object_s3(self.s3_client, bucket, file_name)
assert get_file_hash(file_path_4) == get_file_hash(object_4), "Hashes must be the same"
file_path_5 = generate_file(COMPLEX_OBJ_SIZE)
file_path_5 = generate_file(complex_object_size)
file_name_5 = self.object_key_from_file_path(file_path_5)
with allure.step("Put object with --grant-full-control id=mycanonicaluserid"):
file_path_6 = generate_file_with_content(file_path=file_path_5)
file_path_6 = generate_file_with_content(simple_object_size, file_path=file_path_5)
s3_gate_object.put_object_s3(
self.s3_client,
bucket,
@ -760,7 +771,7 @@ class TestS3GateObject(TestS3GateBase):
with allure.step(
"Put object with --grant-read uri=http://acs.amazonaws.com/groups/global/AllUsers"
):
file_path_7 = generate_file_with_content(file_path=file_path_5)
file_path_7 = generate_file_with_content(simple_object_size, file_path=file_path_5)
s3_gate_object.put_object_s3(
self.s3_client,
bucket,
@ -777,9 +788,9 @@ class TestS3GateObject(TestS3GateBase):
assert get_file_hash(file_path_7) == get_file_hash(object_7), "Hashes must be the same"
@allure.title("Test S3: put object with lock-mode")
def test_s3_put_object_lock_mode(self, bucket):
def test_s3_put_object_lock_mode(self, bucket, complex_object_size, simple_object_size):
file_path_1 = generate_file(COMPLEX_OBJ_SIZE)
file_path_1 = generate_file(complex_object_size)
file_name = self.object_key_from_file_path(file_path_1)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
@ -803,7 +814,7 @@ class TestS3GateObject(TestS3GateBase):
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +3days]"
):
date_obj = datetime.utcnow() + timedelta(days=2)
file_name_1 = generate_file_with_content(file_path=file_path_1)
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3(
self.s3_client,
bucket,
@ -819,7 +830,7 @@ class TestS3GateObject(TestS3GateBase):
"Put new version of object with [--object-lock-mode COMPLIANCE] и [--object-lock-retain-until-date +2days]"
):
date_obj = datetime.utcnow() + timedelta(days=3)
file_name_1 = generate_file_with_content(file_path=file_path_1)
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path_1)
s3_gate_object.put_object_s3(
self.s3_client,
bucket,
@ -857,7 +868,7 @@ class TestS3GateObject(TestS3GateBase):
@allure.title("Test S3 Sync directory")
@pytest.mark.parametrize("sync_type", ["sync", "cp"])
def test_s3_sync_dir(self, sync_type, bucket):
def test_s3_sync_dir(self, sync_type, bucket, simple_object_size):
file_path_1 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_1")
file_path_2 = os.path.join(os.getcwd(), ASSETS_DIR, "test_sync", "test_file_2")
object_metadata = {f"{uuid.uuid4()}": f"{uuid.uuid4()}"}
@ -866,8 +877,8 @@ class TestS3GateObject(TestS3GateBase):
if not isinstance(self.s3_client, AwsCliClient):
pytest.skip("This test is not supported with boto3 client")
generate_file_with_content(file_path=file_path_1)
generate_file_with_content(file_path=file_path_2)
generate_file_with_content(simple_object_size, file_path=file_path_1)
generate_file_with_content(simple_object_size, file_path=file_path_2)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
# TODO: return ACL, when https://github.com/nspcc-dev/neofs-s3-gw/issues/685 will be closed
if sync_type == "sync":
@ -909,10 +920,10 @@ class TestS3GateObject(TestS3GateBase):
# ], "Permission for all groups is FULL_CONTROL"
@allure.title("Test S3 Put 10 nested level object")
def test_s3_put_10_folder(self, bucket, temp_directory):
def test_s3_put_10_folder(self, bucket, temp_directory, simple_object_size):
path = "/".join(["".join(choices(string.ascii_letters, k=3)) for _ in range(10)])
file_path_1 = os.path.join(temp_directory, path, "test_file_1")
generate_file_with_content(file_path=file_path_1)
generate_file_with_content(simple_object_size, file_path=file_path_1)
file_name = self.object_key_from_file_path(file_path_1)
objects_list = s3_gate_object.list_objects_s3(self.s3_client, bucket)
assert not objects_list, f"Expected empty bucket, got {objects_list}"

20
pytest_tests/testsuites/services/s3_gate/test_s3_policy.py
View File

@ -1,21 +1,11 @@
import os
import time
from datetime import datetime, timedelta
from random import choice
from string import ascii_letters
from typing import Tuple
import allure
import pytest
from file_helper import generate_file, generate_file_with_content
from file_helper import generate_file
from python_keywords.container import search_container_by_name
from python_keywords.storage_policy import get_simple_object_copies
from s3_helper import (
assert_object_lock_mode,
check_objects_in_bucket,
object_key_from_file_path,
set_bucket_versioning,
)
from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning
from steps import s3_gate_bucket, s3_gate_object
from steps.s3_gate_base import TestS3GateBase
@ -35,10 +25,10 @@ def pytest_generate_tests(metafunc):
@pytest.mark.s3_gate
class TestS3GatePolicy(TestS3GateBase):
@allure.title("Test S3: Verify bucket creation with retention policy applied")
def test_s3_bucket_location(self):
file_path_1 = generate_file()
def test_s3_bucket_location(self, simple_object_size):
file_path_1 = generate_file(simple_object_size)
file_name_1 = object_key_from_file_path(file_path_1)
file_path_2 = generate_file()
file_path_2 = generate_file(simple_object_size)
file_name_2 = object_key_from_file_path(file_path_2)
with allure.step("Create two buckets with different bucket configuration"):

4
pytest_tests/testsuites/services/s3_gate/test_s3_tagging.py
View File

@ -32,8 +32,8 @@ class TestS3GateTagging(TestS3GateBase):
return tags
@allure.title("Test S3: Object tagging")
def test_s3_object_tagging(self, bucket):
file_path = generate_file()
def test_s3_object_tagging(self, bucket, simple_object_size):
file_path = generate_file(simple_object_size)
file_name = object_key_from_file_path(file_path)
with allure.step("Put with 3 tags object into bucket"):

6
pytest_tests/testsuites/services/s3_gate/test_s3_versioning.py
View File

@ -30,8 +30,8 @@ class TestS3GateVersioning(TestS3GateBase):
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.SUSPENDED)
@allure.title("Test S3: Enable and disable versioning")
def test_s3_version(self):
file_path = generate_file()
def test_s3_version(self, simple_object_size):
file_path = generate_file(simple_object_size)
file_name = self.object_key_from_file_path(file_path)
bucket_objects = [file_name]
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client, False)
@ -61,7 +61,7 @@ class TestS3GateVersioning(TestS3GateBase):
with allure.step("Put several versions of object into bucket"):
version_id_1 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
file_name_1 = generate_file_with_content(file_path=file_path)
file_name_1 = generate_file_with_content(simple_object_size, file_path=file_path)
version_id_2 = s3_gate_object.put_object_s3(self.s3_client, bucket, file_name_1)
with allure.step("Check bucket shows all versions"):

39
pytest_tests/testsuites/services/test_http_gate.py
View File

@ -5,10 +5,9 @@ from time import sleep
import allure
import pytest
from common import COMPLEX_OBJ_SIZE
from container import create_container
from epoch import get_epoch, tick_epoch
from file_helper import generate_file, get_file_hash
from python_keywords.container import create_container
from python_keywords.http_gate import (
get_via_http_curl,
get_via_http_gate,
@ -50,7 +49,7 @@ class TestHttpGate(ClusterTestBase):
TestHttpGate.wallet = default_wallet
@allure.title("Test Put over gRPC, Get over HTTP")
def test_put_grpc_get_http(self):
def test_put_grpc_get_http(self, complex_object_size, simple_object_size):
"""
Test that object can be put using gRPC interface and get using HTTP.
@ -72,7 +71,9 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_1,
basic_acl=PUBLIC_ACL,
)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
with allure.step("Put objects using gRPC"):
oid_simple = put_object_to_random_node(
@ -97,7 +98,7 @@ class TestHttpGate(ClusterTestBase):
@allure.link("https://github.com/nspcc-dev/neofs-http-gw#downloading", name="downloading")
@allure.title("Test Put over HTTP, Get over HTTP")
@pytest.mark.smoke
def test_put_http_get_http(self):
def test_put_http_get_http(self, complex_object_size, simple_object_size):
"""
Test that object can be put and get using HTTP interface.
@ -117,7 +118,9 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_2,
basic_acl=PUBLIC_ACL,
)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
with allure.step("Put objects using HTTP"):
oid_simple = upload_via_http_gate(
@ -143,7 +146,7 @@ class TestHttpGate(ClusterTestBase):
],
ids=["simple", "hyphen", "percent"],
)
def test_put_http_get_http_with_headers(self, attributes: dict):
def test_put_http_get_http_with_headers(self, attributes: dict, simple_object_size):
"""
Test that object can be downloaded using different attributes in HTTP header.
@ -163,7 +166,7 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_2,
basic_acl=PUBLIC_ACL,
)
file_path = generate_file()
file_path = generate_file(simple_object_size)
with allure.step("Put objects using HTTP with attribute"):
headers = self._attr_into_header(attributes)
@ -179,7 +182,7 @@ class TestHttpGate(ClusterTestBase):
self.get_object_by_attr_and_verify_hashes(oid, file_path, cid, attributes)
@allure.title("Test Expiration-Epoch in HTTP header")
def test_expiration_epoch_in_http(self):
def test_expiration_epoch_in_http(self, simple_object_size):
endpoint = self.cluster.default_rpc_endpoint
http_endpoint = self.cluster.default_http_gate_endpoint
@ -190,7 +193,7 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_2,
basic_acl=PUBLIC_ACL,
)
file_path = generate_file()
file_path = generate_file(simple_object_size)
oids = []
curr_epoch = get_epoch(self.shell, self.cluster)
@ -228,7 +231,7 @@ class TestHttpGate(ClusterTestBase):
get_via_http_gate(cid=cid, oid=oid, endpoint=http_endpoint)
@allure.title("Test Zip in HTTP header")
def test_zip_in_http(self):
def test_zip_in_http(self, complex_object_size, simple_object_size):
cid = create_container(
self.wallet,
shell=self.shell,
@ -236,7 +239,9 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_2,
basic_acl=PUBLIC_ACL,
)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
common_prefix = "my_files"
headers1 = {"X-Attribute-FilePath": f"{common_prefix}/file1"}
@ -267,7 +272,7 @@ class TestHttpGate(ClusterTestBase):
@pytest.mark.long
@allure.title("Test Put over HTTP/Curl, Get over HTTP/Curl for large object")
def test_put_http_get_http_large_file(self):
def test_put_http_get_http_large_file(self, complex_object_size):
"""
This test checks upload and download using curl with 'large' object.
Large is object with size up to 20Mb.
@ -280,7 +285,7 @@ class TestHttpGate(ClusterTestBase):
basic_acl=PUBLIC_ACL,
)
obj_size = int(os.getenv("BIG_OBJ_SIZE", COMPLEX_OBJ_SIZE))
obj_size = int(os.getenv("BIG_OBJ_SIZE", complex_object_size))
file_path = generate_file(obj_size)
with allure.step("Put objects using HTTP"):
@ -304,7 +309,7 @@ class TestHttpGate(ClusterTestBase):
)
@allure.title("Test Put/Get over HTTP using Curl utility")
def test_put_http_get_http_curl(self):
def test_put_http_get_http_curl(self, complex_object_size, simple_object_size):
"""
Test checks upload and download over HTTP using curl utility.
"""
@ -315,7 +320,9 @@ class TestHttpGate(ClusterTestBase):
rule=self.PLACEMENT_RULE_2,
basic_acl=PUBLIC_ACL,
)
file_path_simple, file_path_large = generate_file(), generate_file(COMPLEX_OBJ_SIZE)
file_path_simple, file_path_large = generate_file(simple_object_size), generate_file(
complex_object_size
)
with allure.step("Put objects using curl utility"):
oid_simple = upload_via_http_gate_curl(

4
pytest_tests/testsuites/session_token/test_object_session_token.py
View File

@ -3,7 +3,7 @@ import random
import allure
import pytest
from cluster_test_base import ClusterTestBase
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE, WALLET_PASS
from common import WALLET_PASS
from file_helper import generate_file
from grpc_responses import SESSION_NOT_FOUND
from neofs_testlib.utils.wallet import get_last_address_from_wallet
@ -19,7 +19,7 @@ class TestDynamicObjectSession(ClusterTestBase):
@allure.title("Test Object Operations with Session Token")
@pytest.mark.parametrize(
"object_size",
[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE],
[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
)
def test_object_session_token(self, default_wallet, object_size):

48
pytest_tests/testsuites/session_token/test_static_object_session_token.py
View File

@ -4,7 +4,6 @@ import allure
import pytest
from cluster import Cluster
from cluster_test_base import ClusterTestBase
from common import COMPLEX_OBJ_SIZE, SIMPLE_OBJ_SIZE
from epoch import ensure_fresh_epoch
from file_helper import generate_file
from grpc_responses import MALFORMED_REQUEST, OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
@ -13,7 +12,6 @@ from pytest import FixtureRequest
from python_keywords.container import create_container
from python_keywords.neofs_verbs import (
delete_object,
get_netmap_netinfo,
get_object,
get_object_from_random_node,
get_range,
@ -58,7 +56,7 @@ def storage_containers(
@pytest.fixture(
params=[SIMPLE_OBJ_SIZE, COMPLEX_OBJ_SIZE],
params=[pytest.lazy_fixture("simple_object_size"), pytest.lazy_fixture("complex_object_size")],
ids=["simple object", "complex object"],
# Scope module to upload/delete each files set only once
scope="module",
@ -98,16 +96,15 @@ def storage_objects(
@allure.step("Get ranges for test")
def get_ranges(storage_object: StorageObjectInfo, shell: Shell, endpoint: str) -> list[str]:
def get_ranges(
storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, endpoint: str
) -> list[str]:
"""
Returns ranges to test range/hash methods via static session
"""
object_size = storage_object.size
if object_size == COMPLEX_OBJ_SIZE:
net_info = get_netmap_netinfo(storage_object.wallet_file_path, shell, endpoint)
max_object_size = net_info["maximum_object_size"]
# make sure to test multiple parts of complex object
if object_size > max_object_size:
assert object_size >= max_object_size + RANGE_OFFSET_FOR_COMPLEX_OBJECT
return [
"0:10",
@ -160,9 +157,9 @@ class TestObjectStaticSession(ClusterTestBase):
self,
user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo],
static_sessions: list[str],
static_sessions: dict[ObjectVerb, str],
method_under_test,
verb: str,
verb: ObjectVerb,
request: FixtureRequest,
):
"""
@ -175,9 +172,9 @@ class TestObjectStaticSession(ClusterTestBase):
for node in self.cluster.storage_nodes:
for storage_object in storage_objects[0:2]:
method_under_test(
user_wallet.path,
storage_object.cid,
storage_object.oid,
wallet=user_wallet.path,
cid=storage_object.cid,
oid=storage_object.oid,
shell=self.shell,
endpoint=node.get_rpc_endpoint(),
session=static_sessions[verb],
@ -193,10 +190,11 @@ class TestObjectStaticSession(ClusterTestBase):
self,
user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo],
static_sessions: list[str],
static_sessions: dict[ObjectVerb, str],
method_under_test,
verb: str,
verb: ObjectVerb,
request: FixtureRequest,
max_object_size,
):
"""
Validate static session with range operations
@ -205,7 +203,9 @@ class TestObjectStaticSession(ClusterTestBase):
f"Validate static session with range operations for {request.node.callspec.id}"
)
storage_object = storage_objects[0]
ranges_to_test = get_ranges(storage_object, self.shell, self.cluster.default_rpc_endpoint)
ranges_to_test = get_ranges(
storage_object, max_object_size, self.shell, self.cluster.default_rpc_endpoint
)
for range_to_test in ranges_to_test:
with allure.step(f"Check range {range_to_test}"):
@ -227,7 +227,7 @@ class TestObjectStaticSession(ClusterTestBase):
self,
user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo],
static_sessions: list[str],
static_sessions: dict[ObjectVerb, str],
request: FixtureRequest,
):
"""
@ -253,7 +253,7 @@ class TestObjectStaticSession(ClusterTestBase):
self,
user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo],
static_sessions: list[str],
static_sessions: dict[ObjectVerb, str],
request: FixtureRequest,
):
"""
@ -278,7 +278,7 @@ class TestObjectStaticSession(ClusterTestBase):
self,
stranger_wallet: WalletFile,
storage_objects: list[StorageObjectInfo],
static_sessions: list[str],
static_sessions: dict[ObjectVerb, str],
request: FixtureRequest,
):
"""
@ -305,7 +305,7 @@ class TestObjectStaticSession(ClusterTestBase):
self,
user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo],
static_sessions: list[str],
static_sessions: dict[ObjectVerb, str],
request: FixtureRequest,
):
"""
@ -333,7 +333,7 @@ class TestObjectStaticSession(ClusterTestBase):
user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo],
storage_containers: list[str],
static_sessions: list[str],
static_sessions: dict[ObjectVerb, str],
request: FixtureRequest,
):
"""
@ -361,7 +361,7 @@ class TestObjectStaticSession(ClusterTestBase):
owner_wallet: WalletFile,
user_wallet: WalletFile,
stranger_wallet: WalletFile,
storage_containers: list[int],
storage_containers: list[str],
storage_objects: list[StorageObjectInfo],
temp_directory: str,
request: FixtureRequest,
@ -638,7 +638,7 @@ class TestObjectStaticSession(ClusterTestBase):
self,
user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo],
static_sessions: list[str],
static_sessions: dict[ObjectVerb, str],
request: FixtureRequest,
):
"""
@ -663,7 +663,7 @@ class TestObjectStaticSession(ClusterTestBase):
self,
user_wallet: WalletFile,
storage_objects: list[StorageObjectInfo],
static_sessions: list[str],
static_sessions: dict[ObjectVerb, str],
request: FixtureRequest,
):
"""

3
pytest_tests/testsuites/session_token/test_static_session_token_container.py
View File

@ -148,6 +148,7 @@ class TestSessionTokenContainer(ClusterTestBase):
user_wallet: WalletFile,
stranger_wallet: WalletFile,
static_sessions: dict[ContainerVerb, str],
simple_object_size,
):
"""
Validate static session with set eacl operation
@ -159,7 +160,7 @@ class TestSessionTokenContainer(ClusterTestBase):
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
)
file_path = generate_file()
file_path = generate_file(simple_object_size)
assert can_put_object(stranger_wallet.path, cid, file_path, self.shell, self.cluster)
with allure.step(f"Deny all operations for other via eACL"):

1
requirements.txt
View File

@ -55,6 +55,7 @@ pyflakes==2.4.0
pyparsing==3.0.9
pyrsistent==0.18.1
pytest==7.1.2
pytest-lazy-fixture==0.6.3
python-dateutil==2.8.2
pyyaml==6.0
requests==2.28.0

10
robot/resources/lib/python_keywords/storage_group.py
View File

@ -7,7 +7,7 @@ from typing import Optional
import allure
from cluster import Cluster
from common import COMPLEX_OBJ_SIZE, NEOFS_CLI_EXEC, SIMPLE_OBJ_SIZE, WALLET_CONFIG
from common import NEOFS_CLI_EXEC, WALLET_CONFIG
from complex_object_actions import get_link_object
from neofs_testlib.cli import NeofsCli
from neofs_testlib.shell import Shell
@ -201,12 +201,13 @@ def verify_get_storage_group(
gid: str,
obj_list: list,
object_size: int,
max_object_size: int,
bearer: str = None,
wallet_config: str = WALLET_CONFIG,
):
obj_parts = []
endpoint = cluster.default_rpc_endpoint
if object_size == COMPLEX_OBJ_SIZE:
if object_size > max_object_size:
for obj in obj_list:
link_oid = get_link_object(
wallet,
@ -239,11 +240,10 @@ def verify_get_storage_group(
bearer=bearer,
wallet_config=wallet_config,
)
if object_size == SIMPLE_OBJ_SIZE:
exp_size = SIMPLE_OBJ_SIZE * obj_num
exp_size = object_size * obj_num
if object_size < max_object_size:
assert int(storagegroup_data["Group size"]) == exp_size
assert storagegroup_data["Members"] == obj_list
else:
exp_size = COMPLEX_OBJ_SIZE * obj_num
assert int(storagegroup_data["Group size"]) == exp_size
assert storagegroup_data["Members"] == obj_parts

6
robot/variables/common.py
View File

@ -4,9 +4,9 @@ import yaml
CONTAINER_WAIT_INTERVAL = "1m"
# TODO: Get object size data from a node config
SIMPLE_OBJ_SIZE = int(os.getenv("SIMPLE_OBJ_SIZE", "1000"))
COMPLEX_OBJ_SIZE = int(os.getenv("COMPLEX_OBJ_SIZE", "2000"))
SIMPLE_OBJECT_SIZE = os.getenv("SIMPLE_OBJECT_SIZE", "1000")
COMPLEX_OBJECT_CHUNKS_COUNT = os.getenv("COMPLEX_OBJECT_CHUNKS_COUNT", "3")
COMPLEX_OBJECT_TAIL_SIZE = os.getenv("COMPLEX_OBJECT_TAIL_SIZE", "1000")
MAINNET_BLOCK_TIME = os.getenv("MAINNET_BLOCK_TIME", "1s")
MAINNET_TIMEOUT = os.getenv("MAINNET_TIMEOUT", "1min")