2022-07-05 07:18:37 +00:00
|
|
|
import logging
|
2022-11-01 16:11:26 +00:00
|
|
|
import random
|
|
|
|
import sys
|
2022-07-05 07:18:37 +00:00
|
|
|
|
|
|
|
import allure
|
|
|
|
import pytest
|
2023-11-29 13:34:59 +00:00
|
|
|
from frostfs_testlib import reporter
|
2023-05-15 09:59:33 +00:00
|
|
|
from frostfs_testlib.resources.error_patterns import (
|
2022-12-19 15:12:45 +00:00
|
|
|
INVALID_LENGTH_SPECIFIER,
|
|
|
|
INVALID_OFFSET_SPECIFIER,
|
|
|
|
INVALID_RANGE_OVERFLOW,
|
|
|
|
INVALID_RANGE_ZERO_LENGTH,
|
|
|
|
OUT_OF_RANGE,
|
|
|
|
)
|
2023-02-17 15:12:11 +00:00
|
|
|
from frostfs_testlib.shell import Shell
|
2023-05-15 09:59:33 +00:00
|
|
|
from frostfs_testlib.steps.cli.container import create_container
|
|
|
|
from frostfs_testlib.steps.cli.object import (
|
2022-12-05 22:31:45 +00:00
|
|
|
get_object_from_random_node,
|
2022-09-28 12:07:16 +00:00
|
|
|
get_range,
|
|
|
|
get_range_hash,
|
|
|
|
head_object,
|
2022-12-05 22:31:45 +00:00
|
|
|
put_object_to_random_node,
|
2022-09-28 12:07:16 +00:00
|
|
|
search_object,
|
|
|
|
)
|
2023-05-15 09:59:33 +00:00
|
|
|
from frostfs_testlib.steps.complex_object_actions import get_complex_object_split_ranges
|
|
|
|
from frostfs_testlib.steps.storage_object import delete_objects
|
|
|
|
from frostfs_testlib.steps.storage_policy import get_complex_object_copies, get_simple_object_copies
|
|
|
|
from frostfs_testlib.storage.cluster import Cluster
|
2023-08-02 11:54:03 +00:00
|
|
|
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
|
2024-03-11 12:49:37 +00:00
|
|
|
from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy
|
2023-05-15 09:59:33 +00:00
|
|
|
from frostfs_testlib.storage.dataclasses.storage_object_info import StorageObjectInfo
|
2024-03-11 16:34:54 +00:00
|
|
|
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
|
2023-05-15 09:59:33 +00:00
|
|
|
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
|
|
|
|
from frostfs_testlib.utils.file_utils import generate_file, get_file_content, get_file_hash
|
2022-07-05 07:18:37 +00:00
|
|
|
|
2022-09-28 12:07:16 +00:00
|
|
|
logger = logging.getLogger("NeoLogger")
|
2022-07-05 07:18:37 +00:00
|
|
|
|
|
|
|
CLEANUP_TIMEOUT = 10
|
2022-11-01 16:11:26 +00:00
|
|
|
COMMON_ATTRIBUTE = {"common_key": "common_value"}
|
|
|
|
# Will upload object for each attribute set
|
|
|
|
OBJECT_ATTRIBUTES = [
|
|
|
|
None,
|
|
|
|
{"key1": 1, "key2": "abc", "common_key": "common_value"},
|
|
|
|
{"key1": 2, "common_key": "common_value"},
|
|
|
|
]
|
2022-07-05 07:18:37 +00:00
|
|
|
|
2022-11-01 16:11:26 +00:00
|
|
|
# Config for Range tests
|
|
|
|
RANGES_COUNT = 4 # by quarters
|
|
|
|
RANGE_MIN_LEN = 10
|
|
|
|
RANGE_MAX_LEN = 500
|
|
|
|
# Used for static ranges found with issues
|
2022-12-07 12:38:56 +00:00
|
|
|
STATIC_RANGES = {}
|
2022-07-05 07:18:37 +00:00
|
|
|
|
2022-10-04 05:55:09 +00:00
|
|
|
|
2022-12-19 15:12:45 +00:00
|
|
|
def generate_ranges(
|
|
|
|
storage_object: StorageObjectInfo, max_object_size: int, shell: Shell, cluster: Cluster
|
|
|
|
) -> list[(int, int)]:
|
|
|
|
file_range_step = storage_object.size / RANGES_COUNT
|
2022-11-01 16:11:26 +00:00
|
|
|
|
|
|
|
file_ranges = []
|
|
|
|
file_ranges_to_test = []
|
|
|
|
|
|
|
|
for i in range(0, RANGES_COUNT):
|
2022-12-19 15:12:45 +00:00
|
|
|
file_ranges.append((int(file_range_step * i), int(file_range_step)))
|
2022-11-01 16:11:26 +00:00
|
|
|
|
|
|
|
# For simple object we can read all file ranges without too much time for testing
|
2022-12-19 15:12:45 +00:00
|
|
|
if storage_object.size < max_object_size:
|
2022-11-01 16:11:26 +00:00
|
|
|
file_ranges_to_test.extend(file_ranges)
|
|
|
|
# For complex object we need to fetch multiple child objects from different nodes.
|
2022-12-07 12:38:56 +00:00
|
|
|
else:
|
2022-11-01 16:11:26 +00:00
|
|
|
assert (
|
2022-12-19 15:12:45 +00:00
|
|
|
storage_object.size >= RANGE_MAX_LEN + max_object_size
|
|
|
|
), f"Complex object size should be at least {max_object_size + RANGE_MAX_LEN}. Current: {storage_object.size}"
|
|
|
|
file_ranges_to_test.append((RANGE_MAX_LEN, max_object_size - RANGE_MAX_LEN))
|
|
|
|
file_ranges_to_test.extend(get_complex_object_split_ranges(storage_object, shell, cluster))
|
2022-11-01 16:11:26 +00:00
|
|
|
|
|
|
|
# Special cases to read some bytes from start and some bytes from end of object
|
|
|
|
file_ranges_to_test.append((0, RANGE_MIN_LEN))
|
2022-12-19 15:12:45 +00:00
|
|
|
file_ranges_to_test.append((storage_object.size - RANGE_MIN_LEN, RANGE_MIN_LEN))
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2022-12-19 15:12:45 +00:00
|
|
|
for offset, length in file_ranges:
|
2022-11-01 16:11:26 +00:00
|
|
|
range_length = random.randint(RANGE_MIN_LEN, RANGE_MAX_LEN)
|
2022-12-19 15:12:45 +00:00
|
|
|
range_start = random.randint(offset, offset + length)
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2023-11-02 09:19:23 +00:00
|
|
|
file_ranges_to_test.append((range_start, min(range_length, storage_object.size - range_start)))
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2022-12-19 15:12:45 +00:00
|
|
|
file_ranges_to_test.extend(STATIC_RANGES.get(storage_object.size, []))
|
2022-11-01 16:11:26 +00:00
|
|
|
|
|
|
|
return file_ranges_to_test
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture(
|
|
|
|
# Scope session to upload/delete each files set only once
|
2023-09-08 10:35:34 +00:00
|
|
|
scope="module"
|
2022-11-01 16:11:26 +00:00
|
|
|
)
|
|
|
|
def storage_objects(
|
2024-06-04 15:24:21 +00:00
|
|
|
default_wallet: WalletInfo,
|
|
|
|
client_shell: Shell,
|
|
|
|
cluster: Cluster,
|
|
|
|
object_size: ObjectSize,
|
|
|
|
placement_policy: PlacementPolicy,
|
2022-11-01 16:11:26 +00:00
|
|
|
) -> list[StorageObjectInfo]:
|
2022-12-05 22:31:45 +00:00
|
|
|
wallet = default_wallet
|
2022-11-01 16:11:26 +00:00
|
|
|
# Separate containers for complex/simple objects to avoid side-effects
|
2024-06-04 15:24:21 +00:00
|
|
|
cid = create_container(
|
|
|
|
wallet, shell=client_shell, rule=placement_policy.value, endpoint=cluster.default_rpc_endpoint
|
|
|
|
)
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2023-08-02 11:54:03 +00:00
|
|
|
file_path = generate_file(object_size.value)
|
2022-07-05 10:17:36 +00:00
|
|
|
file_hash = get_file_hash(file_path)
|
2022-07-05 07:18:37 +00:00
|
|
|
|
2022-11-01 16:11:26 +00:00
|
|
|
storage_objects = []
|
2022-07-05 07:18:37 +00:00
|
|
|
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step("Put objects"):
|
2022-11-01 16:11:26 +00:00
|
|
|
# We need to upload objects multiple times with different attributes
|
|
|
|
for attributes in OBJECT_ATTRIBUTES:
|
2022-12-05 22:31:45 +00:00
|
|
|
storage_object_id = put_object_to_random_node(
|
2022-10-13 18:53:44 +00:00
|
|
|
wallet=wallet,
|
|
|
|
path=file_path,
|
|
|
|
cid=cid,
|
|
|
|
shell=client_shell,
|
2022-12-05 22:31:45 +00:00
|
|
|
cluster=cluster,
|
2022-11-01 16:11:26 +00:00
|
|
|
attributes=attributes,
|
2022-10-13 18:53:44 +00:00
|
|
|
)
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2022-11-25 12:44:47 +00:00
|
|
|
storage_object = StorageObjectInfo(cid, storage_object_id)
|
2023-08-02 11:54:03 +00:00
|
|
|
storage_object.size = object_size.value
|
2024-03-11 16:34:54 +00:00
|
|
|
storage_object.wallet = wallet
|
2022-11-25 12:44:47 +00:00
|
|
|
storage_object.file_path = file_path
|
|
|
|
storage_object.file_hash = file_hash
|
|
|
|
storage_object.attributes = attributes
|
|
|
|
|
2022-11-01 16:11:26 +00:00
|
|
|
storage_objects.append(storage_object)
|
|
|
|
|
|
|
|
yield storage_objects
|
|
|
|
|
|
|
|
# Teardown after all tests done with current param
|
2022-12-05 22:31:45 +00:00
|
|
|
delete_objects(storage_objects, client_shell, cluster)
|
2022-11-01 16:11:26 +00:00
|
|
|
|
|
|
|
|
2024-06-04 16:22:59 +00:00
|
|
|
@pytest.fixture()
|
|
|
|
def expected_object_copies(placement_policy: PlacementPolicy) -> int:
|
|
|
|
if placement_policy.name == "rep":
|
|
|
|
return 2
|
|
|
|
return 4
|
|
|
|
|
|
|
|
|
2022-11-01 16:11:26 +00:00
|
|
|
@pytest.mark.sanity
|
|
|
|
@pytest.mark.grpc_api
|
2022-12-05 22:31:45 +00:00
|
|
|
class TestObjectApi(ClusterTestBase):
|
2024-03-11 12:49:37 +00:00
|
|
|
@allure.title("Storage policy by native API (obj_size={object_size}, policy={placement_policy})")
|
2022-12-05 22:31:45 +00:00
|
|
|
def test_object_storage_policies(
|
2023-08-02 11:54:03 +00:00
|
|
|
self,
|
|
|
|
storage_objects: list[StorageObjectInfo],
|
|
|
|
simple_object_size: ObjectSize,
|
2024-06-04 15:24:21 +00:00
|
|
|
expected_object_copies: int,
|
2022-12-05 22:31:45 +00:00
|
|
|
):
|
|
|
|
"""
|
|
|
|
Validate object storage policy
|
|
|
|
"""
|
2022-07-05 07:18:37 +00:00
|
|
|
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step("Validate storage policy for objects"):
|
2022-12-05 22:31:45 +00:00
|
|
|
for storage_object in storage_objects:
|
2023-08-02 11:54:03 +00:00
|
|
|
if storage_object.size == simple_object_size.value:
|
2022-12-05 22:31:45 +00:00
|
|
|
copies = get_simple_object_copies(
|
2024-03-11 16:34:54 +00:00
|
|
|
storage_object.wallet,
|
2022-12-05 22:31:45 +00:00
|
|
|
storage_object.cid,
|
|
|
|
storage_object.oid,
|
|
|
|
shell=self.shell,
|
|
|
|
nodes=self.cluster.storage_nodes,
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
copies = get_complex_object_copies(
|
2024-03-11 16:34:54 +00:00
|
|
|
storage_object.wallet,
|
2022-12-05 22:31:45 +00:00
|
|
|
storage_object.cid,
|
|
|
|
storage_object.oid,
|
|
|
|
shell=self.shell,
|
|
|
|
nodes=self.cluster.storage_nodes,
|
|
|
|
)
|
2024-06-04 15:24:21 +00:00
|
|
|
assert copies == expected_object_copies, f"Expected {expected_object_copies} copies"
|
2022-12-05 22:31:45 +00:00
|
|
|
|
2024-03-11 12:49:37 +00:00
|
|
|
@allure.title("Get object by native API (obj_size={object_size}, policy={placement_policy})")
|
2023-09-08 10:35:34 +00:00
|
|
|
def test_get_object_api(self, storage_objects: list[StorageObjectInfo]):
|
2022-12-05 22:31:45 +00:00
|
|
|
"""
|
|
|
|
Validate get object native API
|
|
|
|
"""
|
|
|
|
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step("Get objects and compare hashes"):
|
2022-12-05 22:31:45 +00:00
|
|
|
for storage_object in storage_objects:
|
|
|
|
file_path = get_object_from_random_node(
|
2024-03-11 16:34:54 +00:00
|
|
|
storage_object.wallet,
|
2022-11-01 16:11:26 +00:00
|
|
|
storage_object.cid,
|
|
|
|
storage_object.oid,
|
2022-12-05 22:31:45 +00:00
|
|
|
self.shell,
|
|
|
|
cluster=self.cluster,
|
2022-10-13 18:53:44 +00:00
|
|
|
)
|
2022-12-05 22:31:45 +00:00
|
|
|
file_hash = get_file_hash(file_path)
|
|
|
|
assert storage_object.file_hash == file_hash
|
|
|
|
|
2024-03-11 12:49:37 +00:00
|
|
|
@allure.title("Head object by native API (obj_size={object_size}, policy={placement_policy})")
|
2023-09-08 10:35:34 +00:00
|
|
|
def test_head_object_api(self, storage_objects: list[StorageObjectInfo]):
|
2022-12-05 22:31:45 +00:00
|
|
|
"""
|
|
|
|
Validate head object native API
|
|
|
|
"""
|
|
|
|
|
|
|
|
storage_object_1 = storage_objects[0]
|
|
|
|
storage_object_2 = storage_objects[1]
|
|
|
|
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step("Head object and validate"):
|
2022-12-05 22:31:45 +00:00
|
|
|
head_object(
|
2024-03-11 16:34:54 +00:00
|
|
|
storage_object_1.wallet,
|
2022-12-05 22:31:45 +00:00
|
|
|
storage_object_1.cid,
|
|
|
|
storage_object_1.oid,
|
|
|
|
shell=self.shell,
|
|
|
|
endpoint=self.cluster.default_rpc_endpoint,
|
2022-11-01 16:11:26 +00:00
|
|
|
)
|
2022-12-05 22:31:45 +00:00
|
|
|
head_info = head_object(
|
2024-03-11 16:34:54 +00:00
|
|
|
storage_object_2.wallet,
|
2022-12-05 22:31:45 +00:00
|
|
|
storage_object_2.cid,
|
|
|
|
storage_object_2.oid,
|
|
|
|
shell=self.shell,
|
|
|
|
endpoint=self.cluster.default_rpc_endpoint,
|
|
|
|
)
|
|
|
|
self.check_header_is_presented(head_info, storage_object_2.attributes)
|
|
|
|
|
2024-03-11 12:49:37 +00:00
|
|
|
@allure.title("Search objects by native API (obj_size={object_size}, policy={placement_policy})")
|
2023-09-08 10:35:34 +00:00
|
|
|
def test_search_object_api(self, storage_objects: list[StorageObjectInfo]):
|
2022-12-05 22:31:45 +00:00
|
|
|
"""
|
|
|
|
Validate object search by native API
|
|
|
|
"""
|
|
|
|
|
|
|
|
oids = [storage_object.oid for storage_object in storage_objects]
|
2024-03-11 16:34:54 +00:00
|
|
|
wallet = storage_objects[0].wallet
|
2022-12-05 22:31:45 +00:00
|
|
|
cid = storage_objects[0].cid
|
|
|
|
|
|
|
|
test_table = [
|
|
|
|
(OBJECT_ATTRIBUTES[1], oids[1:2]),
|
|
|
|
(OBJECT_ATTRIBUTES[2], oids[2:3]),
|
|
|
|
(COMMON_ATTRIBUTE, oids[1:3]),
|
|
|
|
]
|
|
|
|
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step("Search objects"):
|
2022-12-05 22:31:45 +00:00
|
|
|
# Search with no attributes
|
2022-11-01 16:11:26 +00:00
|
|
|
result = search_object(
|
|
|
|
wallet,
|
|
|
|
cid,
|
2022-12-05 22:31:45 +00:00
|
|
|
shell=self.shell,
|
|
|
|
endpoint=self.cluster.default_rpc_endpoint,
|
|
|
|
expected_objects_list=oids,
|
2022-11-01 16:11:26 +00:00
|
|
|
root=True,
|
|
|
|
)
|
2022-12-05 22:31:45 +00:00
|
|
|
assert sorted(oids) == sorted(result)
|
|
|
|
|
|
|
|
# search by test table
|
|
|
|
for filter, expected_oids in test_table:
|
|
|
|
result = search_object(
|
|
|
|
wallet,
|
|
|
|
cid,
|
|
|
|
shell=self.shell,
|
|
|
|
endpoint=self.cluster.default_rpc_endpoint,
|
|
|
|
filters=filter,
|
|
|
|
expected_objects_list=expected_oids,
|
|
|
|
root=True,
|
|
|
|
)
|
|
|
|
assert sorted(expected_oids) == sorted(result)
|
2022-07-05 07:18:37 +00:00
|
|
|
|
2023-09-08 10:35:34 +00:00
|
|
|
@allure.title("Search objects with removed items (obj_size={object_size})")
|
2024-03-11 16:34:54 +00:00
|
|
|
def test_object_search_should_return_tombstone_items(self, default_wallet: WalletInfo, object_size: ObjectSize):
|
2022-12-05 22:31:45 +00:00
|
|
|
"""
|
|
|
|
Validate object search with removed items
|
|
|
|
"""
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2022-12-05 22:31:45 +00:00
|
|
|
wallet = default_wallet
|
|
|
|
cid = create_container(wallet, self.shell, self.cluster.default_rpc_endpoint)
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step("Upload file"):
|
2023-08-02 11:54:03 +00:00
|
|
|
file_path = generate_file(object_size.value)
|
2022-12-05 22:31:45 +00:00
|
|
|
file_hash = get_file_hash(file_path)
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2022-12-05 22:31:45 +00:00
|
|
|
storage_object = StorageObjectInfo(
|
|
|
|
cid=cid,
|
|
|
|
oid=put_object_to_random_node(wallet, file_path, cid, self.shell, self.cluster),
|
2023-08-02 11:54:03 +00:00
|
|
|
size=object_size.value,
|
2024-03-11 16:34:54 +00:00
|
|
|
wallet=wallet,
|
2022-12-05 22:31:45 +00:00
|
|
|
file_path=file_path,
|
|
|
|
file_hash=file_hash,
|
|
|
|
)
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step("Search object"):
|
2022-12-05 22:31:45 +00:00
|
|
|
# Root Search object should return root object oid
|
2023-11-02 09:19:23 +00:00
|
|
|
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True)
|
2022-12-05 22:31:45 +00:00
|
|
|
assert result == [storage_object.oid]
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step("Delete file"):
|
2022-12-05 22:31:45 +00:00
|
|
|
delete_objects([storage_object], self.shell, self.cluster)
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step("Search deleted object with --root"):
|
2022-12-05 22:31:45 +00:00
|
|
|
# Root Search object should return nothing
|
2023-11-02 09:19:23 +00:00
|
|
|
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, root=True)
|
2022-12-05 22:31:45 +00:00
|
|
|
assert len(result) == 0
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step("Search deleted object with --phy should return only tombstones"):
|
2022-12-05 22:31:45 +00:00
|
|
|
# Physical Search object should return only tombstones
|
2023-11-02 09:19:23 +00:00
|
|
|
result = search_object(wallet, cid, shell=self.shell, endpoint=self.cluster.default_rpc_endpoint, phy=True)
|
|
|
|
assert storage_object.tombstone in result, "Search result should contain tombstone of removed object"
|
|
|
|
assert storage_object.oid not in result, "Search result should not contain ObjectId of removed object"
|
2022-12-05 22:31:45 +00:00
|
|
|
for tombstone_oid in result:
|
|
|
|
header = head_object(
|
|
|
|
wallet,
|
|
|
|
cid,
|
|
|
|
tombstone_oid,
|
|
|
|
shell=self.shell,
|
|
|
|
endpoint=self.cluster.default_rpc_endpoint,
|
|
|
|
)["header"]
|
|
|
|
object_type = header["objectType"]
|
2022-11-01 16:11:26 +00:00
|
|
|
assert (
|
2022-12-05 22:31:45 +00:00
|
|
|
object_type == "TOMBSTONE"
|
|
|
|
), f"Object wasn't deleted properly. Found object {tombstone_oid} with type {object_type}"
|
|
|
|
|
2024-03-11 12:49:37 +00:00
|
|
|
@allure.title("Get range hash by native API (obj_size={object_size}, policy={placement_policy})")
|
2022-12-05 22:31:45 +00:00
|
|
|
@pytest.mark.grpc_api
|
2023-09-08 10:35:34 +00:00
|
|
|
def test_object_get_range_hash(self, storage_objects: list[StorageObjectInfo], max_object_size):
|
2022-12-05 22:31:45 +00:00
|
|
|
"""
|
2022-12-19 15:12:45 +00:00
|
|
|
Validate get_range_hash for object by native gRPC API
|
2022-12-05 22:31:45 +00:00
|
|
|
"""
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2024-03-11 16:34:54 +00:00
|
|
|
wallet = storage_objects[0].wallet
|
2022-12-05 22:31:45 +00:00
|
|
|
cid = storage_objects[0].cid
|
|
|
|
oids = [storage_object.oid for storage_object in storage_objects[:2]]
|
|
|
|
file_path = storage_objects[0].file_path
|
|
|
|
|
2023-11-02 09:19:23 +00:00
|
|
|
file_ranges_to_test = generate_ranges(storage_objects[0], max_object_size, self.shell, self.cluster)
|
2022-12-05 22:31:45 +00:00
|
|
|
logging.info(f"Ranges used in test {file_ranges_to_test}")
|
|
|
|
|
2022-12-19 15:12:45 +00:00
|
|
|
for range_start, range_len in file_ranges_to_test:
|
2022-12-05 22:31:45 +00:00
|
|
|
range_cut = f"{range_start}:{range_len}"
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step(f"Get range hash ({range_cut})"):
|
2022-12-05 22:31:45 +00:00
|
|
|
for oid in oids:
|
|
|
|
range_hash = get_range_hash(
|
|
|
|
wallet,
|
|
|
|
cid,
|
|
|
|
oid,
|
|
|
|
shell=self.shell,
|
|
|
|
endpoint=self.cluster.default_rpc_endpoint,
|
|
|
|
range_cut=range_cut,
|
|
|
|
)
|
|
|
|
assert (
|
|
|
|
get_file_hash(file_path, range_len, range_start) == range_hash
|
|
|
|
), f"Expected range hash to match {range_cut} slice of file payload"
|
|
|
|
|
2024-03-11 12:49:37 +00:00
|
|
|
@allure.title("Get range by native API (obj_size={object_size}, policy={placement_policy})")
|
2022-12-05 22:31:45 +00:00
|
|
|
@pytest.mark.grpc_api
|
2023-09-08 10:35:34 +00:00
|
|
|
def test_object_get_range(self, storage_objects: list[StorageObjectInfo], max_object_size):
|
2022-12-05 22:31:45 +00:00
|
|
|
"""
|
2022-12-19 15:12:45 +00:00
|
|
|
Validate get_range for object by native gRPC API
|
2022-12-05 22:31:45 +00:00
|
|
|
"""
|
|
|
|
|
2024-03-11 16:34:54 +00:00
|
|
|
wallet = storage_objects[0].wallet
|
2022-12-05 22:31:45 +00:00
|
|
|
cid = storage_objects[0].cid
|
|
|
|
oids = [storage_object.oid for storage_object in storage_objects[:2]]
|
|
|
|
file_path = storage_objects[0].file_path
|
|
|
|
|
2023-11-02 09:19:23 +00:00
|
|
|
file_ranges_to_test = generate_ranges(storage_objects[0], max_object_size, self.shell, self.cluster)
|
2022-12-05 22:31:45 +00:00
|
|
|
logging.info(f"Ranges used in test {file_ranges_to_test}")
|
|
|
|
|
2022-12-19 15:12:45 +00:00
|
|
|
for range_start, range_len in file_ranges_to_test:
|
2022-12-05 22:31:45 +00:00
|
|
|
range_cut = f"{range_start}:{range_len}"
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step(f"Get range ({range_cut})"):
|
2022-12-05 22:31:45 +00:00
|
|
|
for oid in oids:
|
|
|
|
_, range_content = get_range(
|
|
|
|
wallet,
|
|
|
|
cid,
|
|
|
|
oid,
|
|
|
|
shell=self.shell,
|
|
|
|
endpoint=self.cluster.default_rpc_endpoint,
|
|
|
|
range_cut=range_cut,
|
2022-11-01 16:11:26 +00:00
|
|
|
)
|
2022-12-05 22:31:45 +00:00
|
|
|
assert (
|
2023-11-02 09:19:23 +00:00
|
|
|
get_file_content(file_path, content_len=range_len, mode="rb", offset=range_start)
|
2022-12-05 22:31:45 +00:00
|
|
|
== range_content
|
|
|
|
), f"Expected range content to match {range_cut} slice of file payload"
|
|
|
|
|
2024-03-11 12:49:37 +00:00
|
|
|
@allure.title("[NEGATIVE] Get invalid range by native API (obj_size={object_size}, policy={placement_policy})")
|
2022-12-05 22:31:45 +00:00
|
|
|
@pytest.mark.grpc_api
|
|
|
|
def test_object_get_range_negatives(
|
|
|
|
self,
|
|
|
|
storage_objects: list[StorageObjectInfo],
|
|
|
|
):
|
|
|
|
"""
|
2022-12-19 15:12:45 +00:00
|
|
|
Validate get_range negative for object by native gRPC API
|
2022-12-05 22:31:45 +00:00
|
|
|
"""
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2024-03-11 16:34:54 +00:00
|
|
|
wallet = storage_objects[0].wallet
|
2022-12-05 22:31:45 +00:00
|
|
|
cid = storage_objects[0].cid
|
|
|
|
oids = [storage_object.oid for storage_object in storage_objects[:2]]
|
|
|
|
file_size = storage_objects[0].size
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2022-12-05 22:31:45 +00:00
|
|
|
assert (
|
|
|
|
RANGE_MIN_LEN < file_size
|
|
|
|
), f"Incorrect test setup. File size ({file_size}) is less than RANGE_MIN_LEN ({RANGE_MIN_LEN})"
|
|
|
|
|
2022-12-19 15:12:45 +00:00
|
|
|
file_ranges_to_test: list[tuple(int, int, str)] = [
|
2022-12-05 22:31:45 +00:00
|
|
|
# Offset is bigger than the file size, the length is small.
|
2022-12-19 15:12:45 +00:00
|
|
|
(file_size + 1, RANGE_MIN_LEN, OUT_OF_RANGE),
|
2022-12-05 22:31:45 +00:00
|
|
|
# Offset is ok, but offset+length is too big.
|
2022-12-19 15:12:45 +00:00
|
|
|
(file_size - RANGE_MIN_LEN, RANGE_MIN_LEN * 2, OUT_OF_RANGE),
|
2022-12-05 22:31:45 +00:00
|
|
|
# Offset is ok, and length is very-very big (e.g. MaxUint64) so that offset+length is wrapped and still "valid".
|
2022-12-19 15:12:45 +00:00
|
|
|
(RANGE_MIN_LEN, sys.maxsize * 2 + 1, INVALID_RANGE_OVERFLOW),
|
|
|
|
# Length is zero
|
|
|
|
(10, 0, INVALID_RANGE_ZERO_LENGTH),
|
|
|
|
# Negative values
|
|
|
|
(-1, 1, INVALID_OFFSET_SPECIFIER),
|
|
|
|
(10, -5, INVALID_LENGTH_SPECIFIER),
|
2022-12-05 22:31:45 +00:00
|
|
|
]
|
|
|
|
|
2022-12-19 15:12:45 +00:00
|
|
|
for range_start, range_len, expected_error in file_ranges_to_test:
|
2022-12-05 22:31:45 +00:00
|
|
|
range_cut = f"{range_start}:{range_len}"
|
2023-11-02 09:19:23 +00:00
|
|
|
expected_error = expected_error.format(range=range_cut) if "{range}" in expected_error else expected_error
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step(f"Get range ({range_cut})"):
|
2022-12-05 22:31:45 +00:00
|
|
|
for oid in oids:
|
2022-12-19 15:12:45 +00:00
|
|
|
with pytest.raises(Exception, match=expected_error):
|
2022-12-05 22:31:45 +00:00
|
|
|
get_range(
|
|
|
|
wallet,
|
|
|
|
cid,
|
|
|
|
oid,
|
|
|
|
shell=self.shell,
|
|
|
|
endpoint=self.cluster.default_rpc_endpoint,
|
|
|
|
range_cut=range_cut,
|
|
|
|
)
|
|
|
|
|
2024-03-11 12:49:37 +00:00
|
|
|
@allure.title("[NEGATIVE] Get invalid range hash by native API (obj_size={object_size}, policy={placement_policy})")
|
2022-12-05 22:31:45 +00:00
|
|
|
def test_object_get_range_hash_negatives(
|
|
|
|
self,
|
|
|
|
storage_objects: list[StorageObjectInfo],
|
|
|
|
):
|
|
|
|
"""
|
2022-12-19 15:12:45 +00:00
|
|
|
Validate get_range_hash negative for object by native gRPC API
|
2022-12-05 22:31:45 +00:00
|
|
|
"""
|
2022-11-01 16:11:26 +00:00
|
|
|
|
2024-03-11 16:34:54 +00:00
|
|
|
wallet = storage_objects[0].wallet
|
2022-12-05 22:31:45 +00:00
|
|
|
cid = storage_objects[0].cid
|
|
|
|
oids = [storage_object.oid for storage_object in storage_objects[:2]]
|
|
|
|
file_size = storage_objects[0].size
|
2022-07-08 17:24:55 +00:00
|
|
|
|
2022-09-28 12:07:16 +00:00
|
|
|
assert (
|
2022-12-05 22:31:45 +00:00
|
|
|
RANGE_MIN_LEN < file_size
|
|
|
|
), f"Incorrect test setup. File size ({file_size}) is less than RANGE_MIN_LEN ({RANGE_MIN_LEN})"
|
|
|
|
|
2022-12-19 15:12:45 +00:00
|
|
|
file_ranges_to_test: list[tuple(int, int, str)] = [
|
2022-12-05 22:31:45 +00:00
|
|
|
# Offset is bigger than the file size, the length is small.
|
2022-12-19 15:12:45 +00:00
|
|
|
(file_size + 1, RANGE_MIN_LEN, OUT_OF_RANGE),
|
2022-12-05 22:31:45 +00:00
|
|
|
# Offset is ok, but offset+length is too big.
|
2022-12-19 15:12:45 +00:00
|
|
|
(file_size - RANGE_MIN_LEN, RANGE_MIN_LEN * 2, OUT_OF_RANGE),
|
2022-12-05 22:31:45 +00:00
|
|
|
# Offset is ok, and length is very-very big (e.g. MaxUint64) so that offset+length is wrapped and still "valid".
|
2022-12-19 15:12:45 +00:00
|
|
|
(RANGE_MIN_LEN, sys.maxsize * 2 + 1, INVALID_RANGE_OVERFLOW),
|
|
|
|
# Length is zero
|
|
|
|
(10, 0, INVALID_RANGE_ZERO_LENGTH),
|
|
|
|
# Negative values
|
|
|
|
(-1, 1, INVALID_OFFSET_SPECIFIER),
|
|
|
|
(10, -5, INVALID_LENGTH_SPECIFIER),
|
2022-12-05 22:31:45 +00:00
|
|
|
]
|
|
|
|
|
2022-12-19 15:12:45 +00:00
|
|
|
for range_start, range_len, expected_error in file_ranges_to_test:
|
2022-12-05 22:31:45 +00:00
|
|
|
range_cut = f"{range_start}:{range_len}"
|
2023-11-02 09:19:23 +00:00
|
|
|
expected_error = expected_error.format(range=range_cut) if "{range}" in expected_error else expected_error
|
2023-11-29 13:34:59 +00:00
|
|
|
with reporter.step(f"Get range hash ({range_cut})"):
|
2022-12-05 22:31:45 +00:00
|
|
|
for oid in oids:
|
2022-12-19 15:12:45 +00:00
|
|
|
with pytest.raises(Exception, match=expected_error):
|
2022-12-05 22:31:45 +00:00
|
|
|
get_range_hash(
|
|
|
|
wallet,
|
|
|
|
cid,
|
|
|
|
oid,
|
|
|
|
shell=self.shell,
|
|
|
|
endpoint=self.cluster.default_rpc_endpoint,
|
|
|
|
range_cut=range_cut,
|
|
|
|
)
|
|
|
|
|
|
|
|
def check_header_is_presented(self, head_info: dict, object_header: dict) -> None:
|
|
|
|
for key_to_check, val_to_check in object_header.items():
|
2023-11-02 09:19:23 +00:00
|
|
|
assert key_to_check in head_info["header"]["attributes"], f"Key {key_to_check} is found in {head_object}"
|
2022-12-05 22:31:45 +00:00
|
|
|
assert head_info["header"]["attributes"].get(key_to_check) == str(
|
|
|
|
val_to_check
|
|
|
|
), f"Value {val_to_check} is equal"
|