Compare commits

...

11 commits

Author SHA1 Message Date
1e6584f421 [] Fixed run sequence metrics tests
Signed-off-by: Ilyas Niyazov <i.niyazov@yadro.com>
2024-12-24 18:14:17 +03:00
d11abec967 [] Fix imports
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-12-24 05:11:02 +00:00
0ac5edfdfb [] Add test for multipart object in Test_http_object testsuite
Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-12-24 05:07:10 +00:00
f10d899173 [] Update session_token tests
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-12-23 18:01:07 +03:00
9ca70c80e3 [] Move s3 and http directories to avoid conflict with requests
Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-12-18 12:56:30 +03:00
177ef36693 [] Fixed test logs metrics
Signed-off-by: Ilyas Niyazov <i.niyazov@yadro.com>
2024-12-17 14:21:40 +00:00
c2f322daeb [] Add new args for ContainerRequest
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
2024-12-17 14:17:45 +00:00
4eef2f2437 [] Make user fixture parametrized to allow dedicated user creation
Signed-off-by: a.berezin <a.berezin@yadro.com>
2024-12-17 11:03:19 +03:00
c75352e267 [] Automation of PATCH method in S3
Signed-off-by: Kirill Sosnovskikh <k.sosnovskikh@yadro.com>
2024-12-17 07:39:46 +00:00
b36c7e90aa [] Fixed container metrics for tombstone
Signed-off-by: Ilyas Niyazov <i.niyazov@yadro.com>
2024-12-13 17:00:08 +03:00
29aca20956 [] Update error match message for data network test
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
2024-12-05 16:01:19 +03:00
33 changed files with 852 additions and 110 deletions

View file

@ -63,7 +63,15 @@ def _create_container_by_spec(
cluster: Cluster,
endpoint: str,
) -> str:
return create_container(wallet, shell, endpoint, container_request.parsed_rule(cluster), wait_for_creation=False)
return create_container(
wallet,
shell,
endpoint,
container_request.parsed_rule(cluster),
wait_for_creation=False,
nns_zone=container_request.ns_zone,
nns_name=container_request.ns_name,
)
def _apply_ape_rules(cid: str, frostfs_cli: FrostfsCli, endpoint: str, ape_rules: list[ape.Rule]):

View file

@ -21,6 +21,9 @@ class ContainerRequest:
short_name: str | None = None
ns_name: str | None = None
ns_zone: str | None = None
def __post_init__(self):
if self.ape_rules is None:
self.ape_rules = []

View file

@ -20,6 +20,7 @@ from ....helpers.container_access import (
@pytest.mark.sanity
@pytest.mark.bearer
@pytest.mark.ape
@pytest.mark.parametrize("user_tag", ["ApeBearer"], indirect=True) # provide dedicated user with no APE side-policies
class TestApeBearer(ClusterTestBase):
@allure.title("Operations with BearerToken (role={role}, obj_size={object_size})")
@pytest.mark.parametrize("role", [ape.Role.OWNER, ape.Role.OTHERS], indirect=True)

View file

@ -20,6 +20,7 @@ REP1_MSK = ContainerRequest("REP 1 IN MOW CBF 1 SELECT 1 FROM MSK AS MOW FILTER
@pytest.mark.ape_object
@pytest.mark.ape_allow
@pytest.mark.parametrize("container_request", [REP1_MSK], indirect=True)
@pytest.mark.parametrize("user_tag", ["ApeLocalOverrideAllow"], indirect=True) # provide dedicated user with no APE side-policies
class TestApeLocalOverrideAllow(ClusterTestBase):
@allure.title("LocalOverride: Allow to GetObject in root tenant")
def test_local_override_allow_to_get_object_root(

View file

@ -8,18 +8,18 @@ import pytest
from dateutil import parser
from frostfs_testlib import plugins, reporter
from frostfs_testlib.cli import FrostfsCli
from frostfs_testlib.clients import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, S3HttpClient
from frostfs_testlib.clients.s3 import BucketContainerResolver, VersioningStatus
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.healthcheck.interfaces import Healthcheck
from frostfs_testlib.hosting import Hosting
from frostfs_testlib.resources import optionals
from frostfs_testlib.resources.common import COMPLEX_OBJECT_CHUNKS_COUNT, COMPLEX_OBJECT_TAIL_SIZE, SIMPLE_OBJECT_SIZE
from frostfs_testlib.s3 import AwsCliClient, Boto3ClientWrapper, S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.shell import LocalShell, Shell
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.steps.cli.container import DEFAULT_EC_PLACEMENT_RULE, DEFAULT_PLACEMENT_RULE, FROSTFS_CLI_EXEC
from frostfs_testlib.steps.cli.object import get_netmap_netinfo
from frostfs_testlib.steps.epoch import ensure_fresh_epoch
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
@ -206,8 +206,8 @@ def ec_placement_policy() -> PlacementPolicy:
@pytest.fixture(scope="session")
@allure.title("Init Frostfs CLI")
def frostfs_cli(client_shell: Shell, default_wallet: WalletInfo) -> FrostfsCli:
return FrostfsCli(client_shell, FROSTFS_CLI_EXEC, default_wallet.config_path)
def frostfs_cli(client_shell: Shell, wallet: WalletInfo) -> FrostfsCli:
return FrostfsCli(client_shell, FROSTFS_CLI_EXEC, wallet.config_path)
@pytest.fixture(scope="session")
@ -287,20 +287,34 @@ def credentials_provider(cluster: Cluster) -> CredentialsProvider:
],
)
def s3_client(
default_user: User,
user: User,
s3_policy: Optional[str],
cluster: Cluster,
request: pytest.FixtureRequest,
credentials_provider: CredentialsProvider,
) -> S3ClientWrapper:
node = cluster.cluster_nodes[0]
credentials_provider.S3.provide(default_user, node, s3_policy)
credentials_provider.S3.provide(user, node, s3_policy)
s3_client_cls = request.param
client = s3_client_cls(default_user.s3_credentials.access_key, default_user.s3_credentials.secret_key, cluster.default_s3_gate_endpoint)
client = s3_client_cls(user.s3_credentials.access_key, user.s3_credentials.secret_key, cluster.default_s3_gate_endpoint)
return client
@allure.title("[Session] Create S3 http client")
@pytest.fixture(scope="session")
def s3_http_client(
default_user: User, s3_policy: Optional[str], cluster: Cluster, credentials_provider: CredentialsProvider
) -> S3HttpClient:
node = cluster.cluster_nodes[0]
credentials_provider.S3.provide(default_user, node, s3_policy)
return S3HttpClient(
cluster.default_s3_gate_endpoint,
default_user.s3_credentials.access_key,
default_user.s3_credentials.secret_key,
)
@pytest.fixture
def versioning_status(request: pytest.FixtureRequest) -> VersioningStatus:
if "param" in request.__dict__:
@ -443,19 +457,50 @@ def default_user(credentials_provider: CredentialsProvider, cluster: Cluster) ->
return user
@reporter.step("Get wallet for default user")
@pytest.fixture(scope="session")
def default_wallet(default_user: User) -> WalletInfo:
return default_user.wallet
@cached_fixture(optionals.OPTIONAL_CACHE_FIXTURES)
def users_pool(credentials_provider: CredentialsProvider, cluster: Cluster) -> list[User]:
users = [User(string_utils.unique_name("user-")) for _ in range(WALLTETS_IN_POOL)]
parallel(credentials_provider.GRPC.provide, users, cluster_node=cluster.cluster_nodes[0])
return users
@pytest.fixture(scope="session")
def user_tag(request: pytest.FixtureRequest) -> str:
tag = "default"
if "param" in request.__dict__:
tag = request.param
return tag
@pytest.fixture(scope="session")
@cached_fixture(optionals.OPTIONAL_CACHE_FIXTURES)
def wallets_pool(credentials_provider: CredentialsProvider, cluster: Cluster) -> list[WalletInfo]:
users = [User(string_utils.unique_name("user-")) for _ in range(WALLTETS_IN_POOL)]
parallel(credentials_provider.GRPC.provide, users, cluster_node=cluster.cluster_nodes[0])
@reporter.step("Create {user_tag} user")
def user(user_tag: str) -> User:
user = User(string_utils.unique_name("user-"))
user.attributes["tag"] = user_tag
return [user.wallet for user in users]
return user
@pytest.fixture(scope="session")
def wallet(user: User, credentials_provider: CredentialsProvider, cluster: Cluster) -> WalletInfo:
credentials_provider.GRPC.provide(user, cluster.cluster_nodes[0])
return user.wallet
# TODO: Migrate tests to fixture wallet above
@reporter.step("Get wallet for default user")
@pytest.fixture(scope="session")
def default_wallet(wallet: WalletInfo) -> WalletInfo:
return wallet
@pytest.fixture(scope="session")
@cached_fixture(optionals.OPTIONAL_CACHE_FIXTURES)
def wallets_pool(users_pool: list[User]) -> list[WalletInfo]:
return [user.wallet for user in users_pool]
@pytest.fixture(scope="session")
@ -524,26 +569,26 @@ def multiple_containers_request(request: pytest.FixtureRequest) -> ContainerRequ
@pytest.fixture
def container(
default_wallet: WalletInfo,
wallet: WalletInfo,
frostfs_cli: FrostfsCli,
client_shell: Shell,
cluster: Cluster,
rpc_endpoint: str,
container_request: ContainerRequest,
) -> str:
return create_container_with_ape(container_request, frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint)
return create_container_with_ape(container_request, frostfs_cli, wallet, client_shell, cluster, rpc_endpoint)
@pytest.fixture
def containers(
default_wallet: WalletInfo,
wallet: WalletInfo,
frostfs_cli: FrostfsCli,
client_shell: Shell,
cluster: Cluster,
rpc_endpoint: str,
multiple_containers_request: MultipleContainersRequest,
) -> list[str]:
return create_containers_with_ape(frostfs_cli, default_wallet, client_shell, cluster, rpc_endpoint, multiple_containers_request)
return create_containers_with_ape(frostfs_cli, wallet, client_shell, cluster, rpc_endpoint, multiple_containers_request)
@pytest.fixture()

View file

@ -6,9 +6,9 @@ from time import sleep
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.clients.s3 import BucketContainerResolver, S3ClientWrapper, VersioningStatus
from frostfs_testlib.resources.common import MORPH_BLOCK_TIME
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.steps.cli.container import StorageContainer, StorageContainerInfo, create_container
from frostfs_testlib.steps.cli.object import get_object, put_object_to_random_node
from frostfs_testlib.steps.node_management import (
@ -19,8 +19,6 @@ from frostfs_testlib.steps.node_management import (
remove_nodes_from_map_morph,
wait_for_node_to_be_ready,
)
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.steps.s3.s3_helper import search_nodes_with_bucket
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, S3Gate, StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController, ShardsWatcher
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
@ -137,7 +135,7 @@ class TestFailoverStorage(ClusterTestBase):
put_object = s3_client.put_object(bucket, file_path)
s3_helper.check_objects_in_bucket(s3_client, bucket, expected_objects=[file_name])
node_bucket = search_nodes_with_bucket(
node_bucket = s3_helper.search_nodes_with_bucket(
cluster=self.cluster,
bucket_name=bucket,
wallet=default_wallet,

View file

@ -193,7 +193,7 @@ class TestFailoverNetwork(ClusterTestBase):
self.tick_epochs(1, alive_node=nodes_without_an_object[0].storage_node, wait_block=2)
with reporter.step("Get object for target nodes to data interfaces, expect false"):
with pytest.raises(RuntimeError, match="can't create API client: can't init SDK client: gRPC dial: context deadline exceeded"):
with pytest.raises(RuntimeError, match="can't create API client: can't init SDK client: context deadline exceeded"):
get_object(
wallet=default_wallet,
cid=storage_object.cid,

View file

@ -14,10 +14,11 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.parallel import parallel
from frostfs_testlib.utils.file_utils import TestFile, generate_file
from ...helpers.container_request import PUBLIC_WITH_POLICY, ContainerRequest, requires_container
from ...helpers.container_request import PUBLIC_WITH_POLICY, REP_2_1_4_PUBLIC, ContainerRequest, requires_container
from ...helpers.utility import are_numbers_similar
@pytest.mark.order(-5)
@pytest.mark.nightly
@pytest.mark.metrics
class TestContainerMetrics(ClusterTestBase):
@ -133,7 +134,7 @@ class TestContainerMetrics(ClusterTestBase):
@allure.title("Container size metrics put {objects_count} objects (obj_size={object_size})")
@pytest.mark.parametrize("objects_count", [5, 10, 20])
@requires_container
@requires_container(REP_2_1_4_PUBLIC)
def test_container_size_metrics_more_objects(
self, object_size: ObjectSize, default_wallet: WalletInfo, objects_count: int, container: str
):
@ -161,13 +162,22 @@ class TestContainerMetrics(ClusterTestBase):
tombstones_size += int(tombstone["header"]["payloadLength"])
with reporter.step(f"Check container size metrics, 'should be positive in all nodes'"):
futures = parallel(
get_metrics_value, self.cluster.cluster_nodes, command="frostfs_node_engine_container_size_bytes", cid=container
with reporter.step("Search container nodes"):
container_nodes = search_nodes_with_container(
wallet=default_wallet,
cid=container,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
cluster=self.cluster,
)
with reporter.step(f"Get metrics value from container nodes"):
futures = parallel(get_metrics_value, container_nodes, command="frostfs_node_engine_container_size_bytes", cid=container)
metrics_value_nodes = [future.result() for future in futures]
for act_metric in metrics_value_nodes:
assert act_metric >= 0, "Metrics value is negative"
assert sum(metrics_value_nodes) // len(self.cluster.cluster_nodes) == tombstones_size, "tomstone size of objects not correct"
with reporter.step(f"Check container size metrics for tombstone"):
assert sum(metrics_value_nodes) // len(container_nodes) == tombstones_size, "tomstone size of objects not correct"
@allure.title("Container metrics (policy={container_request})")
@pytest.mark.parametrize(

View file

@ -16,6 +16,7 @@ from frostfs_testlib.utils.file_utils import generate_file
from ...helpers.container_request import PUBLIC_WITH_POLICY, requires_container
@pytest.mark.order(-9)
@pytest.mark.nightly
@pytest.mark.metrics
class TestGarbageCollectorMetrics(ClusterTestBase):

View file

@ -18,6 +18,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file
@pytest.mark.order(-6)
@pytest.mark.nightly
@pytest.mark.metrics
class TestGRPCMetrics(ClusterTestBase):

View file

@ -14,6 +14,7 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import wait_for_success
@pytest.mark.order(-10)
@pytest.mark.nightly
@pytest.mark.metrics
class TestLogsMetrics(ClusterTestBase):
@ -34,7 +35,9 @@ class TestLogsMetrics(ClusterTestBase):
current_time = datetime.now(timezone.utc)
counter_metrics = get_metrics_value(cluster_node, **metrics_greps)
counter_logs = self.get_count_logs_by_level(cluster_node, metrics_greps.get("level"), restart_time, current_time, log_priority)
assert counter_logs == counter_metrics, f"counter_logs: {counter_logs}, counter_metrics: {counter_metrics} in node: {cluster_node}"
assert counter_logs == pytest.approx(
counter_metrics, rel=0.02
), f"counter_logs: {counter_logs}, counter_metrics: {counter_metrics} in node: {cluster_node}"
@staticmethod
def get_count_logs_by_level(cluster_node: ClusterNode, log_level: str, after_time: datetime, until_time: datetime, log_priority: str):

View file

@ -17,6 +17,7 @@ from frostfs_testlib.utils.file_utils import TestFile
from ...helpers.container_request import PUBLIC_WITH_POLICY, ContainerRequest, requires_container
@pytest.mark.order(-7)
@pytest.mark.nightly
@pytest.mark.metrics
class TestObjectMetrics(ClusterTestBase):

View file

@ -19,6 +19,7 @@ from frostfs_testlib.utils.file_utils import generate_file
from ...helpers.container_request import PUBLIC_WITH_POLICY, requires_container
@pytest.mark.order(-8)
@pytest.mark.nightly
@pytest.mark.metrics
class TestShardMetrics(ClusterTestBase):

View file

@ -7,10 +7,10 @@ import yaml
from frostfs_testlib import reporter
from frostfs_testlib.cli import FrostfsAdm, FrostfsCli
from frostfs_testlib.cli.netmap_parser import NetmapParser
from frostfs_testlib.clients import AwsCliClient, S3ClientWrapper
from frostfs_testlib.clients.s3 import BucketContainerResolver, VersioningStatus
from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT, FROSTFS_ADM_CONFIG_PATH, FROSTFS_ADM_EXEC, FROSTFS_CLI_EXEC
from frostfs_testlib.resources.common import COMPLEX_OBJECT_CHUNKS_COUNT, COMPLEX_OBJECT_TAIL_SIZE, HOSTING_CONFIG_FILE, MORPH_BLOCK_TIME
from frostfs_testlib.s3 import AwsCliClient, S3ClientWrapper
from frostfs_testlib.s3.interfaces import BucketContainerResolver, VersioningStatus
from frostfs_testlib.storage.cluster import Cluster, ClusterNode, StorageNode
from frostfs_testlib.storage.controllers import ClusterStateController
from frostfs_testlib.storage.controllers.state_managers.config_state_manager import ConfigStateManager

View file

@ -5,7 +5,7 @@ import pytest
from frostfs_testlib import reporter
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
from frostfs_testlib.steps.acl import bearer_token_base64_from_file
from frostfs_testlib.steps.http.http_gate import upload_via_http_gate_curl, verify_object_hash
from frostfs_testlib.steps.http_gate import upload_via_http_gate_curl, verify_object_hash
from frostfs_testlib.storage.cluster import Cluster
from frostfs_testlib.storage.dataclasses import ape
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize

View file

@ -3,7 +3,7 @@ import pytest
from frostfs_testlib import reporter
from frostfs_testlib.steps.cli.object import put_object_to_random_node
from frostfs_testlib.steps.epoch import get_epoch
from frostfs_testlib.steps.http.http_gate import (
from frostfs_testlib.steps.http_gate import (
attr_into_header,
get_object_by_attr_and_verify_hashes,
get_via_http_curl,

View file

@ -6,7 +6,7 @@ import pytest
from frostfs_testlib import reporter
from frostfs_testlib.steps.cli.container import delete_container, list_containers, wait_for_container_deletion
from frostfs_testlib.steps.cli.object import delete_object
from frostfs_testlib.steps.http.http_gate import (
from frostfs_testlib.steps.http_gate import (
attr_into_str_header_curl,
get_object_by_attr_and_verify_hashes,
try_to_get_object_and_expect_error,

View file

@ -3,24 +3,26 @@ import logging
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.s3 import AwsCliClient, S3ClientWrapper
from frostfs_testlib.clients import S3ClientWrapper
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.steps.cli.object import put_object_to_random_node
from frostfs_testlib.steps.http.http_gate import (
from frostfs_testlib.steps.http_gate import (
assert_hashes_are_equal,
get_object_by_attr_and_verify_hashes,
get_via_http_gate,
try_to_get_object_via_passed_request_and_expect_error,
verify_object_hash,
)
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import TestFile
from frostfs_testlib.utils.file_utils import TestFile, generate_file, split_file
from ....helpers.container_request import REP_2_1_4_PUBLIC, requires_container
logger = logging.getLogger("NeoLogger")
PART_SIZE = 5 * 1024 * 1024
@pytest.mark.nightly
@pytest.mark.sanity
@ -66,6 +68,7 @@ class Test_http_object(ClusterTestBase):
cluster=self.cluster,
attributes=f"{key_value1},{key_value2}",
)
with reporter.step("Get object and verify hashes [ get/$CID/$OID ]"):
verify_object_hash(
oid=oid,
@ -91,25 +94,19 @@ class Test_http_object(ClusterTestBase):
)
with reporter.step("Download the object with attribute [get_by_attribute/$CID/chapter1/peace]"):
get_object_by_attr_and_verify_hashes(
oid=oid,
file_name=test_file.path,
cid=container,
attrs=attrs,
node=self.cluster.cluster_nodes[0],
)
get_object_by_attr_and_verify_hashes(oid, test_file, container, attrs, self.cluster.cluster_nodes[0])
with reporter.step("[Negative] try to get object: get_by_attribute/$CID/$OID"):
request = f"/get_by_attribute/{container}/{oid}"
try_to_get_object_via_passed_request_and_expect_error(
cid=container,
oid=oid,
node=self.cluster.cluster_nodes[0],
container,
oid,
self.cluster.cluster_nodes[0],
error_pattern=expected_err_msg,
http_request_path=request,
)
@allure.title("Put over s3, Get over HTTP with bucket name and key (object_size={object_size})")
@pytest.mark.parametrize("s3_client", [AwsCliClient], indirect=True)
@allure.title("Put object over S3, get over HTTP with bucket name and key (s3_client={s3_client}, object_size={object_size})")
def test_object_put_get_bucketname_key(self, test_file: TestFile, s3_client: S3ClientWrapper):
"""
Test that object can be put using s3-gateway interface and got via HTTP with bucket name and object key.
@ -125,17 +122,55 @@ class Test_http_object(ClusterTestBase):
Hashes must be the same.
"""
object_key = s3_helper.object_key_from_file_path(test_file.path)
bucket = s3_client.create_bucket(acl="public-read-write")
s3_client.put_object(bucket=bucket, filepath=test_file.path, key=object_key)
obj_s3 = s3_client.get_object(bucket=bucket, key=object_key)
object_key = s3_helper.object_key_from_file_path(test_file)
request = f"/get/{bucket}/{object_key}"
obj_http = get_via_http_gate(
cid=None,
oid=None,
node=self.cluster.cluster_nodes[0],
request_path=request,
)
with reporter.step("Verify hashes"):
with reporter.step("Create public bucket"):
bucket = s3_client.create_bucket(acl="public-read-write")
with reporter.step("Put object"):
s3_client.put_object(bucket, test_file, object_key)
with reporter.step("Get object via S3 gate"):
obj_s3 = s3_client.get_object(bucket, object_key)
with reporter.step("Get object via HTTP gate"):
obj_http = get_via_http_gate(bucket, object_key, node=self.cluster.cluster_nodes[0])
with reporter.step("Make sure the hashes of both objects are the same"):
assert_hashes_are_equal(test_file.path, obj_http, obj_s3)
@allure.title("Put multipart object over S3, get over HTTP with bucket name and key (s3_client={s3_client})")
def test_object_put_get_bucketname_key_multipart(self, s3_client: S3ClientWrapper):
parts = []
parts_count = 5
original_size = PART_SIZE * parts_count
with reporter.step("Create public container"):
bucket = s3_client.create_bucket(acl="public-read-write")
with reporter.step("Generate original object and split it into parts"):
original_file = generate_file(original_size)
file_parts = split_file(original_file, parts_count)
object_key = s3_helper.object_key_from_file_path(original_file)
with reporter.step("Create multipart and upload parts"):
upload_id = s3_client.create_multipart_upload(bucket, object_key)
for part_id, file_path in enumerate(file_parts, start=1):
etag = s3_client.upload_part(bucket, object_key, upload_id, part_id, file_path)
parts.append((part_id, etag))
with reporter.step("Check all parts are visible in bucket"):
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
assert len(got_parts) == len(file_parts), f"Expected {parts_count} parts, got:\n{got_parts}"
with reporter.step("Complete multipart upload"):
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
with reporter.step("Get multipart object via S3 gate"):
obj_s3 = s3_client.get_object(bucket, object_key)
with reporter.step("Get multipart object via HTTP gate"):
obj_http = get_via_http_gate(bucket, object_key, self.cluster.cluster_nodes[0])
with reporter.step("Make sure the hashes of both objects are the same"):
assert_hashes_are_equal(original_file, obj_http, obj_s3)

View file

@ -3,7 +3,7 @@ import logging
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.steps.http.http_gate import upload_via_http_gate_curl, verify_object_hash
from frostfs_testlib.steps.http_gate import upload_via_http_gate_curl, verify_object_hash
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase

View file

@ -9,7 +9,7 @@ from frostfs_testlib import reporter
from frostfs_testlib.resources.error_patterns import OBJECT_NOT_FOUND
from frostfs_testlib.steps.cli.object import get_netmap_netinfo, get_object_from_random_node, head_object
from frostfs_testlib.steps.epoch import get_epoch, wait_for_epochs_align
from frostfs_testlib.steps.http.http_gate import (
from frostfs_testlib.steps.http_gate import (
attr_into_str_header_curl,
try_to_get_object_and_expect_error,
upload_via_http_gate_curl,

View file

@ -1,10 +1,10 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.clients import S3ClientWrapper
from frostfs_testlib.resources.error_patterns import S3_BUCKET_DOES_NOT_ALLOW_ACL
from frostfs_testlib.resources.s3_acl_grants import PRIVATE_GRANTS, PUBLIC_READ_GRANTS, PUBLIC_READ_WRITE_GRANTS
from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file
@ -33,32 +33,32 @@ class TestS3GateACL:
def test_s3_create_bucket_with_ACL(self, s3_client: S3ClientWrapper):
with reporter.step("Create bucket with ACL private"):
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="private")
bucket_grants = s3_client.get_bucket_acl(bucket)
bucket_grants = s3_client.get_bucket_acl(bucket).get("Grants")
s3_helper.verify_acl_permissions(bucket_grants, PRIVATE_GRANTS)
with reporter.step("Create bucket with ACL public-read"):
read_bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read")
bucket_grants = s3_client.get_bucket_acl(read_bucket)
bucket_grants = s3_client.get_bucket_acl(read_bucket).get("Grants")
s3_helper.verify_acl_permissions(bucket_grants, PUBLIC_READ_GRANTS)
with reporter.step("Create bucket with ACL public-read-write"):
public_rw_bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read-write")
bucket_grants = s3_client.get_bucket_acl(public_rw_bucket)
bucket_grants = s3_client.get_bucket_acl(public_rw_bucket).get("Grants")
s3_helper.verify_acl_permissions(bucket_grants, PUBLIC_READ_WRITE_GRANTS)
@allure.title("Bucket ACL (s3_client={s3_client})")
def test_s3_bucket_ACL(self, s3_client: S3ClientWrapper):
with reporter.step("Create bucket with public-read-write ACL"):
bucket = s3_client.create_bucket(object_lock_enabled_for_bucket=True, acl="public-read-write")
bucket_grants = s3_client.get_bucket_acl(bucket)
bucket_grants = s3_client.get_bucket_acl(bucket).get("Grants")
s3_helper.verify_acl_permissions(bucket_grants, PUBLIC_READ_WRITE_GRANTS)
with reporter.step("Change bucket ACL to private"):
s3_client.put_bucket_acl(bucket, acl="private")
bucket_grants = s3_client.get_bucket_acl(bucket)
bucket_grants = s3_client.get_bucket_acl(bucket).get("Grants")
s3_helper.verify_acl_permissions(bucket_grants, PRIVATE_GRANTS)
with reporter.step("Change bucket ACL to public-read"):
s3_client.put_bucket_acl(bucket, acl="public-read")
bucket_grants = s3_client.get_bucket_acl(bucket)
bucket_grants = s3_client.get_bucket_acl(bucket).get("Grants")
s3_helper.verify_acl_permissions(bucket_grants, PUBLIC_READ_GRANTS)

View file

@ -4,8 +4,8 @@ from datetime import datetime, timedelta
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.clients.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils import string_utils
from frostfs_testlib.utils.file_utils import generate_file

View file

@ -0,0 +1,623 @@
import random
import time
from datetime import datetime
from email.utils import formatdate
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.cli.generic_cli import GenericCli
from frostfs_testlib.clients import Boto3ClientWrapper, S3ClientWrapper, S3HttpClient
from frostfs_testlib.clients.s3 import VersioningStatus
from frostfs_testlib.credentials.interfaces import CredentialsProvider, User
from frostfs_testlib.shell.interfaces import CommandOptions
from frostfs_testlib.shell.local_shell import LocalShell
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.storage.cluster import Cluster, ClusterNode
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.policy import PlacementPolicy
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import TestFile, generate_file, get_file_hash, split_file
from frostfs_testlib.utils.string_utils import unique_name
from ....resources.common import S3_POLICY_FILE_LOCATION
FIVE_GIGABYTES = 5_368_709_120
PART_SIZE_FOR_MULTIPART = 5 * 1024 * 1024
@reporter.step("Allow patch for bucket")
def allow_patch_for_bucket(s3_client: S3ClientWrapper, bucket: str):
s3_client.put_bucket_policy(
bucket,
policy={
"Version": "2012-10-17",
"Id": "aaaa-bbbb-cccc-dddd",
"Statement": [
{
"Sid": "AddPerm",
"Effect": "Allow",
"Principal": "*",
"Action": ["s3:PatchObject"],
"Resource": [f"arn:aws:s3:::{bucket}/*"],
},
],
},
)
def pytest_generate_tests(metafunc: pytest.Metafunc):
if "s3_client" not in metafunc.fixturenames:
return
metafunc.parametrize("s3_policy", [S3_POLICY_FILE_LOCATION], ids=["s3policy"], indirect=True)
@pytest.fixture(scope="session", params=[pytest.param("rep3", marks=pytest.mark.rep), pytest.param("ec3.1", marks=pytest.mark.ec)])
def placement_policy(request: pytest.FixtureRequest) -> PlacementPolicy:
if request.param == "ec3.1":
return PlacementPolicy("ec3.1", "ec3.1")
return PlacementPolicy("rep3", "rep3")
@pytest.fixture(scope="session")
def versioning_status(request: pytest.FixtureRequest) -> VersioningStatus:
if "param" in request.__dict__:
return request.param
return VersioningStatus.UNDEFINED
@allure.title("[Class] Create bucket")
@pytest.fixture(scope="class")
def bucket(s3_client: S3ClientWrapper, versioning_status: VersioningStatus, placement_policy: PlacementPolicy) -> str:
with reporter.step(f"Create bucket with location constraint {placement_policy.value}"):
bucket = s3_client.create_bucket(location_constraint=placement_policy.value)
s3_helper.set_bucket_versioning(s3_client, bucket, versioning_status)
allow_patch_for_bucket(s3_client, bucket)
return bucket
@pytest.fixture(scope="function")
def original_object(s3_client: S3ClientWrapper, bucket: str, test_file: TestFile) -> str:
with reporter.step("Put object"):
key = s3_helper.object_key_from_file_path(test_file)
s3_client.put_object(bucket, test_file, key)
return key
@allure.title("[Session]: Create S3 client for another user")
@pytest.fixture(scope="session")
def another_s3_client(
users_pool: list[User],
s3_policy: str | None,
cluster: Cluster,
credentials_provider: CredentialsProvider,
s3_client: S3ClientWrapper,
) -> S3ClientWrapper:
user = users_pool[0]
node = cluster.cluster_nodes[0]
credentials_provider.S3.provide(user, node, s3_policy)
s3_client_cls = type(s3_client)
return s3_client_cls(user.s3_credentials.access_key, user.s3_credentials.secret_key, cluster.default_s3_gate_endpoint)
@allure.title("[Class] Create bucket under another user")
@pytest.fixture(scope="class")
def another_bucket(another_s3_client: S3ClientWrapper, versioning_status: VersioningStatus, placement_policy: PlacementPolicy) -> str:
with reporter.step(f"Create bucket with location constraint {placement_policy.value}"):
bucket = another_s3_client.create_bucket(location_constraint=placement_policy.value)
s3_helper.set_bucket_versioning(another_s3_client, bucket, versioning_status)
allow_patch_for_bucket(another_s3_client, bucket)
return bucket
@pytest.mark.nightly
@pytest.mark.s3_gate
class TestS3GateHttpObject(ClusterTestBase):
@allure.title("Patch simple object payload (range={patch_range}, s3_client={s3_client}, policy={placement_policy})")
@pytest.mark.parametrize("object_size", ["simple"], indirect=True)
@pytest.mark.parametrize(
"patch_range",
# String "object" denotes size of object.
["0:19", "500:550", "object/2-100:object/2+200", "object-1:object", "object:object", "object:object+123"],
)
def test_patch_simple_object_payload(
self,
s3_client: S3ClientWrapper,
s3_http_client: S3HttpClient,
bucket: str,
original_object: str,
object_size: ObjectSize,
patch_range: str,
):
start, end = s3_helper.get_range_relative_to_object(patch_range, object_size.value, int_values=True)
content_size = end - start + 1
content_range = f"bytes {start}-{end}/*"
with reporter.step("Generate payload object"):
content_file = generate_file(content_size)
with reporter.step("Patch simple object"):
s3_http_client.patch_object(bucket, original_object, content_file, content_range)
with reporter.step("Get patched part of object and make sure it has changed correctly"):
patched_file_part = s3_client.get_object(bucket, original_object, object_range=(start, end))
assert get_file_hash(patched_file_part) == get_file_hash(
content_file
), "Expected content hash did not match actual content hash"
@allure.title("Patch complex object payload (range={patch_range}, s3_client={s3_client}, policy={placement_policy})")
@pytest.mark.parametrize("object_size", ["complex"], indirect=True)
@pytest.mark.parametrize(
"patch_range",
# Strings "object" and "part" denote size of object and its part, respectively.
["part:part+100", "object-part:object", "0:part", "part*2:part*3", "part-1:part*2", "part+1:part*2-1"],
)
def test_patch_complex_object_payload(
self,
s3_client: S3ClientWrapper,
s3_http_client: S3HttpClient,
bucket: str,
original_object: str,
object_size: ObjectSize,
max_object_size: int,
patch_range: str,
):
start, end = s3_helper.get_range_relative_to_object(patch_range, object_size.value, max_object_size, int_values=True)
content_size = end - start + 1
content_range = f"bytes {start}-{end}/*"
with reporter.step("Generate payload object"):
content_file = generate_file(content_size)
with reporter.step("Patch complex object"):
s3_http_client.patch_object(bucket, original_object, content_file, content_range)
with reporter.step("Get patched part of object and make sure it has changed correctly"):
patched_file_part = s3_client.get_object(bucket, original_object, object_range=(start, end))
assert get_file_hash(patched_file_part) == get_file_hash(
content_file
), "Expected content hash did not match actual content hash"
@allure.title(
"Patch object with fulfilled If-Match condition (s3_client={s3_client}, object_size={object_size}, policy={placement_policy})"
)
def test_patch_with_fulfilled_if_match_contidion(
self,
s3_client: S3ClientWrapper,
s3_http_client: S3HttpClient,
bucket: str,
original_object: str,
):
start, end = 100, 199
content_size = end - start + 1
content_range = f"bytes {start}-{end}/*"
with reporter.step("Generate payload object"):
content_file = generate_file(content_size)
expected_hash = get_file_hash(content_file)
with reporter.step("Get object ETag attribute"):
object_info = s3_client.head_object(bucket, original_object)
etag = object_info["ETag"]
with reporter.step("Patch object with If-Match header"):
s3_http_client.patch_object(bucket, original_object, content_file, content_range, if_match=etag)
with reporter.step("Get patched object and make sure it has changed correctly"):
patched_file = s3_client.get_object(bucket, original_object)
patched_hash = get_file_hash(patched_file, offset=start, len=content_size)
assert patched_hash == expected_hash, "Expected content hash did not match actual content hash"
@allure.title(
"[NEGATIVE] Patch cannot be applied with failed If-Match condition "
"(s3_client={s3_client}, object_size={object_size}, policy={placement_policy})"
)
def test_patch_with_failed_if_match_condition(self, s3_http_client: S3HttpClient, bucket: str, original_object: str):
with reporter.step("Try patch object with If-Match header and get exception"):
with pytest.raises(Exception, match="PreconditionFailed"):
s3_http_client.patch_object(bucket, original_object, "content", "bytes 0-6/*", if_match="nonexistentetag")
@allure.title(
"Patch object with fulfilled If-Unmodified-Since condition "
"(s3_client={s3_client}, object_size={object_size}, policy={placement_policy})"
)
def test_patch_with_fulfilled_if_unmodified_since_condition(
self,
s3_client: S3ClientWrapper,
s3_http_client: S3HttpClient,
bucket: str,
original_object: str,
):
start, end = 235, 341
content_size = end - start + 1
content_range = f"bytes {start}-{end}/*"
with reporter.step("Generate payload object"):
content_file = generate_file(content_size)
expected_hash = get_file_hash(content_file)
with reporter.step("Get object LastModified attribute"):
response = s3_client.head_object(bucket, original_object)
if isinstance(response["LastModified"], str):
response["LastModified"] = datetime.fromisoformat(response["LastModified"])
# Convert datetime to RFC 7232 format
last_modified = formatdate(response["LastModified"].timestamp(), localtime=False, usegmt=True)
with reporter.step("Patch object with If-Unmodified-Since header"):
s3_http_client.patch_object(bucket, original_object, content_file, content_range, if_unmodified_since=last_modified)
with reporter.step("Get patched object and make sure it has changed correctly"):
patched_file = s3_client.get_object(bucket, original_object)
patched_hash = get_file_hash(patched_file, offset=start, len=content_size)
assert patched_hash == expected_hash, "Expected content hash did not match actual content hash"
@allure.title(
"[NEGATIVE] Patch cannot be applied with failed If-Unmodified-Since condition "
"(s3_client={s3_client}, object_size={object_size}, policy={placement_policy})"
)
def test_patch_with_failed_if_unmodified_since_condition(
self,
s3_client: S3ClientWrapper,
s3_http_client: S3HttpClient,
bucket: str,
original_object: str,
test_file: TestFile,
):
with reporter.step("Get original object LastModified attribute"):
response = s3_client.head_object(bucket, original_object)
if isinstance(response["LastModified"], str):
response["LastModified"] = datetime.fromisoformat(response["LastModified"])
# Convert datetime to RFC 7232 format
previous_last_modified = formatdate(response["LastModified"].timestamp(), localtime=False, usegmt=True)
with reporter.step("Wait two seconds for LastModified to update"):
# Next PUT for a simple object occurs at the same second the object was initially loaded,
# so the LastModified attribute "as if" does not change after the operation.
time.sleep(2)
with reporter.step("Put new data for existing object"):
s3_client.put_object(bucket, test_file, original_object)
with reporter.step("Get object LastModified attribute with new data and make sure it has changed"):
response = s3_client.head_object(bucket, original_object)
if isinstance(response["LastModified"], str):
response["LastModified"] = datetime.fromisoformat(response["LastModified"])
# Convert datetime to RFC 7232 format
last_modified = formatdate(response["LastModified"].timestamp(), localtime=False, usegmt=True)
assert last_modified != previous_last_modified, f"Attribute LastModified was expected to change: {last_modified}"
with reporter.step("Try patch object with If-Unmodified-Since header and get exception"):
with pytest.raises(Exception, match="PreconditionFailed"):
s3_http_client.patch_object(bucket, original_object, b"modify", "bytes 0-5/*", if_unmodified_since=previous_last_modified)
@allure.title(
"Patch object with fulfilled x-amz-expected-bucket-owner condition "
"(s3_client={s3_client}, object_size={object_size}, policy={placement_policy})"
)
def test_patch_with_fulfilled_if_expected_bucket_owner_condition(
self,
s3_client: S3ClientWrapper,
s3_http_client: S3HttpClient,
bucket: str,
original_object: str,
):
start, end = 512, 749
content_size = end - start + 1
content_range = f"bytes {start}-{end}/*"
with reporter.step("Generate payload object"):
content_file = generate_file(content_size)
expected_hash = get_file_hash(content_file)
with reporter.step("Get bucket owner ID"):
bucket_acl = s3_client.get_bucket_acl(bucket)
expected_bucket_owner = bucket_acl["Owner"]["DisplayName"]
with reporter.step("Patch object with x-amz-expected-bucket-owner header"):
s3_http_client.patch_object(
bucket,
original_object,
content_file,
content_range,
x_amz_expected_bucket_owner=expected_bucket_owner,
)
with reporter.step("Get patched object and make sure it has changed correctly"):
patched_file = s3_client.get_object(bucket, original_object)
patched_hash = get_file_hash(patched_file, offset=start, len=content_size)
assert patched_hash == expected_hash, "Expected content hash did not match actual content hash"
@allure.title(
"[NEGATIVE] Patch cannot be applied with non-existent bucket owner ID in x-amz-expected-bucket-owner header "
"(s3_client={s3_client}, object_size={object_size}, policy={placement_policy})"
)
def test_patch_with_non_existent_bucket_owner_id(
self,
s3_client: S3ClientWrapper,
s3_http_client: S3HttpClient,
bucket: str,
original_object: str,
):
with reporter.step("Get bucket owner ID"):
bucket_acl = s3_client.get_bucket_acl(bucket)
bucket_owner = bucket_acl["Owner"]["DisplayName"]
with reporter.step("Change owner ID to non-existent"):
unexpected_bucket_owner = list(bucket_owner)
random.shuffle(unexpected_bucket_owner)
unexpected_bucket_owner = "".join(unexpected_bucket_owner)
with reporter.step("Try patch object with x-amz-expected-bucket-owner header and get exception"):
with pytest.raises(Exception, match="AccessDenied"):
s3_http_client.patch_object(
bucket,
original_object,
b"blablabla",
"bytes 10-18/*",
x_amz_expected_bucket_owner=unexpected_bucket_owner,
)
# AwsCliClient is not configured correctly for some cases,
# resulting in both buckets being created from the same user, which causes an error.
@allure.title(
"[NEGATIVE] Patch cannot be applied with another bucket owner ID in x-amz-expected-bucket-owner header "
"(s3_client={s3_client}, object_size={object_size}, policy={placement_policy})"
)
@pytest.mark.parametrize("s3_client", [Boto3ClientWrapper], indirect=True)
def test_patch_with_another_bucket_owner_id(
self,
s3_client: S3ClientWrapper,
s3_http_client: S3HttpClient,
bucket: str,
original_object: str,
another_bucket: str,
):
with reporter.step("Get owner ID of another bucket"):
bucket_acl = s3_client.get_bucket_acl(another_bucket)
another_bucket_owner = bucket_acl["Owner"]["DisplayName"]
with reporter.step("Try patch object with x-amz-expected-bucket-owner header and get exception"):
with pytest.raises(Exception, match="AccessDenied"):
s3_http_client.patch_object(
bucket,
original_object,
b"blablabla",
"bytes 10-18/*",
x_amz_expected_bucket_owner=another_bucket_owner,
)
@allure.title(
"[NEGATIVE] Patch cannot be applied with invalid Content-Range header "
"(range={patch_range}, s3_client={s3_client}, object_size={object_size}, policy={placement_policy})"
)
@pytest.mark.parametrize(
"patch_range",
# String "object" denotes size of object.
["object+100:200", "object+10:object+16", "-1:1", "20:100", "0:2", f"0:{FIVE_GIGABYTES}", "0:0"],
)
def test_patch_with_invalid_content_range(
self,
s3_http_client: S3HttpClient,
bucket: str,
original_object: str,
object_size: ObjectSize,
patch_range: str,
):
content_range = s3_helper.get_range_relative_to_object(patch_range, object_size.value)
with reporter.step("Try patch object with invalid Content-Range header and get exception"):
with pytest.raises(Exception, match="InvalidRange"):
s3_http_client.patch_object(bucket, original_object, b"content", content_range)
@allure.title(
"[NEGATIVE] Patch cannot be applied without Content-Range header "
"(s3_client={s3_client}, object_size={object_size}, policy={placement_policy})"
)
def test_patch_without_content_range(self, s3_http_client: S3HttpClient, bucket: str, original_object: str):
with reporter.step("Try patch object without Content-Range header and get exception"):
with pytest.raises(Exception, match="MissingContentRange"):
s3_http_client.patch_object(bucket, original_object, b"content", None)
@allure.title(
"[NEGATIVE] Patch cannot be applied without Content-Length header "
"(s3_client={s3_client}, object_size={object_size}, policy={placement_policy})"
)
def test_patch_without_content_length(
self,
s3_http_client: S3HttpClient,
bucket: str,
original_object: str,
node_under_test: ClusterNode,
):
with reporter.step("Generate headers that comply with AWS specification"):
data = "content"
url = f"{self.cluster.default_s3_gate_endpoint}/{bucket}/{original_object}"
host = self.cluster.default_s3_gate_endpoint[8:]
headers = {"Host": host, "Url": url, "Content-Range": "bytes 0-6/*"}
headers = dict(s3_http_client._create_aws_request("PATCH", url, headers, data).headers)
headers.pop("Content-Length", None)
with reporter.step("Try patch object without Content-Length header and get exception"):
curl = GenericCli("curl", node_under_test.host)
request = f" {url} -X PATCH"
for header, value in headers.items():
request += f" -H '{header}: {value}'"
# Remove Content-Length header
# *Header without a value means to CURL that it should not be inserted into the request.
request += " -H 'Content-Length:'"
request += f" -d '{data}' -k"
response = curl(request, shell=LocalShell(), options=CommandOptions(check=False))
assert "MissingContentLength" in response.stdout, response.stdout
@allure.title("[NEGATIVE] Patch cannot be applied to non-existent bucket")
def test_patch_non_existent_bucket(self, s3_http_client: S3HttpClient):
with reporter.step("Try patch object in non-existent bucket and get exception"):
with pytest.raises(Exception, match="NoSuchBucket"):
s3_http_client.patch_object("fake-bucket", unique_name("object-"), b"content", "bytes 0-6/*")
@allure.title("[NEGATIVE] Patch cannot be applied to non-existent object (s3_client={s3_client}, policy={placement_policy})")
def test_patch_non_existent_object(self, s3_http_client: S3HttpClient, bucket: str):
with reporter.step("Try patch non-existent object and get exception"):
with pytest.raises(Exception, match="NoSuchKey"):
s3_http_client.patch_object(bucket, "fake-object", b"content", "bytes 0-6/*")
@allure.title("Patch object in versioned bucket (s3_client={s3_client}, object_size={object_size}, policy={placement_policy})")
@pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED], indirect=True)
def test_patch_object_in_versioned_bucket(
self,
s3_client: S3ClientWrapper,
s3_http_client: S3HttpClient,
bucket: str,
object_size: ObjectSize,
):
patch_ranges = ["0:35", "40:49", "object-100:object", "object:object+231"]
with reporter.step("Generate original object"):
original_file = generate_file(object_size.value)
original_key = s3_helper.object_key_from_file_path(original_file)
with reporter.step("Put object"):
version = s3_client.put_object(bucket, original_file, original_key)
expected_versions = {version}
with reporter.step("Patch versioned object"):
for rng in patch_ranges:
start, end = s3_helper.get_range_relative_to_object(rng, object_size=object_size.value, int_values=True)
content_size = end - start + 1
content_range = f"bytes {start}-{end}/*"
with reporter.step(f"Generate payload object of {content_size} bytes"):
content_file = generate_file(content_size)
with reporter.step(f"Patch object and get new version"):
response = s3_http_client.patch_object(bucket, original_key, content_file, content_range, version_id=version)
version = response["VersionId"]
expected_versions.add(version)
with reporter.step(f"Get patched part of object and make sure it has changed correctly"):
got_part = s3_client.get_object(bucket, original_key, version_id=version, object_range=(start, end))
assert get_file_hash(got_part) == get_file_hash(content_file), "Expected content hash did not match actual content hash"
with reporter.step("Check that all expected versions are in bucket"):
got_versions = {
version.get("VersionId") for version in s3_client.list_objects_versions(bucket) if version.get("Key") == original_key
}
assert expected_versions == got_versions, f"Expected versions of object are missing from bucket: {expected_versions}"
@allure.title("Patch multipart object (range={patch_range}, s3_client={s3_client}, policy={placement_policy})")
@pytest.mark.parametrize("patch_range", ["0:part-1", "part:part*2-1", "part-100:part*2+200", "object-part-1:object"])
def test_s3_patch_multipart_object(
self,
s3_client: S3ClientWrapper,
s3_http_client: S3HttpClient,
bucket: str,
patch_range: str,
):
parts_count = 5
parts = []
original_size = PART_SIZE_FOR_MULTIPART * parts_count
with reporter.step("Generate original object and split it into parts"):
original_file = generate_file(original_size)
file_parts = split_file(original_file, parts_count)
object_key = s3_helper.object_key_from_file_path(original_file)
start, end = s3_helper.get_range_relative_to_object(
patch_range, object_size=original_size, part_size=PART_SIZE_FOR_MULTIPART, int_values=True
)
content_size = end - start + 1
content_range = f"bytes {start}-{end}/*"
with reporter.step("Generate payload object"):
content_file = generate_file(content_size)
with reporter.step("Create multipart and upload parts"):
upload_id = s3_client.create_multipart_upload(bucket, object_key)
for part_id, file_path in enumerate(file_parts, start=1):
etag = s3_client.upload_part(bucket, object_key, upload_id, part_id, file_path)
parts.append((part_id, etag))
with reporter.step("Check all parts are visible in bucket"):
got_parts = s3_client.list_parts(bucket, object_key, upload_id)
assert len(got_parts) == len(file_parts), f"Expected {parts_count} parts, got:\n{got_parts}"
with reporter.step("Complete multipart upload"):
s3_client.complete_multipart_upload(bucket, object_key, upload_id, parts)
with reporter.step("Patch multipart object"):
s3_http_client.patch_object(bucket, object_key, content_file, content_range, timeout=200)
with reporter.step("Get patched part of object and make sure it has changed correctly"):
got_part = s3_client.get_object(bucket, object_key, object_range=(start, end))
assert get_file_hash(got_part) == get_file_hash(content_file), "Expected content hash did not match actual content hash"
@allure.title("Patch multipart object in versioned bucket (s3_client={s3_client}, policy={placement_policy})")
@pytest.mark.parametrize("versioning_status", [VersioningStatus.ENABLED], indirect=True)
def test_s3_patch_multipart_object_in_versioned_bucket(
self,
s3_client: S3ClientWrapper,
s3_http_client: S3HttpClient,
bucket: str,
):
parts = []
parts_count = 5
original_size = PART_SIZE_FOR_MULTIPART * parts_count
patch_ranges = ["0:part-1", "part:part*2-1", "part-100:part*2+200", "object-part-1:object"]
with reporter.step("Generate original object and split it into parts"):
original_file = generate_file(original_size)
original_key = s3_helper.object_key_from_file_path(original_file)
file_parts = split_file(original_file, parts_count)
with reporter.step("Create multipart and upload parts"):
upload_id = s3_client.create_multipart_upload(bucket, original_key)
for part_id, file_path in enumerate(file_parts, start=1):
etag = s3_client.upload_part(bucket, original_key, upload_id, part_id, file_path)
parts.append((part_id, etag))
with reporter.step("Check all parts are visible in bucket"):
got_parts = s3_client.list_parts(bucket, original_key, upload_id)
assert len(got_parts) == len(file_parts), f"Expected {parts_count} parts, got:\n{got_parts}"
with reporter.step("Complete multipart upload"):
response = s3_client.complete_multipart_upload(bucket, original_key, upload_id, parts)
version = response["VersionId"]
expected_versions = {version}
with reporter.step("Patch versioned multipart object"):
for rng in patch_ranges:
start, end = s3_helper.get_range_relative_to_object(
rng, object_size=original_size, part_size=PART_SIZE_FOR_MULTIPART, int_values=True
)
content_size = end - start + 1
content_range = f"bytes {start}-{end}/*"
with reporter.step("Generate payload object"):
content_file = generate_file(content_size)
with reporter.step("Patch multipart object and get new version"):
response = s3_http_client.patch_object(
bucket, original_key, content_file, content_range, version_id=version, timeout=200
)
version = response["VersionId"]
expected_versions.add(version)
with reporter.step("Get patched part of object and make sure it has changed correctly"):
got_part = s3_client.get_object(bucket, original_key, version_id=version, object_range=(start, end))
assert get_file_hash(got_part) == get_file_hash(content_file), "Expected content hash did not match actual content hash"
with reporter.step("Check that all expected versions are in bucket"):
got_versions = {
version.get("VersionId") for version in s3_client.list_objects_versions(bucket) if version.get("Key") == original_key
}
assert expected_versions == got_versions, f"Expected versions of object are missing from bucket: {expected_versions}"
# TODO: Negative scenario for SSE objects is postponed for now.

View file

@ -4,8 +4,8 @@ from datetime import datetime, timedelta
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.clients import S3ClientWrapper
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content

View file

@ -1,10 +1,9 @@
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.clients.s3 import BucketContainerResolver, S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.steps.cli.container import list_objects
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase

View file

@ -8,11 +8,12 @@ from typing import Literal
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.clients import AwsCliClient, S3ClientWrapper
from frostfs_testlib.clients.s3 import VersioningStatus
from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_PASS
from frostfs_testlib.resources.error_patterns import S3_BUCKET_DOES_NOT_ALLOW_ACL, S3_MALFORMED_XML_REQUEST
from frostfs_testlib.resources.s3_acl_grants import PRIVATE_GRANTS
from frostfs_testlib.s3 import AwsCliClient, S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils import wallet_utils

View file

@ -4,9 +4,8 @@ import allure
import pytest
from botocore.exceptions import ClientError
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.s3.interfaces import BucketContainerResolver
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.clients.s3 import BucketContainerResolver, S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.steps.storage_policy import get_simple_object_copies
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo

View file

@ -5,8 +5,8 @@ from typing import Tuple
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.clients import S3ClientWrapper
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file

View file

@ -3,8 +3,8 @@ import os
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps.s3 import s3_helper
from frostfs_testlib.clients.s3 import S3ClientWrapper, VersioningStatus
from frostfs_testlib.steps import s3_helper
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
from frostfs_testlib.utils.file_utils import generate_file, generate_file_with_content, get_file_content

View file

@ -7,8 +7,8 @@ from frostfs_testlib.utils import string_utils
@pytest.fixture(scope="module")
def owner_wallet(default_wallet: WalletInfo) -> WalletInfo:
return default_wallet
def owner_wallet(wallet: WalletInfo) -> WalletInfo:
return wallet
@pytest.fixture(scope="module")

View file

@ -3,8 +3,8 @@ import random
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
from frostfs_testlib.resources.error_patterns import SESSION_NOT_FOUND
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import delete_object, put_object, put_object_to_random_node
from frostfs_testlib.steps.session_token import create_session_token
from frostfs_testlib.storage.dataclasses.object_size import ObjectSize
@ -12,13 +12,17 @@ from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.utils.file_utils import generate_file
from ...helpers.container_creation import create_container_with_ape
from ...helpers.container_request import APE_OWNER_ALLOW_ALL, ContainerRequest
@pytest.mark.nightly
@pytest.mark.sanity
@pytest.mark.session_token
class TestDynamicObjectSession(ClusterTestBase):
@allure.title("Object Operations with Session Token (obj_size={object_size})")
def test_object_session_token(self, default_wallet: WalletInfo, object_size: ObjectSize):
@pytest.mark.parametrize("user_tag", ["TestDynamicObjectSession"], indirect=True) # provide dedicated user with no APE side-policies
def test_object_session_token(self, wallet: WalletInfo, frostfs_cli: FrostfsCli, object_size: ObjectSize):
"""
Test how operations over objects are executed with a session token
@ -32,17 +36,14 @@ class TestDynamicObjectSession(ClusterTestBase):
with a session token
"""
with reporter.step("Init wallet"):
wallet = default_wallet
with reporter.step("Nodes Settlements"):
session_token_node, container_node, non_container_node = random.sample(self.cluster.storage_nodes, 3)
with reporter.step("Create Session Token"):
session_token = create_session_token(
shell=self.shell,
owner=default_wallet.get_address(),
wallet=default_wallet,
owner=wallet.get_address(),
wallet=wallet,
rpc_endpoint=session_token_node.get_rpc_endpoint(),
)
@ -54,11 +55,13 @@ class TestDynamicObjectSession(ClusterTestBase):
f'AS LOC_{locode}_PLACE FILTER "UN-LOCODE" '
f'EQ "{un_locode}" AS LOC_{locode}'
)
cid = create_container(
cid = create_container_with_ape(
ContainerRequest(placement_policy, APE_OWNER_ALLOW_ALL),
frostfs_cli,
wallet,
shell=self.shell,
endpoint=self.cluster.default_rpc_endpoint,
rule=placement_policy,
self.shell,
self.cluster,
self.cluster.default_rpc_endpoint,
)
with reporter.step("Put Objects"):

View file

@ -3,9 +3,9 @@ import logging
import allure
import pytest
from frostfs_testlib import reporter
from frostfs_testlib.cli.frostfs_cli.cli import FrostfsCli
from frostfs_testlib.resources.error_patterns import EXPIRED_SESSION_TOKEN, MALFORMED_REQUEST, OBJECT_ACCESS_DENIED, OBJECT_NOT_FOUND
from frostfs_testlib.shell import Shell
from frostfs_testlib.steps.cli.container import create_container
from frostfs_testlib.steps.cli.object import (
delete_object,
get_object,
@ -38,16 +38,24 @@ from frostfs_testlib.testing.cluster_test_base import ClusterTestBase
from frostfs_testlib.testing.test_control import expect_not_raises
from frostfs_testlib.utils.file_utils import generate_file
from ...helpers.container_creation import create_containers_with_ape
from ...helpers.container_request import OWNER_ALLOW_ALL, MultipleContainersRequest
logger = logging.getLogger("NeoLogger")
RANGE_OFFSET_FOR_COMPLEX_OBJECT = 200
@pytest.fixture(scope="module")
def storage_containers(owner_wallet: WalletInfo, client_shell: Shell, cluster: Cluster) -> list[str]:
cid = create_container(owner_wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
other_cid = create_container(owner_wallet, shell=client_shell, endpoint=cluster.default_rpc_endpoint)
yield [cid, other_cid]
def storage_containers(owner_wallet: WalletInfo, frostfs_cli: FrostfsCli, client_shell: Shell, cluster: Cluster) -> list[str]:
return create_containers_with_ape(
frostfs_cli,
owner_wallet,
client_shell,
cluster,
cluster.default_rpc_endpoint,
MultipleContainersRequest([OWNER_ALLOW_ALL, OWNER_ALLOW_ALL]),
)
@pytest.fixture(
@ -135,6 +143,7 @@ def static_sessions(
@pytest.mark.nightly
@pytest.mark.static_session
@pytest.mark.parametrize("user_tag", ["TestObjectStaticSession"], indirect=True) # provide dedicated user with no APE side-policies
class TestObjectStaticSession(ClusterTestBase):
@allure.title("Read operations with static session (method={method_under_test.__name__}, obj_size={object_size})")
@pytest.mark.parametrize(