Add deletion 1001 objects test

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-04-05 19:02:42 +03:00
parent b995bfca41
commit d355eccfd8
2 changed files with 58 additions and 3 deletions

View file

@ -14,13 +14,12 @@ from botocore.exceptions import ClientError
from frostfs_testlib.shell import Shell from frostfs_testlib.shell import Shell
from pytest import FixtureRequest from pytest import FixtureRequest
from pytest_tests.steps import s3_gate_bucket
from pytest_tests.steps import s3_gate_object
from pytest_tests.helpers.aws_cli_client import AwsCliClient from pytest_tests.helpers.aws_cli_client import AwsCliClient
from pytest_tests.helpers.cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd from pytest_tests.helpers.cli_helpers import _cmd_run, _configure_aws_cli, _run_with_passwd
from pytest_tests.helpers.cluster import Cluster from pytest_tests.helpers.cluster import Cluster
from pytest_tests.helpers.container import list_containers from pytest_tests.helpers.container import list_containers
from pytest_tests.resources.common import FROSTFS_AUTHMATE_EXEC from pytest_tests.resources.common import FROSTFS_AUTHMATE_EXEC
from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.cluster_test_base import ClusterTestBase from pytest_tests.steps.cluster_test_base import ClusterTestBase
# Disable warnings on self-signed certificate which the # Disable warnings on self-signed certificate which the
@ -34,6 +33,9 @@ CREDENTIALS_CREATE_TIMEOUT = "1m"
# without any retries) # without any retries)
MAX_REQUEST_ATTEMPTS = 1 MAX_REQUEST_ATTEMPTS = 1
RETRY_MODE = "standard" RETRY_MODE = "standard"
S3_MALFORMED_XML_REQUEST = (
"The XML you provided was not well-formed or did not validate against our published schema."
)
class TestS3GateBase(ClusterTestBase): class TestS3GateBase(ClusterTestBase):

View file

@ -21,9 +21,10 @@ from pytest_tests.helpers.s3_helper import (
check_objects_in_bucket, check_objects_in_bucket,
set_bucket_versioning, set_bucket_versioning,
) )
from pytest_tests.helpers.test_control import expect_not_raises
from pytest_tests.resources.common import ASSETS_DIR, WALLET_PASS from pytest_tests.resources.common import ASSETS_DIR, WALLET_PASS
from pytest_tests.steps import s3_gate_bucket, s3_gate_object from pytest_tests.steps import s3_gate_bucket, s3_gate_object
from pytest_tests.steps.s3_gate_base import TestS3GateBase from pytest_tests.steps.s3_gate_base import S3_MALFORMED_XML_REQUEST, TestS3GateBase
def pytest_generate_tests(metafunc): def pytest_generate_tests(metafunc):
@ -39,6 +40,31 @@ class TestS3GateObject(TestS3GateBase):
def object_key_from_file_path(full_path: str) -> str: def object_key_from_file_path(full_path: str) -> str:
return os.path.basename(full_path) return os.path.basename(full_path)
@allure.title("Set object size for current test")
@pytest.fixture
def object_size(self, request: pytest.FixtureRequest) -> int:
object_size = request.param
return object_size
@allure.title("Put objects in a bucket")
@pytest.fixture
def objects_in_bucket(
self, bucket: str, object_size: int, request: pytest.FixtureRequest
) -> list[str]:
objects: list[str] = []
objects_count = int(request.param)
with allure.step(
f"Put {objects_count} objects of size '{object_size}' bytes into bucket '{bucket}'"
):
for _ in range(objects_count):
file_path = generate_file(object_size)
file_name = self.object_key_from_file_path(file_path)
objects.append(file_name)
s3_gate_object.put_object_s3(self.s3_client, bucket, file_path)
return objects
@pytest.fixture @pytest.fixture
def second_wallet_public_key(self): def second_wallet_public_key(self):
second_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json") second_wallet = os.path.join(os.getcwd(), ASSETS_DIR, f"{str(uuid.uuid4())}.json")
@ -481,6 +507,33 @@ class TestS3GateObject(TestS3GateBase):
con_file = concat_files([object_3_part_1, object_3_part_2, object_3_part_3]) con_file = concat_files([object_3_part_1, object_3_part_2, object_3_part_3])
assert get_file_hash(con_file) == get_file_hash(file_name_1), "Hashes must be the same" assert get_file_hash(con_file) == get_file_hash(file_name_1), "Hashes must be the same"
def copy_extend_list(self, original_list: list[str], n: int) -> list[str]:
"""Extend the list with own elements up to n elements"""
multiplier = n // len(original_list)
result_list = original_list.copy()
result_list = result_list * multiplier
for i in range(n - len(result_list)):
result_list.append(result_list[i])
return result_list
@allure.title("Test S3: Bulk deletion should be limited to 1000 objects")
@pytest.mark.parametrize(
"objects_in_bucket, object_size",
[(3, 10)],
indirect=True,
)
def test_s3_bulk_deletion_limit(self, bucket: str, objects_in_bucket: list[str]):
# Extend deletion list to 1001 elements with same keys for test speed
objects_to_delete = self.copy_extend_list(objects_in_bucket, 1001)
with allure.step("Delete 1001 objects and expect error"):
with pytest.raises(Exception, match=S3_MALFORMED_XML_REQUEST):
s3_gate_object.delete_objects_s3(self.s3_client, bucket, objects_to_delete)
with allure.step("Delete 1000 objects without error"):
with expect_not_raises():
s3_gate_object.delete_objects_s3(self.s3_client, bucket, objects_to_delete[:1000])
@allure.title("Test S3: Copy object with metadata") @allure.title("Test S3: Copy object with metadata")
@pytest.mark.smoke @pytest.mark.smoke
def test_s3_head_object(self, bucket, complex_object_size, simple_object_size): def test_s3_head_object(self, bucket, complex_object_size, simple_object_size):