forked from TrueCloudLab/frostfs-testcases
[#312] Add new multipart upload test
Signed-off-by: Yulia Kovshova <y.kovshova@yadro.com>
This commit is contained in:
Julia Kovshova
parent
7e30006623
commit
6734cd70e6
3 changed files with 165 additions and 0 deletions
|
|
@ -426,6 +426,17 @@ class AwsCliClient:
|
|||
output = _cmd_run(cmd, LONG_TIMEOUT)
|
||||
return self._to_json(output)
|
||||
|
||||
def upload_part_copy(
|
||||
self, UploadId: str, Bucket: str, Key: str, PartNumber: int, CopySource: str
|
||||
) -> dict:
|
||||
cmd = (
|
||||
f"aws {self.common_flags} s3api upload-part-copy --bucket {Bucket} --key {Key} "
|
||||
f"--upload-id {UploadId} --part-number {PartNumber} --copy-source {CopySource} "
|
||||
f"--endpoint-url {S3_GATE}"
|
||||
)
|
||||
output = _cmd_run(cmd, LONG_TIMEOUT)
|
||||
return self._to_json(output)
|
||||
|
||||
def list_parts(self, UploadId: str, Bucket: str, Key: str) -> dict:
|
||||
cmd = (
|
||||
f"aws {self.common_flags} s3api list-parts --bucket {Bucket} --key {Key} "
|
||||
|
|
|
|||
|
|
@ -385,6 +385,30 @@ def upload_part_s3(
|
|||
) from err
|
||||
|
||||
|
||||
@allure.step("Upload copy part S3")
|
||||
def upload_part_copy_s3(
|
||||
s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, copy_source: str
|
||||
) -> str:
|
||||
|
||||
try:
|
||||
response = s3_client.upload_part_copy(
|
||||
UploadId=upload_id,
|
||||
Bucket=bucket_name,
|
||||
Key=object_key,
|
||||
PartNumber=part_num,
|
||||
CopySource=copy_source,
|
||||
)
|
||||
log_command_execution("S3 Upload copy part", response)
|
||||
assert response.get("CopyPartResult").get("ETag"), f"Expected ETag in response:\n{response}"
|
||||
|
||||
return response.get("CopyPartResult").get("ETag")
|
||||
except ClientError as err:
|
||||
raise Exception(
|
||||
f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
||||
) from err
|
||||
|
||||
|
||||
@allure.step("List parts S3")
|
||||
def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> list[dict]:
|
||||
try:
|
||||
|
|
|
|||
130
pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py
Normal file
130
pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
import allure
|
||||
import pytest
|
||||
from file_helper import generate_file, get_file_hash, split_file
|
||||
from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning
|
||||
|
||||
from steps import s3_gate_bucket, s3_gate_object
|
||||
from steps.s3_gate_base import TestS3GateBase
|
||||
|
||||
PART_SIZE = 5 * 1024 * 1024
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc):
|
||||
if "s3_client" in metafunc.fixturenames:
|
||||
metafunc.parametrize("s3_client", ["aws cli", "boto3"], indirect=True)
|
||||
|
||||
|
||||
@pytest.mark.s3_gate
|
||||
class TestS3GateMultipart(TestS3GateBase):
|
||||
@allure.title("Test S3 Object Multipart API")
|
||||
def test_s3_object_multipart(self):
|
||||
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
|
||||
parts_count = 5
|
||||
file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part
|
||||
object_key = object_key_from_file_path(file_name_large)
|
||||
part_files = split_file(file_name_large, parts_count)
|
||||
parts = []
|
||||
|
||||
with allure.step("Upload first part"):
|
||||
upload_id = s3_gate_object.create_multipart_upload_s3(
|
||||
self.s3_client, bucket, object_key
|
||||
)
|
||||
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
|
||||
etag = s3_gate_object.upload_part_s3(
|
||||
self.s3_client, bucket, object_key, upload_id, 1, part_files[0]
|
||||
)
|
||||
parts.append((1, etag))
|
||||
got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
|
||||
assert len(got_parts) == 1, f"Expected {1} parts, got\n{got_parts}"
|
||||
|
||||
with allure.step("Upload last parts"):
|
||||
for part_id, file_path in enumerate(part_files[1:], start=2):
|
||||
etag = s3_gate_object.upload_part_s3(
|
||||
self.s3_client, bucket, object_key, upload_id, part_id, file_path
|
||||
)
|
||||
parts.append((part_id, etag))
|
||||
got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
|
||||
s3_gate_object.complete_multipart_upload_s3(
|
||||
self.s3_client, bucket, object_key, upload_id, parts
|
||||
)
|
||||
assert len(got_parts) == len(
|
||||
part_files
|
||||
), f"Expected {parts_count} parts, got\n{got_parts}"
|
||||
|
||||
with allure.step("Check upload list is empty"):
|
||||
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
|
||||
assert not uploads, f"Expected there is no uploads in bucket {bucket}"
|
||||
|
||||
with allure.step("Check we can get whole object from bucket"):
|
||||
got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, object_key)
|
||||
assert get_file_hash(got_object) == get_file_hash(file_name_large)
|
||||
|
||||
@allure.title("Test S3 Multipart abord")
|
||||
def test_s3_abort_multipart(self):
|
||||
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
|
||||
parts_count = 5
|
||||
file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part
|
||||
object_key = object_key_from_file_path(file_name_large)
|
||||
part_files = split_file(file_name_large, parts_count)
|
||||
parts = []
|
||||
|
||||
with allure.step("Upload first part"):
|
||||
upload_id = s3_gate_object.create_multipart_upload_s3(
|
||||
self.s3_client, bucket, object_key
|
||||
)
|
||||
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
|
||||
etag = s3_gate_object.upload_part_s3(
|
||||
self.s3_client, bucket, object_key, upload_id, 1, part_files[0]
|
||||
)
|
||||
parts.append((1, etag))
|
||||
got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
|
||||
assert len(got_parts) == 1, f"Expected {1} parts, got\n{got_parts}"
|
||||
|
||||
with allure.step("Abort multipart upload"):
|
||||
s3_gate_object.abort_multipart_uploads_s3(self.s3_client, bucket, object_key, upload_id)
|
||||
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
|
||||
assert not uploads, f"Expected there is no uploads in bucket {bucket}"
|
||||
|
||||
@allure.title("Test S3 Upload Part Copy")
|
||||
def test_s3_multipart_copy(self):
|
||||
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
|
||||
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
|
||||
parts_count = 3
|
||||
file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part
|
||||
object_key = object_key_from_file_path(file_name_large)
|
||||
part_files = split_file(file_name_large, parts_count)
|
||||
parts = []
|
||||
objs = []
|
||||
|
||||
with allure.step(f"Put {parts_count} objec in bucket"):
|
||||
for part in part_files:
|
||||
s3_gate_object.put_object_s3(self.s3_client, bucket, part)
|
||||
objs.append(object_key_from_file_path(part))
|
||||
check_objects_in_bucket(self.s3_client, bucket, objs)
|
||||
|
||||
with allure.step("Create multipart upload object"):
|
||||
upload_id = s3_gate_object.create_multipart_upload_s3(
|
||||
self.s3_client, bucket, object_key
|
||||
)
|
||||
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
|
||||
assert uploads, f"Expected there are uploads in bucket {bucket}"
|
||||
|
||||
with allure.step("Start multipart upload"):
|
||||
for part_id, obj_key in enumerate(objs, start=1):
|
||||
etag = s3_gate_object.upload_part_copy_s3(
|
||||
self.s3_client, bucket, object_key, upload_id, part_id, f"{bucket}/{obj_key}"
|
||||
)
|
||||
parts.append((part_id, etag))
|
||||
got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
|
||||
s3_gate_object.complete_multipart_upload_s3(
|
||||
self.s3_client, bucket, object_key, upload_id, parts
|
||||
)
|
||||
assert len(got_parts) == len(
|
||||
part_files
|
||||
), f"Expected {parts_count} parts, got\n{got_parts}"
|
||||
|
||||
with allure.step("Check we can get whole object from bucket"):
|
||||
got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, object_key)
|
||||
assert get_file_hash(got_object) == get_file_hash(file_name_large)
|
||||
Loading…
Reference in a new issue