[#312] Add new multipart upload test

Signed-off-by: Yulia Kovshova <y.kovshova@yadro.com>
This commit is contained in:
Юлия Ковшова 2022-10-04 11:22:04 +03:00 committed by Julia Kovshova
parent 7e30006623
commit 6734cd70e6
3 changed files with 165 additions and 0 deletions

View file

@ -426,6 +426,17 @@ class AwsCliClient:
output = _cmd_run(cmd, LONG_TIMEOUT) output = _cmd_run(cmd, LONG_TIMEOUT)
return self._to_json(output) return self._to_json(output)
def upload_part_copy(
self, UploadId: str, Bucket: str, Key: str, PartNumber: int, CopySource: str
) -> dict:
cmd = (
f"aws {self.common_flags} s3api upload-part-copy --bucket {Bucket} --key {Key} "
f"--upload-id {UploadId} --part-number {PartNumber} --copy-source {CopySource} "
f"--endpoint-url {S3_GATE}"
)
output = _cmd_run(cmd, LONG_TIMEOUT)
return self._to_json(output)
def list_parts(self, UploadId: str, Bucket: str, Key: str) -> dict: def list_parts(self, UploadId: str, Bucket: str, Key: str) -> dict:
cmd = ( cmd = (
f"aws {self.common_flags} s3api list-parts --bucket {Bucket} --key {Key} " f"aws {self.common_flags} s3api list-parts --bucket {Bucket} --key {Key} "

View file

@ -385,6 +385,30 @@ def upload_part_s3(
) from err ) from err
@allure.step("Upload copy part S3")
def upload_part_copy_s3(
s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, copy_source: str
) -> str:
try:
response = s3_client.upload_part_copy(
UploadId=upload_id,
Bucket=bucket_name,
Key=object_key,
PartNumber=part_num,
CopySource=copy_source,
)
log_command_execution("S3 Upload copy part", response)
assert response.get("CopyPartResult").get("ETag"), f"Expected ETag in response:\n{response}"
return response.get("CopyPartResult").get("ETag")
except ClientError as err:
raise Exception(
f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
) from err
@allure.step("List parts S3") @allure.step("List parts S3")
def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> list[dict]: def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> list[dict]:
try: try:

View file

@ -0,0 +1,130 @@
import allure
import pytest
from file_helper import generate_file, get_file_hash, split_file
from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning
from steps import s3_gate_bucket, s3_gate_object
from steps.s3_gate_base import TestS3GateBase
PART_SIZE = 5 * 1024 * 1024
def pytest_generate_tests(metafunc):
if "s3_client" in metafunc.fixturenames:
metafunc.parametrize("s3_client", ["aws cli", "boto3"], indirect=True)
@pytest.mark.s3_gate
class TestS3GateMultipart(TestS3GateBase):
@allure.title("Test S3 Object Multipart API")
def test_s3_object_multipart(self):
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
parts_count = 5
file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part
object_key = object_key_from_file_path(file_name_large)
part_files = split_file(file_name_large, parts_count)
parts = []
with allure.step("Upload first part"):
upload_id = s3_gate_object.create_multipart_upload_s3(
self.s3_client, bucket, object_key
)
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
etag = s3_gate_object.upload_part_s3(
self.s3_client, bucket, object_key, upload_id, 1, part_files[0]
)
parts.append((1, etag))
got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
assert len(got_parts) == 1, f"Expected {1} parts, got\n{got_parts}"
with allure.step("Upload last parts"):
for part_id, file_path in enumerate(part_files[1:], start=2):
etag = s3_gate_object.upload_part_s3(
self.s3_client, bucket, object_key, upload_id, part_id, file_path
)
parts.append((part_id, etag))
got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
s3_gate_object.complete_multipart_upload_s3(
self.s3_client, bucket, object_key, upload_id, parts
)
assert len(got_parts) == len(
part_files
), f"Expected {parts_count} parts, got\n{got_parts}"
with allure.step("Check upload list is empty"):
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
assert not uploads, f"Expected there is no uploads in bucket {bucket}"
with allure.step("Check we can get whole object from bucket"):
got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, object_key)
assert get_file_hash(got_object) == get_file_hash(file_name_large)
@allure.title("Test S3 Multipart abord")
def test_s3_abort_multipart(self):
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
parts_count = 5
file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part
object_key = object_key_from_file_path(file_name_large)
part_files = split_file(file_name_large, parts_count)
parts = []
with allure.step("Upload first part"):
upload_id = s3_gate_object.create_multipart_upload_s3(
self.s3_client, bucket, object_key
)
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
etag = s3_gate_object.upload_part_s3(
self.s3_client, bucket, object_key, upload_id, 1, part_files[0]
)
parts.append((1, etag))
got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
assert len(got_parts) == 1, f"Expected {1} parts, got\n{got_parts}"
with allure.step("Abort multipart upload"):
s3_gate_object.abort_multipart_uploads_s3(self.s3_client, bucket, object_key, upload_id)
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
assert not uploads, f"Expected there is no uploads in bucket {bucket}"
@allure.title("Test S3 Upload Part Copy")
def test_s3_multipart_copy(self):
bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
parts_count = 3
file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part
object_key = object_key_from_file_path(file_name_large)
part_files = split_file(file_name_large, parts_count)
parts = []
objs = []
with allure.step(f"Put {parts_count} objec in bucket"):
for part in part_files:
s3_gate_object.put_object_s3(self.s3_client, bucket, part)
objs.append(object_key_from_file_path(part))
check_objects_in_bucket(self.s3_client, bucket, objs)
with allure.step("Create multipart upload object"):
upload_id = s3_gate_object.create_multipart_upload_s3(
self.s3_client, bucket, object_key
)
uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
assert uploads, f"Expected there are uploads in bucket {bucket}"
with allure.step("Start multipart upload"):
for part_id, obj_key in enumerate(objs, start=1):
etag = s3_gate_object.upload_part_copy_s3(
self.s3_client, bucket, object_key, upload_id, part_id, f"{bucket}/{obj_key}"
)
parts.append((part_id, etag))
got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
s3_gate_object.complete_multipart_upload_s3(
self.s3_client, bucket, object_key, upload_id, parts
)
assert len(got_parts) == len(
part_files
), f"Expected {parts_count} parts, got\n{got_parts}"
with allure.step("Check we can get whole object from bucket"):
got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, object_key)
assert get_file_hash(got_object) == get_file_hash(file_name_large)