From 6734cd70e6b7ee71e62f37723e20a8a96430621e Mon Sep 17 00:00:00 2001 From: Yulia Kovshova Date: Tue, 4 Oct 2022 11:22:04 +0300 Subject: [PATCH] [#312] Add new multipart upload test Signed-off-by: Yulia Kovshova --- pytest_tests/steps/aws_cli_client.py | 11 ++ pytest_tests/steps/s3_gate_object.py | 24 ++++ .../services/s3_gate/test_s3_multipart.py | 130 ++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py diff --git a/pytest_tests/steps/aws_cli_client.py b/pytest_tests/steps/aws_cli_client.py index 5ab0742..8ed1ba1 100644 --- a/pytest_tests/steps/aws_cli_client.py +++ b/pytest_tests/steps/aws_cli_client.py @@ -426,6 +426,17 @@ class AwsCliClient: output = _cmd_run(cmd, LONG_TIMEOUT) return self._to_json(output) + def upload_part_copy( + self, UploadId: str, Bucket: str, Key: str, PartNumber: int, CopySource: str + ) -> dict: + cmd = ( + f"aws {self.common_flags} s3api upload-part-copy --bucket {Bucket} --key {Key} " + f"--upload-id {UploadId} --part-number {PartNumber} --copy-source {CopySource} " + f"--endpoint-url {S3_GATE}" + ) + output = _cmd_run(cmd, LONG_TIMEOUT) + return self._to_json(output) + def list_parts(self, UploadId: str, Bucket: str, Key: str) -> dict: cmd = ( f"aws {self.common_flags} s3api list-parts --bucket {Bucket} --key {Key} " diff --git a/pytest_tests/steps/s3_gate_object.py b/pytest_tests/steps/s3_gate_object.py index 3f603f2..e35229b 100644 --- a/pytest_tests/steps/s3_gate_object.py +++ b/pytest_tests/steps/s3_gate_object.py @@ -385,6 +385,30 @@ def upload_part_s3( ) from err +@allure.step("Upload copy part S3") +def upload_part_copy_s3( + s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, copy_source: str +) -> str: + + try: + response = s3_client.upload_part_copy( + UploadId=upload_id, + Bucket=bucket_name, + Key=object_key, + PartNumber=part_num, + CopySource=copy_source, + ) + log_command_execution("S3 Upload copy part", response) + assert response.get("CopyPartResult").get("ETag"), f"Expected ETag in response:\n{response}" + + return response.get("CopyPartResult").get("ETag") + except ClientError as err: + raise Exception( + f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}' + ) from err + + @allure.step("List parts S3") def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> list[dict]: try: diff --git a/pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py b/pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py new file mode 100644 index 0000000..13061db --- /dev/null +++ b/pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py @@ -0,0 +1,130 @@ +import allure +import pytest +from file_helper import generate_file, get_file_hash, split_file +from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning + +from steps import s3_gate_bucket, s3_gate_object +from steps.s3_gate_base import TestS3GateBase + +PART_SIZE = 5 * 1024 * 1024 + + +def pytest_generate_tests(metafunc): + if "s3_client" in metafunc.fixturenames: + metafunc.parametrize("s3_client", ["aws cli", "boto3"], indirect=True) + + +@pytest.mark.s3_gate +class TestS3GateMultipart(TestS3GateBase): + @allure.title("Test S3 Object Multipart API") + def test_s3_object_multipart(self): + bucket = s3_gate_bucket.create_bucket_s3(self.s3_client) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + parts_count = 5 + file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part + object_key = object_key_from_file_path(file_name_large) + part_files = split_file(file_name_large, parts_count) + parts = [] + + with allure.step("Upload first part"): + upload_id = s3_gate_object.create_multipart_upload_s3( + self.s3_client, bucket, object_key + ) + uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket) + etag = s3_gate_object.upload_part_s3( + self.s3_client, bucket, object_key, upload_id, 1, part_files[0] + ) + parts.append((1, etag)) + got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id) + assert len(got_parts) == 1, f"Expected {1} parts, got\n{got_parts}" + + with allure.step("Upload last parts"): + for part_id, file_path in enumerate(part_files[1:], start=2): + etag = s3_gate_object.upload_part_s3( + self.s3_client, bucket, object_key, upload_id, part_id, file_path + ) + parts.append((part_id, etag)) + got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id) + s3_gate_object.complete_multipart_upload_s3( + self.s3_client, bucket, object_key, upload_id, parts + ) + assert len(got_parts) == len( + part_files + ), f"Expected {parts_count} parts, got\n{got_parts}" + + with allure.step("Check upload list is empty"): + uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket) + assert not uploads, f"Expected there is no uploads in bucket {bucket}" + + with allure.step("Check we can get whole object from bucket"): + got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, object_key) + assert get_file_hash(got_object) == get_file_hash(file_name_large) + + @allure.title("Test S3 Multipart abord") + def test_s3_abort_multipart(self): + bucket = s3_gate_bucket.create_bucket_s3(self.s3_client) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + parts_count = 5 + file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part + object_key = object_key_from_file_path(file_name_large) + part_files = split_file(file_name_large, parts_count) + parts = [] + + with allure.step("Upload first part"): + upload_id = s3_gate_object.create_multipart_upload_s3( + self.s3_client, bucket, object_key + ) + uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket) + etag = s3_gate_object.upload_part_s3( + self.s3_client, bucket, object_key, upload_id, 1, part_files[0] + ) + parts.append((1, etag)) + got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id) + assert len(got_parts) == 1, f"Expected {1} parts, got\n{got_parts}" + + with allure.step("Abort multipart upload"): + s3_gate_object.abort_multipart_uploads_s3(self.s3_client, bucket, object_key, upload_id) + uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket) + assert not uploads, f"Expected there is no uploads in bucket {bucket}" + + @allure.title("Test S3 Upload Part Copy") + def test_s3_multipart_copy(self): + bucket = s3_gate_bucket.create_bucket_s3(self.s3_client) + set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED) + parts_count = 3 + file_name_large = generate_file(PART_SIZE * parts_count) # 5Mb - min part + object_key = object_key_from_file_path(file_name_large) + part_files = split_file(file_name_large, parts_count) + parts = [] + objs = [] + + with allure.step(f"Put {parts_count} objec in bucket"): + for part in part_files: + s3_gate_object.put_object_s3(self.s3_client, bucket, part) + objs.append(object_key_from_file_path(part)) + check_objects_in_bucket(self.s3_client, bucket, objs) + + with allure.step("Create multipart upload object"): + upload_id = s3_gate_object.create_multipart_upload_s3( + self.s3_client, bucket, object_key + ) + uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket) + assert uploads, f"Expected there are uploads in bucket {bucket}" + + with allure.step("Start multipart upload"): + for part_id, obj_key in enumerate(objs, start=1): + etag = s3_gate_object.upload_part_copy_s3( + self.s3_client, bucket, object_key, upload_id, part_id, f"{bucket}/{obj_key}" + ) + parts.append((part_id, etag)) + got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id) + s3_gate_object.complete_multipart_upload_s3( + self.s3_client, bucket, object_key, upload_id, parts + ) + assert len(got_parts) == len( + part_files + ), f"Expected {parts_count} parts, got\n{got_parts}" + + with allure.step("Check we can get whole object from bucket"): + got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, object_key) + assert get_file_hash(got_object) == get_file_hash(file_name_large)