[#312] Add new multipart upload test

Signed-off-by: Yulia Kovshova <y.kovshova@yadro.com>

pytest_tests/steps/aws_cli_client.py

@@ -426,6 +426,17 @@ class AwsCliClient:
         output = _cmd_run(cmd, LONG_TIMEOUT)
         return self._to_json(output)
 
+    def upload_part_copy(
+        self, UploadId: str, Bucket: str, Key: str, PartNumber: int, CopySource: str
+    ) -> dict:
+        cmd = (
+            f"aws {self.common_flags} s3api upload-part-copy --bucket {Bucket} --key {Key} "
+            f"--upload-id {UploadId} --part-number {PartNumber} --copy-source {CopySource} "
+            f"--endpoint-url {S3_GATE}"
+        )
+        output = _cmd_run(cmd, LONG_TIMEOUT)
+        return self._to_json(output)
+
     def list_parts(self, UploadId: str, Bucket: str, Key: str) -> dict:
         cmd = (
             f"aws {self.common_flags} s3api list-parts --bucket {Bucket} --key {Key} "

pytest_tests/steps/s3_gate_object.py

@@ -385,6 +385,30 @@ def upload_part_s3(
         ) from err
 
 
+@allure.step("Upload copy part S3")
+def upload_part_copy_s3(
+    s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, copy_source: str
+) -> str:
+
+    try:
+        response = s3_client.upload_part_copy(
+            UploadId=upload_id,
+            Bucket=bucket_name,
+            Key=object_key,
+            PartNumber=part_num,
+            CopySource=copy_source,
+        )
+        log_command_execution("S3 Upload copy part", response)
+        assert response.get("CopyPartResult").get("ETag"), f"Expected ETag in response:\n{response}"
+
+        return response.get("CopyPartResult").get("ETag")
+    except ClientError as err:
+        raise Exception(
+            f'Error Message: {err.response["Error"]["Message"]}\n'
+            f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
+        ) from err
+
+
 @allure.step("List parts S3")
 def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> list[dict]:
     try:

pytest_tests/testsuites/services/s3_gate/test_s3_multipart.py

@@ -0,0 +1,130 @@
+import allure
+import pytest
+from file_helper import generate_file, get_file_hash, split_file
+from s3_helper import check_objects_in_bucket, object_key_from_file_path, set_bucket_versioning
+
+from steps import s3_gate_bucket, s3_gate_object
+from steps.s3_gate_base import TestS3GateBase
+
+PART_SIZE = 5 * 1024 * 1024
+
+
+def pytest_generate_tests(metafunc):
+    if "s3_client" in metafunc.fixturenames:
+        metafunc.parametrize("s3_client", ["aws cli", "boto3"], indirect=True)
+
+
+@pytest.mark.s3_gate
+class TestS3GateMultipart(TestS3GateBase):
+    @allure.title("Test S3 Object Multipart API")
+    def test_s3_object_multipart(self):
+        bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
+        set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
+        parts_count = 5
+        file_name_large = generate_file(PART_SIZE * parts_count)  # 5Mb - min part
+        object_key = object_key_from_file_path(file_name_large)
+        part_files = split_file(file_name_large, parts_count)
+        parts = []
+
+        with allure.step("Upload first part"):
+            upload_id = s3_gate_object.create_multipart_upload_s3(
+                self.s3_client, bucket, object_key
+            )
+            uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
+            etag = s3_gate_object.upload_part_s3(
+                self.s3_client, bucket, object_key, upload_id, 1, part_files[0]
+            )
+            parts.append((1, etag))
+            got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
+            assert len(got_parts) == 1, f"Expected {1} parts, got\n{got_parts}"
+
+        with allure.step("Upload last parts"):
+            for part_id, file_path in enumerate(part_files[1:], start=2):
+                etag = s3_gate_object.upload_part_s3(
+                    self.s3_client, bucket, object_key, upload_id, part_id, file_path
+                )
+                parts.append((part_id, etag))
+            got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
+            s3_gate_object.complete_multipart_upload_s3(
+                self.s3_client, bucket, object_key, upload_id, parts
+            )
+            assert len(got_parts) == len(
+                part_files
+            ), f"Expected {parts_count} parts, got\n{got_parts}"
+
+        with allure.step("Check upload list is empty"):
+            uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
+            assert not uploads, f"Expected there is no uploads in bucket {bucket}"
+
+        with allure.step("Check we can get whole object from bucket"):
+            got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, object_key)
+            assert get_file_hash(got_object) == get_file_hash(file_name_large)
+
+    @allure.title("Test S3 Multipart abord")
+    def test_s3_abort_multipart(self):
+        bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
+        set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
+        parts_count = 5
+        file_name_large = generate_file(PART_SIZE * parts_count)  # 5Mb - min part
+        object_key = object_key_from_file_path(file_name_large)
+        part_files = split_file(file_name_large, parts_count)
+        parts = []
+
+        with allure.step("Upload first part"):
+            upload_id = s3_gate_object.create_multipart_upload_s3(
+                self.s3_client, bucket, object_key
+            )
+            uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
+            etag = s3_gate_object.upload_part_s3(
+                self.s3_client, bucket, object_key, upload_id, 1, part_files[0]
+            )
+            parts.append((1, etag))
+            got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
+            assert len(got_parts) == 1, f"Expected {1} parts, got\n{got_parts}"
+
+        with allure.step("Abort multipart upload"):
+            s3_gate_object.abort_multipart_uploads_s3(self.s3_client, bucket, object_key, upload_id)
+            uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
+            assert not uploads, f"Expected there is no uploads in bucket {bucket}"
+
+    @allure.title("Test S3 Upload Part Copy")
+    def test_s3_multipart_copy(self):
+        bucket = s3_gate_bucket.create_bucket_s3(self.s3_client)
+        set_bucket_versioning(self.s3_client, bucket, s3_gate_bucket.VersioningStatus.ENABLED)
+        parts_count = 3
+        file_name_large = generate_file(PART_SIZE * parts_count)  # 5Mb - min part
+        object_key = object_key_from_file_path(file_name_large)
+        part_files = split_file(file_name_large, parts_count)
+        parts = []
+        objs = []
+
+        with allure.step(f"Put {parts_count} objec in bucket"):
+            for part in part_files:
+                s3_gate_object.put_object_s3(self.s3_client, bucket, part)
+                objs.append(object_key_from_file_path(part))
+            check_objects_in_bucket(self.s3_client, bucket, objs)
+
+        with allure.step("Create multipart upload object"):
+            upload_id = s3_gate_object.create_multipart_upload_s3(
+                self.s3_client, bucket, object_key
+            )
+            uploads = s3_gate_object.list_multipart_uploads_s3(self.s3_client, bucket)
+            assert uploads, f"Expected there are uploads in bucket {bucket}"
+
+        with allure.step("Start multipart upload"):
+            for part_id, obj_key in enumerate(objs, start=1):
+                etag = s3_gate_object.upload_part_copy_s3(
+                    self.s3_client, bucket, object_key, upload_id, part_id, f"{bucket}/{obj_key}"
+                )
+                parts.append((part_id, etag))
+            got_parts = s3_gate_object.list_parts_s3(self.s3_client, bucket, object_key, upload_id)
+            s3_gate_object.complete_multipart_upload_s3(
+                self.s3_client, bucket, object_key, upload_id, parts
+            )
+            assert len(got_parts) == len(
+                part_files
+            ), f"Expected {parts_count} parts, got\n{got_parts}"
+
+        with allure.step("Check we can get whole object from bucket"):
+            got_object = s3_gate_object.get_object_s3(self.s3_client, bucket, object_key)
+            assert get_file_hash(got_object) == get_file_hash(file_name_large)