From 2f04775fce2ab9d65291d70367d068081ca82d36 Mon Sep 17 00:00:00 2001 From: anikeev-yadro Date: Mon, 12 Dec 2022 13:34:52 +0300 Subject: [PATCH] Fix delete all objects in bucket Signed-off-by: anikeev-yadro --- pytest_tests/steps/s3_gate_base.py | 21 ++++++++++++++-- pytest_tests/steps/s3_gate_object.py | 36 ++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/pytest_tests/steps/s3_gate_base.py b/pytest_tests/steps/s3_gate_base.py index c6a35397..b523632f 100644 --- a/pytest_tests/steps/s3_gate_base.py +++ b/pytest_tests/steps/s3_gate_base.py @@ -88,15 +88,32 @@ class TestS3GateBase(ClusterTestBase): def delete_all_object_in_bucket(self, bucket): versioning_status = s3_gate_bucket.get_bucket_versioning_status(self.s3_client, bucket) if versioning_status == s3_gate_bucket.VersioningStatus.ENABLED.value: - # From versioned bucket we should delete all versions of all objects + # From versioned bucket we should delete all versions and delete markers of all objects objects_versions = s3_gate_object.list_objects_versions_s3(self.s3_client, bucket) if objects_versions: - s3_gate_object.delete_object_versions_s3(self.s3_client, bucket, objects_versions) + s3_gate_object.delete_object_versions_s3_without_dm( + self.s3_client, bucket, objects_versions + ) + objects_delete_markers = s3_gate_object.list_objects_delete_markers_s3( + self.s3_client, bucket + ) + if objects_delete_markers: + s3_gate_object.delete_object_versions_s3_without_dm( + self.s3_client, bucket, objects_delete_markers + ) + else: # From non-versioned bucket it's sufficient to delete objects by key objects = s3_gate_object.list_objects_s3(self.s3_client, bucket) if objects: s3_gate_object.delete_objects_s3(self.s3_client, bucket, objects) + objects_delete_markers = s3_gate_object.list_objects_delete_markers_s3( + self.s3_client, bucket + ) + if objects_delete_markers: + s3_gate_object.delete_object_versions_s3_without_dm( + self.s3_client, bucket, objects_delete_markers + ) # Delete the bucket itself s3_gate_bucket.delete_bucket_s3(self.s3_client, bucket) diff --git a/pytest_tests/steps/s3_gate_object.py b/pytest_tests/steps/s3_gate_object.py index 1266bce6..1b30baef 100644 --- a/pytest_tests/steps/s3_gate_object.py +++ b/pytest_tests/steps/s3_gate_object.py @@ -85,6 +85,21 @@ def list_objects_versions_s3(s3_client, bucket: str, full_output: bool = False) ) from err +@allure.step("List objects delete markers S3") +def list_objects_delete_markers_s3(s3_client, bucket: str, full_output: bool = False) -> list: + try: + response = s3_client.list_object_versions(Bucket=bucket) + delete_markers = response.get("DeleteMarkers", []) + log_command_execution("S3 List objects delete markers result", response) + return response if full_output else delete_markers + + except ClientError as err: + raise Exception( + f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}' + ) from err + + @allure.step("Put object S3") def put_object_s3(s3_client, bucket: str, filepath: str, **kwargs): filename = os.path.basename(filepath) @@ -185,6 +200,27 @@ def delete_object_versions_s3(s3_client, bucket: str, object_versions: list): ) from err +@allure.step("Delete object versions S3 without delete markers") +def delete_object_versions_s3_without_dm(s3_client, bucket: str, object_versions: list): + try: + # Delete objects without creating delete markers + for object_version in object_versions: + params = { + "Bucket": bucket, + "Key": object_version["Key"], + "VersionId": object_version["VersionId"], + } + response = s3_client.delete_object(**params) + log_command_execution("S3 Delete object result", response) + return response + + except ClientError as err: + raise Exception( + f'Error Message: {err.response["Error"]["Message"]}\n' + f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}' + ) from err + + @allure.step("Put object ACL") def put_object_acl_s3( s3_client,