nuke_prefixed_buckets deletes objects in batches

speed up the cleanup by using delete_objects() with batches of 128

Signed-off-by: Casey Bodley <cbodley@redhat.com>
(cherry picked from commit bb995c2aeb)

s3tests_boto3/functional/__init__.py

@@ -75,38 +75,38 @@ def get_objects_list(bucket, client=None, prefix=None):
 
     return objects_list
 
-def get_versioned_objects_list(bucket, client=None):
+# generator function that returns object listings in batches, where each
-    if client == None:
+# batch is a list of dicts compatible with delete_objects()
-        client = get_client()
+def list_versions(client, bucket, batch_size):
-    response = client.list_object_versions(Bucket=bucket)
+    key_marker = ''
-    versioned_objects_list = []
+    version_marker = ''
+    truncated = True
+    while truncated:
+        listing = client.list_object_versions(
+                Bucket=bucket,
+                KeyMarker=key_marker,
+                VersionIdMarker=version_marker,
+                MaxKeys=batch_size)
 
-    if 'Versions' in response:
+        key_marker = listing.get('NextKeyMarker')
-        contents = response['Versions']
+        version_marker = listing.get('NextVersionIdMarker')
-        for obj in contents:
+        truncated = listing['IsTruncated']
-            key = obj['Key']
-            version_id = obj['VersionId']
-            versioned_obj = (key,version_id)
-            versioned_objects_list.append(versioned_obj)
 
-    return versioned_objects_list
+        objs = listing.get('Versions', []) + listing.get('DeleteMarkers', [])
+        if len(objs):
+            yield [{'Key': o['Key'], 'VersionId': o['VersionId']} for o in objs]
 
-def get_delete_markers_list(bucket, client=None):
+def nuke_bucket(client, bucket):
-    if client == None:
+    batch_size = 128
-        client = get_client()
+    max_retain_date = None
-    response = client.list_object_versions(Bucket=bucket)
-    delete_markers = []
 
-    if 'DeleteMarkers' in response:
+    # list and delete objects in batches
-        contents = response['DeleteMarkers']
+    for objects in list_versions(client, bucket, batch_size):
-        for obj in contents:
+        client.delete_objects(Bucket=bucket,
-            key = obj['Key']
+                Delete={'Objects': objects, 'Quiet': True},
-            version_id = obj['VersionId']
+                BypassGovernanceRetention=True)
-            versioned_obj = (key,version_id)
-            delete_markers.append(versioned_obj)
-
-    return delete_markers
 
+    client.delete_bucket(Bucket=bucket)
 
 def nuke_prefixed_buckets(prefix, client=None):
     if client == None:
@@ -115,20 +115,10 @@ def nuke_prefixed_buckets(prefix, client=None):
     buckets = get_buckets_list(client, prefix)
 
     err = None
-    if buckets != []:
     for bucket_name in buckets:
-            objects_list = get_objects_list(bucket_name, client)
-            for obj in objects_list:
-                response = client.delete_object(Bucket=bucket_name,Key=obj)
-            versioned_objects_list = get_versioned_objects_list(bucket_name, client)
-            for obj in versioned_objects_list:
-                response = client.delete_object(Bucket=bucket_name,Key=obj[0],VersionId=obj[1])
-            delete_markers = get_delete_markers_list(bucket_name, client)
-            for obj in delete_markers:
-                response = client.delete_object(Bucket=bucket_name,Key=obj[0],VersionId=obj[1])
         try:
-                response = client.delete_bucket(Bucket=bucket_name)
+            nuke_bucket(client, bucket_name)
-            except ClientError as e:
+        except Exception as e:
             # The exception shouldn't be raised when doing cleanup. Pass and continue
             # the bucket cleanup process. Otherwise left buckets wouldn't be cleared
             # resulting in some kind of resource leak. err is used to hint user some