dkirillov/s3-tests
1
0
Fork 0
forked from TrueCloudLab/s3-tests
Code Pull requests Activity

nuke_prefixed_buckets deletes objects in batches

Browse source 
speed up the cleanup by using delete_objects() with batches of 128

Signed-off-by: Casey Bodley <cbodley@redhat.com>
(cherry picked from commit bb995c2aeb)
This commit is contained in:
Casey Bodley 2021-08-04 13:56:39 -04:00 committed by Ali Maredia
parent df426ea041
commit 8090ea4629
1 changed files with 39 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
s3tests_boto3/functional/__init__.py
View file

@ -75,38 +75,38 @@ def get_objects_list(bucket, client=None, prefix=None):
return objects_list return objects_list
def get_versioned_objects_list(bucket, client=None): # generator function that returns object listings in batches, where each
if client == None: # batch is a list of dicts compatible with delete_objects()
client = get_client() def list_versions(client, bucket, batch_size):
response = client.list_object_versions(Bucket=bucket) key_marker = ''
versioned_objects_list = [] version_marker = ''
truncated = True
while truncated:
listing = client.list_object_versions(
Bucket=bucket,
KeyMarker=key_marker,
VersionIdMarker=version_marker,
MaxKeys=batch_size)
if 'Versions' in response: key_marker = listing.get('NextKeyMarker')
contents = response['Versions'] version_marker = listing.get('NextVersionIdMarker')
for obj in contents: truncated = listing['IsTruncated']
key = obj['Key']
version_id = obj['VersionId']
versioned_obj = (key,version_id)
versioned_objects_list.append(versioned_obj)
return versioned_objects_list objs = listing.get('Versions', []) + listing.get('DeleteMarkers', [])
if len(objs):
yield [{'Key': o['Key'], 'VersionId': o['VersionId']} for o in objs]
def get_delete_markers_list(bucket, client=None): def nuke_bucket(client, bucket):
if client == None: batch_size = 128
client = get_client() max_retain_date = None
response = client.list_object_versions(Bucket=bucket)
delete_markers = []
if 'DeleteMarkers' in response: # list and delete objects in batches
contents = response['DeleteMarkers'] for objects in list_versions(client, bucket, batch_size):
for obj in contents: client.delete_objects(Bucket=bucket,
key = obj['Key'] Delete={'Objects': objects, 'Quiet': True},
version_id = obj['VersionId'] BypassGovernanceRetention=True)
versioned_obj = (key,version_id)
delete_markers.append(versioned_obj)
return delete_markers
client.delete_bucket(Bucket=bucket)
def nuke_prefixed_buckets(prefix, client=None): def nuke_prefixed_buckets(prefix, client=None):
if client == None: if client == None:
@ -115,28 +115,18 @@ def nuke_prefixed_buckets(prefix, client=None):
buckets = get_buckets_list(client, prefix) buckets = get_buckets_list(client, prefix)
err = None err = None
if buckets != []: for bucket_name in buckets:
for bucket_name in buckets: try:
objects_list = get_objects_list(bucket_name, client) nuke_bucket(client, bucket_name)
for obj in objects_list: except Exception as e:
response = client.delete_object(Bucket=bucket_name,Key=obj) # The exception shouldn't be raised when doing cleanup. Pass and continue
versioned_objects_list = get_versioned_objects_list(bucket_name, client) # the bucket cleanup process. Otherwise left buckets wouldn't be cleared
for obj in versioned_objects_list: # resulting in some kind of resource leak. err is used to hint user some
response = client.delete_object(Bucket=bucket_name,Key=obj[0],VersionId=obj[1]) # exception once occurred.
delete_markers = get_delete_markers_list(bucket_name, client) err = e
for obj in delete_markers: pass
response = client.delete_object(Bucket=bucket_name,Key=obj[0],VersionId=obj[1]) if err:
try: raise err
response = client.delete_bucket(Bucket=bucket_name)
except ClientError as e:
# The exception shouldn't be raised when doing cleanup. Pass and continue
# the bucket cleanup process. Otherwise left buckets wouldn't be cleared
# resulting in some kind of resource leak. err is used to hint user some
# exception once occurred.
err = e
pass
if err:
raise err
print('Done with cleanup of buckets in tests.') print('Done with cleanup of buckets in tests.')