TrueCloudLab/s3-tests
7
0
Fork 1
mirror of https://github.com/ceph/s3-tests.git synced 2024-11-22 09:29:43 +00:00
Code Issues 1 Projects Releases Packages Wiki Activity

rgw: add more multipart upload tests

Browse source 
Check for issue #8846.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
This commit is contained in:
Yehuda Sadeh 2014-07-16 13:22:38 -07:00
parent 05f45c0fe9
commit b07fce46a0
1 changed files with 40 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
s3tests/functional/test_s3.py
View file

@ -4068,34 +4068,34 @@ def transfer_part(bucket, mp_id, mp_keyname, i, part):
part_out = StringIO(part) part_out = StringIO(part)
mp.upload_part_from_file(part_out, i+1) mp.upload_part_from_file(part_out, i+1)
def generate_random(mb_size): def generate_random(size):
""" """
Generate the specified number of megabytes of random data. Generate the specified number of megabytes of random data.
(actually each MB is a repetition of the first KB) (actually each MB is a repetition of the first KB)
""" """
mb = 1024 * 1024 mb = 1024 * 1024
chunk = 1024 chunk = 1024
part_size_mb = 5 part_size = 5 * mb
allowed = string.ascii_letters allowed = string.ascii_letters
for x in range(0, mb_size, part_size_mb): for x in range(0, size, part_size):
strpart = ''.join([allowed[random.randint(0, len(allowed) - 1)] for _ in xrange(chunk)]) strpart = ''.join([allowed[random.randint(0, len(allowed) - 1)] for _ in xrange(chunk)])
s = '' s = ''
left = mb_size - x left = size - x
this_part_size = min(left, part_size_mb) this_part_size = min(left, part_size)
for y in range(this_part_size * mb / chunk): for y in range(this_part_size / chunk):
s = s + strpart s = s + strpart
yield s yield s
if (x == mb_size): if (x == size):
return return
def _multipart_upload(bucket, s3_key_name, mb_size, do_list=None, headers=None, metadata=None): def _multipart_upload(bucket, s3_key_name, size, do_list=None, headers=None, metadata=None):
""" """
generate a multi-part upload for a random file of specifed size, generate a multi-part upload for a random file of specifed size,
if requested, generate a list of the parts if requested, generate a list of the parts
return the upload descriptor return the upload descriptor
""" """
upload = bucket.initiate_multipart_upload(s3_key_name, headers=headers, metadata=metadata) upload = bucket.initiate_multipart_upload(s3_key_name, headers=headers, metadata=metadata)
for i, part in enumerate(generate_random(mb_size)): for i, part in enumerate(generate_random(size)):
transfer_part(bucket, upload.id, upload.key_name, i, part) transfer_part(bucket, upload.id, upload.key_name, i, part)
if do_list is not None: if do_list is not None:
@ -4112,7 +4112,7 @@ def test_multipart_upload():
bucket = get_new_bucket() bucket = get_new_bucket()
key="mymultipart" key="mymultipart"
content_type='text/bla' content_type='text/bla'
upload = _multipart_upload(bucket, key, 30, headers={'Content-Type': content_type}, metadata={'foo': 'bar'}) upload = _multipart_upload(bucket, key, 30 * 1024 * 1024, headers={'Content-Type': content_type}, metadata={'foo': 'bar'})
upload.complete_upload() upload.complete_upload()
(obj_count, bytes_used) = _head_bucket(bucket) (obj_count, bytes_used) = _head_bucket(bucket)
@ -4124,6 +4124,31 @@ def test_multipart_upload():
eq(k.metadata['foo'], 'bar') eq(k.metadata['foo'], 'bar')
eq(k.content_type, content_type) eq(k.content_type, content_type)
@attr(resource='object')
@attr(method='put')
@attr(operation='complete multiple multi-part upload with different sizes')
@attr(assertion='successful')
def test_multipart_upload_multiple_sizes():
bucket = get_new_bucket()
key="mymultipart"
upload = _multipart_upload(bucket, key, 5 * 1024 * 1024)
upload.complete_upload()
upload = _multipart_upload(bucket, key, 5 * 1024 * 1024 + 100 * 1024)
upload.complete_upload()
upload = _multipart_upload(bucket, key, 5 * 1024 * 1024 + 600 * 1024)
upload.complete_upload()
upload = _multipart_upload(bucket, key, 10 * 1024 * 1024 + 100 * 1024)
upload.complete_upload()
upload = _multipart_upload(bucket, key, 10 * 1024 * 1024 + 600 * 1024)
upload.complete_upload()
upload = _multipart_upload(bucket, key, 10 * 1024 * 1024)
upload.complete_upload()
@attr(resource='object') @attr(resource='object')
@attr(method='put') @attr(method='put')
@attr(operation='check contents of multi-part upload') @attr(operation='check contents of multi-part upload')
@ -4174,7 +4199,7 @@ def test_multipart_upload_overwrite_existing_object():
def test_abort_multipart_upload(): def test_abort_multipart_upload():
bucket = get_new_bucket() bucket = get_new_bucket()
key="mymultipart" key="mymultipart"
upload = _multipart_upload(bucket, key, 10) upload = _multipart_upload(bucket, key, 10 * 1024 * 1024)
upload.cancel_upload() upload.cancel_upload()
(obj_count, bytes_used) = _head_bucket(bucket) (obj_count, bytes_used) = _head_bucket(bucket)
@ -4189,11 +4214,12 @@ def test_abort_multipart_upload():
def test_list_multipart_upload(): def test_list_multipart_upload():
bucket = get_new_bucket() bucket = get_new_bucket()
key="mymultipart" key="mymultipart"
upload1 = _multipart_upload(bucket, key, 5, 1) mb = 1024 * 1024
upload2 = _multipart_upload(bucket, key, 6, 1) upload1 = _multipart_upload(bucket, key, 5 * mb, 1)
upload2 = _multipart_upload(bucket, key, 6 * mb, 1)
key2="mymultipart2" key2="mymultipart2"
upload3 = _multipart_upload(bucket, key2, 5, 1) upload3 = _multipart_upload(bucket, key2, 5 * mb, 1)
l = bucket.list_multipart_uploads() l = bucket.list_multipart_uploads()
l = list(l) l = list(l)