dkirillov/s3-tests
1
0
Fork 0
forked from TrueCloudLab/s3-tests
Code Pull requests Activity

Merge pull request #75 from jmunhoz/test-multipart-cp

Browse source 
Test multipart copy part feature

Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>
This commit is contained in:
Yehuda Sadeh 2015-10-27 08:19:30 -07:00
commit 001972a51c
1 changed files with 71 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

71
s3tests/functional/test_s3.py
View file

@ -4579,6 +4579,14 @@ def transfer_part(bucket, mp_id, mp_keyname, i, part):
part_out = StringIO(part) part_out = StringIO(part)
mp.upload_part_from_file(part_out, i+1) mp.upload_part_from_file(part_out, i+1)
def copy_part(src_bucket, src_keyname, dst_bucket, dst_keyname, mp_id, i, start=None, end=None):
"""Copy a part of a multipart upload from other bucket.
"""
mp = boto.s3.multipart.MultiPartUpload(dst_bucket)
mp.key_name = dst_keyname
mp.id = mp_id
mp.copy_part_from_key(src_bucket, src_keyname, i+1, start, end)
def generate_random(size, part_size=5*1024*1024): def generate_random(size, part_size=5*1024*1024):
""" """
Generate the specified number random data. Generate the specified number random data.
@ -4619,6 +4627,29 @@ def _multipart_upload(bucket, s3_key_name, size, part_size=5*1024*1024, do_list=
return (upload, s) return (upload, s)
def _multipart_copy(src_bucketname, src_keyname, dst_bucket, dst_keyname, size, part_size=5*1024*1024, do_list=None, headers=None, metadata=None, resend_parts=[]):
upload = dst_bucket.initiate_multipart_upload(dst_keyname, headers=headers, metadata=metadata)
i = 0
for start_offset in range(0, size, part_size):
end_offset = min(start_offset + part_size - 1, size - 1)
copy_part(src_bucketname, src_keyname, dst_bucket, dst_keyname, upload.id, i, start_offset, end_offset)
if i in resend_parts:
copy_part(src_bucketname, src_keyname, dst_bucket, dst_name, upload.id, i, start_offset, end_offset)
i = i + 1
if do_list is not None:
l = dst_bucket.list_multipart_uploads()
l = list(l)
return upload
def _create_key_with_random_content(keyname, size=7*1024*1024):
bucket = get_new_bucket()
key = bucket.new_key(keyname)
data = StringIO(str(generate_random(size, size).next()))
key.set_contents_from_file(fp=data)
return (bucket, key)
@attr(resource='object') @attr(resource='object')
@attr(method='put') @attr(method='put')
@attr(operation='check multipart upload without parts') @attr(operation='check multipart upload without parts')
@ -4642,6 +4673,19 @@ def test_multipart_upload_small():
key2 = bucket.get_key(key) key2 = bucket.get_key(key)
eq(key2.size, size) eq(key2.size, size)
@attr(resource='object')
@attr(method='put')
@attr(operation='check multipart copies with single small part')
def test_multipart_copy_small():
(src_bucket, src_key) = _create_key_with_random_content('foo')
dst_bucket = get_new_bucket()
dst_keyname = "mymultipart"
size = 1
copy = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, size)
copy.complete_upload()
key2 = dst_bucket.get_key(dst_keyname)
eq(key2.size, size)
def _check_content_using_range(k, data, step): def _check_content_using_range(k, data, step):
objlen = k.size objlen = k.size
for ofs in xrange(0, k.size, step): for ofs in xrange(0, k.size, step):
@ -4736,6 +4780,33 @@ def test_multipart_upload_multiple_sizes():
(upload, data) = _multipart_upload(bucket, key, 10 * 1024 * 1024) (upload, data) = _multipart_upload(bucket, key, 10 * 1024 * 1024)
upload.complete_upload() upload.complete_upload()
@attr(assertion='successful')
def test_multipart_copy_multiple_sizes():
(src_bucket, src_key) = _create_key_with_random_content('foo', 12 * 1024 * 1024)
dst_bucket = get_new_bucket()
dst_keyname="mymultipart"
k = src_bucket.get_key(src_key.name)
s = k.get_contents_as_string()
upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 5 * 1024 * 1024)
upload.complete_upload()
upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 5 * 1024 * 1024 + 100 * 1024)
upload.complete_upload()
upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 5 * 1024 * 1024 + 600 * 1024)
upload.complete_upload()
upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 10 * 1024 * 1024 + 100 * 1024)
upload.complete_upload()
upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 10 * 1024 * 1024 + 600 * 1024)
upload.complete_upload()
upload = _multipart_copy(src_bucket.name, src_key.name, dst_bucket, dst_keyname, 10 * 1024 * 1024)
upload.complete_upload()
@attr(resource='object') @attr(resource='object')
@attr(method='put') @attr(method='put')
@attr(operation='check failure on multiple multi-part upload with size too small') @attr(operation='check failure on multiple multi-part upload with size too small')