From fd33286494a221030eb9128f446337e7f3cfdef9 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 19 Feb 2013 17:58:54 -0800 Subject: [PATCH 1/4] test_s3: fix test_list_multipart_upload test wasn't really testing what it should have Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 29085cf..0c3e23c 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -3889,11 +3889,22 @@ def test_list_multipart_upload(): bucket = get_new_bucket() key="mymultipart" upload1 = _multipart_upload(bucket, key, 5, 1) - upload2 = _multipart_upload(bucket, key, 5, 1) + upload2 = _multipart_upload(bucket, key, 6, 1) key2="mymultipart2" upload3 = _multipart_upload(bucket, key2, 5, 1) + l = bucket.list_multipart_uploads() + l = list(l) + + index = dict([(key, 2), (key2, 1)]) + + for upload in l: + index[upload.key_name] -= 1; + + for k, c in index.items(): + eq(c, 0) + upload1.cancel_upload() upload2.cancel_upload() upload3.cancel_upload() From 27581ccf6a6c4bb383a95388ea94c5cf0e60f0d4 Mon Sep 17 00:00:00 2001 From: caleb miles Date: Thu, 7 Feb 2013 15:58:32 -0500 Subject: [PATCH 2/4] test_s3: Add test of ACL grants through HTTP headers. Signed-off-by: caleb miles --- s3tests/functional/test_s3.py | 138 ++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 0c3e23c..fca7e1e 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -38,6 +38,7 @@ from ordereddict import OrderedDict from . import ( nuke_prefixed_buckets, get_new_bucket, + get_new_bucket_name, s3, config, get_prefix, @@ -3289,6 +3290,143 @@ def test_bucket_acl_no_grants(): # can write acl bucket.set_acl('private') +def _get_acl_header(user=None, perms=None): + all_headers = ["read", "write", "read-acp", "write-acp", "full-control"] + headers = {} + + if user == None: + user = config.alt.user_id + + if perms != None: + for perm in perms: + headers["x-amz-grant-{perm}".format(perm=perm)] = "id={uid}".format(uid=user) + + else: + for perm in all_headers: + headers["x-amz-grant-{perm}".format(perm=perm)] = "id={uid}".format(uid=user) + + return headers + +@attr(resource='object') +@attr(method='PUT') +@attr(operation='add all grants to user through headers') +@attr(assertion='adds all grants individually to second user') +@attr('fails_on_dho') +def test_object_header_acl_grants(): + bucket = get_new_bucket() + headers = _get_acl_header() + k = bucket.new_key("foo_key") + k.set_contents_from_string("bar", headers=headers) + + policy = k.get_acl() + check_grants( + policy.acl.grants, + [ + dict( + permission='READ', + id=config.alt.user_id, + display_name=config.alt.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + dict( + permission='WRITE', + id=config.alt.user_id, + display_name=config.alt.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + dict( + permission='READ_ACP', + id=config.alt.user_id, + display_name=config.alt.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + dict( + permission='WRITE_ACP', + id=config.alt.user_id, + display_name=config.alt.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + dict( + permission='FULL_CONTROL', + id=config.alt.user_id, + display_name=config.alt.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + ], + ) + + +@attr(resource='bucket') +@attr(method='PUT') +@attr(operation='add all grants to user through headers') +@attr(assertion='adds all grants individually to second user') +@attr('fails_on_dho') +def test_bucket_header_acl_grants(): + headers = _get_acl_header() + bucket = s3.main.create_bucket(get_prefix(), headers=headers) + + policy = bucket.get_acl() + check_grants( + policy.acl.grants, + [ + dict( + permission='READ', + id=config.alt.user_id, + display_name=config.alt.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + dict( + permission='WRITE', + id=config.alt.user_id, + display_name=config.alt.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + dict( + permission='READ_ACP', + id=config.alt.user_id, + display_name=config.alt.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + dict( + permission='WRITE_ACP', + id=config.alt.user_id, + display_name=config.alt.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + dict( + permission='FULL_CONTROL', + id=config.alt.user_id, + display_name=config.alt.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + ], + ) + + # alt user can write + bucket2 = s3.alt.get_bucket(bucket.name) + key = bucket2.new_key('foo') + key.set_contents_from_string('bar') + # This test will fail on DH Objects. DHO allows multiple users with one account, which # would violate the uniqueness requirement of a user's email. As such, DHO users are From 409567582772efd5f98bc40bb48cb07e3b034f14 Mon Sep 17 00:00:00 2001 From: caleb miles Date: Wed, 31 Oct 2012 15:32:03 -0400 Subject: [PATCH 3/4] test_s3: test multi-part uploads using boto provided functionality. Tests the implementation of multi-part upload and verifies written object. Signed-off-by: caleb miles --- s3tests/functional/test_s3.py | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index fca7e1e..4e79532 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -4004,6 +4004,46 @@ def test_multipart_upload(): eq(obj_count, 1) eq(bytes_used, 30 * 1024 * 1024) +@attr(resource='object') +@attr(method='put') +@attr(operation='check contents of multi-part upload') +@attr(assertion='successful') +def test_multipart_upload_contents(): + bucket = get_new_bucket() + key_name="mymultipart" + num_parts=5 + payload='foo'*10*1024*1024 + mp=bucket.initiate_multipart_upload(key_name) + for i in range(0, num_parts): + mp.upload_part_from_file(StringIO(payload), i+1) + + mp.complete_upload() + key=bucket.get_key(key_name) + test_string=key.get_contents_as_string() + assert test_string == payload*num_parts + + +@attr(resource='object') +@attr(method='put') +@attr(operation=' multi-part upload overwrites existing key') +@attr(assertion='successful') +def test_multipart_upload_overwrite_existing_object(): + bucket = get_new_bucket() + key_name="mymultipart" + payload='bar'*10*1024*1024 + num_parts=5 + key=bucket.new_key(key_name) + key.set_contents_from_string(payload) + + mp=bucket.initiate_multipart_upload(key_name) + for i in range(0, num_parts): + mp.upload_part_from_file(StringIO(payload), i+1) + + mp.complete_upload() + key=bucket.get_key(key_name) + test_string=key.get_contents_as_string() + assert test_string == payload*num_parts + @attr(resource='object') @attr(method='put') @attr(operation='abort multi-part upload') From 10a6caf4eee1e7af7a6e5147a256fed8ee65c339 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Wed, 20 Feb 2013 13:55:35 -0800 Subject: [PATCH 4/4] test_s3: reduce multipart tests sizes Tests were dominating run time Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 4e79532..1059301 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -4011,8 +4011,8 @@ def test_multipart_upload(): def test_multipart_upload_contents(): bucket = get_new_bucket() key_name="mymultipart" - num_parts=5 - payload='foo'*10*1024*1024 + num_parts=3 + payload='12345'*1024*1024 mp=bucket.initiate_multipart_upload(key_name) for i in range(0, num_parts): mp.upload_part_from_file(StringIO(payload), i+1) @@ -4030,8 +4030,8 @@ def test_multipart_upload_contents(): def test_multipart_upload_overwrite_existing_object(): bucket = get_new_bucket() key_name="mymultipart" - payload='bar'*10*1024*1024 - num_parts=5 + payload='12345'*1024*1024 + num_parts=2 key=bucket.new_key(key_name) key.set_contents_from_string(payload)