From d4e9e40792b935f1d8695f849cb65775c94a80ec Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 28 Oct 2014 15:20:58 -0700 Subject: [PATCH 01/15] teardown: clean object versions also don't set acls on buckets, objects before removing, no need for that. Signed-off-by: Yehuda Sadeh --- s3tests/functional/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/s3tests/functional/__init__.py b/s3tests/functional/__init__.py index 51dd506..0b0561f 100644 --- a/s3tests/functional/__init__.py +++ b/s3tests/functional/__init__.py @@ -59,17 +59,19 @@ def nuke_prefixed_buckets_on_conn(prefix, name, conn): name=name, prefix=prefix, ) + for bucket in conn.get_all_buckets(): + print 'prefix=',prefix if bucket.name.startswith(prefix): print 'Cleaning bucket {bucket}'.format(bucket=bucket) try: - bucket.set_canned_acl('private') - for key in bucket.list(): + # bucket.set_canned_acl('private') + for key in bucket.list_versions(): print 'Cleaning bucket {bucket} key {key}'.format( bucket=bucket, key=key, ) - key.set_canned_acl('private') + # key.set_canned_acl('private') key.delete() bucket.delete() except boto.exception.S3ResponseError as e: From 032f41f6f3cfa081a8632de01712f8584af49e0a Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 28 Oct 2014 16:04:39 -0700 Subject: [PATCH 02/15] functional: add test_versioning_bucket_create_suspend() Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index a7e5c89..8eb2789 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -5175,4 +5175,31 @@ def test_region_copy_object(): e = assert_raises(boto.exception.S3ResponseError, conn.get_bucket, dest_bucket.name) eq(e.status, 404) +def check_versioning(bucket, status): + try: + eq(bucket.get_versioning_status()['Versioning'], status) + except KeyError: + eq(status, None) + +@attr(resource='bucket') +@attr(method='create') +@attr(operation='create versioned bucket') +@attr(assertion='can create and suspend bucket versioning') +@attr('versioning') +def test_versioning_bucket_create_suspend(): + bucket = get_new_bucket() + print bucket.get_versioning_status() + check_versioning(bucket, None) + + bucket.configure_versioning(False) + check_versioning(bucket, "Suspended") + + bucket.configure_versioning(True) + check_versioning(bucket, "Enabled") + + bucket.configure_versioning(True) + check_versioning(bucket, "Enabled") + + bucket.configure_versioning(False) + check_versioning(bucket, "Suspended") From 0f0e83380124079f9d9bf2ebfaea429f1006bdee Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Wed, 29 Oct 2014 15:26:57 -0700 Subject: [PATCH 03/15] s3tests: add test_versioning_obj_create_read_remove() Signed-off-by: Yehuda Sadeh --- s3tests/functional/__init__.py | 2 +- s3tests/functional/test_s3.py | 58 +++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/s3tests/functional/__init__.py b/s3tests/functional/__init__.py index 0b0561f..6d0d680 100644 --- a/s3tests/functional/__init__.py +++ b/s3tests/functional/__init__.py @@ -72,7 +72,7 @@ def nuke_prefixed_buckets_on_conn(prefix, name, conn): key=key, ) # key.set_canned_acl('private') - key.delete() + bucket.delete_key(key.name, version_id = key.version_id) bucket.delete() except boto.exception.S3ResponseError as e: if e.error_code != 'AccessDenied': diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 8eb2789..f82c5b2 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -5188,7 +5188,6 @@ def check_versioning(bucket, status): @attr('versioning') def test_versioning_bucket_create_suspend(): bucket = get_new_bucket() - print bucket.get_versioning_status() check_versioning(bucket, None) bucket.configure_versioning(False) @@ -5203,3 +5202,60 @@ def test_versioning_bucket_create_suspend(): bucket.configure_versioning(False) check_versioning(bucket, "Suspended") +@attr(resource='object') +@attr(method='create') +@attr(operation='create versioned object') +@attr(assertion='can create access and remove appropriate versions') +@attr('versioning') +def test_versioning_obj_create_read_remove(): + bucket = get_new_bucket() + objname = 'testobj' + + total = 5 + + c = [] + for i in xrange(total): + c.append('content-{i}'.format(i=i)) + + key = bucket.new_key(objname) + key.set_contents_from_string(c[i]) + + if i == 0: + bucket.configure_versioning(True) + check_versioning(bucket, "Enabled") + + # check to see if object is pointing at correct version + key = bucket.get_key(objname) + eq(key.get_contents_as_string(), c[-1]) + print 'total', total + + k = [] + for o in bucket.list_versions(): + k.insert(0, o) + + for j in xrange(total): + print j, k[j], k[j].version_id + + for j in xrange(total): + # check by versioned key + rmkey = k.pop(-1) + eq(rmkey.get_contents_as_string(), c[-1]) + + # remove version + print 'removing version_id=', rmkey.version_id + bucket.delete_key(rmkey.name, version_id = rmkey.version_id) + c.pop(-1) + + # check to see if object is pointing at correct version + key = bucket.get_key(objname) + + if len(c) > 0: + print c[-1] + eq(key.get_contents_as_string(), c[-1]) + i = len(c) + for key in bucket.list_versions(): + i -= 1 + eq(key.get_contents_as_string(), c[i]) + else: + eq(key, None) + From 5d67c7f5df98276081c6bae2931c906fcaddf1b6 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 30 Oct 2014 13:19:59 -0700 Subject: [PATCH 04/15] s3tests: extend object versions test Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 95 ++++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 36 deletions(-) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index f82c5b2..9468ca8 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -5202,19 +5202,25 @@ def test_versioning_bucket_create_suspend(): bucket.configure_versioning(False) check_versioning(bucket, "Suspended") -@attr(resource='object') -@attr(method='create') -@attr(operation='create versioned object') -@attr(assertion='can create access and remove appropriate versions') -@attr('versioning') -def test_versioning_obj_create_read_remove(): - bucket = get_new_bucket() - objname = 'testobj' +def test_obj_versions(bucket, objname, contents): + # check to see if object is pointing at correct version + key = bucket.get_key(objname) - total = 5 + if len(contents) > 0: + print contents[-1] + print 'testing obj head', objname + eq(key.get_contents_as_string(), contents[-1]) + i = len(contents) + for key in bucket.list_versions(): + i -= 1 + print 'testing obj version-id=', key.version_id + eq(key.get_contents_as_string(), contents[i]) + else: + eq(key, None) +def create_multiple_versions(bucket, objname, num_versions): c = [] - for i in xrange(total): + for i in xrange(num_versions): c.append('content-{i}'.format(i=i)) key = bucket.new_key(objname) @@ -5223,39 +5229,56 @@ def test_versioning_obj_create_read_remove(): if i == 0: bucket.configure_versioning(True) check_versioning(bucket, "Enabled") - - # check to see if object is pointing at correct version - key = bucket.get_key(objname) - eq(key.get_contents_as_string(), c[-1]) - print 'total', total - k = [] for o in bucket.list_versions(): + print o, o.version_id k.insert(0, o) + print 'created obj name=', objname, 'version-id=', o.version_id - for j in xrange(total): + eq(len(k), len(c)) + + for j in xrange(num_versions): print j, k[j], k[j].version_id - for j in xrange(total): - # check by versioned key - rmkey = k.pop(-1) - eq(rmkey.get_contents_as_string(), c[-1]) + return (k, c) - # remove version - print 'removing version_id=', rmkey.version_id - bucket.delete_key(rmkey.name, version_id = rmkey.version_id) - c.pop(-1) - # check to see if object is pointing at correct version - key = bucket.get_key(objname) +def remove_obj_version(bucket, k, c, i): + # check by versioned key + i = i % len(k) + rmkey = k.pop(i) + eq(rmkey.get_contents_as_string(), c.pop(i)) - if len(c) > 0: - print c[-1] - eq(key.get_contents_as_string(), c[-1]) - i = len(c) - for key in bucket.list_versions(): - i -= 1 - eq(key.get_contents_as_string(), c[i]) - else: - eq(key, None) + # remove version + print 'removing version_id=', rmkey.version_id + bucket.delete_key(rmkey.name, version_id = rmkey.version_id) + +def do_test_create_remove_versions(bucket, objname, num_versions, remove_start_idx, idx_inc): + (k, c) = create_multiple_versions(bucket, objname, num_versions) + + test_obj_versions(bucket, objname, c) + + idx = remove_start_idx + + for j in xrange(num_versions): + remove_obj_version(bucket, k, c, idx) + idx += idx_inc + test_obj_versions(bucket, objname, c) + +@attr(resource='object') +@attr(method='create') +@attr(operation='create versioned object') +@attr(assertion='can create access and remove appropriate versions') +@attr('versioning') +def test_versioning_obj_create_read_remove(): + bucket = get_new_bucket() + objname = 'testobj' + num_vers = 5 + + do_test_create_remove_versions(bucket, objname, num_vers, -1, 0) + do_test_create_remove_versions(bucket, objname, num_vers, -1, 0) + do_test_create_remove_versions(bucket, objname, num_vers, 0, 0) + do_test_create_remove_versions(bucket, objname, num_vers, 1, 0) + do_test_create_remove_versions(bucket, objname, num_vers, 4, -1) + do_test_create_remove_versions(bucket, objname, num_vers, 3, 3) From a3ce6be105dfde17fe220e57fabc951980d5ff96 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 30 Oct 2014 17:05:13 -0700 Subject: [PATCH 05/15] s3tests: object version removal and delete marker test Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 74 +++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 9468ca8..1027baf 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -5202,19 +5202,33 @@ def test_versioning_bucket_create_suspend(): bucket.configure_versioning(False) check_versioning(bucket, "Suspended") -def test_obj_versions(bucket, objname, contents): + +def check_head_obj_content(key, content): + if content is not None: + eq(key.get_contents_as_string(), content) + else: + print 'check head', key + eq(key, None) + +def check_obj_content(key, content): + if content is not None: + eq(key.get_contents_as_string(), content) + else: + eq(isinstance(key, boto.s3.deletemarker.DeleteMarker), True) + + +def check_obj_versions(bucket, objname, contents): # check to see if object is pointing at correct version key = bucket.get_key(objname) if len(contents) > 0: - print contents[-1] print 'testing obj head', objname - eq(key.get_contents_as_string(), contents[-1]) + check_head_obj_content(key, contents[-1]) i = len(contents) for key in bucket.list_versions(): i -= 1 print 'testing obj version-id=', key.version_id - eq(key.get_contents_as_string(), contents[i]) + check_obj_content(key, contents[i]) else: eq(key, None) @@ -5247,27 +5261,59 @@ def remove_obj_version(bucket, k, c, i): # check by versioned key i = i % len(k) rmkey = k.pop(i) - eq(rmkey.get_contents_as_string(), c.pop(i)) + content = c.pop(i) + if (not rmkey.delete_marker): + eq(rmkey.get_contents_as_string(), content) # remove version print 'removing version_id=', rmkey.version_id bucket.delete_key(rmkey.name, version_id = rmkey.version_id) +def remove_obj_head(bucket, objname, k, c): + print 'removing obj=', objname + key = bucket.delete_key(objname) + + k.append(key) + c.append(None) + + eq(key.delete_marker, True) + + check_obj_versions(bucket, objname, c) + def do_test_create_remove_versions(bucket, objname, num_versions, remove_start_idx, idx_inc): (k, c) = create_multiple_versions(bucket, objname, num_versions) - test_obj_versions(bucket, objname, c) + check_obj_versions(bucket, objname, c) idx = remove_start_idx for j in xrange(num_versions): remove_obj_version(bucket, k, c, idx) idx += idx_inc - test_obj_versions(bucket, objname, c) + check_obj_versions(bucket, objname, c) + +def do_test_create_remove_versions_and_head(bucket, objname, num_versions, num_ops, remove_start_idx, idx_inc, head_rm_ratio): + (k, c) = create_multiple_versions(bucket, objname, num_versions) + + check_obj_versions(bucket, objname, c) + + idx = remove_start_idx + + r = 0 + + for j in xrange(num_ops): + r += head_rm_ratio + if r >= 1: + r %= 1 + remove_obj_head(bucket, objname, k, c) + else: + remove_obj_version(bucket, k, c, idx) + idx += idx_inc + check_obj_versions(bucket, objname, c) @attr(resource='object') @attr(method='create') -@attr(operation='create versioned object') +@attr(operation='create and remove versioned object') @attr(assertion='can create access and remove appropriate versions') @attr('versioning') def test_versioning_obj_create_read_remove(): @@ -5282,3 +5328,15 @@ def test_versioning_obj_create_read_remove(): do_test_create_remove_versions(bucket, objname, num_vers, 4, -1) do_test_create_remove_versions(bucket, objname, num_vers, 3, 3) +@attr(resource='object') +@attr(method='create') +@attr(operation='create and remove versioned object and head') +@attr(assertion='can create access and remove appropriate versions') +@attr('versioning') +def test_versioning_obj_create_read_remove_head(): + bucket = get_new_bucket() + objname = 'testobj' + num_vers = 5 + + do_test_create_remove_versions_and_head(bucket, objname, num_vers, num_vers * 2, -1, 0, 0.5) + From 3c297038f63aee27c4d491f40ade074a5c887596 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Fri, 14 Nov 2014 16:57:51 -0800 Subject: [PATCH 06/15] s3tests: more object versioning tests Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 212 +++++++++++++++++++++++++++++----- 1 file changed, 183 insertions(+), 29 deletions(-) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 1027baf..d3b9290 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -5181,6 +5181,26 @@ def check_versioning(bucket, status): except KeyError: eq(status, None) +# amazon is eventual consistent, retry a bit if failed +def check_configure_versioning_retry(bucket, status, expected_string): + bucket.configure_versioning(status) + + read_status = None + + for i in xrange(5): + try: + read_status = bucket.get_versioning_status()['Versioning'] + except KeyError: + read_status = None + + if (expected_string == read_status): + break + + time.sleep(1) + + eq(expected_string, read_status) + + @attr(resource='bucket') @attr(method='create') @attr(operation='create versioned bucket') @@ -5190,17 +5210,10 @@ def test_versioning_bucket_create_suspend(): bucket = get_new_bucket() check_versioning(bucket, None) - bucket.configure_versioning(False) - check_versioning(bucket, "Suspended") - - bucket.configure_versioning(True) - check_versioning(bucket, "Enabled") - - bucket.configure_versioning(True) - check_versioning(bucket, "Enabled") - - bucket.configure_versioning(False) - check_versioning(bucket, "Suspended") + check_configure_versioning_retry(bucket, False, "Suspended") + check_configure_versioning_retry(bucket, True, "Enabled") + check_configure_versioning_retry(bucket, True, "Enabled") + check_configure_versioning_retry(bucket, False, "Suspended") def check_head_obj_content(key, content): @@ -5217,7 +5230,7 @@ def check_obj_content(key, content): eq(isinstance(key, boto.s3.deletemarker.DeleteMarker), True) -def check_obj_versions(bucket, objname, contents): +def check_obj_versions(bucket, objname, keys, contents): # check to see if object is pointing at correct version key = bucket.get_key(objname) @@ -5227,13 +5240,15 @@ def check_obj_versions(bucket, objname, contents): i = len(contents) for key in bucket.list_versions(): i -= 1 + eq(keys[i].version_id or 'null', key.version_id) print 'testing obj version-id=', key.version_id check_obj_content(key, contents[i]) else: eq(key, None) -def create_multiple_versions(bucket, objname, num_versions): - c = [] +def create_multiple_versions(bucket, objname, num_versions, k = None, c = None): + c = c or [] + k = k or [] for i in xrange(num_versions): c.append('content-{i}'.format(i=i)) @@ -5241,12 +5256,17 @@ def create_multiple_versions(bucket, objname, num_versions): key.set_contents_from_string(c[i]) if i == 0: - bucket.configure_versioning(True) - check_versioning(bucket, "Enabled") - k = [] + check_configure_versioning_retry(bucket, True, "Enabled") + + k_pos = len(k) + i = 0 for o in bucket.list_versions(): + i += 1 + if i > num_versions: + break + print o, o.version_id - k.insert(0, o) + k.insert(k_pos, o) print 'created obj name=', objname, 'version-id=', o.version_id eq(len(k), len(c)) @@ -5254,6 +5274,8 @@ def create_multiple_versions(bucket, objname, num_versions): for j in xrange(num_versions): print j, k[j], k[j].version_id + check_obj_versions(bucket, objname, k, c) + return (k, c) @@ -5268,6 +5290,7 @@ def remove_obj_version(bucket, k, c, i): # remove version print 'removing version_id=', rmkey.version_id bucket.delete_key(rmkey.name, version_id = rmkey.version_id) + check_obj_versions(bucket, rmkey.name, k, c) def remove_obj_head(bucket, objname, k, c): print 'removing obj=', objname @@ -5278,30 +5301,25 @@ def remove_obj_head(bucket, objname, k, c): eq(key.delete_marker, True) - check_obj_versions(bucket, objname, c) + check_obj_versions(bucket, objname, k, c) def do_test_create_remove_versions(bucket, objname, num_versions, remove_start_idx, idx_inc): (k, c) = create_multiple_versions(bucket, objname, num_versions) - check_obj_versions(bucket, objname, c) - idx = remove_start_idx for j in xrange(num_versions): remove_obj_version(bucket, k, c, idx) idx += idx_inc - check_obj_versions(bucket, objname, c) - -def do_test_create_remove_versions_and_head(bucket, objname, num_versions, num_ops, remove_start_idx, idx_inc, head_rm_ratio): - (k, c) = create_multiple_versions(bucket, objname, num_versions) - - check_obj_versions(bucket, objname, c) +def do_remove_versions(bucket, objname, remove_start_idx, idx_inc, head_rm_ratio, k, c): idx = remove_start_idx r = 0 - for j in xrange(num_ops): + total = len(k) + + for j in xrange(total): r += head_rm_ratio if r >= 1: r %= 1 @@ -5309,7 +5327,13 @@ def do_test_create_remove_versions_and_head(bucket, objname, num_versions, num_o else: remove_obj_version(bucket, k, c, idx) idx += idx_inc - check_obj_versions(bucket, objname, c) + + check_obj_versions(bucket, objname, k, c) + +def do_test_create_remove_versions_and_head(bucket, objname, num_versions, num_ops, remove_start_idx, idx_inc, head_rm_ratio): + (k, c) = create_multiple_versions(bucket, objname, num_versions) + + do_remove_versions(bucket, objname, remove_start_idx, idx_inc, head_rm_ratio, k, c) @attr(resource='object') @attr(method='create') @@ -5340,3 +5364,133 @@ def test_versioning_obj_create_read_remove_head(): do_test_create_remove_versions_and_head(bucket, objname, num_vers, num_vers * 2, -1, 0, 0.5) +def is_null_key(k): + return (k.version_id is None) or (k.version_id == 'null') + +def delete_suspended_versioning_obj(bucket, objname, k, c): + key = bucket.delete_key(objname) + + i = 0 + while i < len(k): + if is_null_key(k[i]): + k.pop(i) + c.pop(i) + else: + i += 1 + + key.version_id = "null" + k.append(key) + c.append(None) + + check_obj_versions(bucket, objname, k, c) + +def overwrite_suspended_versioning_obj(bucket, objname, k, c, content): + key = bucket.new_key(objname) + key.set_contents_from_string(content) + + i = 0 + while i < len(k): + print 'kkk', i, k[i], k[i].version_id + if is_null_key(k[i]): + print 'null key!' + k.pop(i) + c.pop(i) + else: + i += 1 + + k.append(key) + c.append(content) + + check_obj_versions(bucket, objname, k, c) + +@attr(resource='object') +@attr(method='create') +@attr(operation='suspend versioned bucket') +@attr(assertion='suspended versioning behaves correctly') +@attr('versioning') +def test_versioning_obj_suspend_versions(): + bucket = get_new_bucket() + check_versioning(bucket, None) + + check_configure_versioning_retry(bucket, True, "Enabled") + + num_versions = 5 + objname = 'testobj' + + (k, c) = create_multiple_versions(bucket, objname, num_versions) + + check_configure_versioning_retry(bucket, False, "Suspended") + + delete_suspended_versioning_obj(bucket, objname, k, c) + delete_suspended_versioning_obj(bucket, objname, k, c) + overwrite_suspended_versioning_obj(bucket, objname, k, c, 'null content 1') + overwrite_suspended_versioning_obj(bucket, objname, k, c, 'null content 2') + delete_suspended_versioning_obj(bucket, objname, k, c) + overwrite_suspended_versioning_obj(bucket, objname, k, c, 'null content 3') + delete_suspended_versioning_obj(bucket, objname, k, c) + + check_configure_versioning_retry(bucket, True, "Enabled") + + (k, c) = create_multiple_versions(bucket, objname, 3, k, c) + + do_remove_versions(bucket, objname, 0, 5, 0.5, k, c) + do_remove_versions(bucket, objname, 0, 5, 0, k, c) + + eq(len(k), 0) + eq(len(k), len(c)) + +@attr(resource='object') +@attr(method='create') +@attr(operation='suspend versioned bucket') +@attr(assertion='suspended versioning behaves correctly') +@attr('versioning') +def test_versioning_obj_suspend_versions_simple(): + bucket = get_new_bucket() + check_versioning(bucket, None) + + check_configure_versioning_retry(bucket, True, "Enabled") + + num_versions = 1 + objname = 'testobj' + + (k, c) = create_multiple_versions(bucket, objname, num_versions) + + check_configure_versioning_retry(bucket, False, "Suspended") + + delete_suspended_versioning_obj(bucket, objname, k, c) + + check_configure_versioning_retry(bucket, True, "Enabled") + + (k, c) = create_multiple_versions(bucket, objname, 1, k, c) + + for i in xrange(len(k)): + print 'JJJ: ', k[i].version_id, c[i] + + do_remove_versions(bucket, objname, 0, 0, 0.5, k, c) + do_remove_versions(bucket, objname, 0, 0, 0, k, c) + + eq(len(k), 0) + eq(len(k), len(c)) + +@attr(resource='object') +@attr(method='remove') +@attr(operation='create and remove versions') +@attr(assertion='everything works') +@attr('versioning') +def test_versioning_obj_create_versions_remove_all(): + bucket = get_new_bucket() + check_versioning(bucket, None) + + check_configure_versioning_retry(bucket, True, "Enabled") + + num_versions = 10 + objname = 'testobj' + + (k, c) = create_multiple_versions(bucket, objname, num_versions) + + do_remove_versions(bucket, objname, 0, 5, 0.5, k, c) + do_remove_versions(bucket, objname, 0, 5, 0, k, c) + + eq(len(k), 0) + eq(len(k), len(c)) + From b4d406bcf73a1b51b9570d5310673c0ec14cf251 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Mon, 17 Nov 2014 11:08:13 -0800 Subject: [PATCH 07/15] test_s3: add versioned multipart upload test Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 62 ++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index d3b9290..f78c4bf 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -4380,26 +4380,36 @@ def test_multipart_upload_size_too_small(): eq(e.status, 400) eq(e.error_code, u'EntityTooSmall') -@attr(resource='object') -@attr(method='put') -@attr(operation='check contents of multi-part upload') -@attr(assertion='successful') -def test_multipart_upload_contents(): - bucket = get_new_bucket() - key_name="mymultipart" - num_parts=3 - payload='12345'*1024*1024 +def gen_rand_string(size, chars=string.ascii_uppercase + string.digits): + return ''.join(random.choice(chars) for _ in range(size)) + +def do_test_multipart_upload_contents(bucket, key_name, num_parts): + payload=gen_rand_string(5)*1024*1024 mp=bucket.initiate_multipart_upload(key_name) for i in range(0, num_parts): mp.upload_part_from_file(StringIO(payload), i+1) last_payload='123'*1024*1024 - mp.upload_part_from_file(StringIO(last_payload), 4) + mp.upload_part_from_file(StringIO(last_payload), num_parts + 1) mp.complete_upload() key=bucket.get_key(key_name) test_string=key.get_contents_as_string() - assert test_string == payload*num_parts+last_payload + + all_payload = payload*num_parts + last_payload + print 'JJJ', key_name, len(all_payload), len(test_string) + + assert test_string == all_payload + + return all_payload + + +@attr(resource='object') +@attr(method='put') +@attr(operation='check contents of multi-part upload') +@attr(assertion='successful') +def test_multipart_upload_contents(): + do_test_multipart_upload_contents(get_new_bucket(), 'mymultipart', 3) @attr(resource='object') @@ -5494,3 +5504,33 @@ def test_versioning_obj_create_versions_remove_all(): eq(len(k), 0) eq(len(k), len(c)) +@attr(resource='object') +@attr(method='multipart') +@attr(operation='create and test multipart object') +@attr(assertion='everything works') +@attr('versioning') +def test_versioning_obj_create_overwrite_multipart(): + bucket = get_new_bucket() + check_configure_versioning_retry(bucket, True, "Enabled") + + objname = 'testobj' + + c = [] + + num_vers = 3 + + for i in xrange(num_vers): + c.append(do_test_multipart_upload_contents(bucket, objname, 3)) + + k = [] + for key in bucket.list_versions(): + k.insert(0, key) + + eq(len(k), num_vers) + check_obj_versions(bucket, objname, k, c) + + do_remove_versions(bucket, objname, 0, 3, 0.5, k, c) + do_remove_versions(bucket, objname, 0, 3, 0, k, c) + + eq(len(k), 0) + eq(len(k), len(c)) From 67bb05c87e3f456e91be980c69e49828d08ba1a1 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Mon, 24 Nov 2014 10:47:16 -0800 Subject: [PATCH 08/15] s3test: test copy version Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index f78c4bf..b40b182 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -5534,3 +5534,33 @@ def test_versioning_obj_create_overwrite_multipart(): eq(len(k), 0) eq(len(k), len(c)) + + +def test_versioning_copy_obj_version(): + bucket = get_new_bucket() + + check_configure_versioning_retry(bucket, True, "Enabled") + + num_versions = 3 + objname = 'testobj' + + (k, c) = create_multiple_versions(bucket, objname, num_versions) + + # copy into the same bucket + for i in xrange(num_versions): + new_key_name = 'key_{i}'.format(i=i) + new_key = bucket.copy_key(new_key_name, bucket.name, k[i].name, src_version_id=k[i].version_id) + eq(new_key.get_contents_as_string(), c[i]) + + another_bucket = get_new_bucket() + + # copy into a different bucket + for i in xrange(num_versions): + new_key_name = 'key_{i}'.format(i=i) + new_key = another_bucket.copy_key(new_key_name, bucket.name, k[i].name, src_version_id=k[i].version_id) + eq(new_key.get_contents_as_string(), c[i]) + + # test copy of head object + new_key = another_bucket.copy_key('new_key', bucket.name, objname) + eq(new_key.get_contents_as_string(), c[num_versions - 1]) + From ec6ea92443eb204629adf011488d631a3dc40637 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Fri, 12 Dec 2014 10:14:37 -0800 Subject: [PATCH 09/15] test: fixes Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 44 +++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index b40b182..03f641c 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -4383,7 +4383,7 @@ def test_multipart_upload_size_too_small(): def gen_rand_string(size, chars=string.ascii_uppercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) -def do_test_multipart_upload_contents(bucket, key_name, num_parts): +def _do_test_multipart_upload_contents(bucket, key_name, num_parts): payload=gen_rand_string(5)*1024*1024 mp=bucket.initiate_multipart_upload(key_name) for i in range(0, num_parts): @@ -4409,7 +4409,7 @@ def do_test_multipart_upload_contents(bucket, key_name, num_parts): @attr(operation='check contents of multi-part upload') @attr(assertion='successful') def test_multipart_upload_contents(): - do_test_multipart_upload_contents(get_new_bucket(), 'mymultipart', 3) + _do_test_multipart_upload_contents(get_new_bucket(), 'mymultipart', 3) @attr(resource='object') @@ -5313,7 +5313,7 @@ def remove_obj_head(bucket, objname, k, c): check_obj_versions(bucket, objname, k, c) -def do_test_create_remove_versions(bucket, objname, num_versions, remove_start_idx, idx_inc): +def _do_test_create_remove_versions(bucket, objname, num_versions, remove_start_idx, idx_inc): (k, c) = create_multiple_versions(bucket, objname, num_versions) idx = remove_start_idx @@ -5322,7 +5322,7 @@ def do_test_create_remove_versions(bucket, objname, num_versions, remove_start_i remove_obj_version(bucket, k, c, idx) idx += idx_inc -def do_remove_versions(bucket, objname, remove_start_idx, idx_inc, head_rm_ratio, k, c): +def _do_remove_versions(bucket, objname, remove_start_idx, idx_inc, head_rm_ratio, k, c): idx = remove_start_idx r = 0 @@ -5340,10 +5340,10 @@ def do_remove_versions(bucket, objname, remove_start_idx, idx_inc, head_rm_ratio check_obj_versions(bucket, objname, k, c) -def do_test_create_remove_versions_and_head(bucket, objname, num_versions, num_ops, remove_start_idx, idx_inc, head_rm_ratio): +def _do_test_create_remove_versions_and_head(bucket, objname, num_versions, num_ops, remove_start_idx, idx_inc, head_rm_ratio): (k, c) = create_multiple_versions(bucket, objname, num_versions) - do_remove_versions(bucket, objname, remove_start_idx, idx_inc, head_rm_ratio, k, c) + _do_remove_versions(bucket, objname, remove_start_idx, idx_inc, head_rm_ratio, k, c) @attr(resource='object') @attr(method='create') @@ -5355,12 +5355,12 @@ def test_versioning_obj_create_read_remove(): objname = 'testobj' num_vers = 5 - do_test_create_remove_versions(bucket, objname, num_vers, -1, 0) - do_test_create_remove_versions(bucket, objname, num_vers, -1, 0) - do_test_create_remove_versions(bucket, objname, num_vers, 0, 0) - do_test_create_remove_versions(bucket, objname, num_vers, 1, 0) - do_test_create_remove_versions(bucket, objname, num_vers, 4, -1) - do_test_create_remove_versions(bucket, objname, num_vers, 3, 3) + _do_test_create_remove_versions(bucket, objname, num_vers, -1, 0) + _do_test_create_remove_versions(bucket, objname, num_vers, -1, 0) + _do_test_create_remove_versions(bucket, objname, num_vers, 0, 0) + _do_test_create_remove_versions(bucket, objname, num_vers, 1, 0) + _do_test_create_remove_versions(bucket, objname, num_vers, 4, -1) + _do_test_create_remove_versions(bucket, objname, num_vers, 3, 3) @attr(resource='object') @attr(method='create') @@ -5372,7 +5372,7 @@ def test_versioning_obj_create_read_remove_head(): objname = 'testobj' num_vers = 5 - do_test_create_remove_versions_and_head(bucket, objname, num_vers, num_vers * 2, -1, 0, 0.5) + _do_test_create_remove_versions_and_head(bucket, objname, num_vers, num_vers * 2, -1, 0, 0.5) def is_null_key(k): return (k.version_id is None) or (k.version_id == 'null') @@ -5443,8 +5443,8 @@ def test_versioning_obj_suspend_versions(): (k, c) = create_multiple_versions(bucket, objname, 3, k, c) - do_remove_versions(bucket, objname, 0, 5, 0.5, k, c) - do_remove_versions(bucket, objname, 0, 5, 0, k, c) + _do_remove_versions(bucket, objname, 0, 5, 0.5, k, c) + _do_remove_versions(bucket, objname, 0, 5, 0, k, c) eq(len(k), 0) eq(len(k), len(c)) @@ -5476,8 +5476,8 @@ def test_versioning_obj_suspend_versions_simple(): for i in xrange(len(k)): print 'JJJ: ', k[i].version_id, c[i] - do_remove_versions(bucket, objname, 0, 0, 0.5, k, c) - do_remove_versions(bucket, objname, 0, 0, 0, k, c) + _do_remove_versions(bucket, objname, 0, 0, 0.5, k, c) + _do_remove_versions(bucket, objname, 0, 0, 0, k, c) eq(len(k), 0) eq(len(k), len(c)) @@ -5498,8 +5498,8 @@ def test_versioning_obj_create_versions_remove_all(): (k, c) = create_multiple_versions(bucket, objname, num_versions) - do_remove_versions(bucket, objname, 0, 5, 0.5, k, c) - do_remove_versions(bucket, objname, 0, 5, 0, k, c) + _do_remove_versions(bucket, objname, 0, 5, 0.5, k, c) + _do_remove_versions(bucket, objname, 0, 5, 0, k, c) eq(len(k), 0) eq(len(k), len(c)) @@ -5520,7 +5520,7 @@ def test_versioning_obj_create_overwrite_multipart(): num_vers = 3 for i in xrange(num_vers): - c.append(do_test_multipart_upload_contents(bucket, objname, 3)) + c.append(_do_test_multipart_upload_contents(bucket, objname, 3)) k = [] for key in bucket.list_versions(): @@ -5529,8 +5529,8 @@ def test_versioning_obj_create_overwrite_multipart(): eq(len(k), num_vers) check_obj_versions(bucket, objname, k, c) - do_remove_versions(bucket, objname, 0, 3, 0.5, k, c) - do_remove_versions(bucket, objname, 0, 3, 0, k, c) + _do_remove_versions(bucket, objname, 0, 3, 0.5, k, c) + _do_remove_versions(bucket, objname, 0, 3, 0, k, c) eq(len(k), 0) eq(len(k), len(c)) From 53d5857e83d723695ec49853302dbd18f815d136 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Mon, 22 Dec 2014 15:47:13 -0800 Subject: [PATCH 10/15] s3test: add versioned multi delete Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 95 +++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 03f641c..065d316 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -5536,6 +5536,12 @@ def test_versioning_obj_create_overwrite_multipart(): eq(len(k), len(c)) + +@attr(resource='object') +@attr(method='multipart') +@attr(operation='create and test versioned object copying') +@attr(assertion='everything works') +@attr('versioning') def test_versioning_copy_obj_version(): bucket = get_new_bucket() @@ -5564,3 +5570,92 @@ def test_versioning_copy_obj_version(): new_key = another_bucket.copy_key('new_key', bucket.name, objname) eq(new_key.get_contents_as_string(), c[num_versions - 1]) +def _count_bucket_objs(bucket): + k = [] + for key in bucket.list_versions(): + k.insert(0, key) + return len(k) + + +@attr(resource='object') +@attr(method='delete') +@attr(operation='delete multiple versions') +@attr(assertion='deletes multiple versions of an object with a single call') +@attr('versioning') +def test_versioning_multi_object_delete(): + bucket = get_new_bucket() + + check_configure_versioning_retry(bucket, True, "Enabled") + + keyname = 'key' + + key0 = bucket.new_key(keyname) + key0.set_contents_from_string('foo') + key1 = bucket.new_key(keyname) + key1.set_contents_from_string('bar') + + stored_keys = [] + for key in bucket.list_versions(): + stored_keys.insert(0, key) + + eq(len(stored_keys), 2) + + result = bucket.delete_keys(stored_keys) + eq(len(result.deleted), 2) + eq(len(result.errors), 0) + + eq(_count_bucket_objs(bucket), 0) + + # now remove again, should all succeed due to idempotency + result = bucket.delete_keys(stored_keys) + eq(len(result.deleted), 2) + eq(len(result.errors), 0) + + eq(_count_bucket_objs(bucket), 0) + +@attr(resource='object') +@attr(method='delete') +@attr(operation='delete multiple versions') +@attr(assertion='deletes multiple versions of an object and delete marker with a single call') +@attr('versioning') +def test_versioning_multi_object_delete_with_marker(): + bucket = get_new_bucket() + + check_configure_versioning_retry(bucket, True, "Enabled") + + keyname = 'key' + + key0 = bucket.new_key(keyname) + key0.set_contents_from_string('foo') + key1 = bucket.new_key(keyname) + key1.set_contents_from_string('bar') + + key2 = bucket.delete_key(keyname) + eq(key2.delete_marker, True) + + stored_keys = [] + for key in bucket.list_versions(): + stored_keys.insert(0, key) + + eq(len(stored_keys), 3) + + result = bucket.delete_keys(stored_keys) + eq(len(result.deleted), 3) + eq(len(result.errors), 0) + eq(_count_bucket_objs(bucket), 0) + + delete_markers = [] + for o in result.deleted: + if o.delete_marker: + delete_markers.insert(0, o) + + eq(len(delete_markers), 1) + eq(key2.version_id, delete_markers[0].version_id) + + # now remove again, should all succeed due to idempotency + result = bucket.delete_keys(stored_keys) + eq(len(result.deleted), 3) + eq(len(result.errors), 0) + + eq(_count_bucket_objs(bucket), 0) + From e109c5c7b736382f8e2190b9dc2debafd1fc3064 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Mon, 22 Dec 2014 17:00:43 -0800 Subject: [PATCH 11/15] s3test: add a new multi obj delete test Validate creation of delete marker. Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 065d316..2da4310 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -5659,3 +5659,34 @@ def test_versioning_multi_object_delete_with_marker(): eq(_count_bucket_objs(bucket), 0) +@attr(resource='object') +@attr(method='delete') +@attr(operation='multi delete create marker') +@attr(assertion='returns correct marker version id') +@attr('versioning') +def test_versioning_multi_object_delete_with_marker_create(): + bucket = get_new_bucket() + + check_configure_versioning_retry(bucket, True, "Enabled") + + keyname = 'key' + + rmkeys = { bucket.new_key(keyname) } + + eq(_count_bucket_objs(bucket), 0) + + result = bucket.delete_keys(rmkeys) + eq(len(result.deleted), 1) + eq(_count_bucket_objs(bucket), 1) + + delete_markers = [] + for o in result.deleted: + if o.delete_marker: + delete_markers.insert(0, o) + + eq(len(delete_markers), 1) + + for o in bucket.list_versions(): + eq(o.name, keyname) + eq(o.version_id, delete_markers[0].delete_marker_version_id) + From 5be969486350e99a33f1eacb7a441d10b96cf591 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 23 Dec 2014 10:14:38 -0800 Subject: [PATCH 12/15] s3tests: versioned get_acl, put_acl modify a specific version acl, check this specific version, check it didn't affect object head acls. Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 70 +++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 2da4310..d2ae862 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -5690,3 +5690,73 @@ def test_versioning_multi_object_delete_with_marker_create(): eq(o.name, keyname) eq(o.version_id, delete_markers[0].delete_marker_version_id) +@attr(resource='object') +@attr(method='put') +@attr(operation='change acl on an object version changes specific version') +@attr(assertion='works') +@attr('versioning') +def test_versioned_object_acl(): + bucket = get_new_bucket() + + check_configure_versioning_retry(bucket, True, "Enabled") + + keyname = 'foo' + + key0 = bucket.new_key(keyname) + key0.set_contents_from_string('bar') + key1 = bucket.new_key(keyname) + key1.set_contents_from_string('bla') + key2 = bucket.new_key(keyname) + key2.set_contents_from_string('zxc') + + stored_keys = [] + for key in bucket.list_versions(): + stored_keys.insert(0, key) + + k1 = stored_keys[1] + + policy = bucket.get_acl(key_name=k1.name, version_id=k1.version_id) + + default_policy = [ + dict( + permission='FULL_CONTROL', + id=policy.owner.id, + display_name=policy.owner.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + ] + + print repr(policy) + check_grants(policy.acl.grants, default_policy) + + bucket.set_canned_acl('public-read', key_name=k1.name, version_id=k1.version_id) + + policy = bucket.get_acl(key_name=k1.name, version_id=k1.version_id) + print repr(policy) + check_grants( + policy.acl.grants, + [ + dict( + permission='FULL_CONTROL', + id=policy.owner.id, + display_name=policy.owner.display_name, + uri=None, + email_address=None, + type='CanonicalUser', + ), + dict( + permission='READ', + id=None, + display_name=None, + uri='http://acs.amazonaws.com/groups/global/AllUsers', + email_address=None, + type='Group', + ), + ], + ) + + k = bucket.new_key(keyname) + check_grants(k.get_acl().acl.grants, default_policy) + From 460e3f1f1e2eb0e5c9581cf0f73d6bf2634a2089 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 23 Dec 2014 16:26:47 -0800 Subject: [PATCH 13/15] s3tests: concurrent object versioning tests test concurrent creation and removal Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 102 +++++++++++++++++++++++++++++++--- 1 file changed, 95 insertions(+), 7 deletions(-) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index d2ae862..3cbe74d 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -21,6 +21,7 @@ import sha import pytz import json import httplib2 +import threading import xml.etree.ElementTree as ET @@ -5570,7 +5571,7 @@ def test_versioning_copy_obj_version(): new_key = another_bucket.copy_key('new_key', bucket.name, objname) eq(new_key.get_contents_as_string(), c[num_versions - 1]) -def _count_bucket_objs(bucket): +def _count_bucket_versioned_objs(bucket): k = [] for key in bucket.list_versions(): k.insert(0, key) @@ -5604,14 +5605,14 @@ def test_versioning_multi_object_delete(): eq(len(result.deleted), 2) eq(len(result.errors), 0) - eq(_count_bucket_objs(bucket), 0) + eq(_count_bucket_versioned_objs(bucket), 0) # now remove again, should all succeed due to idempotency result = bucket.delete_keys(stored_keys) eq(len(result.deleted), 2) eq(len(result.errors), 0) - eq(_count_bucket_objs(bucket), 0) + eq(_count_bucket_versioned_objs(bucket), 0) @attr(resource='object') @attr(method='delete') @@ -5642,7 +5643,7 @@ def test_versioning_multi_object_delete_with_marker(): result = bucket.delete_keys(stored_keys) eq(len(result.deleted), 3) eq(len(result.errors), 0) - eq(_count_bucket_objs(bucket), 0) + eq(_count_bucket_versioned_objs(bucket), 0) delete_markers = [] for o in result.deleted: @@ -5657,7 +5658,7 @@ def test_versioning_multi_object_delete_with_marker(): eq(len(result.deleted), 3) eq(len(result.errors), 0) - eq(_count_bucket_objs(bucket), 0) + eq(_count_bucket_versioned_objs(bucket), 0) @attr(resource='object') @attr(method='delete') @@ -5673,11 +5674,11 @@ def test_versioning_multi_object_delete_with_marker_create(): rmkeys = { bucket.new_key(keyname) } - eq(_count_bucket_objs(bucket), 0) + eq(_count_bucket_versioned_objs(bucket), 0) result = bucket.delete_keys(rmkeys) eq(len(result.deleted), 1) - eq(_count_bucket_objs(bucket), 1) + eq(_count_bucket_versioned_objs(bucket), 1) delete_markers = [] for o in result.deleted: @@ -5760,3 +5761,90 @@ def test_versioned_object_acl(): k = bucket.new_key(keyname) check_grants(k.get_acl().acl.grants, default_policy) + +def _do_create_object(bucket, objname, i): + k = bucket.new_key(objname) + k.set_contents_from_string('data {i}'.format(i=i)) + +def _do_remove_ver(bucket, obj): + bucket.delete_key(obj.name, version_id = obj.version_id) + +def _do_create_versioned_obj_concurrent(bucket, objname, num): + t = [] + for i in range(num): + thr = threading.Thread(target = _do_create_object, args=(bucket, objname, i)) + thr.start() + t.append(thr) + return t + +def _do_clear_versioned_bucket_concurrent(bucket): + t = [] + for o in bucket.list_versions(): + thr = threading.Thread(target = _do_remove_ver, args=(bucket, o)) + thr.start() + t.append(thr) + return t + +def _do_wait_completion(t): + for thr in t: + thr.join() + +@attr(resource='object') +@attr(method='put') +@attr(operation='concurrent creation of objects, concurrent removal') +@attr(assertion='works') +@attr('versioning') +def test_versioned_concurrent_object_create_concurrent_remove(): + bucket = get_new_bucket() + + check_configure_versioning_retry(bucket, True, "Enabled") + + keyname = 'myobj' + + num_objs = 5 + + for i in xrange(5): + t = _do_create_versioned_obj_concurrent(bucket, keyname, num_objs) + _do_wait_completion(t) + + eq(_count_bucket_versioned_objs(bucket), num_objs) + eq(len(bucket.get_all_keys()), 1) + + t = _do_clear_versioned_bucket_concurrent(bucket) + _do_wait_completion(t) + + eq(_count_bucket_versioned_objs(bucket), 0) + eq(len(bucket.get_all_keys()), 0) + +@attr(resource='object') +@attr(method='put') +@attr(operation='concurrent creation and removal of objects') +@attr(assertion='works') +@attr('versioning') +def test_versioned_concurrent_object_create_and_remove(): + bucket = get_new_bucket() + + check_configure_versioning_retry(bucket, True, "Enabled") + + keyname = 'myobj' + + num_objs = 3 + + all_threads = [] + + for i in xrange(3): + t = _do_create_versioned_obj_concurrent(bucket, keyname, num_objs) + all_threads.append(t) + + t = _do_clear_versioned_bucket_concurrent(bucket) + all_threads.append(t) + + + for t in all_threads: + _do_wait_completion(t) + + t = _do_clear_versioned_bucket_concurrent(bucket) + _do_wait_completion(t) + + eq(_count_bucket_versioned_objs(bucket), 0) + eq(len(bucket.get_all_keys()), 0) From 7a0932fe02b0216ff118b8a31f20f499831dde5a Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 8 Jan 2015 16:11:23 -0800 Subject: [PATCH 14/15] s3tests: test list bucket versions Signed-off-by: Yehuda Sadeh --- s3tests/functional/test_s3.py | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 3cbe74d..98475c9 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -22,6 +22,7 @@ import pytz import json import httplib2 import threading +import itertools import xml.etree.ElementTree as ET @@ -5250,6 +5251,9 @@ def check_obj_versions(bucket, objname, keys, contents): check_head_obj_content(key, contents[-1]) i = len(contents) for key in bucket.list_versions(): + if key.name != objname: + continue + i -= 1 eq(keys[i].version_id or 'null', key.version_id) print 'testing obj version-id=', key.version_id @@ -5272,6 +5276,8 @@ def create_multiple_versions(bucket, objname, num_versions, k = None, c = None): k_pos = len(k) i = 0 for o in bucket.list_versions(): + if o.name != objname: + continue i += 1 if i > num_versions: break @@ -5538,6 +5544,43 @@ def test_versioning_obj_create_overwrite_multipart(): +@attr(resource='object') +@attr(method='multipart') +@attr(operation='list versioned objects') +@attr(assertion='everything works') +@attr('versioning') +def test_versioning_obj_list_marker(): + bucket = get_new_bucket() + check_configure_versioning_retry(bucket, True, "Enabled") + + objname = 'testobj' + objname2 = 'testobj-1' + + num_vers = 5 + + (k, c) = create_multiple_versions(bucket, objname, num_vers) + (k2, c2) = create_multiple_versions(bucket, objname2, num_vers) + + k.reverse() + k2.reverse() + + allkeys = k + k2 + + names = [] + + for key1, key2 in itertools.izip_longest(bucket.list_versions(), allkeys): + eq(key1.version_id, key2.version_id) + names.append(key1.name) + + for i in xrange(len(allkeys)): + for key1, key2 in itertools.izip_longest(bucket.list_versions(key_marker=names[i], version_id_marker=allkeys[i].version_id), allkeys[i+1:]): + eq(key1.version_id, key2.version_id) + + # with nonexisting version id, skip to next object + for key1, key2 in itertools.izip_longest(bucket.list_versions(key_marker=objname, version_id_marker='nosuchversion'), allkeys[5:]): + eq(key1.version_id, key2.version_id) + + @attr(resource='object') @attr(method='multipart') @attr(operation='create and test versioned object copying') From 76e8bed4bd48f050f1de6dc38da072ecd38d3286 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 13 Jan 2015 16:46:33 -0800 Subject: [PATCH 15/15] s3tests: send raw http requests to test redirects Since we fixed the rgw redirect code, it turns out that boto actually follows on them. We just want to check if redirect is sent. Signed-off-by: Yehuda Sadeh --- requirements.txt | 1 + s3tests/functional/test_s3.py | 33 ++++++++++++++++++++++++--------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index 81e93e9..9c294dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ isodate >=0.4.4 requests ==0.14.0 pytz >=2011k ordereddict +httplib2 diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 0d7eff3..ab02f1d 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -20,6 +20,7 @@ import hmac import sha import pytz import json +import httplib2 import xml.etree.ElementTree as ET @@ -5029,6 +5030,21 @@ def check_can_test_multiregion(): if not targets.main.master or len(targets.main.secondaries) == 0: raise SkipTest +def create_presigned_url(conn, method, bucket_name, key_name, expiration): + return conn.generate_url(expires_in=expiration, + method=method, + bucket=bucket_name, + key=key_name, + query_auth=True, + ) + +def send_raw_http_request(conn, method, bucket_name, key_name, follow_redirects = False): + url = create_presigned_url(conn, method, bucket_name, key_name, 3600) + print url + h = httplib2.Http() + h.follow_redirects = follow_redirects + return h.request(url, method) + @attr(resource='bucket') @attr(method='get') @attr(operation='create on one region, access in another') @@ -5043,12 +5059,11 @@ def test_region_bucket_create_secondary_access_remove_master(): conn = r.connection bucket = get_new_bucket(r) - e = assert_raises(boto.exception.S3ResponseError, master_conn.get_bucket, bucket.name) - eq(e.status, 301) - - e = assert_raises(boto.exception.S3ResponseError, master_conn.delete_bucket, bucket.name) - eq(e.status, 301) + r, content = send_raw_http_request(master_conn, 'GET', bucket.name, '', follow_redirects = False) + eq(r.status, 301) + r, content = send_raw_http_request(master_conn, 'DELETE', bucket.name, '', follow_redirects = False) + eq(r.status, 301) conn.delete_bucket(bucket) @@ -5069,11 +5084,11 @@ def test_region_bucket_create_master_access_remove_secondary(): region_sync_meta(targets.main, master) - e = assert_raises(boto.exception.S3ResponseError, conn.get_bucket, bucket.name) - eq(e.status, 301) + r, content = send_raw_http_request(conn, 'GET', bucket.name, '', follow_redirects = False) + eq(r.status, 301) - e = assert_raises(boto.exception.S3ResponseError, conn.delete_bucket, bucket.name) - eq(e.status, 301) + r, content = send_raw_http_request(conn, 'DELETE', bucket.name, '', follow_redirects = False) + eq(r.status, 301) master_conn.delete_bucket(bucket) region_sync_meta(targets.main, master)