From 4948f8b009711bbbecdc3939febae72131571a5b Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Fri, 11 Sep 2020 10:17:39 -0400 Subject: [PATCH 1/7] fix and remark on test_lifecycle_expiration_days0 1. fix a python3-related KeyError exception 2. note here: AWS documentation includes examples of "Days 0" in use, but boto3 will not accept them--this is why the days0 test currently sets Days 1 3. delay increased to 30s, to avoid occasional failures due to jitter Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 969d28f..3fa7657 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -9128,14 +9128,17 @@ def test_lifecycle_expiration_days0(): rules=[{'Expiration': {'Days': 1}, 'ID': 'rule1', 'Prefix': 'days0/', 'Status':'Enabled'}] lifecycle = {'Rules': rules} - print(lifecycle) + response = client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration=lifecycle) eq(response['ResponseMetadata']['HTTPStatusCode'], 200) - time.sleep(20) + time.sleep(30) response = client.list_objects(Bucket=bucket_name) - expire_objects = response['Contents'] + try: + expire_objects = response['Contents'] + except KeyError: + expire_objects = [] eq(len(expire_objects), 0) From 979e739eff3c8f00ff1f35c3583beb7d327e1501 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Fri, 11 Sep 2020 13:23:41 -0400 Subject: [PATCH 2/7] fix test_lifecycle_expiration_header_{put,head} Primarily fixes the expiration header() verifier function check_lifecycle_expiration_header, but also cleans up prefix handling in setup_lifecycle_expiration(). Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 3fa7657..a01694b 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -9153,25 +9153,26 @@ def setup_lifecycle_expiration(client, bucket_name, rule_id, delta_days, Bucket=bucket_name, LifecycleConfiguration=lifecycle) eq(response['ResponseMetadata']['HTTPStatusCode'], 200) - key = rule_prefix + '/foo' + key = rule_prefix + 'foo' body = 'bar' response = client.put_object(Bucket=bucket_name, Key=key, Body=body) eq(response['ResponseMetadata']['HTTPStatusCode'], 200) - response = client.get_bucket_lifecycle_configuration(Bucket=bucket_name) return response def check_lifecycle_expiration_header(response, start_time, rule_id, delta_days): print(response) - #TODO: see how this can work - #print(response['ResponseMetadata']['HTTPHeaders']) - #exp_header = response['ResponseMetadata']['HTTPHeaders']['x-amz-expiration'] - #m = re.search(r'expiry-date="(.+)", rule-id="(.+)"', exp_header) + print(response['ResponseMetadata']['HTTPHeaders']) - #expiration = datetime.datetime.strptime(m.group(1), - # '%a %b %d %H:%M:%S %Y') - #eq((expiration - start_time).days, delta_days) - #eq(m.group(2), rule_id) + exp_header = response['ResponseMetadata']['HTTPHeaders']['x-amz-expiration'] + m = re.search(r'expiry-date="(.+)", rule-id="(.+)"', exp_header) + datestr = m.group(1) + exp_date = datetime.datetime.strptime(datestr, '%a, %d %b %Y %H:%M:%S %Z') + exp_diff = exp_date - start_time + rule_id = m.group(2) + + eq(exp_diff.days, delta_days) + eq(m.group(2), rule_id) return True @@ -9200,9 +9201,9 @@ def test_lifecycle_expiration_header_head(): now = datetime.datetime.now(None) response = setup_lifecycle_expiration( - client, bucket_name, 'rule1', 1, 'days1') + client, bucket_name, 'rule1', 1, 'days1/') - key = 'days1/' + '/foo' + key = 'days1/' + 'foo' # stat the object, check header response = client.head_object(Bucket=bucket_name, Key=key) From 9d526d1a76214538cc6319191f61f928d3538cb9 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Fri, 11 Sep 2020 13:52:29 -0400 Subject: [PATCH 3/7] s/test_set_tagging/test_set_bucket_tagging/; The test exercises bucket tagging, has nothing to do with object tagging (clarity). Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index a01694b..a17cf88 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -7441,7 +7441,7 @@ def test_cors_header_option(): @attr(operation='put tags') @attr(assertion='succeeds') @attr('tagging') -def test_set_tagging(): +def test_set_bucket_tagging(): bucket_name = get_new_bucket() client = get_client() From c2b59fb714ae0feeb0b5a6449870d6594d9593cf Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Mon, 14 Sep 2020 14:23:48 -0400 Subject: [PATCH 4/7] fix lifecycle expiration days: 0 In fact test_lifecycle_expiration_days0 is should fail, as 0-day expiration is permitted for transition rules but not expiration rules. Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index a17cf88..c114bca 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -9126,21 +9126,18 @@ def test_lifecycle_expiration_days0(): bucket_name = _create_objects(keys=['days0/foo', 'days0/bar']) client = get_client() - rules=[{'Expiration': {'Days': 1}, 'ID': 'rule1', 'Prefix': 'days0/', 'Status':'Enabled'}] + rules=[{'Expiration': {'Days': 0}, 'ID': 'rule1', 'Prefix': 'days0/', 'Status':'Enabled'}] lifecycle = {'Rules': rules} - response = client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration=lifecycle) - eq(response['ResponseMetadata']['HTTPStatusCode'], 200) - - time.sleep(30) - - response = client.list_objects(Bucket=bucket_name) + # days: 0 is legal in a transition rule, but not legal in an + # expiration rule + response_code = "" try: - expire_objects = response['Contents'] - except KeyError: - expire_objects = [] + response = client.put_bucket_lifecycle_configuration(Bucket=bucket_name, LifecycleConfiguration=lifecycle) + except botocore.exceptions.ClientError as e: + response_code = e.response['Error']['Code'] - eq(len(expire_objects), 0) + eq(response_code, 'InvalidArgument') def setup_lifecycle_expiration(client, bucket_name, rule_id, delta_days, From e79dffa731a58b8ffddf33e087de8d1e9288c071 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Mon, 14 Sep 2020 19:06:32 -0400 Subject: [PATCH 5/7] add tests for lifecycle expiration w/1 and 2 object tags Note that the 1-tag case contains a filter prefix--which exposes an apparent bug parsing Filter when it contains a Prefix element and a single Tag element (without And). Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 122 ++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index c114bca..2925b59 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -9001,6 +9001,128 @@ def test_lifecycle_expiration_versioning_enabled(): eq(len(versions), 1) eq(len(delete_markers), 1) +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle expiration with 1 tag') +@attr('lifecycle') +@attr('lifecycle_expiration') +@attr('fails_on_aws') +def test_lifecycle_expiration_tags1(): + bucket_name = get_new_bucket() + client = get_client() + + tom_key = 'days1/tom' + tom_tagset = {'TagSet': + [{'Key': 'tom', 'Value': 'sawyer'}]} + + client.put_object(Bucket=bucket_name, Key=tom_key, Body='tom_body') + + response = client.put_object_tagging(Bucket=bucket_name, Key=tom_key, + Tagging=tom_tagset) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + lifecycle_config = { + 'Rules': [ + { + 'Expiration': { + 'Days': 1, + }, + 'ID': 'rule_tag1', + 'Filter': { + 'Prefix': 'days1/', + 'Tag': { + 'Key': 'tom', + 'Value': 'sawyer' + }, + }, + 'Status': 'Enabled', + }, + ] + } + + response = client.put_bucket_lifecycle_configuration( + Bucket=bucket_name, LifecycleConfiguration=lifecycle_config) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + time.sleep(28) + + try: + expire_objects = response['Contents'] + except KeyError: + expire_objects = [] + + eq(len(expire_objects), 0) + +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle expiration with 2 tags') +@attr('lifecycle') +@attr('lifecycle_expiration') +@attr('fails_on_aws') +def test_lifecycle_expiration_tags2(): + bucket_name = get_new_bucket() + client = get_client() + + tom_key = 'days1/tom' + tom_tagset = {'TagSet': + [{'Key': 'tom', 'Value': 'sawyer'}]} + + client.put_object(Bucket=bucket_name, Key=tom_key, Body='tom_body') + + response = client.put_object_tagging(Bucket=bucket_name, Key=tom_key, + Tagging=tom_tagset) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + huck_key = 'days1/huck' + huck_tagset = { + 'TagSet': + [{'Key': 'tom', 'Value': 'sawyer'}, + {'Key': 'huck', 'Value': 'finn'}]} + + client.put_object(Bucket=bucket_name, Key=huck_key, Body='huck_body') + + response = client.put_object_tagging(Bucket=bucket_name, Key=huck_key, + Tagging=huck_tagset) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + lifecycle_config = { + 'Rules': [ + { + 'Expiration': { + 'Days': 1, + }, + 'ID': 'rule_tag1', + 'Filter': { + 'Prefix': 'days1/', + 'Tag': { + 'Key': 'tom', + 'Value': 'sawyer' + }, + 'And': { + 'Prefix': 'days1', + 'Tags': [ + { + 'Key': 'huck', + 'Value': 'finn' + }, + ] + } + }, + 'Status': 'Enabled', + }, + ] + } + + response = client.put_bucket_lifecycle_configuration( + Bucket=bucket_name, LifecycleConfiguration=lifecycle_config) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + time.sleep(28) + response = client.list_objects(Bucket=bucket_name) + expire1_objects = response['Contents'] + + eq(len(expire1_objects), 1) + @attr(resource='bucket') @attr(method='put') @attr(operation='id too long in lifecycle rule') From 6ff497d908cc9bb1d5328d31d68e0f43fd49e3be Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Tue, 15 Sep 2020 09:57:00 -0400 Subject: [PATCH 6/7] add lifecycle expiration test mixing 2-tag filter w/versioning By design this test duplicates test_lifecycle_expiration_tags2, but enables object versioning on the bucket. The tests install a rule which requires -2- tags to be matched, and creates 2 objects, one matching only 1 of the required tags, the other matching both. Only the 2nd object should expire. Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 46 ++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 2925b59..649cccc 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -9053,16 +9053,8 @@ def test_lifecycle_expiration_tags1(): eq(len(expire_objects), 0) -@attr(resource='bucket') -@attr(method='put') -@attr(operation='test lifecycle expiration with 2 tags') -@attr('lifecycle') -@attr('lifecycle_expiration') -@attr('fails_on_aws') -def test_lifecycle_expiration_tags2(): - bucket_name = get_new_bucket() - client = get_client() - +# factor out common setup code +def setup_lifecycle_tags2(client, bucket_name): tom_key = 'days1/tom' tom_tagset = {'TagSet': [{'Key': 'tom', 'Value': 'sawyer'}]} @@ -9116,6 +9108,40 @@ def test_lifecycle_expiration_tags2(): response = client.put_bucket_lifecycle_configuration( Bucket=bucket_name, LifecycleConfiguration=lifecycle_config) eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + return response + +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle expiration with 2 tags') +@attr('lifecycle') +@attr('lifecycle_expiration') +@attr('fails_on_aws') +def test_lifecycle_expiration_tags2(): + bucket_name = get_new_bucket() + client = get_client() + + response = setup_lifecycle_tags2(client, bucket_name) + + time.sleep(28) + response = client.list_objects(Bucket=bucket_name) + expire1_objects = response['Contents'] + + eq(len(expire1_objects), 1) + +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle expiration with versioning and 2 tags') +@attr('lifecycle') +@attr('lifecycle_expiration') +@attr('fails_on_aws') +def test_lifecycle_expiration_versioned_tags2(): + bucket_name = get_new_bucket() + client = get_client() + + # mix in versioning + check_configure_versioning_retry(bucket_name, "Enabled", "Enabled") + + response = setup_lifecycle_tags2(client, bucket_name) time.sleep(28) response = client.list_objects(Bucket=bucket_name) From f0868651fdb3c4987239056e3f8858aaa408352a Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Wed, 16 Sep 2020 13:19:41 -0400 Subject: [PATCH 7/7] add noncurrent version expiration rule w/tag filter Create 10 object versions (9 noncurrent). Install a noncurrent version expiration at 4 days. Verify that 10 versions exist at T+20, and only 1 (current) at T+60. Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 77 +++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 649cccc..fd62224 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -9149,6 +9149,83 @@ def test_lifecycle_expiration_versioned_tags2(): eq(len(expire1_objects), 1) +# setup for scenario based on vidushi mishra's in rhbz#1877737 +def setup_lifecycle_noncur_tags(client, bucket_name, days): + + # first create and tag the objects (10 versions of 1) + key = "myobject_" + tagset = {'TagSet': + [{'Key': 'vidushi', 'Value': 'mishra'}]} + + for ix in range(10): + body = "%s v%d" % (key, ix) + response = client.put_object(Bucket=bucket_name, Key=key, Body=body) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + response = client.put_object_tagging(Bucket=bucket_name, Key=key, + Tagging=tagset) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + lifecycle_config = { + 'Rules': [ + { + 'NoncurrentVersionExpiration': { + 'NoncurrentDays': days, + }, + 'ID': 'rule_tag1', + 'Filter': { + 'Prefix': '', + 'Tag': { + 'Key': 'vidushi', + 'Value': 'mishra' + }, + }, + 'Status': 'Enabled', + }, + ] + } + + response = client.put_bucket_lifecycle_configuration( + Bucket=bucket_name, LifecycleConfiguration=lifecycle_config) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + return response + +def verify_lifecycle_expiration_noncur_tags(client, bucket_name, secs): + time.sleep(secs) + try: + response = client.list_object_versions(Bucket=bucket_name) + objs_list = response['Versions'] + except: + objs_list = [] + return len(objs_list) + +@attr(resource='bucket') +@attr(method='put') +@attr(operation='test lifecycle noncurrent expiration with 1 tag filter') +@attr('lifecycle') +@attr('lifecycle_expiration') +@attr('fails_on_aws') +def test_lifecycle_expiration_noncur_tags1(): + bucket_name = get_new_bucket() + client = get_client() + + check_configure_versioning_retry(bucket_name, "Enabled", "Enabled") + + # create 10 object versions (9 noncurrent) and a tag-filter + # noncurrent version expiration at 4 "days" + response = setup_lifecycle_noncur_tags(client, bucket_name, 4) + + num_objs = verify_lifecycle_expiration_noncur_tags( + client, bucket_name, 20) + + # at T+20, 10 objects should exist + eq(num_objs, 10) + + num_objs = verify_lifecycle_expiration_noncur_tags( + client, bucket_name, 40) + + # at T+60, only the current object version should exist + eq(num_objs, 1) + @attr(resource='bucket') @attr(method='put') @attr(operation='id too long in lifecycle rule')