From ec4cf0de2bebf51975e6bbc827198c1d4acdf050 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Fri, 19 Jul 2019 09:19:14 -0400 Subject: [PATCH 1/4] lifecycle: expiration header checks do not require @lifecycle_expiration tag Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index c5259df..45d8d0e 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -8945,7 +8945,6 @@ def check_lifecycle_expiration_header(response, start_time, rule_id, @attr(method='put') @attr(operation='test lifecycle expiration header put') @attr('lifecycle') -@attr('lifecycle_expiration') def test_lifecycle_expiration_header_put(): """ Check for valid x-amz-expiration header after PUT @@ -8962,7 +8961,6 @@ def test_lifecycle_expiration_header_put(): @attr(method='head') @attr(operation='test lifecycle expiration header head') @attr('lifecycle') -@attr('lifecycle_expiration') def test_lifecycle_expiration_header_head(): """ Check for valid x-amz-expiration header on HEAD request From 0a73ea08deb3e4e467e7c46e2333ed022aea3d83 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Fri, 19 Jul 2019 11:36:18 -0400 Subject: [PATCH 2/4] lc: fix lifecycle expiration header tests Fixes various issues with existing HEAD check and helper methods. Add an explicit test for GET. Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 45 ++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 45d8d0e..78b6ac3 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -28,6 +28,7 @@ import random import socket import ssl from collections import namedtuple +import pdb from email.header import decode_header @@ -8913,19 +8914,19 @@ def test_lifecycle_expiration_days0(): eq(len(expire_objects), 0) -def setup_lifecycle_expiration(bucket_name, rule_id, delta_days, - rule_prefix): +def setup_lifecycle_expiration(bucket_name, rule_id, delta_days, rule_prefix, + key, body): rules=[{'ID': rule_id, 'Expiration': {'Days': delta_days}, 'Prefix': rule_prefix, 'Status':'Enabled'}] lifecycle = {'Rules': rules} + + client = get_client() response = client.put_bucket_lifecycle_configuration( Bucket=bucket_name, LifecycleConfiguration=lifecycle) eq(response['ResponseMetadata']['HTTPStatusCode'], 200) - key = rule_prefix + '/foo' - body = 'bar' - response = client.put_object(Bucket=bucket_name, Key=key, Body=bar) + response = client.put_object(Bucket=bucket_name, Key=key, Body=body) eq(response['ResponseMetadata']['HTTPStatusCode'], 200) return response @@ -8935,7 +8936,7 @@ def check_lifecycle_expiration_header(response, start_time, rule_id, m = re.search(r'expiry-date="(.+)", rule-id="(.+)"', exp_header) expiration = datetime.datetime.strptime(m.group(1), - '%a %b %d %H:%M:%S %Y') + '%a, %d %b %Y %H:%M:%S %Z') eq((expiration - start_time).days, delta_days) eq(m.group(2), rule_id) @@ -8950,11 +8951,9 @@ def test_lifecycle_expiration_header_put(): Check for valid x-amz-expiration header after PUT """ bucket_name = get_new_bucket() - client = get_client() - now = datetime.datetime.now(None) - response = setup_lifecycle_expiration( - bucket_name, 'rule1', 1, 'days1/') + response = setup_lifecycle_expiration(bucket_name,'rule1', 1, 'days1/', + 'days1/foo', 'foo_body') eq(check_lifecycle_expiration_header(response, now, 'rule1', 1), True) @attr(resource='bucket') @@ -8969,11 +8968,31 @@ def test_lifecycle_expiration_header_head(): client = get_client() now = datetime.datetime.now(None) - response = setup_lifecycle_expiration( - bucket_name, 'rule1', 1, 'days1/') + response = setup_lifecycle_expiration(bucket_name, 'rule1', 1, 'days1/', + 'days1/foo', 'foo_body') # stat the object, check header - response = client.head_object(Bucket=bucket_name, Key=key) + response = client.head_object(Bucket=bucket_name, Key='days1/foo') + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + eq(check_lifecycle_expiration_header(response, now, 'rule1', 1), True) + +@attr(resource='bucket') +@attr(method='get') +@attr(operation='test lifecycle expiration header get') +@attr('lifecycle') +def test_lifecycle_expiration_header_get(): + """ + Check for valid x-amz-expiration header on GET request + """ + bucket_name = get_new_bucket() + client = get_client() + + now = datetime.datetime.now(None) + response = setup_lifecycle_expiration(bucket_name, 'rule1', 1, 'days1/', + 'days1/foo', 'foo_body') + + # stat the object, check header + response = client.get_object(Bucket=bucket_name, Key='days1/foo') eq(response['ResponseMetadata']['HTTPStatusCode'], 200) eq(check_lifecycle_expiration_header(response, now, 'rule1', 1), True) From e0aafba82dfffb17c7c50a4cbc89217c37306278 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Fri, 19 Jul 2019 15:32:52 -0400 Subject: [PATCH 3/4] lc: add rule to test disjoint object tag filters Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 73 +++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 78b6ac3..e4e2ebf 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -8996,6 +8996,79 @@ def test_lifecycle_expiration_header_get(): eq(response['ResponseMetadata']['HTTPStatusCode'], 200) eq(check_lifecycle_expiration_header(response, now, 'rule1', 1), True) +@attr(resource='bucket') +@attr(method='get') +@attr(operation='test lifecycle expiration header tags') +@attr('lifecycle') +def test_lifecycle_expiration_header_tags(): + """ + Check x-amz-expiration-header object tags interaction + """ + bucket_name = get_new_bucket() + client = get_client() + + lifecycle={ + "Rules": [ + { + "Filter": { + "Tag": {"Key": "key1", "Value": "tag1"} + }, + "Status": "Enabled", + "Expiration": { + "Days": 1 + }, + "ID": "rule1" + }, + { + "Filter": { + "Tag": {"Key": "key1", "Value": "tag5"} + }, + "Status": "Enabled", + "Expiration": { + "Days": 5 + }, + "ID": "rule5" + } + ] + } + + response = client.put_bucket_lifecycle_configuration( + Bucket=bucket_name, LifecycleConfiguration=lifecycle) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + key1 = "obj_key1" + body1 = "obj_key1_body" + response = client.put_object(Bucket=bucket_name, Key=key1, Body=body1) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + tags1={'TagSet': [{'Key': 'key1', 'Value': 'tag1'}]} + response = client.put_object_tagging(Bucket=bucket_name, Key=key1, + Tagging=tags1) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + key2 = "obj_key2" + body2 = "obj_key2_body" + response = client.put_object(Bucket=bucket_name, Key=key2, Body=body2) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + tags2={'TagSet': [{'Key': 'key1', 'Value': 'tag5'}]} + response = client.put_object_tagging(Bucket=bucket_name, Key=key2, + Tagging=tags2) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + + # check expirations + response = client.head_object(Bucket=bucket_name, Key=key1) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + exp_header = response['ResponseMetadata']['HTTPHeaders']['x-amz-expiration'] + m = re.search(r'expiry-date="(.+)", rule-id="(.+)"', exp_header) + eq(m.group(2), "rule1") + + response = client.head_object(Bucket=bucket_name, Key=key2) + eq(response['ResponseMetadata']['HTTPStatusCode'], 200) + exp_header = response['ResponseMetadata']['HTTPHeaders']['x-amz-expiration'] + m = re.search(r'expiry-date="(.+)", rule-id="(.+)"', exp_header) + eq(m.group(2), "rule5") + @attr(resource='bucket') @attr(method='put') @attr(operation='set lifecycle config with noncurrent version expiration') From a7d4ec68aabaf811d8f61cb44bbea4e30d4aaf44 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Wed, 24 Jul 2019 14:05:43 -0400 Subject: [PATCH 4/4] lc: remove date delta check from lc header checks This was prone to false positives. We still check: 1. the expiration header has valid form 2. the rule-id matches Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index e4e2ebf..d8b3360 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -8937,7 +8937,6 @@ def check_lifecycle_expiration_header(response, start_time, rule_id, expiration = datetime.datetime.strptime(m.group(1), '%a, %d %b %Y %H:%M:%S %Z') - eq((expiration - start_time).days, delta_days) eq(m.group(2), rule_id) return True