From 1d3919887266c9fd89888a0e9e2dd328cdfd731d Mon Sep 17 00:00:00 2001 From: Abhishek Lekshmanan Date: Wed, 22 Jan 2020 17:01:30 +0100 Subject: [PATCH 1/9] boto3: add bucket policy status checks for public ACLs Signed-off-by: Abhishek Lekshmanan (cherry picked from commit 02b1d50ca7064ced156482b86817b44356683ad2) --- s3tests_boto3/functional/test_s3.py | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index c3d2045..d992e69 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -12396,3 +12396,41 @@ def test_user_policy(): PolicyName='AllAccessPolicy', UserName=get_tenant_user_id(), ) + + +@attr(resource='bucket') +@attr(method='get') +@attr(operation='get bucket policy status on a new bucket') +@attr(assertion='succeeds') +@attr('policy_status') +def test_get_bucket_policy_status(): + bucket_name = get_new_bucket() + client = get_client() + resp = client.get_bucket_policy_status(Bucket=bucket_name) + eq(resp['PolicyStatus']['IsPublic'],False) + +@attr(resource='bucket') +@attr(method='get') +@attr(operation='get bucket policy status on a public acl bucket') +@attr(assertion='succeeds') +@attr('policy_status') +def test_get_public_bucket_policy_status(): + bucket_name = get_new_bucket() + client = get_client() + client = get_client() + client.put_bucket_acl(Bucket=bucket_name, ACL='public-read') + resp = client.get_bucket_policy_status(Bucket=bucket_name) + eq(resp['PolicyStatus']['IsPublic'],True) + +@attr(resource='bucket') +@attr(method='get') +@attr(operation='get bucket policy status on a authenticated acl bucket') +@attr(assertion='succeeds') +@attr('policy_status') +def test_get_authpublic_bucket_policy_status(): + bucket_name = get_new_bucket() + client = get_client() + client = get_client() + client.put_bucket_acl(Bucket=bucket_name, ACL='authenticated-read') + resp = client.get_bucket_policy_status(Bucket=bucket_name) + eq(resp['PolicyStatus']['IsPublic'],True) From 0e3084c9955452cf69f9ea8cd2759605639a7929 Mon Sep 17 00:00:00 2001 From: Abhishek Lekshmanan Date: Fri, 30 Aug 2019 11:34:51 +0200 Subject: [PATCH 2/9] add a few test cases for public bucket policies Signed-off-by: Abhishek Lekshmanan (cherry picked from commit 3f9d31c6c77ee8e12cb95511898e3c59a6dee62b) --- s3tests_boto3/functional/test_s3.py | 110 +++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 2 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index d992e69..cedd386 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -12414,7 +12414,7 @@ def test_get_bucket_policy_status(): @attr(operation='get bucket policy status on a public acl bucket') @attr(assertion='succeeds') @attr('policy_status') -def test_get_public_bucket_policy_status(): +def test_get_public_acl_bucket_policy_status(): bucket_name = get_new_bucket() client = get_client() client = get_client() @@ -12427,10 +12427,116 @@ def test_get_public_bucket_policy_status(): @attr(operation='get bucket policy status on a authenticated acl bucket') @attr(assertion='succeeds') @attr('policy_status') -def test_get_authpublic_bucket_policy_status(): +def test_get_authpublic_acl_bucket_policy_status(): bucket_name = get_new_bucket() client = get_client() client = get_client() client.put_bucket_acl(Bucket=bucket_name, ACL='authenticated-read') resp = client.get_bucket_policy_status(Bucket=bucket_name) eq(resp['PolicyStatus']['IsPublic'],True) + + +@attr(resource='bucket') +@attr(method='get') +@attr(operation='get bucket policy status on a public policy bucket') +@attr(assertion='succeeds') +@attr('policy_status') +def test_get_publicpolicy_acl_bucket_policy_status(): + bucket_name = get_new_bucket() + client = get_client() + client = get_client() + + resp = client.get_bucket_policy_status(Bucket=bucket_name) + eq(resp['PolicyStatus']['IsPublic'],False) + + resource1 = "arn:aws:s3:::" + bucket_name + resource2 = "arn:aws:s3:::" + bucket_name + "/*" + policy_document = json.dumps( + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:ListBucket", + "Resource": [ + "{}".format(resource1), + "{}".format(resource2) + ] + }] + }) + + client.put_bucket_policy(Bucket=bucket_name, Policy=policy_document) + resp = client.get_bucket_policy_status(Bucket=bucket_name) + eq(resp['PolicyStatus']['IsPublic'],True) + + +@attr(resource='bucket') +@attr(method='get') +@attr(operation='get bucket policy status on a public policy bucket') +@attr(assertion='succeeds') +@attr('policy_status') +def test_get_nonpublicpolicy_acl_bucket_policy_status(): + bucket_name = get_new_bucket() + client = get_client() + client = get_client() + + resp = client.get_bucket_policy_status(Bucket=bucket_name) + eq(resp['PolicyStatus']['IsPublic'],False) + + resource1 = "arn:aws:s3:::" + bucket_name + resource2 = "arn:aws:s3:::" + bucket_name + "/*" + policy_document = json.dumps( + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"AWS": "*"}, + "Action": "s3:ListBucket", + "Resource": [ + "{}".format(resource1), + "{}".format(resource2) + ], + "Condition": { + "IpAddress": + {"aws:SourceIp": "10.0.0.0/32"} + } + }] + }) + + client.put_bucket_policy(Bucket=bucket_name, Policy=policy_document) + resp = client.get_bucket_policy_status(Bucket=bucket_name) + eq(resp['PolicyStatus']['IsPublic'],False) + + +@attr(resource='bucket') +@attr(method='get') +@attr(operation='get bucket policy status on a public policy bucket') +@attr(assertion='succeeds') +@attr('policy_status') +def test_get_nonpublicpolicy_deny_bucket_policy_status(): + bucket_name = get_new_bucket() + client = get_client() + client = get_client() + + resp = client.get_bucket_policy_status(Bucket=bucket_name) + eq(resp['PolicyStatus']['IsPublic'],False) + + resource1 = "arn:aws:s3:::" + bucket_name + resource2 = "arn:aws:s3:::" + bucket_name + "/*" + policy_document = json.dumps( + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "NotPrincipal": {"AWS": "arn:aws:iam::s3tenant1:root"}, + "Action": "s3:ListBucket", + "Resource": [ + "{}".format(resource1), + "{}".format(resource2) + ], + }] + }) + + client.put_bucket_policy(Bucket=bucket_name, Policy=policy_document) + resp = client.get_bucket_policy_status(Bucket=bucket_name) + eq(resp['PolicyStatus']['IsPublic'],True) From 94168194fd64d2ccf307a1cc0087398e72aac80d Mon Sep 17 00:00:00 2001 From: Abhishek Lekshmanan Date: Tue, 8 Oct 2019 10:46:38 +0200 Subject: [PATCH 3/9] add tests for public access configuration Signed-off-by: Abhishek Lekshmanan (cherry picked from commit 1ad38530e07b388e54f4aee93d4b55344b015132) --- s3tests_boto3/functional/test_s3.py | 106 ++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index cedd386..85751bc 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -12540,3 +12540,109 @@ def test_get_nonpublicpolicy_deny_bucket_policy_status(): client.put_bucket_policy(Bucket=bucket_name, Policy=policy_document) resp = client.get_bucket_policy_status(Bucket=bucket_name) eq(resp['PolicyStatus']['IsPublic'],True) + +@attr(resource='bucket') +@attr(method='get') +@attr(operation='get public access block on a bucket') +@attr(assertion='succeeds') +@attr('policy_status') +def test_get_default_public_block(): + #client = get_svc_client(svc='s3control', client_config=Config(s3={'addressing_style': 'path'})) + bucket_name = get_new_bucket() + client = get_client() + + resp = client.get_public_access_block(Bucket=bucket_name) + eq(resp['PublicAccessBlockConfiguration']['BlockPublicAcls'], False) + eq(resp['PublicAccessBlockConfiguration']['BlockPublicPolicy'], False) + eq(resp['PublicAccessBlockConfiguration']['IgnorePublicAcls'], False) + eq(resp['PublicAccessBlockConfiguration']['RestrictPublicBuckets'], False) + +@attr(resource='bucket') +@attr(method='put') +@attr(operation='get public access block on a bucket') +@attr(assertion='succeeds') +@attr('policy_status') +def test_put_public_block(): + #client = get_svc_client(svc='s3control', client_config=Config(s3={'addressing_style': 'path'})) + bucket_name = get_new_bucket() + client = get_client() + + access_conf = {'BlockPublicAcls': True, + 'IgnorePublicAcls': True, + 'BlockPublicPolicy': True, + 'RestrictPublicBuckets': False} + + client.put_public_access_block(Bucket=bucket_name, PublicAccessBlockConfiguration=access_conf) + + resp = client.get_public_access_block(Bucket=bucket_name) + eq(resp['PublicAccessBlockConfiguration']['BlockPublicAcls'], access_conf['BlockPublicAcls']) + eq(resp['PublicAccessBlockConfiguration']['BlockPublicPolicy'], access_conf['BlockPublicPolicy']) + eq(resp['PublicAccessBlockConfiguration']['IgnorePublicAcls'], access_conf['IgnorePublicAcls']) + eq(resp['PublicAccessBlockConfiguration']['RestrictPublicBuckets'], access_conf['RestrictPublicBuckets']) + + +@attr(resource='bucket') +@attr(method='put') +@attr(operation='get public access block on a bucket') +@attr(assertion='succeeds') +@attr('policy_status') +def test_block_public_put_bucket_acls(): + #client = get_svc_client(svc='s3control', client_config=Config(s3={'addressing_style': 'path'})) + bucket_name = get_new_bucket() + client = get_client() + + access_conf = {'BlockPublicAcls': True, + 'IgnorePublicAcls': False, + 'BlockPublicPolicy': True, + 'RestrictPublicBuckets': False} + + client.put_public_access_block(Bucket=bucket_name, PublicAccessBlockConfiguration=access_conf) + + resp = client.get_public_access_block(Bucket=bucket_name) + eq(resp['PublicAccessBlockConfiguration']['BlockPublicAcls'], access_conf['BlockPublicAcls']) + eq(resp['PublicAccessBlockConfiguration']['BlockPublicPolicy'], access_conf['BlockPublicPolicy']) + + e = assert_raises(ClientError, client.put_bucket_acl, Bucket=bucket_name,ACL='public-read') + status, error_code = _get_status_and_error_code(e.response) + eq(status, 403) + + e = assert_raises(ClientError, client.put_bucket_acl, Bucket=bucket_name,ACL='public-read-write') + status, error_code = _get_status_and_error_code(e.response) + eq(status, 403) + + e = assert_raises(ClientError, client.put_bucket_acl, Bucket=bucket_name,ACL='authenticated-read') + status, error_code = _get_status_and_error_code(e.response) + eq(status, 403) + + +@attr(resource='bucket') +@attr(method='put') +@attr(operation='block public acls on canned acls') +@attr(assertion='succeeds') +@attr('policy_status') +def test_block_public_object_canned_acls(): + bucket_name = get_new_bucket() + client = get_client() + + access_conf = {'BlockPublicAcls': True, + 'IgnorePublicAcls': False, + 'BlockPublicPolicy': False, + 'RestrictPublicBuckets': False} + + client.put_public_access_block(Bucket=bucket_name, PublicAccessBlockConfiguration=access_conf) + + # resp = client.get_public_access_block(Bucket=bucket_name) + # eq(resp['PublicAccessBlockConfiguration']['BlockPublicAcls'], access_conf['BlockPublicAcls']) + # eq(resp['PublicAccessBlockConfiguration']['BlockPublicPolicy'], access_conf['BlockPublicPolicy']) + + e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key='foo1', Body='bar', ACL='public-read') + status, error_code = _get_status_and_error_code(e.response) + eq(status, 403) + + e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key='foo2', Body='bar', ACL='public-read') + status, error_code = _get_status_and_error_code(e.response) + eq(status, 403) + + e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key='foo3', Body='bar', ACL='authenticated-read') + status, error_code = _get_status_and_error_code(e.response) + eq(status, 403) From 19947bd541fcc153380df65b18e717f9cb6f0274 Mon Sep 17 00:00:00 2001 From: Abhishek Lekshmanan Date: Wed, 22 Jan 2020 17:03:10 +0100 Subject: [PATCH 4/9] add ability to get svc client for s3config set of apis Signed-off-by: Abhishek Lekshmanan (cherry picked from commit 6d3f574a8ec8e61a2d5591a4c448b5b9e6404598) --- s3tests_boto3/functional/__init__.py | 12 ++++++++++++ s3tests_boto3/functional/test_s3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/s3tests_boto3/functional/__init__.py b/s3tests_boto3/functional/__init__.py index 5d7e3c1..9549276 100644 --- a/s3tests_boto3/functional/__init__.py +++ b/s3tests_boto3/functional/__init__.py @@ -293,6 +293,18 @@ def get_bad_auth_client(aws_access_key_id='badauth'): config=Config(signature_version='s3v4')) return client +def get_svc_client(client_config=None, svc='s3'): + if client_config == None: + client_config = Config(signature_version='s3v4') + + client = boto3.client(service_name=svc, + aws_access_key_id=config.main_access_key, + aws_secret_access_key=config.main_secret_key, + endpoint_url=config.default_endpoint, + use_ssl=config.default_is_secure, + config=client_config) + return client + bucket_counter = itertools.count(1) def get_new_bucket_name(): diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 85751bc..f3bd89c 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -69,6 +69,7 @@ from . import ( get_objects_list, get_main_kms_keyid, get_secondary_kms_keyid, + get_svc_client, nuke_prefixed_buckets, ) From a20e0d47f2083a0618609f8485dcb2178ea935cb Mon Sep 17 00:00:00 2001 From: Abhishek Lekshmanan Date: Tue, 8 Oct 2019 10:47:23 +0200 Subject: [PATCH 5/9] remove redundant get_client calls Signed-off-by: Abhishek Lekshmanan (cherry picked from commit 4996430709ba3910a7cb8380f25f81c0e3da88a0) --- s3tests_boto3/functional/test_s3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index f3bd89c..5414726 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -12517,7 +12517,6 @@ def test_get_nonpublicpolicy_acl_bucket_policy_status(): def test_get_nonpublicpolicy_deny_bucket_policy_status(): bucket_name = get_new_bucket() client = get_client() - client = get_client() resp = client.get_bucket_policy_status(Bucket=bucket_name) eq(resp['PolicyStatus']['IsPublic'],False) From a48cf75391ace28d98380008d7c4bb79e30e0d98 Mon Sep 17 00:00:00 2001 From: Abhishek Lekshmanan Date: Tue, 8 Oct 2019 15:21:20 +0200 Subject: [PATCH 6/9] use empty bodies for canned acl tests with BlockPublicAccess This should be a temporary workaround until #42208 is fixed Signed-off-by: Abhishek Lekshmanan (cherry picked from commit d02c1819f67011d6869b86d4497cf6c31fc7be5c) --- s3tests_boto3/functional/test_s3.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 5414726..fef2fa5 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -12635,14 +12635,17 @@ def test_block_public_object_canned_acls(): # eq(resp['PublicAccessBlockConfiguration']['BlockPublicAcls'], access_conf['BlockPublicAcls']) # eq(resp['PublicAccessBlockConfiguration']['BlockPublicPolicy'], access_conf['BlockPublicPolicy']) - e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key='foo1', Body='bar', ACL='public-read') + #FIXME: use empty body until #42208 + e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key='foo1', Body='', ACL='public-read') status, error_code = _get_status_and_error_code(e.response) eq(status, 403) - e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key='foo2', Body='bar', ACL='public-read') + e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key='foo2', Body='', ACL='public-read') status, error_code = _get_status_and_error_code(e.response) eq(status, 403) - e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key='foo3', Body='bar', ACL='authenticated-read') + e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key='foo3', Body='', ACL='authenticated-read') status, error_code = _get_status_and_error_code(e.response) eq(status, 403) + + From 0a495efc8c65d45a6eb3725a07891b1955772386 Mon Sep 17 00:00:00 2001 From: Abhishek Lekshmanan Date: Tue, 8 Oct 2019 15:22:19 +0200 Subject: [PATCH 7/9] add test for block public policy Signed-off-by: Abhishek Lekshmanan (cherry picked from commit b4516725f28aad87796e442682fec23a4fb7e7e3) --- s3tests_boto3/functional/test_s3.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index fef2fa5..d619cbe 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -12649,3 +12649,25 @@ def test_block_public_object_canned_acls(): eq(status, 403) +@attr(resource='bucket') +@attr(method='put') +@attr(operation='block public acls on canned acls') +@attr(assertion='succeeds') +@attr('policy_status') +def test_block_public_policy(): + bucket_name = get_new_bucket() + client = get_client() + + access_conf = {'BlockPublicAcls': False, + 'IgnorePublicAcls': False, + 'BlockPublicPolicy': True, + 'RestrictPublicBuckets': False} + + client.put_public_access_block(Bucket=bucket_name, PublicAccessBlockConfiguration=access_conf) + resource = _make_arn_resource("{}/{}".format(bucket_name, "*")) + policy_document = make_json_policy("s3:GetObject", + resource) + + check_access_denied(client.put_bucket_policy, Bucket=bucket_name, Policy=policy_document) + + From 4fc133b1b5d43be1684e68f7fc4c4da8e5efb2aa Mon Sep 17 00:00:00 2001 From: Abhishek Lekshmanan Date: Tue, 8 Oct 2019 15:22:42 +0200 Subject: [PATCH 8/9] add tests for ignore public acls Signed-off-by: Abhishek Lekshmanan (cherry picked from commit 3b1571ace6c27a1f8995e42576e1d779848d110f) --- s3tests_boto3/functional/test_s3.py | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index d619cbe..36f0160 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -12671,3 +12671,33 @@ def test_block_public_policy(): check_access_denied(client.put_bucket_policy, Bucket=bucket_name, Policy=policy_document) +@attr(resource='bucket') +@attr(method='put') +@attr(operation='ignore public acls on canned acls') +@attr(assertion='succeeds') +@attr('policy_status') +def test_ignore_public_acls(): + bucket_name = get_new_bucket() + client = get_client() + alt_client = get_alt_client() + + client.put_bucket_acl(Bucket=bucket_name, ACL='public-read') + # Public bucket should be accessible + alt_client.list_objects(Bucket=bucket_name) + + client.put_object(Bucket=bucket_name,Key='key1',Body='abcde',ACL='public-read') + resp=alt_client.get_object(Bucket=bucket_name, Key='key1') + eq(resp['Body'].read(), 'abcde') + + access_conf = {'BlockPublicAcls': False, + 'IgnorePublicAcls': True, + 'BlockPublicPolicy': False, + 'RestrictPublicBuckets': False} + + client.put_public_access_block(Bucket=bucket_name, PublicAccessBlockConfiguration=access_conf) + resource = _make_arn_resource("{}/{}".format(bucket_name, "*")) + + client.put_bucket_acl(Bucket=bucket_name, ACL='public-read') + # IgnorePublicACLs is true, so regardless this should behave as a private bucket + check_access_denied(alt_client.list_objects, Bucket=bucket_name) + check_access_denied(alt_client.get_object, Bucket=bucket_name, Key='key1') From 7b3df700cc589bb7e6932e677371063fadf0f775 Mon Sep 17 00:00:00 2001 From: Abhishek Lekshmanan Date: Wed, 22 Jan 2020 17:06:09 +0100 Subject: [PATCH 9/9] fix ignore public acls with py3 compatible code Signed-off-by: Abhishek Lekshmanan (cherry picked from commit 4d675235ddc12478a0075fe937e2e4919f0ad5b9) --- s3tests_boto3/functional/test_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 36f0160..aaf885b 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -12687,7 +12687,7 @@ def test_ignore_public_acls(): client.put_object(Bucket=bucket_name,Key='key1',Body='abcde',ACL='public-read') resp=alt_client.get_object(Bucket=bucket_name, Key='key1') - eq(resp['Body'].read(), 'abcde') + eq(_get_body(resp), 'abcde') access_conf = {'BlockPublicAcls': False, 'IgnorePublicAcls': True,