From 88daca5095c6163b2a98f524a094b788418bf64c Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Wed, 22 Jan 2025 11:15:38 -0500 Subject: [PATCH 1/9] requirements: unpin botocore version Signed-off-by: Casey Bodley --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 724e990..585bb9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ PyYAML boto >=2.6.0 boto3 >=1.0.0 -# botocore-1.28 broke v2 signatures, see https://tracker.ceph.com/issues/58059 -botocore <1.28.0 +botocore munch >=2.0.0 # 0.14 switches to libev, that means bootstrap needs to change too gevent >=1.0 From e786aa4f4279e23cd510dd404321c575c64a3425 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Wed, 22 Jan 2025 11:16:13 -0500 Subject: [PATCH 2/9] s3: mark sigv2 tests with fails_on_rgw Signed-off-by: Casey Bodley --- s3tests_boto3/functional/test_headers.py | 2 ++ s3tests_boto3/functional/test_s3.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/s3tests_boto3/functional/test_headers.py b/s3tests_boto3/functional/test_headers.py index 66cabe5..557f234 100644 --- a/s3tests_boto3/functional/test_headers.py +++ b/s3tests_boto3/functional/test_headers.py @@ -504,12 +504,14 @@ def test_bucket_create_bad_authorization_invalid_aws2(): assert error_code == 'InvalidArgument' @pytest.mark.auth_aws2 +@pytest.mark.fails_on_rgw def test_bucket_create_bad_ua_empty_aws2(): v2_client = get_v2_client() headers = {'User-Agent': ''} _add_header_create_bucket(headers, v2_client) @pytest.mark.auth_aws2 +@pytest.mark.fails_on_rgw def test_bucket_create_bad_ua_none_aws2(): v2_client = get_v2_client() remove = 'User-Agent' diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 152d02d..5602d50 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -6917,12 +6917,14 @@ def test_cors_presigned_get_object_tenant(): method='get_object', ) +@pytest.mark.fails_on_rgw def test_cors_presigned_get_object_v2(): _test_cors_options_presigned_method( client=get_v2_client(), method='get_object', ) +@pytest.mark.fails_on_rgw def test_cors_presigned_get_object_tenant_v2(): _test_cors_options_presigned_method( client=get_v2_tenant_client(), @@ -6942,12 +6944,14 @@ def test_cors_presigned_put_object_with_acl(): cannedACL='private', ) +@pytest.mark.fails_on_rgw def test_cors_presigned_put_object_v2(): _test_cors_options_presigned_method( client=get_v2_client(), method='put_object', ) +@pytest.mark.fails_on_rgw def test_cors_presigned_put_object_tenant_v2(): _test_cors_options_presigned_method( client=get_v2_tenant_client(), From 25a502872b7043222ae5c37e1a2ce31e4cef5000 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Wed, 22 Jan 2025 12:04:40 -0500 Subject: [PATCH 3/9] s3: sse-kms policy tests use v4 client Signed-off-by: Casey Bodley --- s3tests_boto3/functional/test_s3.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 5602d50..ee63526 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -11939,7 +11939,7 @@ def test_bucket_policy_put_obj_grant(): @pytest.mark.encryption def test_put_obj_enc_conflict_c_s3(): bucket_name = get_new_bucket() - client = get_v2_client() + client = get_client() # boto3.set_stream_logger(name='botocore') @@ -11965,7 +11965,7 @@ def test_put_obj_enc_conflict_c_kms(): if kms_keyid is None: kms_keyid = 'fool-me-once' bucket_name = get_new_bucket() - client = get_v2_client() + client = get_client() # boto3.set_stream_logger(name='botocore') @@ -11992,7 +11992,7 @@ def test_put_obj_enc_conflict_s3_kms(): if kms_keyid is None: kms_keyid = 'fool-me-once' bucket_name = get_new_bucket() - client = get_v2_client() + client = get_client() # boto3.set_stream_logger(name='botocore') @@ -12016,7 +12016,7 @@ def test_put_obj_enc_conflict_bad_enc_kms(): if kms_keyid is None: kms_keyid = 'fool-me-once' bucket_name = get_new_bucket() - client = get_v2_client() + client = get_client() # boto3.set_stream_logger(name='botocore') @@ -12039,7 +12039,7 @@ def test_put_obj_enc_conflict_bad_enc_kms(): @pytest.mark.fails_on_dbstore def test_bucket_policy_put_obj_s3_noenc(): bucket_name = get_new_bucket() - client = get_v2_client() + client = get_client() deny_unencrypted_obj = { "Null" : { @@ -12068,7 +12068,7 @@ def test_bucket_policy_put_obj_s3_noenc(): @pytest.mark.fails_on_dbstore def test_bucket_policy_put_obj_s3_incorrect_algo_sse_s3(): bucket_name = get_new_bucket() - client = get_v2_client() + client = get_client() deny_incorrect_algo = { "StringNotEquals": { @@ -12099,7 +12099,7 @@ def test_bucket_policy_put_obj_s3_kms(): if kms_keyid is None: kms_keyid = 'fool-me-twice' bucket_name = get_new_bucket() - client = get_v2_client() + client = get_client() deny_incorrect_algo = { "StringNotEquals": { @@ -12145,7 +12145,7 @@ def test_bucket_policy_put_obj_kms_noenc(): if kms_keyid is None: pytest.skip('[s3 main] section missing kms_keyid') bucket_name = get_new_bucket() - client = get_v2_client() + client = get_client() deny_incorrect_algo = { "StringNotEquals": { @@ -12188,7 +12188,7 @@ def test_bucket_policy_put_obj_kms_noenc(): @pytest.mark.bucket_policy def test_bucket_policy_put_obj_kms_s3(): bucket_name = get_new_bucket() - client = get_v2_client() + client = get_client() deny_incorrect_algo = { "StringNotEquals": { From 5aff597245ed31cccab7de6a37823f20b049965d Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 6 Sep 2024 08:34:44 -0400 Subject: [PATCH 4/9] s3: clean up use of tenanted bucket names the botocore.handlers.validate_bucket_name validation is enabled by default, but we can disable it with unregister(). this avoids having to mess with the signature or url Signed-off-by: Casey Bodley --- s3tests_boto3/functional/test_s3.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index ee63526..f6b06d1 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -2,6 +2,7 @@ import boto3 import botocore.session from botocore.exceptions import ClientError from botocore.exceptions import ParamValidationError +from botocore.handlers import validate_bucket_name import isodate import email.utils import datetime @@ -10781,20 +10782,6 @@ def test_bucketv2_policy_acl(): client.delete_bucket_policy(Bucket=bucket_name) client.put_bucket_acl(Bucket=bucket_name, ACL='public-read') -def tenanted_bucket_name(tenant): - def change_bucket_name(params, **kwargs): - old_name = params['context']['signing']['bucket'] - new_name = "{}:{}".format(tenant, old_name) - params['Bucket'] = new_name - params['context']['signing']['bucket'] = new_name - - # the : needs to be url-encoded for urls - new_name_url = "{}%3A{}".format(tenant, old_name) - params['url'] = params['url'].replace(old_name, new_name_url) - params['url_path'] = params['url_path'].replace(old_name, new_name_url) - - return change_bucket_name - @pytest.mark.bucket_policy def test_bucket_policy_different_tenant(): bucket_name = get_new_bucket() @@ -10822,8 +10809,8 @@ def test_bucket_policy_different_tenant(): # use the tenanted client to list the global tenant's bucket tenant_client = get_tenant_client() - tenant_client.meta.events.register('before-call.s3.ListObjects', tenanted_bucket_name('')) - response = tenant_client.list_objects(Bucket=bucket_name) + tenant_client.meta.events.unregister("before-parameter-build.s3", validate_bucket_name) + response = tenant_client.list_objects(Bucket=":{}".format(bucket_name)) assert len(response['Contents']) == 1 @@ -10856,9 +10843,9 @@ def test_bucket_policy_tenanted_bucket(): # use the global tenant's client to list the tenanted bucket client = get_client() - client.meta.events.register('before-call.s3.ListObjects', tenanted_bucket_name(tenant)) + client.meta.events.unregister("before-parameter-build.s3", validate_bucket_name) - response = client.list_objects(Bucket=bucket_name) + response = client.list_objects(Bucket="{}:{}".format(tenant, bucket_name)) assert len(response['Contents']) == 1 @pytest.mark.bucket_policy From dfb8fc5b16f585e07898922d732a2d5777f851be Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Wed, 12 Feb 2025 14:44:45 -0500 Subject: [PATCH 5/9] s3: disable request_checksum_calculation for test_multipart_checksum_upload_fallback the test sends create_multipart_upload() with a requested checksum algorithm, and upload_part() requests without. newer boto automatically adds `x-amz-sdk-checksum-algorithm: CRC32` when nothing is specified, so we have to explicitly disable that behavior via botocore config Signed-off-by: Casey Bodley --- s3tests_boto3/functional/test_s3.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index f6b06d1..81298f8 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -1,5 +1,6 @@ import boto3 import botocore.session +import botocore.config from botocore.exceptions import ClientError from botocore.exceptions import ParamValidationError from botocore.handlers import validate_bucket_name @@ -14041,7 +14042,11 @@ def test_multipart_checksum_3parts(): @pytest.mark.fails_on_dbstore def test_multipart_checksum_upload_fallback(): bucket = get_new_bucket() - client = get_client() + + # to test client.upload_part() without a checksum header, + # we have to disable its calculation in botocore config + config = botocore.config.Config(request_checksum_calculation = 'when_required') + client = get_client(config) key = "mpu_cksum_fallback" alg = 'SHA256' From cc56036255d3e97443a741f54f0de08ae2668b20 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Wed, 12 Feb 2025 16:06:49 -0500 Subject: [PATCH 6/9] s3: disable multipart/ranged tests with checksum failures boto update exposed some bugs in our checksum feature around multipart copy and ranged requests. disabling them with fails_on_rgw until https://tracker.ceph.com/issues/69936 is resolved Signed-off-by: Casey Bodley --- s3tests_boto3/functional/test_s3.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 81298f8..40249fa 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -5943,6 +5943,7 @@ def _check_key_content(src_key, src_bucket_name, dest_key, dest_bucket_name, ver assert src_data == dest_data @pytest.mark.fails_on_dbstore +@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936 def test_multipart_copy_small(): src_key = 'foo' src_bucket_name = _create_key_with_random_content(src_key) @@ -6035,6 +6036,7 @@ def test_multipart_copy_without_range(): _check_key_content(src_key, src_bucket_name, dest_key, dest_bucket_name) @pytest.mark.fails_on_dbstore +@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936 def test_multipart_copy_special_names(): src_bucket_name = get_new_bucket() @@ -6214,6 +6216,7 @@ def test_multipart_upload_multiple_sizes(): client.complete_multipart_upload(Bucket=bucket_name, Key=key, UploadId=upload_id, MultipartUpload={'Parts': parts}) @pytest.mark.fails_on_dbstore +@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936 def test_multipart_copy_multiple_sizes(): src_key = 'foo' src_bucket_name = _create_key_with_random_content(src_key, 12*1024*1024) @@ -7390,6 +7393,7 @@ def test_multipart_resend_first_finishes_last(): _verify_atomic_key_data(bucket_name, key_name, file_size, 'A') @pytest.mark.fails_on_dbstore +@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936 def test_ranged_request_response_code(): content = 'testcontent' @@ -7408,6 +7412,7 @@ def _generate_random_string(size): return ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(size)) @pytest.mark.fails_on_dbstore +@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936 def test_ranged_big_request_response_code(): content = _generate_random_string(8*1024*1024) @@ -7423,6 +7428,7 @@ def test_ranged_big_request_response_code(): assert response['ResponseMetadata']['HTTPStatusCode'] == 206 @pytest.mark.fails_on_dbstore +@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936 def test_ranged_request_skip_leading_bytes_response_code(): content = 'testcontent' @@ -7438,6 +7444,7 @@ def test_ranged_request_skip_leading_bytes_response_code(): assert response['ResponseMetadata']['HTTPStatusCode'] == 206 @pytest.mark.fails_on_dbstore +@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936 def test_ranged_request_return_trailing_bytes_response_code(): content = 'testcontent' From 41b7297e7af6c1c794c075249732ebd6d8845f98 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Thu, 13 Feb 2025 11:45:42 -0500 Subject: [PATCH 7/9] iam: remove invalid Bucket param from list_buckets() > botocore.exceptions.ParamValidationError: Parameter validation failed: > Unknown parameter in input: "Bucket", must be one of: MaxBuckets, ContinuationToken, Prefix, BucketRegion Signed-off-by: Casey Bodley --- s3tests_boto3/functional/test_iam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3tests_boto3/functional/test_iam.py b/s3tests_boto3/functional/test_iam.py index e5a78dc..d102fb4 100644 --- a/s3tests_boto3/functional/test_iam.py +++ b/s3tests_boto3/functional/test_iam.py @@ -507,7 +507,7 @@ def test_deny_bucket_actions_in_user_policy(): UserName=get_alt_user_id()) assert response['ResponseMetadata']['HTTPStatusCode'] == 200 - e = assert_raises(ClientError, s3_client.list_buckets, Bucket=bucket) + e = assert_raises(ClientError, s3_client.list_buckets) status, error_code = _get_status_and_error_code(e.response) assert status == 403 assert error_code == 'AccessDenied' From a40c28fcc35a363f3320de9ce6586c4ca7fe09ef Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 14 Feb 2025 11:35:54 -0500 Subject: [PATCH 8/9] s3: disable checksum calculation for test_object_create_bad_contentlength_negative if tls is enabled, boto will switch to STREAMING-UNSIGNED-PAYLOAD-TRAILER and omit the provided content-length header. this leads to test failure: > ________________ test_object_create_bad_contentlength_negative _________________ > e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key=key_name, ContentLength=-1) > AssertionError: ClientError not raised Signed-off-by: Casey Bodley --- s3tests_boto3/functional/test_headers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/s3tests_boto3/functional/test_headers.py b/s3tests_boto3/functional/test_headers.py index 557f234..360a2e6 100644 --- a/s3tests_boto3/functional/test_headers.py +++ b/s3tests_boto3/functional/test_headers.py @@ -1,5 +1,6 @@ import boto3 import pytest +import botocore.config from botocore.exceptions import ClientError from email.utils import formatdate @@ -209,7 +210,10 @@ def test_object_create_bad_contentlength_empty(): @pytest.mark.auth_common @pytest.mark.fails_on_mod_proxy_fcgi def test_object_create_bad_contentlength_negative(): - client = get_client() + # to test Content-Length=-1, we have to prevent the checksum calculation + # from switching to STREAMING-UNSIGNED-PAYLOAD-TRAILER + config = botocore.config.Config(request_checksum_calculation = 'when_required') + client = get_client(config) bucket_name = get_new_bucket() key_name = 'foo' e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key=key_name, ContentLength=-1) From 48068f427dfbac7bf4f9286cee26e9d19ac43e6e Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 14 Feb 2025 15:31:32 -0500 Subject: [PATCH 9/9] sns: clear region_name for s3 clients work around sns failures due to default region_name='': > ERROR s3tests_boto3/functional/test_sns.py::test_account_topic_publish - botocore.exceptions.EndpointResolutionError: Invalid region: region was not a valid DNS name. > ERROR s3tests_boto3/functional/test_sns.py::test_cross_account_topic_publish - botocore.exceptions.EndpointResolutionError: Invalid region: region was not a valid DNS name. Signed-off-by: Casey Bodley --- s3tests_boto3/functional/__init__.py | 8 ++++---- s3tests_boto3/functional/test_sns.py | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/s3tests_boto3/functional/__init__.py b/s3tests_boto3/functional/__init__.py index 7ca874b..9339403 100644 --- a/s3tests_boto3/functional/__init__.py +++ b/s3tests_boto3/functional/__init__.py @@ -415,25 +415,25 @@ def get_v2_client(): return client def get_sts_client(**kwargs): + kwargs.setdefault('region_name', '') kwargs.setdefault('aws_access_key_id', config.alt_access_key) kwargs.setdefault('aws_secret_access_key', config.alt_secret_key) kwargs.setdefault('config', Config(signature_version='s3v4')) client = boto3.client(service_name='sts', endpoint_url=config.default_endpoint, - region_name='', use_ssl=config.default_is_secure, verify=config.default_ssl_verify, **kwargs) return client def get_iam_client(**kwargs): + kwargs.setdefault('region_name', '') kwargs.setdefault('aws_access_key_id', config.iam_access_key) kwargs.setdefault('aws_secret_access_key', config.iam_secret_key) client = boto3.client(service_name='iam', endpoint_url=config.default_endpoint, - region_name='', use_ssl=config.default_is_secure, verify=config.default_ssl_verify, **kwargs) @@ -453,22 +453,22 @@ def get_iam_s3client(**kwargs): def get_iam_root_client(**kwargs): kwargs.setdefault('service_name', 'iam') + kwargs.setdefault('region_name', '') kwargs.setdefault('aws_access_key_id', config.iam_root_access_key) kwargs.setdefault('aws_secret_access_key', config.iam_root_secret_key) return boto3.client(endpoint_url=config.default_endpoint, - region_name='', use_ssl=config.default_is_secure, verify=config.default_ssl_verify, **kwargs) def get_iam_alt_root_client(**kwargs): kwargs.setdefault('service_name', 'iam') + kwargs.setdefault('region_name', '') kwargs.setdefault('aws_access_key_id', config.iam_alt_root_access_key) kwargs.setdefault('aws_secret_access_key', config.iam_alt_root_secret_key) return boto3.client(endpoint_url=config.default_endpoint, - region_name='', use_ssl=config.default_is_secure, verify=config.default_ssl_verify, **kwargs) diff --git a/s3tests_boto3/functional/test_sns.py b/s3tests_boto3/functional/test_sns.py index 589343b..60d0662 100644 --- a/s3tests_boto3/functional/test_sns.py +++ b/s3tests_boto3/functional/test_sns.py @@ -41,13 +41,15 @@ def sns_alt(iam_alt_root): @pytest.fixture def s3(iam_root): - client = get_iam_root_client(service_name='s3') + # clear region_name to work around Invalid region: region was not a valid DNS name. + client = get_iam_root_client(service_name='s3', region_name=None) yield client nuke_prefixed_buckets(get_prefix(), client) @pytest.fixture def s3_alt(iam_alt_root): - client = get_iam_alt_root_client(service_name='s3') + # clear region_name to work around Invalid region: region was not a valid DNS name. + client = get_iam_alt_root_client(service_name='s3', region_name=None) yield client nuke_prefixed_buckets(get_prefix(), client)