mirror of
https://github.com/ceph/s3-tests.git
synced 2025-03-22 15:19:28 +00:00
Merge pull request #619 from cbodley/wip-unpin-botocore
unpin botocore, disable failing sigv2/checksum tests
This commit is contained in:
Casey Bodley
GitHub
commit
dc155b5c26
6 changed files with 48 additions and 38 deletions
|
|
@ -1,8 +1,7 @@
|
|||
PyYAML
|
||||
boto >=2.6.0
|
||||
boto3 >=1.0.0
|
||||
# botocore-1.28 broke v2 signatures, see https://tracker.ceph.com/issues/58059
|
||||
botocore <1.28.0
|
||||
botocore
|
||||
munch >=2.0.0
|
||||
# 0.14 switches to libev, that means bootstrap needs to change too
|
||||
gevent >=1.0
|
||||
|
|
|
|||
|
|
@ -415,25 +415,25 @@ def get_v2_client():
|
|||
return client
|
||||
|
||||
def get_sts_client(**kwargs):
|
||||
kwargs.setdefault('region_name', '')
|
||||
kwargs.setdefault('aws_access_key_id', config.alt_access_key)
|
||||
kwargs.setdefault('aws_secret_access_key', config.alt_secret_key)
|
||||
kwargs.setdefault('config', Config(signature_version='s3v4'))
|
||||
|
||||
client = boto3.client(service_name='sts',
|
||||
endpoint_url=config.default_endpoint,
|
||||
region_name='',
|
||||
use_ssl=config.default_is_secure,
|
||||
verify=config.default_ssl_verify,
|
||||
**kwargs)
|
||||
return client
|
||||
|
||||
def get_iam_client(**kwargs):
|
||||
kwargs.setdefault('region_name', '')
|
||||
kwargs.setdefault('aws_access_key_id', config.iam_access_key)
|
||||
kwargs.setdefault('aws_secret_access_key', config.iam_secret_key)
|
||||
|
||||
client = boto3.client(service_name='iam',
|
||||
endpoint_url=config.default_endpoint,
|
||||
region_name='',
|
||||
use_ssl=config.default_is_secure,
|
||||
verify=config.default_ssl_verify,
|
||||
**kwargs)
|
||||
|
|
@ -453,22 +453,22 @@ def get_iam_s3client(**kwargs):
|
|||
|
||||
def get_iam_root_client(**kwargs):
|
||||
kwargs.setdefault('service_name', 'iam')
|
||||
kwargs.setdefault('region_name', '')
|
||||
kwargs.setdefault('aws_access_key_id', config.iam_root_access_key)
|
||||
kwargs.setdefault('aws_secret_access_key', config.iam_root_secret_key)
|
||||
|
||||
return boto3.client(endpoint_url=config.default_endpoint,
|
||||
region_name='',
|
||||
use_ssl=config.default_is_secure,
|
||||
verify=config.default_ssl_verify,
|
||||
**kwargs)
|
||||
|
||||
def get_iam_alt_root_client(**kwargs):
|
||||
kwargs.setdefault('service_name', 'iam')
|
||||
kwargs.setdefault('region_name', '')
|
||||
kwargs.setdefault('aws_access_key_id', config.iam_alt_root_access_key)
|
||||
kwargs.setdefault('aws_secret_access_key', config.iam_alt_root_secret_key)
|
||||
|
||||
return boto3.client(endpoint_url=config.default_endpoint,
|
||||
region_name='',
|
||||
use_ssl=config.default_is_secure,
|
||||
verify=config.default_ssl_verify,
|
||||
**kwargs)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import boto3
|
||||
import pytest
|
||||
import botocore.config
|
||||
from botocore.exceptions import ClientError
|
||||
from email.utils import formatdate
|
||||
|
||||
|
|
@ -209,7 +210,10 @@ def test_object_create_bad_contentlength_empty():
|
|||
@pytest.mark.auth_common
|
||||
@pytest.mark.fails_on_mod_proxy_fcgi
|
||||
def test_object_create_bad_contentlength_negative():
|
||||
client = get_client()
|
||||
# to test Content-Length=-1, we have to prevent the checksum calculation
|
||||
# from switching to STREAMING-UNSIGNED-PAYLOAD-TRAILER
|
||||
config = botocore.config.Config(request_checksum_calculation = 'when_required')
|
||||
client = get_client(config)
|
||||
bucket_name = get_new_bucket()
|
||||
key_name = 'foo'
|
||||
e = assert_raises(ClientError, client.put_object, Bucket=bucket_name, Key=key_name, ContentLength=-1)
|
||||
|
|
@ -504,12 +508,14 @@ def test_bucket_create_bad_authorization_invalid_aws2():
|
|||
assert error_code == 'InvalidArgument'
|
||||
|
||||
@pytest.mark.auth_aws2
|
||||
@pytest.mark.fails_on_rgw
|
||||
def test_bucket_create_bad_ua_empty_aws2():
|
||||
v2_client = get_v2_client()
|
||||
headers = {'User-Agent': ''}
|
||||
_add_header_create_bucket(headers, v2_client)
|
||||
|
||||
@pytest.mark.auth_aws2
|
||||
@pytest.mark.fails_on_rgw
|
||||
def test_bucket_create_bad_ua_none_aws2():
|
||||
v2_client = get_v2_client()
|
||||
remove = 'User-Agent'
|
||||
|
|
|
|||
|
|
@ -507,7 +507,7 @@ def test_deny_bucket_actions_in_user_policy():
|
|||
UserName=get_alt_user_id())
|
||||
assert response['ResponseMetadata']['HTTPStatusCode'] == 200
|
||||
|
||||
e = assert_raises(ClientError, s3_client.list_buckets, Bucket=bucket)
|
||||
e = assert_raises(ClientError, s3_client.list_buckets)
|
||||
status, error_code = _get_status_and_error_code(e.response)
|
||||
assert status == 403
|
||||
assert error_code == 'AccessDenied'
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
import boto3
|
||||
import botocore.session
|
||||
import botocore.config
|
||||
from botocore.exceptions import ClientError
|
||||
from botocore.exceptions import ParamValidationError
|
||||
from botocore.handlers import validate_bucket_name
|
||||
import isodate
|
||||
import email.utils
|
||||
import datetime
|
||||
|
|
@ -5941,6 +5943,7 @@ def _check_key_content(src_key, src_bucket_name, dest_key, dest_bucket_name, ver
|
|||
assert src_data == dest_data
|
||||
|
||||
@pytest.mark.fails_on_dbstore
|
||||
@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936
|
||||
def test_multipart_copy_small():
|
||||
src_key = 'foo'
|
||||
src_bucket_name = _create_key_with_random_content(src_key)
|
||||
|
|
@ -6033,6 +6036,7 @@ def test_multipart_copy_without_range():
|
|||
_check_key_content(src_key, src_bucket_name, dest_key, dest_bucket_name)
|
||||
|
||||
@pytest.mark.fails_on_dbstore
|
||||
@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936
|
||||
def test_multipart_copy_special_names():
|
||||
src_bucket_name = get_new_bucket()
|
||||
|
||||
|
|
@ -6212,6 +6216,7 @@ def test_multipart_upload_multiple_sizes():
|
|||
client.complete_multipart_upload(Bucket=bucket_name, Key=key, UploadId=upload_id, MultipartUpload={'Parts': parts})
|
||||
|
||||
@pytest.mark.fails_on_dbstore
|
||||
@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936
|
||||
def test_multipart_copy_multiple_sizes():
|
||||
src_key = 'foo'
|
||||
src_bucket_name = _create_key_with_random_content(src_key, 12*1024*1024)
|
||||
|
|
@ -6917,12 +6922,14 @@ def test_cors_presigned_get_object_tenant():
|
|||
method='get_object',
|
||||
)
|
||||
|
||||
@pytest.mark.fails_on_rgw
|
||||
def test_cors_presigned_get_object_v2():
|
||||
_test_cors_options_presigned_method(
|
||||
client=get_v2_client(),
|
||||
method='get_object',
|
||||
)
|
||||
|
||||
@pytest.mark.fails_on_rgw
|
||||
def test_cors_presigned_get_object_tenant_v2():
|
||||
_test_cors_options_presigned_method(
|
||||
client=get_v2_tenant_client(),
|
||||
|
|
@ -6942,12 +6949,14 @@ def test_cors_presigned_put_object_with_acl():
|
|||
cannedACL='private',
|
||||
)
|
||||
|
||||
@pytest.mark.fails_on_rgw
|
||||
def test_cors_presigned_put_object_v2():
|
||||
_test_cors_options_presigned_method(
|
||||
client=get_v2_client(),
|
||||
method='put_object',
|
||||
)
|
||||
|
||||
@pytest.mark.fails_on_rgw
|
||||
def test_cors_presigned_put_object_tenant_v2():
|
||||
_test_cors_options_presigned_method(
|
||||
client=get_v2_tenant_client(),
|
||||
|
|
@ -7384,6 +7393,7 @@ def test_multipart_resend_first_finishes_last():
|
|||
_verify_atomic_key_data(bucket_name, key_name, file_size, 'A')
|
||||
|
||||
@pytest.mark.fails_on_dbstore
|
||||
@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936
|
||||
def test_ranged_request_response_code():
|
||||
content = 'testcontent'
|
||||
|
||||
|
|
@ -7402,6 +7412,7 @@ def _generate_random_string(size):
|
|||
return ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(size))
|
||||
|
||||
@pytest.mark.fails_on_dbstore
|
||||
@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936
|
||||
def test_ranged_big_request_response_code():
|
||||
content = _generate_random_string(8*1024*1024)
|
||||
|
||||
|
|
@ -7417,6 +7428,7 @@ def test_ranged_big_request_response_code():
|
|||
assert response['ResponseMetadata']['HTTPStatusCode'] == 206
|
||||
|
||||
@pytest.mark.fails_on_dbstore
|
||||
@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936
|
||||
def test_ranged_request_skip_leading_bytes_response_code():
|
||||
content = 'testcontent'
|
||||
|
||||
|
|
@ -7432,6 +7444,7 @@ def test_ranged_request_skip_leading_bytes_response_code():
|
|||
assert response['ResponseMetadata']['HTTPStatusCode'] == 206
|
||||
|
||||
@pytest.mark.fails_on_dbstore
|
||||
@pytest.mark.fails_on_rgw # https://tracker.ceph.com/issues/69936
|
||||
def test_ranged_request_return_trailing_bytes_response_code():
|
||||
content = 'testcontent'
|
||||
|
||||
|
|
@ -10777,20 +10790,6 @@ def test_bucketv2_policy_acl():
|
|||
client.delete_bucket_policy(Bucket=bucket_name)
|
||||
client.put_bucket_acl(Bucket=bucket_name, ACL='public-read')
|
||||
|
||||
def tenanted_bucket_name(tenant):
|
||||
def change_bucket_name(params, **kwargs):
|
||||
old_name = params['context']['signing']['bucket']
|
||||
new_name = "{}:{}".format(tenant, old_name)
|
||||
params['Bucket'] = new_name
|
||||
params['context']['signing']['bucket'] = new_name
|
||||
|
||||
# the : needs to be url-encoded for urls
|
||||
new_name_url = "{}%3A{}".format(tenant, old_name)
|
||||
params['url'] = params['url'].replace(old_name, new_name_url)
|
||||
params['url_path'] = params['url_path'].replace(old_name, new_name_url)
|
||||
|
||||
return change_bucket_name
|
||||
|
||||
@pytest.mark.bucket_policy
|
||||
def test_bucket_policy_different_tenant():
|
||||
bucket_name = get_new_bucket()
|
||||
|
|
@ -10818,8 +10817,8 @@ def test_bucket_policy_different_tenant():
|
|||
|
||||
# use the tenanted client to list the global tenant's bucket
|
||||
tenant_client = get_tenant_client()
|
||||
tenant_client.meta.events.register('before-call.s3.ListObjects', tenanted_bucket_name(''))
|
||||
response = tenant_client.list_objects(Bucket=bucket_name)
|
||||
tenant_client.meta.events.unregister("before-parameter-build.s3", validate_bucket_name)
|
||||
response = tenant_client.list_objects(Bucket=":{}".format(bucket_name))
|
||||
|
||||
assert len(response['Contents']) == 1
|
||||
|
||||
|
|
@ -10852,9 +10851,9 @@ def test_bucket_policy_tenanted_bucket():
|
|||
|
||||
# use the global tenant's client to list the tenanted bucket
|
||||
client = get_client()
|
||||
client.meta.events.register('before-call.s3.ListObjects', tenanted_bucket_name(tenant))
|
||||
client.meta.events.unregister("before-parameter-build.s3", validate_bucket_name)
|
||||
|
||||
response = client.list_objects(Bucket=bucket_name)
|
||||
response = client.list_objects(Bucket="{}:{}".format(tenant, bucket_name))
|
||||
assert len(response['Contents']) == 1
|
||||
|
||||
@pytest.mark.bucket_policy
|
||||
|
|
@ -11935,7 +11934,7 @@ def test_bucket_policy_put_obj_grant():
|
|||
@pytest.mark.encryption
|
||||
def test_put_obj_enc_conflict_c_s3():
|
||||
bucket_name = get_new_bucket()
|
||||
client = get_v2_client()
|
||||
client = get_client()
|
||||
|
||||
# boto3.set_stream_logger(name='botocore')
|
||||
|
||||
|
|
@ -11961,7 +11960,7 @@ def test_put_obj_enc_conflict_c_kms():
|
|||
if kms_keyid is None:
|
||||
kms_keyid = 'fool-me-once'
|
||||
bucket_name = get_new_bucket()
|
||||
client = get_v2_client()
|
||||
client = get_client()
|
||||
|
||||
# boto3.set_stream_logger(name='botocore')
|
||||
|
||||
|
|
@ -11988,7 +11987,7 @@ def test_put_obj_enc_conflict_s3_kms():
|
|||
if kms_keyid is None:
|
||||
kms_keyid = 'fool-me-once'
|
||||
bucket_name = get_new_bucket()
|
||||
client = get_v2_client()
|
||||
client = get_client()
|
||||
|
||||
# boto3.set_stream_logger(name='botocore')
|
||||
|
||||
|
|
@ -12012,7 +12011,7 @@ def test_put_obj_enc_conflict_bad_enc_kms():
|
|||
if kms_keyid is None:
|
||||
kms_keyid = 'fool-me-once'
|
||||
bucket_name = get_new_bucket()
|
||||
client = get_v2_client()
|
||||
client = get_client()
|
||||
|
||||
# boto3.set_stream_logger(name='botocore')
|
||||
|
||||
|
|
@ -12035,7 +12034,7 @@ def test_put_obj_enc_conflict_bad_enc_kms():
|
|||
@pytest.mark.fails_on_dbstore
|
||||
def test_bucket_policy_put_obj_s3_noenc():
|
||||
bucket_name = get_new_bucket()
|
||||
client = get_v2_client()
|
||||
client = get_client()
|
||||
|
||||
deny_unencrypted_obj = {
|
||||
"Null" : {
|
||||
|
|
@ -12064,7 +12063,7 @@ def test_bucket_policy_put_obj_s3_noenc():
|
|||
@pytest.mark.fails_on_dbstore
|
||||
def test_bucket_policy_put_obj_s3_incorrect_algo_sse_s3():
|
||||
bucket_name = get_new_bucket()
|
||||
client = get_v2_client()
|
||||
client = get_client()
|
||||
|
||||
deny_incorrect_algo = {
|
||||
"StringNotEquals": {
|
||||
|
|
@ -12095,7 +12094,7 @@ def test_bucket_policy_put_obj_s3_kms():
|
|||
if kms_keyid is None:
|
||||
kms_keyid = 'fool-me-twice'
|
||||
bucket_name = get_new_bucket()
|
||||
client = get_v2_client()
|
||||
client = get_client()
|
||||
|
||||
deny_incorrect_algo = {
|
||||
"StringNotEquals": {
|
||||
|
|
@ -12141,7 +12140,7 @@ def test_bucket_policy_put_obj_kms_noenc():
|
|||
if kms_keyid is None:
|
||||
pytest.skip('[s3 main] section missing kms_keyid')
|
||||
bucket_name = get_new_bucket()
|
||||
client = get_v2_client()
|
||||
client = get_client()
|
||||
|
||||
deny_incorrect_algo = {
|
||||
"StringNotEquals": {
|
||||
|
|
@ -12184,7 +12183,7 @@ def test_bucket_policy_put_obj_kms_noenc():
|
|||
@pytest.mark.bucket_policy
|
||||
def test_bucket_policy_put_obj_kms_s3():
|
||||
bucket_name = get_new_bucket()
|
||||
client = get_v2_client()
|
||||
client = get_client()
|
||||
|
||||
deny_incorrect_algo = {
|
||||
"StringNotEquals": {
|
||||
|
|
@ -14050,7 +14049,11 @@ def test_multipart_checksum_3parts():
|
|||
@pytest.mark.fails_on_dbstore
|
||||
def test_multipart_checksum_upload_fallback():
|
||||
bucket = get_new_bucket()
|
||||
client = get_client()
|
||||
|
||||
# to test client.upload_part() without a checksum header,
|
||||
# we have to disable its calculation in botocore config
|
||||
config = botocore.config.Config(request_checksum_calculation = 'when_required')
|
||||
client = get_client(config)
|
||||
|
||||
key = "mpu_cksum_fallback"
|
||||
alg = 'SHA256'
|
||||
|
|
|
|||
|
|
@ -41,13 +41,15 @@ def sns_alt(iam_alt_root):
|
|||
|
||||
@pytest.fixture
|
||||
def s3(iam_root):
|
||||
client = get_iam_root_client(service_name='s3')
|
||||
# clear region_name to work around Invalid region: region was not a valid DNS name.
|
||||
client = get_iam_root_client(service_name='s3', region_name=None)
|
||||
yield client
|
||||
nuke_prefixed_buckets(get_prefix(), client)
|
||||
|
||||
@pytest.fixture
|
||||
def s3_alt(iam_alt_root):
|
||||
client = get_iam_alt_root_client(service_name='s3')
|
||||
# clear region_name to work around Invalid region: region was not a valid DNS name.
|
||||
client = get_iam_alt_root_client(service_name='s3', region_name=None)
|
||||
yield client
|
||||
nuke_prefixed_buckets(get_prefix(), client)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue