diff --git a/.gitignore b/.gitignore index bcbae80..2a246a8 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,6 @@ /*.egg-info /virtualenv +/venv config.yaml diff --git a/README.rst b/README.rst index f2b9818..cf9e702 100644 --- a/README.rst +++ b/README.rst @@ -52,6 +52,15 @@ You can run only the boto3 tests with:: This section contains some basic tests for the AssumeRole, GetSessionToken and AssumeRoleWithWebIdentity API's. The test file is located under ``s3tests_boto3/functional``. +To run the STS tests, the vstart cluster should be started with the following parameter (in addition to any parameters already used with it):: + + vstart.sh -o rgw_sts_key=abcdefghijklmnop -o rgw_s3_auth_use_sts=true + +Note that the ``rgw_sts_key`` can be set to anything that is 128 bits in length. +After the cluster is up the following command should be executed:: + + radosgw-admin caps add --tenant=testx --uid="9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef" --caps="roles=*" + You can run only the sts tests (all the three API's) with:: S3TEST_CONF=your.conf tox s3tests_boto3/functional/test_sts.py diff --git a/requirements.txt b/requirements.txt index ac1d18f..7742d8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ pytz >=2011k httplib2 lxml pytest +tox diff --git a/s3tests/functional/test_s3_website.py b/s3tests/functional/test_s3_website.py index 8cc04ed..641cd71 100644 --- a/s3tests/functional/test_s3_website.py +++ b/s3tests/functional/test_s3_website.py @@ -1,5 +1,5 @@ import sys -import collections +from collections.abc import Container import pytest import string import random @@ -159,9 +159,9 @@ def _test_website_prep(bucket, xml_template, hardcoded_fields = {}, expect_fail= return f def __website_expected_reponse_status(res, status, reason): - if not isinstance(status, collections.Container): + if not isinstance(status, Container): status = set([status]) - if not isinstance(reason, collections.Container): + if not isinstance(reason, Container): reason = set([reason]) if status is not IGNORE_FIELD: @@ -179,7 +179,7 @@ def _website_expected_default_html(**kwargs): v = kwargs[k] if isinstance(v, str): v = [v] - elif not isinstance(v, collections.Container): + elif not isinstance(v, Container): v = [v] for v2 in v: s = '
  • %s: %s
  • ' % (k,v2) @@ -199,7 +199,7 @@ def _website_expected_error_response(res, bucket_name, status, reason, code, con if code is not IGNORE_FIELD: assert errorcode == code - if not isinstance(content, collections.Container): + if not isinstance(content, Container): content = set([content]) for f in content: if f is not IGNORE_FIELD and f is not None: diff --git a/s3tests_boto3/functional/__init__.py b/s3tests_boto3/functional/__init__.py index 22b136c..a65b54c 100644 --- a/s3tests_boto3/functional/__init__.py +++ b/s3tests_boto3/functional/__init__.py @@ -82,18 +82,13 @@ def get_objects_list(bucket, client=None, prefix=None): # generator function that returns object listings in batches, where each # batch is a list of dicts compatible with delete_objects() def list_versions(client, bucket, batch_size): - key_marker = '' - version_marker = '' + kwargs = {'Bucket': bucket, 'MaxKeys': batch_size} truncated = True while truncated: - listing = client.list_object_versions( - Bucket=bucket, - KeyMarker=key_marker, - VersionIdMarker=version_marker, - MaxKeys=batch_size) + listing = client.list_object_versions(**kwargs) - key_marker = listing.get('NextKeyMarker') - version_marker = listing.get('NextVersionIdMarker') + kwargs['KeyMarker'] = listing.get('NextKeyMarker') + kwargs['VersionIdMarker'] = listing.get('NextVersionIdMarker') truncated = listing['IsTruncated'] objs = listing.get('Versions', []) + listing.get('DeleteMarkers', []) diff --git a/s3tests_boto3/functional/test_iam.py b/s3tests_boto3/functional/test_iam.py index 0995f97..fa44357 100644 --- a/s3tests_boto3/functional/test_iam.py +++ b/s3tests_boto3/functional/test_iam.py @@ -477,6 +477,7 @@ def test_allow_bucket_actions_in_user_policy(): @pytest.mark.user_policy @pytest.mark.test_of_iam +@pytest.mark.fails_on_dbstore def test_deny_bucket_actions_in_user_policy(): client = get_iam_client() s3_client = get_alt_client() @@ -551,6 +552,7 @@ def test_allow_object_actions_in_user_policy(): @pytest.mark.user_policy @pytest.mark.test_of_iam +@pytest.mark.fails_on_dbstore def test_deny_object_actions_in_user_policy(): client = get_iam_client() s3_client_alt = get_alt_client() @@ -625,6 +627,7 @@ def test_allow_multipart_actions_in_user_policy(): @pytest.mark.user_policy @pytest.mark.test_of_iam +@pytest.mark.fails_on_dbstore def test_deny_multipart_actions_in_user_policy(): client = get_iam_client() s3_client = get_alt_client() @@ -667,6 +670,7 @@ def test_deny_multipart_actions_in_user_policy(): @pytest.mark.user_policy @pytest.mark.test_of_iam +@pytest.mark.fails_on_dbstore def test_allow_tagging_actions_in_user_policy(): client = get_iam_client() s3_client_alt = get_alt_client() @@ -712,6 +716,7 @@ def test_allow_tagging_actions_in_user_policy(): @pytest.mark.user_policy @pytest.mark.test_of_iam +@pytest.mark.fails_on_dbstore def test_deny_tagging_actions_in_user_policy(): client = get_iam_client() s3_client = get_alt_client() @@ -763,6 +768,7 @@ def test_deny_tagging_actions_in_user_policy(): @pytest.mark.user_policy @pytest.mark.test_of_iam +@pytest.mark.fails_on_dbstore def test_verify_conflicting_user_policy_statements(): s3client = get_alt_client() bucket = get_new_bucket(client=s3client) @@ -794,6 +800,7 @@ def test_verify_conflicting_user_policy_statements(): @pytest.mark.user_policy @pytest.mark.test_of_iam +@pytest.mark.fails_on_dbstore def test_verify_conflicting_user_policies(): s3client = get_alt_client() bucket = get_new_bucket(client=s3client) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index e309c48..0d47f1c 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -1573,6 +1573,19 @@ def test_object_write_to_nonexist_bucket(): assert error_code == 'NoSuchBucket' +def _ev_add_te_header(request, **kwargs): + request.headers.add_header('Transfer-Encoding', 'chunked') + +def test_object_write_with_chunked_transfer_encoding(): + bucket_name = get_new_bucket() + client = get_client() + + client.meta.events.register_first('before-sign.*.*', _ev_add_te_header) + response = client.put_object(Bucket=bucket_name, Key='foo', Body='bar') + + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + + def test_bucket_create_delete(): bucket_name = get_new_bucket() client = get_client() @@ -1624,6 +1637,39 @@ def _make_objs_dict(key_names): objs_dict = {'Objects': objs_list} return objs_dict +def test_versioning_concurrent_multi_object_delete(): + num_objects = 5 + num_threads = 5 + bucket_name = get_new_bucket() + + check_configure_versioning_retry(bucket_name, "Enabled", "Enabled") + + key_names = ["key_{:d}".format(x) for x in range(num_objects)] + bucket = _create_objects(bucket_name=bucket_name, keys=key_names) + + client = get_client() + versions = client.list_object_versions(Bucket=bucket_name)['Versions'] + assert len(versions) == num_objects + objs_dict = {'Objects': [dict((k, v[k]) for k in ["Key", "VersionId"]) for v in versions]} + results = [None] * num_threads + + def do_request(n): + results[n] = client.delete_objects(Bucket=bucket_name, Delete=objs_dict) + + t = [] + for i in range(num_threads): + thr = threading.Thread(target = do_request, args=[i]) + thr.start() + t.append(thr) + _do_wait_completion(t) + + for response in results: + assert len(response['Deleted']) == num_objects + assert 'Errors' not in response + + response = client.list_objects(Bucket=bucket_name) + assert 'Contents' not in response + def test_multi_object_delete(): key_names = ['key0', 'key1', 'key2'] bucket_name = _create_objects(keys=key_names) @@ -2818,6 +2864,53 @@ def test_post_object_upload_size_below_minimum(): r = requests.post(url, files=payload, verify=get_config_ssl_verify()) assert r.status_code == 400 +def test_post_object_upload_size_rgw_chunk_size_bug(): + # Test for https://tracker.ceph.com/issues/58627 + # TODO: if this value is different in Teuthology runs, this would need tuning + # https://github.com/ceph/ceph/blob/main/qa/suites/rgw/verify/striping%24/stripe-greater-than-chunk.yaml + _rgw_max_chunk_size = 4 * 2**20 # 4MiB + min_size = _rgw_max_chunk_size + max_size = _rgw_max_chunk_size * 3 + # [(chunk),(small)] + test_payload_size = _rgw_max_chunk_size + 200 # extra bit to push it over the chunk boundary + # it should be valid when we run this test! + assert test_payload_size > min_size + assert test_payload_size < max_size + + bucket_name = get_new_bucket() + client = get_client() + + url = _get_post_url(bucket_name) + utc = pytz.utc + expires = datetime.datetime.now(utc) + datetime.timedelta(seconds=+6000) + + policy_document = {"expiration": expires.strftime("%Y-%m-%dT%H:%M:%SZ"),\ + "conditions": [\ + {"bucket": bucket_name},\ + ["starts-with", "$key", "foo"],\ + {"acl": "private"},\ + ["starts-with", "$Content-Type", "text/plain"],\ + ["content-length-range", min_size, max_size],\ + ]\ + } + + test_payload = 'x' * test_payload_size + + json_policy_document = json.JSONEncoder().encode(policy_document) + bytes_json_policy_document = bytes(json_policy_document, 'utf-8') + policy = base64.b64encode(bytes_json_policy_document) + aws_secret_access_key = get_main_aws_secret_key() + aws_access_key_id = get_main_aws_access_key() + + signature = base64.b64encode(hmac.new(bytes(aws_secret_access_key, 'utf-8'), policy, hashlib.sha1).digest()) + + payload = OrderedDict([ ("key" , "foo.txt"),("AWSAccessKeyId" , aws_access_key_id),\ + ("acl" , "private"),("signature" , signature),("policy" , policy),\ + ("Content-Type" , "text/plain"),('file', (test_payload))]) + + r = requests.post(url, files=payload, verify=get_config_ssl_verify()) + assert r.status_code == 204 + def test_post_object_empty_conditions(): bucket_name = get_new_bucket() client = get_client() @@ -7412,20 +7505,17 @@ def test_versioning_multi_object_delete(): num_versions = 2 (version_ids, contents) = create_multiple_versions(client, bucket_name, key, num_versions) + assert len(version_ids) == 2 - response = client.list_object_versions(Bucket=bucket_name) - versions = response['Versions'] - versions.reverse() - - for version in versions: - client.delete_object(Bucket=bucket_name, Key=key, VersionId=version['VersionId']) + # delete both versions + objects = [{'Key': key, 'VersionId': v} for v in version_ids] + client.delete_objects(Bucket=bucket_name, Delete={'Objects': objects}) response = client.list_object_versions(Bucket=bucket_name) assert not 'Versions' in response # now remove again, should all succeed due to idempotency - for version in versions: - client.delete_object(Bucket=bucket_name, Key=key, VersionId=version['VersionId']) + client.delete_objects(Bucket=bucket_name, Delete={'Objects': objects}) response = client.list_object_versions(Bucket=bucket_name) assert not 'Versions' in response @@ -7440,33 +7530,24 @@ def test_versioning_multi_object_delete_with_marker(): num_versions = 2 (version_ids, contents) = create_multiple_versions(client, bucket_name, key, num_versions) + assert len(version_ids) == num_versions + objects = [{'Key': key, 'VersionId': v} for v in version_ids] - client.delete_object(Bucket=bucket_name, Key=key) - response = client.list_object_versions(Bucket=bucket_name) - versions = response['Versions'] - delete_markers = response['DeleteMarkers'] + # create a delete marker + response = client.delete_object(Bucket=bucket_name, Key=key) + assert response['DeleteMarker'] + objects += [{'Key': key, 'VersionId': response['VersionId']}] - version_ids.append(delete_markers[0]['VersionId']) - assert len(version_ids) == 3 - assert len(delete_markers) == 1 - - for version in versions: - client.delete_object(Bucket=bucket_name, Key=key, VersionId=version['VersionId']) - - for delete_marker in delete_markers: - client.delete_object(Bucket=bucket_name, Key=key, VersionId=delete_marker['VersionId']) + # delete all versions + client.delete_objects(Bucket=bucket_name, Delete={'Objects': objects}) response = client.list_object_versions(Bucket=bucket_name) assert not 'Versions' in response assert not 'DeleteMarkers' in response - for version in versions: - client.delete_object(Bucket=bucket_name, Key=key, VersionId=version['VersionId']) - - for delete_marker in delete_markers: - client.delete_object(Bucket=bucket_name, Key=key, VersionId=delete_marker['VersionId']) - # now remove again, should all succeed due to idempotency + client.delete_objects(Bucket=bucket_name, Delete={'Objects': objects}) + response = client.list_object_versions(Bucket=bucket_name) assert not 'Versions' in response assert not 'DeleteMarkers' in response @@ -7480,8 +7561,11 @@ def test_versioning_multi_object_delete_with_marker_create(): key = 'key' - response = client.delete_object(Bucket=bucket_name, Key=key) - delete_marker_version_id = response['VersionId'] + # use delete_objects() to create a delete marker + response = client.delete_objects(Bucket=bucket_name, Delete={'Objects': [{'Key': key}]}) + assert len(response['Deleted']) == 1 + assert response['Deleted'][0]['DeleteMarker'] + delete_marker_version_id = response['Deleted'][0]['DeleteMarkerVersionId'] response = client.list_object_versions(Bucket=bucket_name) delete_markers = response['DeleteMarkers'] @@ -8299,10 +8383,11 @@ def test_lifecycle_expiration_header_tags_head(): # stat the object, check header response = client.head_object(Bucket=bucket_name, Key=key1) assert response['ResponseMetadata']['HTTPStatusCode'] == 200 - assert check_lifecycle_expiration_header(response, datetime.datetime.now(None), 'rule1', 1) + assert check_lifecycle_expiration_header(response, datetime.datetime.now(None), 'rule1', 1) == False @pytest.mark.lifecycle @pytest.mark.lifecycle_expiration +@pytest.mark.fails_on_dbstore def test_lifecycle_expiration_header_and_tags_head(): now = datetime.datetime.now(None) bucket_name = get_new_bucket() @@ -8344,7 +8429,7 @@ def test_lifecycle_expiration_header_and_tags_head(): # stat the object, check header response = client.head_object(Bucket=bucket_name, Key=key1) assert response['ResponseMetadata']['HTTPStatusCode'] == 200 - assert check_lifecycle_expiration_header(response, datetime.datetime.now(None), 'rule1', 1) + assert check_lifecycle_expiration_header(response, datetime.datetime.now(None), 'rule1', 1) == False @pytest.mark.lifecycle def test_lifecycle_set_noncurrent(): @@ -12594,6 +12679,7 @@ def test_sse_s3_default_multipart_upload(): assert response['Metadata'] == metadata assert response['ResponseMetadata']['HTTPHeaders']['content-type'] == content_type + assert response['ResponseMetadata']['HTTPHeaders']['x-amz-server-side-encryption'] == 'AES256' body = _get_body(response) assert body == data @@ -12738,3 +12824,24 @@ def test_sse_s3_encrypted_upload_1mb(): @pytest.mark.fails_on_dbstore def test_sse_s3_encrypted_upload_8mb(): _test_sse_s3_encrypted_upload(8*1024*1024) + +def test_get_object_torrent(): + client = get_client() + bucket_name = get_new_bucket() + key = 'Avatar.mpg' + + file_size = 7 * 1024 * 1024 + data = 'A' * file_size + + client.put_object(Bucket=bucket_name, Key=key, Body=data) + + response = None + try: + response = client.get_object_torrent(Bucket=bucket_name, Key=key) + # if successful, verify the torrent contents are different from the body + assert data != _get_body(response) + except ClientError as e: + # accept 404 errors - torrent support may not be configured + status, error_code = _get_status_and_error_code(e.response) + assert status == 404 + assert error_code == 'NoSuchKey' diff --git a/s3tests_boto3/functional/test_s3select.py b/s3tests_boto3/functional/test_s3select.py index 5c18073..1ce4fa3 100644 --- a/s3tests_boto3/functional/test_s3select.py +++ b/s3tests_boto3/functional/test_s3select.py @@ -403,6 +403,7 @@ def test_count_operation(): def test_count_json_operation(): json_obj_name = get_random_string() bucket_name = get_new_bucket_name() + num_of_rows = 1 obj_to_load = create_random_json_object(num_of_rows,10) upload_object(bucket_name,json_obj_name,obj_to_load) @@ -425,6 +426,7 @@ def test_json_column_sum_min_max(): json_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,json_obj_name,json_obj) json_obj_name_2 = get_random_string() @@ -491,6 +493,7 @@ def test_json_nullif_expressions(): json_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,json_obj_name,json_obj) res_s3select_nullif = remove_xml_tags_from_result( run_s3select_json(bucket_name,json_obj_name,"select count(0) from s3object[*].root where nullif(_1.c1,_1.c2) is null ;") ).replace("\n","") @@ -530,6 +533,7 @@ def test_column_sum_min_max(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) csv_obj_name_2 = get_random_string() @@ -596,6 +600,7 @@ def test_nullif_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where nullif(_1,_2) is null ;") ).replace("\n","") @@ -651,6 +656,7 @@ def test_nulliftrue_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from s3object where (nullif(_1,_2) is null) = true ;") ).replace("\n","") @@ -678,6 +684,7 @@ def test_is_not_null_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_null = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(*) from s3object where nullif(_1,_2) is not null ;") ).replace("\n","") @@ -699,6 +706,7 @@ def test_lowerupper_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select lower("AB12cd$$") from s3object ;') ).replace("\n","") @@ -717,6 +725,7 @@ def test_in_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where int(_1) in(1);')).replace("\n","") @@ -786,6 +795,7 @@ def test_true_false_in_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from s3object where (int(_1) in(1)) = true;')).replace("\n","") @@ -831,6 +841,7 @@ def test_like_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where _1 like "%aeio%";')).replace("\n","") @@ -918,6 +929,7 @@ def test_truefalselike_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_like = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where (_1 like "%aeio%") = true;')).replace("\n","") @@ -963,6 +975,7 @@ def test_nullif_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_nullif = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select count(0) from stdin where nullif(_1,_2) is null ;") ).replace("\n","") @@ -990,6 +1003,7 @@ def test_lowerupper_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select lower("AB12cd$$") from stdin ;') ).replace("\n","") @@ -1008,6 +1022,7 @@ def test_in_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select int(_1) from stdin where int(_1) in(1);')).replace("\n","") @@ -1047,6 +1062,7 @@ def test_like_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_in = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from stdin where _1 like "%aeio%";')).replace("\n","") @@ -1094,6 +1110,7 @@ def test_complex_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select min(int(_1)),max(int(_2)),min(int(_3))+1 from s3object;")).replace("\n","") @@ -1130,6 +1147,7 @@ def test_alias(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select int(_1) as a1, int(_2) as a2 , (a1+a2) as a3 from s3object where a3>100 and a3<300;") ).replace(",","") @@ -1149,6 +1167,7 @@ def test_alias_cyclic_refernce(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_alias = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,"select int(_1) as a1,int(_2) as a2, a1+a4 as a3, a5+a1 as a4, int(_3)+a3 as a5 from s3object;") ) @@ -1334,6 +1353,7 @@ def test_when_then_else_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select case when cast(_1 as int)>100 and cast(_1 as int)<200 then "(100-200)" when cast(_1 as int)>200 and cast(_1 as int)<300 then "(200-300)" else "NONE" end from s3object;') ).replace("\n","") @@ -1363,6 +1383,7 @@ def test_coalesce_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where char_length(_3)>2 and char_length(_4)>2 and cast(substring(_3,1,2) as int) = cast(substring(_4,1,2) as int);') ).replace("\n","") @@ -1385,6 +1406,7 @@ def test_cast_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where cast(_3 as int)>999;') ).replace("\n","") @@ -1424,6 +1446,7 @@ def test_trim_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(_1) = "aeiou";')).replace("\n","") @@ -1463,6 +1486,7 @@ def test_truefalse_trim_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_trim = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where trim(_1) = "aeiou" = true;')).replace("\n","") @@ -1502,6 +1526,7 @@ def test_escape_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_escape = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where _1 like "%_ar" escape "%";')).replace("\n","") @@ -1523,6 +1548,7 @@ def test_case_value_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_case = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select case cast(_1 as int) when cast(_2 as int) then "case_1_1" else "case_2_2" end from s3object;')).replace("\n","") @@ -1538,6 +1564,7 @@ def test_bool_cast_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_cast = remove_xml_tags_from_result( run_s3select(bucket_name,csv_obj_name,'select count(*) from s3object where cast(int(_1) as bool) = true ;')).replace("\n","") @@ -1553,6 +1580,7 @@ def test_progress_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) obj_size = len(csv_obj.encode('utf-8')) @@ -1586,6 +1614,7 @@ def test_output_serial_expressions(): csv_obj_name = get_random_string() bucket_name = get_new_bucket_name() + upload_object(bucket_name,csv_obj_name,csv_obj) res_s3select_1 = remove_xml_tags_from_result( run_s3select_output(bucket_name,csv_obj_name,"select _1, _2 from s3object where nullif(_1,_2) is null ;", "ALWAYS") ).replace("\n",",").replace(",","") diff --git a/s3tests_boto3/functional/test_sts.py b/s3tests_boto3/functional/test_sts.py index 0229dbd..8969167 100644 --- a/s3tests_boto3/functional/test_sts.py +++ b/s3tests_boto3/functional/test_sts.py @@ -56,6 +56,7 @@ log = logging.getLogger(__name__) def create_role(iam_client,path,rolename,policy_document,description,sessionduration,permissionboundary,tag_list=None): role_err=None + role_response = None if rolename is None: rolename=get_parameter_name() if tag_list is None: @@ -68,6 +69,7 @@ def create_role(iam_client,path,rolename,policy_document,description,sessiondura def put_role_policy(iam_client,rolename,policyname,role_policy): role_err=None + role_response = None if policyname is None: policyname=get_parameter_name() try: @@ -78,6 +80,7 @@ def put_role_policy(iam_client,rolename,policyname,role_policy): def put_user_policy(iam_client,username,policyname,policy_document): role_err=None + role_response = None if policyname is None: policyname=get_parameter_name() try: @@ -222,11 +225,17 @@ def test_assume_role_allow(): policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/"+sts_user_id+"\"]},\"Action\":[\"sts:AssumeRole\"]}]}" (role_error,role_response,general_role_name)=create_role(iam_client,'/',None,policy_document,None,None,None) - assert role_response['Role']['Arn'] == 'arn:aws:iam:::role/'+general_role_name+'' + if role_response: + assert role_response['Role']['Arn'] == 'arn:aws:iam:::role/'+general_role_name+'' + else: + assert False, role_error role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}}" (role_err,response)=put_role_policy(iam_client,general_role_name,None,role_policy) - assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + if response: + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + else: + assert False, role_err resp=sts_client.assume_role(RoleArn=role_response['Role']['Arn'],RoleSessionName=role_session_name) assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 @@ -256,11 +265,17 @@ def test_assume_role_deny(): policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/"+sts_user_id+"\"]},\"Action\":[\"sts:AssumeRole\"]}]}" (role_error,role_response,general_role_name)=create_role(iam_client,'/',None,policy_document,None,None,None) - assert role_response['Role']['Arn'] == 'arn:aws:iam:::role/'+general_role_name+'' + if role_response: + assert role_response['Role']['Arn'] == 'arn:aws:iam:::role/'+general_role_name+'' + else: + assert False, role_error role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Deny\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}}" (role_err,response)=put_role_policy(iam_client,general_role_name,None,role_policy) - assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + if response: + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + else: + assert False, role_err resp=sts_client.assume_role(RoleArn=role_response['Role']['Arn'],RoleSessionName=role_session_name) assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 @@ -290,11 +305,17 @@ def test_assume_role_creds_expiry(): policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/"+sts_user_id+"\"]},\"Action\":[\"sts:AssumeRole\"]}]}" (role_error,role_response,general_role_name)=create_role(iam_client,'/',None,policy_document,None,None,None) - assert role_response['Role']['Arn'] == 'arn:aws:iam:::role/'+general_role_name+'' + if role_response: + assert role_response['Role']['Arn'] == 'arn:aws:iam:::role/'+general_role_name+'' + else: + assert False, role_error role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}}" (role_err,response)=put_role_policy(iam_client,general_role_name,None,role_policy) - assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + if response: + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + else: + assert False, role_err resp=sts_client.assume_role(RoleArn=role_response['Role']['Arn'],RoleSessionName=role_session_name,DurationSeconds=900) assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 @@ -329,12 +350,18 @@ def test_assume_role_deny_head_nonexistent(): policy_document = '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/'+sts_user_id+'"]},"Action":["sts:AssumeRole"]}]}' (role_error,role_response,general_role_name)=create_role(iam_client,'/',None,policy_document,None,None,None) - assert role_response['Role']['Arn'] == 'arn:aws:iam:::role/'+general_role_name + if role_response: + assert role_response['Role']['Arn'] == 'arn:aws:iam:::role/'+general_role_name + else: + assert False, role_error # allow GetObject but deny ListBucket role_policy = '{"Version":"2012-10-17","Statement":{"Effect":"Allow","Action":"s3:GetObject","Principal":"*","Resource":"arn:aws:s3:::*"}}' (role_err,response)=put_role_policy(iam_client,general_role_name,None,role_policy) - assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + if response: + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + else: + assert False, role_err resp=sts_client.assume_role(RoleArn=role_response['Role']['Arn'],RoleSessionName=role_session_name) assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 @@ -367,12 +394,18 @@ def test_assume_role_allow_head_nonexistent(): policy_document = '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/'+sts_user_id+'"]},"Action":["sts:AssumeRole"]}]}' (role_error,role_response,general_role_name)=create_role(iam_client,'/',None,policy_document,None,None,None) - assert role_response['Role']['Arn'] == 'arn:aws:iam:::role/'+general_role_name + if role_response: + assert role_response['Role']['Arn'] == 'arn:aws:iam:::role/'+general_role_name + else: + assert False, role_error # allow GetObject and ListBucket role_policy = '{"Version":"2012-10-17","Statement":{"Effect":"Allow","Action":["s3:GetObject","s3:ListBucket"],"Principal":"*","Resource":"arn:aws:s3:::*"}}' (role_err,response)=put_role_policy(iam_client,general_role_name,None,role_policy) - assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + if response: + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + else: + assert False, role_err resp=sts_client.assume_role(RoleArn=role_response['Role']['Arn'],RoleSessionName=role_session_name) assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 @@ -418,7 +451,10 @@ def test_assume_role_with_web_identity(): role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}}" (role_err,response)=put_role_policy(iam_client,general_role_name,None,role_policy) - assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + if response: + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + else: + assert False, role_err resp=sts_client.assume_role_with_web_identity(RoleArn=role_response['Role']['Arn'],RoleSessionName=role_session_name,WebIdentityToken=token) assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 diff --git a/tox.ini b/tox.ini index 93fde38..e4a30e5 100644 --- a/tox.ini +++ b/tox.ini @@ -3,5 +3,7 @@ envlist = py [testenv] deps = -rrequirements.txt -passenv = S3TEST_CONF S3_USE_SIGV4 +passenv = + S3TEST_CONF + S3_USE_SIGV4 commands = pytest {posargs}