From d92031ef1688fb86fafc2c2fa4bd01fbb57d2be1 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Thu, 2 Jun 2016 18:13:56 +0200 Subject: [PATCH 1/7] Additional tests for server side encryption, S3 SSE-C mode. All tests belong to group 'encryption'. Signed-off-by: Adam Kupczyk --- s3tests/functional/test_s3.py | 342 +++++++++++++++++++++++++++++++++- 1 file changed, 338 insertions(+), 4 deletions(-) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 0bf900f..5c0c272 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -5119,14 +5119,14 @@ def test_object_copy_versioning_multipart_upload(): got = key6.get_contents_as_string() eq(got, data) -def transfer_part(bucket, mp_id, mp_keyname, i, part): +def transfer_part(bucket, mp_id, mp_keyname, i, part, headers=None): """Transfer a part of a multipart upload. Designed to be run in parallel. """ mp = boto.s3.multipart.MultiPartUpload(bucket) mp.key_name = mp_keyname mp.id = mp_id part_out = StringIO(part) - mp.upload_part_from_file(part_out, i+1) + mp.upload_part_from_file(part_out, i+1, headers=headers) def copy_part(src_bucket, src_keyname, dst_bucket, dst_keyname, mp_id, i, start=None, end=None, src_version_id=None): """Copy a part of a multipart upload from other bucket. @@ -5167,9 +5167,9 @@ def _multipart_upload(bucket, s3_key_name, size, part_size=5*1024*1024, do_list= s = '' for i, part in enumerate(generate_random(size, part_size)): s += part - transfer_part(bucket, upload.id, upload.key_name, i, part) + transfer_part(bucket, upload.id, upload.key_name, i, part, headers) if i in resend_parts: - transfer_part(bucket, upload.id, upload.key_name, i, part) + transfer_part(bucket, upload.id, upload.key_name, i, part, headers) if do_list is not None: l = bucket.list_multipart_uploads() @@ -7533,3 +7533,337 @@ def test_lifecycle_rules_conflicted(): eq(e.status, 400) eq(e.error_code, 'InvalidRequest') +def _test_encryption_sse_customer_write(file_size): + """ + Tests Create a file of A's, use it to set_contents_from_file. + Create a file of B's, use it to re-set_contents_from_file. + Re-read the contents, and confirm we get B's + """ + bucket = get_new_bucket() + sse_client_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==' + } + key = bucket.new_key('testobj') + data = 'A'*file_size + key.set_contents_from_string(data, headers=sse_client_headers) + rdata = key.get_contents_as_string(headers=sse_client_headers) + eq(data, rdata) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-C encrypted transfer 1 byte') +@attr(assertion='success') +@attr('encryption') +def test_encrypted_transfer_1b(): + _test_encryption_sse_customer_write(1) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-C encrypted transfer 1KB') +@attr(assertion='success') +@attr('encryption') +def test_encrypted_transfer_1kb(): + _test_encryption_sse_customer_write(1024) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-C encrypted transfer 1MB') +@attr(assertion='success') +@attr('encryption') +def test_encrypted_transfer_1MB(): + _test_encryption_sse_customer_write(1024*1024) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-C encrypted transfer 13 bytes') +@attr(assertion='success') +@attr('encryption') +def test_encrypted_transfer_13b(): + _test_encryption_sse_customer_write(13) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='write encrypted with SSE-C and read without SSE-C') +@attr(assertion='operation fails') +@attr('encryption') +def test_encryption_sse_c_present(): + bucket = get_new_bucket() + sse_client_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==' + } + key = bucket.new_key('testobj') + data = 'A'*100 + key.set_contents_from_string(data, headers=sse_client_headers) + e = assert_raises(boto.exception.S3ResponseError, key.get_contents_as_string) + eq(e.status, 400) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='write encrypted with SSE-C but read with other key') +@attr(assertion='operation fails') +@attr('encryption') +def test_encryption_sse_c_other_key(): + bucket = get_new_bucket() + sse_client_headers_A = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==' + } + sse_client_headers_B = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': '6b+WOZ1T3cqZMxgThRcXAQBrS5mXKdDUphvpxptl9/4=', + 'x-amz-server-side-encryption-customer-key-md5': 'arxBvwY2V4SiOne6yppVPQ==' + } + key = bucket.new_key('testobj') + data = 'A'*100 + key.set_contents_from_string(data, headers=sse_client_headers_A) + e = assert_raises(boto.exception.S3ResponseError, + key.get_contents_as_string, headers=sse_client_headers_B) + eq(e.status, 400) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='write encrypted with SSE-C, but md5 is bad') +@attr(assertion='operation fails') +@attr('encryption') +def test_encryption_sse_c_invalid_md5(): + bucket = get_new_bucket() + sse_client_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'AAAAAAAAAAAAAAAAAAAAAA==' + } + key = bucket.new_key('testobj') + data = 'A'*100 + e = assert_raises(boto.exception.S3ResponseError, + key.set_contents_from_string, data, headers=sse_client_headers) + eq(e.status, 400) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='write encrypted with SSE-C, but dont provide MD5') +@attr(assertion='operation fails') +@attr('encryption') +def test_encryption_sse_c_no_md5(): + bucket = get_new_bucket() + sse_client_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=' + } + key = bucket.new_key('testobj') + data = 'A'*100 + e = assert_raises(boto.exception.S3ResponseError, + key.set_contents_from_string, data, headers=sse_client_headers) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='declare SSE-C but do not provide key') +@attr(assertion='operation fails') +@attr('encryption') +def test_encryption_sse_c_no_key(): + bucket = get_new_bucket() + sse_client_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256' + } + key = bucket.new_key('testobj') + data = 'A'*100 + e = assert_raises(boto.exception.S3ResponseError, + key.set_contents_from_string, data, headers=sse_client_headers) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Do not declare SSE-C but provide key and MD5') +@attr(assertion='operation successfull, no encryption') +@attr('encryption') +def test_encryption_key_no_sse_c(): + bucket = get_new_bucket() + sse_client_headers = { + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==' + } + key = bucket.new_key('testobj') + data = 'A'*100 + key.set_contents_from_string(data, headers=sse_client_headers) + rdata = key.get_contents_as_string() + eq(data, rdata) + + +def _multipart_upload_enc(bucket, s3_key_name, size, part_size=5*1024*1024, + do_list=None, init_headers=None, part_headers=None, + metadata=None, resend_parts=[]): + """ + generate a multi-part upload for a random file of specifed size, + if requested, generate a list of the parts + return the upload descriptor + """ + upload = bucket.initiate_multipart_upload(s3_key_name, headers=init_headers, metadata=metadata) + s = '' + for i, part in enumerate(generate_random(size, part_size)): + s += part + transfer_part(bucket, upload.id, upload.key_name, i, part, part_headers) + if i in resend_parts: + transfer_part(bucket, upload.id, upload.key_name, i, part, part_headers) + + if do_list is not None: + l = bucket.list_multipart_uploads() + l = list(l) + + return (upload, s) + + +def _check_content_using_range_enc(k, data, step, enc_headers=None): + objlen = k.size + for ofs in xrange(0, k.size, step): + toread = k.size - ofs + if toread > step: + toread = step + end = ofs + toread - 1 + read_range = k.get_contents_as_string( + headers=dict({'Range': 'bytes={s}-{e}'.format(s=ofs, e=end)}, **enc_headers)) + eq(len(read_range), toread) + eq(read_range, data[ofs:end+1]) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='complete multi-part upload') +@attr(assertion='successful') +@attr('encryption') +def test_encryption_sse_c_multipart_upload(): + bucket = get_new_bucket() + key = "multipart_enc" + content_type = 'text/plain' + objlen = 30 * 1024 * 1024 + enc_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==', + 'Content-Type': content_type + } + (upload, data) = _multipart_upload_enc(bucket, key, objlen, + init_headers=enc_headers, part_headers=enc_headers, + metadata={'foo': 'bar'}) + upload.complete_upload() + result = _head_bucket(bucket) + + eq(result.get('x-rgw-object-count', 1), 1) + eq(result.get('x-rgw-bytes-used', 30 * 1024 * 1024), 30 * 1024 * 1024) + + k = bucket.get_key(key) + eq(k.metadata['foo'], 'bar') + eq(k.content_type, content_type) + test_string = k.get_contents_as_string(headers=enc_headers) + eq(len(test_string), k.size) + eq(data, test_string) + eq(test_string, data) + + _check_content_using_range_enc(k, data, 1000000, enc_headers=enc_headers) + _check_content_using_range_enc(k, data, 10000000, enc_headers=enc_headers) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='multipart upload with bad key for uploading chunks') +@attr(assertion='successful') +@attr('encryption') +def test_encryption_sse_c_multipart_invalid_chunks_1(): + bucket = get_new_bucket() + key = "multipart_enc" + content_type = 'text/bla' + objlen = 30 * 1024 * 1024 + init_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==', + 'Content-Type': content_type + } + part_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': '6b+WOZ1T3cqZMxgThRcXAQBrS5mXKdDUphvpxptl9/4=', + 'x-amz-server-side-encryption-customer-key-md5': 'arxBvwY2V4SiOne6yppVPQ==' + } + e = assert_raises(boto.exception.S3ResponseError, + _multipart_upload_enc, bucket, key, objlen, + init_headers=init_headers, part_headers=part_headers, + metadata={'foo': 'bar'}) + eq(e.status, 400) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='multipart upload with bad md5 for chunks') +@attr(assertion='successful') +@attr('encryption') +def test_encryption_sse_c_multipart_invalid_chunks_2(): + bucket = get_new_bucket() + key = "multipart_enc" + content_type = 'text/plain' + objlen = 30 * 1024 * 1024 + init_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==', + 'Content-Type': content_type + } + part_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'AAAAAAAAAAAAAAAAAAAAAA==' + } + e = assert_raises(boto.exception.S3ResponseError, + _multipart_upload_enc, bucket, key, objlen, + init_headers=init_headers, part_headers=part_headers, + metadata={'foo': 'bar'}) + eq(e.status, 400) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='complete multi-part upload and download with bad key') +@attr(assertion='successful') +@attr('encryption') +def test_encryption_sse_c_multipart_bad_download(): + bucket = get_new_bucket() + key = "multipart_enc" + content_type = 'text/plain' + objlen = 30 * 1024 * 1024 + put_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==', + 'Content-Type': content_type + } + get_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': '6b+WOZ1T3cqZMxgThRcXAQBrS5mXKdDUphvpxptl9/4=', + 'x-amz-server-side-encryption-customer-key-md5': 'arxBvwY2V4SiOne6yppVPQ==' + } + + (upload, data) = _multipart_upload_enc(bucket, key, objlen, + init_headers=put_headers, part_headers=put_headers, + metadata={'foo': 'bar'}) + upload.complete_upload() + result = _head_bucket(bucket) + + eq(result.get('x-rgw-object-count', 1), 1) + eq(result.get('x-rgw-bytes-used', 30 * 1024 * 1024), 30 * 1024 * 1024) + + k = bucket.get_key(key) + eq(k.metadata['foo'], 'bar') + eq(k.content_type, content_type) + e = assert_raises(boto.exception.S3ResponseError, + k.get_contents_as_string, headers=get_headers) From f8761f855b393f8c8a338f0a72db3df97ec9e606 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Mon, 11 Jul 2016 15:43:52 +0200 Subject: [PATCH 2/7] Additional tests for server side encryption, S3 SSE-KMS mode. All tests belong to group 'encryption'. Signed-off-by: Adam Kupczyk --- s3tests/functional/test_s3.py | 312 ++++++++++++++++++++++++++++++++++ 1 file changed, 312 insertions(+) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 5c0c272..3f3f5f5 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -7867,3 +7867,315 @@ def test_encryption_sse_c_multipart_bad_download(): eq(k.content_type, content_type) e = assert_raises(boto.exception.S3ResponseError, k.get_contents_as_string, headers=get_headers) + + +@attr(resource='object') +@attr(method='post') +@attr(operation='authenticated browser based upload via POST request') +@attr(assertion='succeeds and returns written data') +@attr('encryption') +def test_encryption_sse_c_post_object_authenticated_request(): + bucket = get_new_bucket() + + url = _get_post_url(s3.main, bucket) + + utc = pytz.utc + expires = datetime.datetime.now(utc) + datetime.timedelta(seconds=+6000) + + policy_document = {"expiration": expires.strftime("%Y-%m-%dT%H:%M:%SZ"), \ + "conditions": [ \ + {"bucket": bucket.name}, \ + ["starts-with", "$key", "foo"], \ + {"acl": "private"}, \ + ["starts-with", "$Content-Type", "text/plain"], \ + ["starts-with", "$x-amz-server-side-encryption-customer-algorithm", ""], \ + ["starts-with", "$x-amz-server-side-encryption-customer-key", ""], \ + ["starts-with", "$x-amz-server-side-encryption-customer-key-md5", ""], \ + ["content-length-range", 0, 1024] \ + ] \ + } + + json_policy_document = json.JSONEncoder().encode(policy_document) + policy = base64.b64encode(json_policy_document) + conn = s3.main + signature = base64.b64encode(hmac.new(conn.aws_secret_access_key, policy, sha).digest()) + + payload = OrderedDict([ ("key" , "foo.txt"),("AWSAccessKeyId" , conn.aws_access_key_id), \ + ("acl" , "private"),("signature" , signature),("policy" , policy), \ + ("Content-Type" , "text/plain"), \ + ('x-amz-server-side-encryption-customer-algorithm', 'AES256'), \ + ('x-amz-server-side-encryption-customer-key', 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs='), \ + ('x-amz-server-side-encryption-customer-key-md5', 'DWygnHRtgiJ77HCm+1rvHw=='), \ + ('file', ('bar'),), ]) + + r = requests.post(url, files = payload) + eq(r.status_code, 204) + get_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==' + } + + key = bucket.get_key("foo.txt") + got = key.get_contents_as_string(headers=get_headers) + eq(got, 'bar') + + +def _test_sse_kms_customer_write(file_size): + """ + Tests Create a file of A's, use it to set_contents_from_file. + Create a file of B's, use it to re-set_contents_from_file. + Re-read the contents, and confirm we get B's + """ + bucket = get_new_bucket() + sse_kms_client_headers = { + 'x-amz-server-side-encryption': 'aws:kms', + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-1' + } + key = bucket.new_key('testobj') + data = 'A'*file_size + key.set_contents_from_string(data, headers=sse_kms_client_headers) + rdata = key.get_contents_as_string(headers=sse_kms_client_headers) + eq(data, rdata) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-KMS encrypted transfer 1 byte') +@attr(assertion='success') +@attr('encryption') +def test_sse_kms_transfer_1b(): + _test_sse_kms_customer_write(1) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-KMS encrypted transfer 1KB') +@attr(assertion='success') +@attr('encryption') +def test_sse_kms_transfer_1kb(): + _test_sse_kms_customer_write(1024) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-KMS encrypted transfer 1MB') +@attr(assertion='success') +@attr('encryption') +def test_sse_kms_transfer_1MB(): + _test_sse_kms_customer_write(1024*1024) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-KMS encrypted transfer 13 bytes') +@attr(assertion='success') +@attr('encryption') +def test_sse_kms_transfer_13b(): + _test_sse_kms_customer_write(13) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='write encrypted with SSE-KMS and read without SSE-KMS') +@attr(assertion='operation success') +@attr('encryption') +def test_sse_kms_present(): + bucket = get_new_bucket() + sse_kms_client_headers = { + 'x-amz-server-side-encryption': 'aws:kms', + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-1' + } + key = bucket.new_key('testobj') + data = 'A'*100 + key.set_contents_from_string(data, headers=sse_kms_client_headers) + result = key.get_contents_as_string() + eq(data, result) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='write encrypted with SSE-KMS but read with other key') +@attr(assertion='operation fails') +@attr('encryption') +def test_sse_kms_other_key(): + bucket = get_new_bucket() + sse_kms_client_headers_A = { + 'x-amz-server-side-encryption': 'aws:kms', + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-1' + } + sse_kms_client_headers_B = { + 'x-amz-server-side-encryption': 'aws:kms', + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-2' + } + key = bucket.new_key('testobj') + data = 'A'*100 + key.set_contents_from_string(data, headers=sse_kms_client_headers_A) + result = key.get_contents_as_string(headers=sse_kms_client_headers_B) + eq(data, result) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='declare SSE-KMS but do not provide key_id') +@attr(assertion='operation fails') +@attr('encryption') +def test_sse_kms_no_key(): + bucket = get_new_bucket() + sse_kms_client_headers = { + 'x-amz-server-side-encryption': 'aws:kms' + } + key = bucket.new_key('testobj') + data = 'A'*100 + e = assert_raises(boto.exception.S3ResponseError, + key.set_contents_from_string, data, headers=sse_kms_client_headers) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Do not declare SSE-KMS but provide key_id') +@attr(assertion='operation successfull, no encryption') +@attr('encryption') +def test_sse_kms_not_declared(): + bucket = get_new_bucket() + sse_kms_client_headers = { + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-2' + } + key = bucket.new_key('testobj') + data = 'A'*100 + key.set_contents_from_string(data, headers=sse_kms_client_headers) + rdata = key.get_contents_as_string() + eq(data, rdata) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='complete KMS multi-part upload') +@attr(assertion='successful') +@attr('encryption') +def test_sse_kms_multipart_upload(): + bucket = get_new_bucket() + key = "multipart_enc" + content_type = 'text/plain' + objlen = 30 * 1024 * 1024 + enc_headers = { + 'x-amz-server-side-encryption': 'aws:kms', + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-2', + 'Content-Type': content_type + } + (upload, data) = _multipart_upload_enc(bucket, key, objlen, + init_headers=enc_headers, part_headers=enc_headers, + metadata={'foo': 'bar'}) + upload.complete_upload() + result = _head_bucket(bucket) + + eq(result.get('x-rgw-object-count', 1), 1) + eq(result.get('x-rgw-bytes-used', 30 * 1024 * 1024), 30 * 1024 * 1024) + + k = bucket.get_key(key) + eq(k.metadata['foo'], 'bar') + eq(k.content_type, content_type) + test_string = k.get_contents_as_string(headers=enc_headers) + eq(len(test_string), k.size) + eq(data, test_string) + eq(test_string, data) + + _check_content_using_range_enc(k, data, 1000000, enc_headers=enc_headers) + _check_content_using_range_enc(k, data, 10000000, enc_headers=enc_headers) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='multipart KMS upload with bad key_id for uploading chunks') +@attr(assertion='successful') +@attr('encryption') +def test_sse_kms_multipart_invalid_chunks_1(): + bucket = get_new_bucket() + key = "multipart_enc" + content_type = 'text/bla' + objlen = 30 * 1024 * 1024 + init_headers = { + 'x-amz-server-side-encryption': 'aws:kms', + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-1', + 'Content-Type': content_type + } + part_headers = { + 'x-amz-server-side-encryption': 'aws:kms', + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-2' + } + _multipart_upload_enc(bucket, key, objlen, + init_headers=init_headers, part_headers=part_headers, + metadata={'foo': 'bar'}) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='multipart KMS upload with unexistent key_id for chunks') +@attr(assertion='successful') +@attr('encryption') +def test_sse_kms_multipart_invalid_chunks_2(): + bucket = get_new_bucket() + key = "multipart_enc" + content_type = 'text/plain' + objlen = 30 * 1024 * 1024 + init_headers = { + 'x-amz-server-side-encryption': 'aws:kms', + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-1', + 'Content-Type': content_type + } + part_headers = { + 'x-amz-server-side-encryption': 'aws:kms', + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-not-present' + } + _multipart_upload_enc(bucket, key, objlen, + init_headers=init_headers, part_headers=part_headers, + metadata={'foo': 'bar'}) + + +@attr(resource='object') +@attr(method='post') +@attr(operation='authenticated KMS browser based upload via POST request') +@attr(assertion='succeeds and returns written data') +@attr('encryption') +def test_sse_kms_post_object_authenticated_request(): + bucket = get_new_bucket() + + url = _get_post_url(s3.main, bucket) + + utc = pytz.utc + expires = datetime.datetime.now(utc) + datetime.timedelta(seconds=+6000) + + policy_document = {"expiration": expires.strftime("%Y-%m-%dT%H:%M:%SZ"), \ + "conditions": [ \ + {"bucket": bucket.name}, \ + ["starts-with", "$key", "foo"], \ + {"acl": "private"}, \ + ["starts-with", "$Content-Type", "text/plain"], \ + ["starts-with", "$x-amz-server-side-encryption", ""], \ + ["starts-with", "$x-amz-server-side-encryption-aws-kms-key-id", ""], \ + ["content-length-range", 0, 1024] \ + ] \ + } + + json_policy_document = json.JSONEncoder().encode(policy_document) + policy = base64.b64encode(json_policy_document) + conn = s3.main + signature = base64.b64encode(hmac.new(conn.aws_secret_access_key, policy, sha).digest()) + + payload = OrderedDict([ ("key" , "foo.txt"),("AWSAccessKeyId" , conn.aws_access_key_id), \ + ("acl" , "private"),("signature" , signature),("policy" , policy), \ + ("Content-Type" , "text/plain"), \ + ('x-amz-server-side-encryption', 'aws:kms'), \ + ('x-amz-server-side-encryption-aws-kms-key-id', 'testkey-1'), \ + ('file', ('bar'),), ]) + + r = requests.post(url, files = payload) + eq(r.status_code, 204) + get_headers = { + 'x-amz-server-side-encryption': 'aws:kms', + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-1', + } + + key = bucket.get_key("foo.txt") + got = key.get_contents_as_string(headers=get_headers) + eq(got, 'bar') From 33deec9a770cf92695f423212288608907fd82a9 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Fri, 16 Sep 2016 17:37:15 +0200 Subject: [PATCH 3/7] A bit more test for kms Signed-off-by: Adam Kupczyk --- s3tests/functional/test_s3.py | 62 +++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index 3f3f5f5..bf9baa5 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -7921,7 +7921,7 @@ def test_encryption_sse_c_post_object_authenticated_request(): eq(got, 'bar') -def _test_sse_kms_customer_write(file_size): +def _test_sse_kms_customer_write(file_size, key_id = 'testkey-1'): """ Tests Create a file of A's, use it to set_contents_from_file. Create a file of B's, use it to re-set_contents_from_file. @@ -7930,7 +7930,7 @@ def _test_sse_kms_customer_write(file_size): bucket = get_new_bucket() sse_kms_client_headers = { 'x-amz-server-side-encryption': 'aws:kms', - 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-1' + 'x-amz-server-side-encryption-aws-kms-key-id': key_id } key = bucket.new_key('testobj') data = 'A'*file_size @@ -8071,7 +8071,7 @@ def test_sse_kms_multipart_upload(): eq(result.get('x-rgw-object-count', 1), 1) eq(result.get('x-rgw-bytes-used', 30 * 1024 * 1024), 30 * 1024 * 1024) - + k = bucket.get_key(key) eq(k.metadata['foo'], 'bar') eq(k.content_type, content_type) @@ -8179,3 +8179,59 @@ def test_sse_kms_post_object_authenticated_request(): key = bucket.get_key("foo.txt") got = key.get_contents_as_string(headers=get_headers) eq(got, 'bar') + +keyid = "d8f40c6d-1c2c-4314-b435-4ef445c6971f" +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-KMS encrypted transfer 1 byte') +@attr(assertion='success') +@attr('encryption') +def test_sse_kms_barb_transfer_1b(): + _test_sse_kms_customer_write(48, key_id = keyid) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-KMS encrypted transfer 1KB') +@attr(assertion='success') +@attr('encryption') +def test_sse_kms_barb_transfer_1kb(): + _test_sse_kms_customer_write(1024, key_id = keyid) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-KMS encrypted transfer 1MB') +@attr(assertion='success') +@attr('encryption') +def test_sse_kms_barb_transfer_1MB(): + _test_sse_kms_customer_write(1024*1024, key_id = keyid) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='Test SSE-KMS encrypted transfer 13 bytes') +@attr(assertion='success') +@attr('encryption') +def test_sse_kms_barb_transfer_13b(): + _test_sse_kms_customer_write(13, key_id = keyid) + + +@attr(resource='object') +@attr(method='put') +@attr(operation='data write from file (w/100-Continue)') +@attr(assertion='succeeds and returns written data') +@attr('stress') +def test_object_write_file_prrr(): + # boto Key.set_contents_from_file / .send_file uses Expect: + # 100-Continue, so this test exercises that (though a bit too + # subtly) + bucket = get_new_bucket() + for rozmiar in range(1,1000,1): + key = bucket.new_key('plikus') + data = StringIO('bar') + dataarr = [random.randint(0,255) for _ in xrange(rozmiar)] + data = "".join( chr( val ) for val in dataarr ) + key.set_contents_from_string(data) + got = key.get_contents_as_string() + eq(got, data) From 05543b85b1fd762a35e51bab54be426e951ab59a Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Thu, 15 Dec 2016 11:09:36 +0100 Subject: [PATCH 4/7] Modified aws kms tests to use config variable kms_keyid Signed-off-by: Adam Kupczyk --- s3tests/functional/__init__.py | 1 + s3tests/functional/test_s3.py | 37 +++++++++++----------------------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/s3tests/functional/__init__.py b/s3tests/functional/__init__.py index 8fb4845..7a6eaee 100644 --- a/s3tests/functional/__init__.py +++ b/s3tests/functional/__init__.py @@ -318,6 +318,7 @@ def setup(): 'host', 'port', 'is_secure', + 'kms_keyid', ]: try: config[name][var] = cfg.get(section, var) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index bf9baa5..e4e7b65 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -8180,14 +8180,15 @@ def test_sse_kms_post_object_authenticated_request(): got = key.get_contents_as_string(headers=get_headers) eq(got, 'bar') -keyid = "d8f40c6d-1c2c-4314-b435-4ef445c6971f" @attr(resource='object') @attr(method='put') @attr(operation='Test SSE-KMS encrypted transfer 1 byte') @attr(assertion='success') @attr('encryption') def test_sse_kms_barb_transfer_1b(): - _test_sse_kms_customer_write(48, key_id = keyid) + if 'kms_keyid' not in config['main']: + raise SkipTest + _test_sse_kms_customer_write(1, key_id = config['main']['kms_keyid']) @attr(resource='object') @@ -8196,7 +8197,9 @@ def test_sse_kms_barb_transfer_1b(): @attr(assertion='success') @attr('encryption') def test_sse_kms_barb_transfer_1kb(): - _test_sse_kms_customer_write(1024, key_id = keyid) + if 'kms_keyid' not in config['main']: + raise SkipTest + _test_sse_kms_customer_write(1024, key_id = config['main']['kms_keyid']) @attr(resource='object') @@ -8205,7 +8208,9 @@ def test_sse_kms_barb_transfer_1kb(): @attr(assertion='success') @attr('encryption') def test_sse_kms_barb_transfer_1MB(): - _test_sse_kms_customer_write(1024*1024, key_id = keyid) + if 'kms_keyid' not in config['main']: + raise SkipTest + _test_sse_kms_customer_write(1024*1024, key_id = config['main']['kms_keyid']) @attr(resource='object') @@ -8214,24 +8219,6 @@ def test_sse_kms_barb_transfer_1MB(): @attr(assertion='success') @attr('encryption') def test_sse_kms_barb_transfer_13b(): - _test_sse_kms_customer_write(13, key_id = keyid) - - -@attr(resource='object') -@attr(method='put') -@attr(operation='data write from file (w/100-Continue)') -@attr(assertion='succeeds and returns written data') -@attr('stress') -def test_object_write_file_prrr(): - # boto Key.set_contents_from_file / .send_file uses Expect: - # 100-Continue, so this test exercises that (though a bit too - # subtly) - bucket = get_new_bucket() - for rozmiar in range(1,1000,1): - key = bucket.new_key('plikus') - data = StringIO('bar') - dataarr = [random.randint(0,255) for _ in xrange(rozmiar)] - data = "".join( chr( val ) for val in dataarr ) - key.set_contents_from_string(data) - got = key.get_contents_as_string() - eq(got, data) + if 'kms_keyid' not in config['main']: + raise SkipTest + _test_sse_kms_customer_write(13, key_id = config['main']['kms_keyid']) From fa597a5bf16322ab71662b4cb38ebec2cf508631 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Thu, 15 Dec 2016 11:27:49 +0100 Subject: [PATCH 5/7] Added micro instuction how to configure tests for KMS (barbican) integration Signed-off-by: Adam Kupczyk --- README.rst | 3 +++ config.yaml.SAMPLE | 3 +++ 2 files changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 0ce6584..65294fa 100644 --- a/README.rst +++ b/README.rst @@ -56,6 +56,9 @@ service and two different credentials, something like this:: access_key = ABCDEFGHIJKLMNOPQRST secret_key = abcdefghijklmnopqrstuvwxyzabcdefghijklmn + ## replace with key id obtained when secret is created, or delete if KMS not tested + kms_keyid = 01234567-89ab-cdef-0123-456789abcdef + [s3 alt] ## another user account, used for ACL-related tests user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 diff --git a/config.yaml.SAMPLE b/config.yaml.SAMPLE index 169acab..f18096e 100644 --- a/config.yaml.SAMPLE +++ b/config.yaml.SAMPLE @@ -71,6 +71,9 @@ s3: access_key: AWS_ACCESS_KEY secret_key: AWS_SECRET_KEY +## If KMS is tested, this if barbican key id. Optional. + kms_keyid: barbican_key_id + alt: ## Another user accout, used for ACL-related tests. From b27ddae9c49e66ad95f1003e04b3b2a3849a84ea Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Wed, 22 Feb 2017 11:26:28 +0100 Subject: [PATCH 6/7] Added tests for HEAD method for encrypted objects. Signed-off-by: Adam Kupczyk --- s3tests/functional/test_s3.py | 44 +++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index e4e7b65..a6ed33c 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -7588,6 +7588,29 @@ def test_encrypted_transfer_13b(): _test_encryption_sse_customer_write(13) +@attr(resource='object') +@attr(method='head') +@attr(operation='Test SSE-C encrypted does perform head properly') +@attr(assertion='success') +@attr('encryption') +def test_encryption_sse_c_method_head(): + bucket = get_new_bucket() + sse_client_headers = { + 'x-amz-server-side-encryption-customer-algorithm': 'AES256', + 'x-amz-server-side-encryption-customer-key': 'pO3upElrwuEXSoFwCfnZPdSsmt/xWeFa0N9KgDijwVs=', + 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==' + } + key = bucket.new_key('testobj') + data = 'A'*1000 + key.set_contents_from_string(data, headers=sse_client_headers) + + res = _make_request('HEAD', bucket, key, authenticated=True) + eq(res.status, 400) + + res = _make_request('HEAD', bucket, key, authenticated=True, request_headers=sse_client_headers) + eq(res.status, 200) + + @attr(resource='object') @attr(method='put') @attr(operation='write encrypted with SSE-C and read without SSE-C') @@ -7975,6 +7998,27 @@ def test_sse_kms_transfer_13b(): _test_sse_kms_customer_write(13) +@attr(resource='object') +@attr(method='head') +@attr(operation='Test SSE-KMS encrypted does perform head properly') +@attr(assertion='success') +@attr('encryption') +def test_sse_kms_method_head(): + bucket = get_new_bucket() + sse_kms_client_headers = { + 'x-amz-server-side-encryption': 'aws:kms', + 'x-amz-server-side-encryption-aws-kms-key-id': 'testkey-1' + } + key = bucket.new_key('testobj') + data = 'A'*1000 + key.set_contents_from_string(data, headers=sse_kms_client_headers) + + res = _make_request('HEAD', bucket, key, authenticated=True) + eq(res.status, 200) + eq(res.getheader('x-amz-server-side-encryption'), 'aws:kms') + eq(res.getheader('x-amz-server-side-encryption-aws-kms-key-id'), 'testkey-1') + + @attr(resource='object') @attr(method='put') @attr(operation='write encrypted with SSE-KMS and read without SSE-KMS') From e49cf3ddab75d90525e6b07bf53acf2d8eabd497 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Tue, 28 Feb 2017 16:58:17 -0500 Subject: [PATCH 7/7] crypto: add key headers to valid HEAD requests Signed-off-by: Casey Bodley --- s3tests/functional/test_s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/s3tests/functional/test_s3.py b/s3tests/functional/test_s3.py index a6ed33c..8f9119d 100644 --- a/s3tests/functional/test_s3.py +++ b/s3tests/functional/test_s3.py @@ -7786,7 +7786,7 @@ def test_encryption_sse_c_multipart_upload(): eq(result.get('x-rgw-object-count', 1), 1) eq(result.get('x-rgw-bytes-used', 30 * 1024 * 1024), 30 * 1024 * 1024) - k = bucket.get_key(key) + k = bucket.get_key(key, headers=enc_headers) eq(k.metadata['foo'], 'bar') eq(k.content_type, content_type) test_string = k.get_contents_as_string(headers=enc_headers) @@ -7885,7 +7885,7 @@ def test_encryption_sse_c_multipart_bad_download(): eq(result.get('x-rgw-object-count', 1), 1) eq(result.get('x-rgw-bytes-used', 30 * 1024 * 1024), 30 * 1024 * 1024) - k = bucket.get_key(key) + k = bucket.get_key(key, headers=put_headers) eq(k.metadata['foo'], 'bar') eq(k.content_type, content_type) e = assert_raises(boto.exception.S3ResponseError, @@ -7939,7 +7939,7 @@ def test_encryption_sse_c_post_object_authenticated_request(): 'x-amz-server-side-encryption-customer-key-md5': 'DWygnHRtgiJ77HCm+1rvHw==' } - key = bucket.get_key("foo.txt") + key = bucket.get_key("foo.txt", headers=get_headers) got = key.get_contents_as_string(headers=get_headers) eq(got, 'bar')