From 73ed9121f41944f15620031ca4428261510b6bc0 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Wed, 1 May 2024 13:59:09 -0400 Subject: [PATCH 1/6] add "checksum" marker, since new checksum tests reference it this removes a Pytest warning during execution Signed-off-by: Matt Benjamin --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 73d1563..1a7d9a8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,6 +7,7 @@ markers = auth_common bucket_policy bucket_encryption + checksum cloud_transition encryption fails_on_aws From a3dbac711542e2af3674f6b6d555397746624aee Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Wed, 1 May 2024 14:05:52 -0400 Subject: [PATCH 2/6] test_multipart_upload_sha256: work around failures re-trying complete-multipart As described in https://tracker.ceph.com/issues/65746, retrying complete-multipart after having attempted to complete the same upload with a bad checksum argument fails with an internal error. The status code is 500, but I'm unsure if it can be retried again, or whether the upload can be aborted later. Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 88 +++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 47cc525..a8fa059 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -13434,3 +13434,91 @@ def test_get_object_torrent(): status, error_code = _get_status_and_error_code(e.response) assert status == 404 assert error_code == 'NoSuchKey' + +@pytest.mark.checksum +def test_object_checksum_sha256(): + bucket = get_new_bucket() + client = get_client() + + key = "myobj" + size = 1024 + body = FakeWriteFile(size, 'A') + sha256sum = 'arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0=' + response = client.put_object(Bucket=bucket, Key=key, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=sha256sum) + assert sha256sum == response['ChecksumSHA256'] + + response = client.head_object(Bucket=bucket, Key=key) + assert 'ChecksumSHA256' not in response + response = client.head_object(Bucket=bucket, Key=key, ChecksumMode='ENABLED') + assert sha256sum == response['ChecksumSHA256'] + + e = assert_raises(ClientError, client.put_object, Bucket=bucket, Key=key, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256='bad') + status, error_code = _get_status_and_error_code(e.response) + assert status == 400 + assert error_code == 'InvalidRequest' + +@pytest.mark.checksum +def test_multipart_checksum_sha256(): + bucket = get_new_bucket() + client = get_client() + + key = "mymultipart" + response = client.create_multipart_upload(Bucket=bucket, Key=key, ChecksumAlgorithm='SHA256') + assert 'SHA256' == response['ChecksumAlgorithm'] + upload_id = response['UploadId'] + + size = 1024 + body = FakeWriteFile(size, 'A') + part_sha256sum = 'arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0=' + response = client.upload_part(UploadId=upload_id, Bucket=bucket, Key=key, PartNumber=1, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=part_sha256sum) + + # should reject the bad request checksum + e = assert_raises(ClientError, client.complete_multipart_upload, Bucket=bucket, Key=key, UploadId=upload_id, ChecksumSHA256='bad', MultipartUpload={'Parts': [ + {'ETag': response['ETag'].strip('"'), 'ChecksumSHA256': response['ChecksumSHA256'], 'PartNumber': 1}]}) + status, error_code = _get_status_and_error_code(e.response) + assert status == 400 + assert error_code == 'InvalidRequest' + + # XXXX re-trying the complete is failing in RGW due to an internal error that appears not caused + # checksums; + # 2024-04-25T17:47:47.991-0400 7f78e3a006c0 0 req 4931907640780566174 0.011000143s s3:complete_multipart check_previously_completed() ERROR: get_obj_attrs() returned ret=-2 + # 2024-04-25T17:47:47.991-0400 7f78e3a006c0 2 req 4931907640780566174 0.011000143s s3:complete_multipart completing + # 2024-04-25T17:47:47.991-0400 7f78e3a006c0 1 req 4931907640780566174 0.011000143s s3:complete_multipart ERROR: either op_ret is negative (execute failed) or target_obj is null, op_ret: -2200 + # -2200 turns into 500, InternalError + + key = "mymultipart2" + response = client.create_multipart_upload(Bucket=bucket, Key=key, ChecksumAlgorithm='SHA256') + assert 'SHA256' == response['ChecksumAlgorithm'] + upload_id = response['UploadId'] + + size = 1024 + body = FakeWriteFile(size, 'A') + part_sha256sum = 'arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0=' + response = client.upload_part(UploadId=upload_id, Bucket=bucket, Key=key, PartNumber=1, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=part_sha256sum) + + # should reject the missing part checksum + e = assert_raises(ClientError, client.complete_multipart_upload, Bucket=bucket, Key=key, UploadId=upload_id, ChecksumSHA256='bad', MultipartUpload={'Parts': [ + {'ETag': response['ETag'].strip('"'), 'PartNumber': 1}]}) + status, error_code = _get_status_and_error_code(e.response) + assert status == 400 + assert error_code == 'InvalidRequest' + + key = "mymultipart3" + response = client.create_multipart_upload(Bucket=bucket, Key=key, ChecksumAlgorithm='SHA256') + assert 'SHA256' == response['ChecksumAlgorithm'] + upload_id = response['UploadId'] + + size = 1024 + body = FakeWriteFile(size, 'A') + part_sha256sum = 'arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0=' + response = client.upload_part(UploadId=upload_id, Bucket=bucket, Key=key, PartNumber=1, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=part_sha256sum) + + composite_sha256sum = 'Ok6Cs5b96ux6+MWQkJO7UBT5sKPBeXBLwvj/hK89smg=-1' + response = client.complete_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id, ChecksumSHA256=composite_sha256sum, MultipartUpload={'Parts': [ + {'ETag': response['ETag'].strip('"'), 'ChecksumSHA256': response['ChecksumSHA256'], 'PartNumber': 1}]}) + assert composite_sha256sum == response['ChecksumSHA256'] + + response = client.head_object(Bucket=bucket, Key=key) + assert 'ChecksumSHA256' not in response + response = client.head_object(Bucket=bucket, Key=key, ChecksumMode='ENABLED') + assert composite_sha256sum == response['ChecksumSHA256'] From 9577cde013ed0b6a08bfc3d710db24e4add02fc3 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Wed, 1 May 2024 14:15:36 -0400 Subject: [PATCH 3/6] add test_multipart_checksum_3parts tests a full multipart upload cycle with 3 unique parts, which verifies composite checksum computation and the logic to propagate parts_count to ComleteMultipart Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index a8fa059..1a085bd 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -13522,3 +13522,41 @@ def test_multipart_checksum_sha256(): assert 'ChecksumSHA256' not in response response = client.head_object(Bucket=bucket, Key=key, ChecksumMode='ENABLED') assert composite_sha256sum == response['ChecksumSHA256'] + +@pytest.mark.checksum +def test_multipart_checksum_3parts(): + bucket = get_new_bucket() + client = get_client() + + key = "mymultipart3" + response = client.create_multipart_upload(Bucket=bucket, Key=key, ChecksumAlgorithm='SHA256') + assert 'SHA256' == response['ChecksumAlgorithm'] + upload_id = response['UploadId'] + + size = 5 * 1024 * 1024 # each part but the last must be at least 5M + body = FakeWriteFile(size, 'A') + part1_sha256sum = '275VF5loJr1YYawit0XSHREhkFXYkkPKGuoK0x9VKxI=' + response = client.upload_part(UploadId=upload_id, Bucket=bucket, Key=key, PartNumber=1, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=part1_sha256sum) + etag1 = response['ETag'].strip('"') + + body = FakeWriteFile(size, 'B') + part2_sha256sum = 'mrHwOfjTL5Zwfj74F05HOQGLdUb7E5szdCbxgUSq6NM=' + response = client.upload_part(UploadId=upload_id, Bucket=bucket, Key=key, PartNumber=2, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=part2_sha256sum) + etag2 = response['ETag'].strip('"') + + body = FakeWriteFile(size, 'C') + part3_sha256sum = 'Vw7oB/nKQ5xWb3hNgbyfkvDiivl+U+/Dft48nfJfDow=' + response = client.upload_part(UploadId=upload_id, Bucket=bucket, Key=key, PartNumber=3, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=part3_sha256sum) + etag3 = response['ETag'].strip('"') + + composite_sha256sum = 'uWBwpe1dxI4Vw8Gf0X9ynOdw/SS6VBzfWm9giiv1sf4=-3' + response = client.complete_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id, ChecksumSHA256=composite_sha256sum, MultipartUpload={'Parts': [ + {'ETag': etag1, 'ChecksumSHA256': response['ChecksumSHA256'], 'PartNumber': 1}, + {'ETag': etag2, 'ChecksumSHA256': response['ChecksumSHA256'], 'PartNumber': 2}, + {'ETag': etag3, 'ChecksumSHA256': response['ChecksumSHA256'], 'PartNumber': 3}]}) + assert composite_sha256sum == response['ChecksumSHA256'] + + response = client.head_object(Bucket=bucket, Key=key) + assert 'ChecksumSHA256' not in response + response = client.head_object(Bucket=bucket, Key=key, ChecksumMode='ENABLED') + assert composite_sha256sum == response['ChecksumSHA256'] From 95df503ced29ec0457d572f548b990713f7ae9c1 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Fri, 3 May 2024 16:25:19 -0400 Subject: [PATCH 4/6] add test_post_object_upload_checksum this tests a two-megabyte binary upload with validated (awscli-computed) SHA256 checksum, and also verifies failure when a bad checksum is provided Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 53 +++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 1a085bd..b8a6142 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -13560,3 +13560,56 @@ def test_multipart_checksum_3parts(): assert 'ChecksumSHA256' not in response response = client.head_object(Bucket=bucket, Key=key, ChecksumMode='ENABLED') assert composite_sha256sum == response['ChecksumSHA256'] + +def test_post_object_upload_checksum(): + megabytes = 1024 * 1024 + min_size = 0 + max_size = 5 * megabytes + test_payload_size = 2 * megabytes + + bucket_name = get_new_bucket() + client = get_client() + + url = _get_post_url(bucket_name) + utc = pytz.utc + expires = datetime.datetime.now(utc) + datetime.timedelta(seconds=+6000) + + policy_document = {"expiration": expires.strftime("%Y-%m-%dT%H:%M:%SZ"),\ + "conditions": [\ + {"bucket": bucket_name},\ + ["starts-with", "$key", "foo_cksum_test"],\ + {"acl": "private"},\ + ["starts-with", "$Content-Type", "text/plain"],\ + ["content-length-range", min_size, max_size],\ + ]\ + } + + test_payload = b'x' * test_payload_size + + json_policy_document = json.JSONEncoder().encode(policy_document) + bytes_json_policy_document = bytes(json_policy_document, 'utf-8') + policy = base64.b64encode(bytes_json_policy_document) + aws_secret_access_key = get_main_aws_secret_key() + aws_access_key_id = get_main_aws_access_key() + + signature = base64.b64encode(hmac.new(bytes(aws_secret_access_key, 'utf-8'), policy, hashlib.sha1).digest()) + + # good checksum payload (checked via upload from awscli) + payload = OrderedDict([ ("key" , "foo_cksum_test.txt"),("AWSAccessKeyId" , aws_access_key_id),\ + ("acl" , "private"),("signature" , signature),("policy" , policy),\ + ("Content-Type" , "text/plain"),\ + ('x-amz-checksum-sha256', 'aTL9MeXa9HObn6eP93eygxsJlcwdCwCTysgGAZAgE7w='),\ + ('file', (test_payload)),]) + + r = requests.post(url, files=payload, verify=get_config_ssl_verify()) + assert r.status_code == 204 + + # bad checksum payload + payload = OrderedDict([ ("key" , "foo_cksum_test.txt"),("AWSAccessKeyId" , aws_access_key_id),\ + ("acl" , "private"),("signature" , signature),("policy" , policy),\ + ("Content-Type" , "text/plain"),\ + ('x-amz-checksum-sha256', 'sailorjerry'),\ + ('file', (test_payload)),]) + + r = requests.post(url, files=payload, verify=get_config_ssl_verify()) + assert r.status_code == 400 From c0f0b679db7f20c44553d6ddf9470afa649d5e37 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Sat, 22 Jun 2024 17:42:21 -0400 Subject: [PATCH 5/6] remove duplicate size assigment [rkhudov review] Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index b8a6142..24301ba 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -13491,7 +13491,6 @@ def test_multipart_checksum_sha256(): assert 'SHA256' == response['ChecksumAlgorithm'] upload_id = response['UploadId'] - size = 1024 body = FakeWriteFile(size, 'A') part_sha256sum = 'arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0=' response = client.upload_part(UploadId=upload_id, Bucket=bucket, Key=key, PartNumber=1, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=part_sha256sum) @@ -13508,7 +13507,6 @@ def test_multipart_checksum_sha256(): assert 'SHA256' == response['ChecksumAlgorithm'] upload_id = response['UploadId'] - size = 1024 body = FakeWriteFile(size, 'A') part_sha256sum = 'arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0=' response = client.upload_part(UploadId=upload_id, Bucket=bucket, Key=key, PartNumber=1, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=part_sha256sum) From 8277a9fb9aa66ab8b24f603c4df545928a813f81 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Wed, 3 Jul 2024 09:42:37 -0400 Subject: [PATCH 6/6] mark two tests that fail on dbstore also add @pytest.mark.checksum for new checksum tests Signed-off-by: Matt Benjamin --- s3tests_boto3/functional/test_s3.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/s3tests_boto3/functional/test_s3.py b/s3tests_boto3/functional/test_s3.py index 24301ba..20cc7c3 100644 --- a/s3tests_boto3/functional/test_s3.py +++ b/s3tests_boto3/functional/test_s3.py @@ -13458,6 +13458,7 @@ def test_object_checksum_sha256(): assert error_code == 'InvalidRequest' @pytest.mark.checksum +@pytest.mark.fails_on_dbstore def test_multipart_checksum_sha256(): bucket = get_new_bucket() client = get_client() @@ -13522,6 +13523,7 @@ def test_multipart_checksum_sha256(): assert composite_sha256sum == response['ChecksumSHA256'] @pytest.mark.checksum +@pytest.mark.fails_on_dbstore def test_multipart_checksum_3parts(): bucket = get_new_bucket() client = get_client() @@ -13559,6 +13561,7 @@ def test_multipart_checksum_3parts(): response = client.head_object(Bucket=bucket, Key=key, ChecksumMode='ENABLED') assert composite_sha256sum == response['ChecksumSHA256'] +@pytest.mark.checksum def test_post_object_upload_checksum(): megabytes = 1024 * 1024 min_size = 0