dkirillov/s3-tests
1
0
Fork 0
forked from TrueCloudLab/s3-tests
Code Pull requests Activity

test_multipart_upload_sha256: work around failures re-trying complete-multipart

Browse source 
As described in https://tracker.ceph.com/issues/65746, retrying complete-multipart
after having attempted to complete the same upload with a bad checksum argument
fails with an internal error.

The status code is 500, but I'm unsure if it can be retried again, or whether
the upload can be aborted later.

Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>
(cherry picked from commit a3dbac7115)
This commit is contained in:
Matt Benjamin 2024-05-01 14:05:52 -04:00 committed by Casey Bodley
parent 2495db2447
commit 9de20e91f2
1 changed files with 88 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
s3tests_boto3/functional/test_s3.py
View file

@ -13434,3 +13434,91 @@ def test_get_object_torrent():
status, error_code = _get_status_and_error_code(e.response)
assert status == 404
assert error_code == 'NoSuchKey'
@pytest.mark.checksum
def test_object_checksum_sha256():
bucket = get_new_bucket()
client = get_client()
key = "myobj"
size = 1024
body = FakeWriteFile(size, 'A')
sha256sum = 'arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0='
response = client.put_object(Bucket=bucket, Key=key, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=sha256sum)
assert sha256sum == response['ChecksumSHA256']
response = client.head_object(Bucket=bucket, Key=key)
assert 'ChecksumSHA256' not in response
response = client.head_object(Bucket=bucket, Key=key, ChecksumMode='ENABLED')
assert sha256sum == response['ChecksumSHA256']
e = assert_raises(ClientError, client.put_object, Bucket=bucket, Key=key, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256='bad')
status, error_code = _get_status_and_error_code(e.response)
assert status == 400
assert error_code == 'InvalidRequest'
@pytest.mark.checksum
def test_multipart_checksum_sha256():
bucket = get_new_bucket()
client = get_client()
key = "mymultipart"
response = client.create_multipart_upload(Bucket=bucket, Key=key, ChecksumAlgorithm='SHA256')
assert 'SHA256' == response['ChecksumAlgorithm']
upload_id = response['UploadId']
size = 1024
body = FakeWriteFile(size, 'A')
part_sha256sum = 'arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0='
response = client.upload_part(UploadId=upload_id, Bucket=bucket, Key=key, PartNumber=1, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=part_sha256sum)
# should reject the bad request checksum
e = assert_raises(ClientError, client.complete_multipart_upload, Bucket=bucket, Key=key, UploadId=upload_id, ChecksumSHA256='bad', MultipartUpload={'Parts': [
{'ETag': response['ETag'].strip('"'), 'ChecksumSHA256': response['ChecksumSHA256'], 'PartNumber': 1}]})
status, error_code = _get_status_and_error_code(e.response)
assert status == 400
assert error_code == 'InvalidRequest'
# XXXX re-trying the complete is failing in RGW due to an internal error that appears not caused
# checksums;
# 2024-04-25T17:47:47.991-0400 7f78e3a006c0 0 req 4931907640780566174 0.011000143s s3:complete_multipart check_previously_completed() ERROR: get_obj_attrs() returned ret=-2
# 2024-04-25T17:47:47.991-0400 7f78e3a006c0 2 req 4931907640780566174 0.011000143s s3:complete_multipart completing
# 2024-04-25T17:47:47.991-0400 7f78e3a006c0 1 req 4931907640780566174 0.011000143s s3:complete_multipart ERROR: either op_ret is negative (execute failed) or target_obj is null, op_ret: -2200
# -2200 turns into 500, InternalError
key = "mymultipart2"
response = client.create_multipart_upload(Bucket=bucket, Key=key, ChecksumAlgorithm='SHA256')
assert 'SHA256' == response['ChecksumAlgorithm']
upload_id = response['UploadId']
size = 1024
body = FakeWriteFile(size, 'A')
part_sha256sum = 'arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0='
response = client.upload_part(UploadId=upload_id, Bucket=bucket, Key=key, PartNumber=1, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=part_sha256sum)
# should reject the missing part checksum
e = assert_raises(ClientError, client.complete_multipart_upload, Bucket=bucket, Key=key, UploadId=upload_id, ChecksumSHA256='bad', MultipartUpload={'Parts': [
{'ETag': response['ETag'].strip('"'), 'PartNumber': 1}]})
status, error_code = _get_status_and_error_code(e.response)
assert status == 400
assert error_code == 'InvalidRequest'
key = "mymultipart3"
response = client.create_multipart_upload(Bucket=bucket, Key=key, ChecksumAlgorithm='SHA256')
assert 'SHA256' == response['ChecksumAlgorithm']
upload_id = response['UploadId']
size = 1024
body = FakeWriteFile(size, 'A')
part_sha256sum = 'arcu6553sHVAiX4MjW0j7I7vD4w6R+Gz9Ok0Q9lTa+0='
response = client.upload_part(UploadId=upload_id, Bucket=bucket, Key=key, PartNumber=1, Body=body, ChecksumAlgorithm='SHA256', ChecksumSHA256=part_sha256sum)
composite_sha256sum = 'Ok6Cs5b96ux6+MWQkJO7UBT5sKPBeXBLwvj/hK89smg=-1'
response = client.complete_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id, ChecksumSHA256=composite_sha256sum, MultipartUpload={'Parts': [
{'ETag': response['ETag'].strip('"'), 'ChecksumSHA256': response['ChecksumSHA256'], 'PartNumber': 1}]})
assert composite_sha256sum == response['ChecksumSHA256']
response = client.head_object(Bucket=bucket, Key=key)
assert 'ChecksumSHA256' not in response
response = client.head_object(Bucket=bucket, Key=key, ChecksumMode='ENABLED')
assert composite_sha256sum == response['ChecksumSHA256']