frostfs-testcases/robot/resources/lib/python_keywords/s3_gate_object.py

354 lines
14 KiB
Python

#!/usr/bin/python3.9
import allure
import os
import uuid
from enum import Enum
from time import sleep
from typing import Optional
import urllib3
from botocore.exceptions import ClientError
from robot.api import logger
from cli_helpers import log_command_execution
from python_keywords.aws_cli_client import AwsCliClient
from python_keywords.s3_gate_bucket import S3_SYNC_WAIT_TIME
##########################################################
# Disabling warnings on self-signed certificate which the
# boto library produces on requests to S3-gate in dev-env.
urllib3.disable_warnings()
##########################################################
ROBOT_AUTO_KEYWORDS = False
CREDENTIALS_CREATE_TIMEOUT = '30s'
ASSETS_DIR = os.getenv('ASSETS_DIR', 'TemporaryDir/')
class VersioningStatus(Enum):
ENABLED = 'Enabled'
SUSPENDED = 'Suspended'
@allure.step('List objects S3 v2')
def list_objects_s3_v2(s3_client, bucket: str) -> list:
try:
response = s3_client.list_objects_v2(Bucket=bucket)
content = response.get('Contents', [])
log_command_execution('S3 v2 List objects result', response)
obj_list = []
for obj in content:
obj_list.append(obj['Key'])
logger.info(f'Found s3 objects: {obj_list}')
return obj_list
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('List objects S3')
def list_objects_s3(s3_client, bucket: str) -> list:
try:
response = s3_client.list_objects(Bucket=bucket)
content = response.get('Contents', [])
log_command_execution('S3 List objects result', response)
obj_list = []
for obj in content:
obj_list.append(obj['Key'])
logger.info(f'Found s3 objects: {obj_list}')
return obj_list
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('List objects versions S3')
def list_objects_versions_s3(s3_client, bucket: str) -> list:
try:
response = s3_client.list_object_versions(Bucket=bucket)
versions = response.get('Versions', [])
log_command_execution('S3 List objects versions result', response)
return versions
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Put object S3')
def put_object_s3(s3_client, bucket: str, filepath: str):
filename = os.path.basename(filepath)
if isinstance(s3_client, AwsCliClient):
file_content = filepath
else:
with open(filepath, 'rb') as put_file:
file_content = put_file.read()
try:
response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename)
log_command_execution('S3 Put object result', response)
return response.get('VersionId')
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Head object S3')
def head_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None):
try:
params = {'Bucket': bucket, 'Key': object_key}
if version_id:
params['VersionId'] = version_id
response = s3_client.head_object(**params)
log_command_execution('S3 Head object result', response)
return response
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Delete object S3')
def delete_object_s3(s3_client, bucket, object_key, version_id: str = None):
try:
params = {'Bucket': bucket, 'Key': object_key}
if version_id:
params['VersionId'] = version_id
response = s3_client.delete_object(**params)
log_command_execution('S3 Delete object result', response)
sleep(S3_SYNC_WAIT_TIME)
return response
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Delete objects S3')
def delete_objects_s3(s3_client, bucket: str, object_keys: list):
try:
response = s3_client.delete_objects(Bucket=bucket, Delete=_make_objs_dict(object_keys))
log_command_execution('S3 Delete objects result', response)
sleep(S3_SYNC_WAIT_TIME)
return response
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Delete object versions S3')
def delete_object_versions_s3(s3_client, bucket: str, object_versions: list):
try:
# Build deletion list in S3 format
delete_list = {
"Objects": [
{
"Key": object_version["Key"],
"VersionId": object_version["VersionId"],
}
for object_version in object_versions
]
}
response = s3_client.delete_objects(Bucket=bucket, Delete=delete_list)
log_command_execution('S3 Delete objects result', response)
return response
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Copy object S3')
def copy_object_s3(s3_client, bucket, object_key, bucket_dst=None):
filename = f'{os.getcwd()}/{uuid.uuid4()}'
try:
response = s3_client.copy_object(Bucket=bucket_dst or bucket,
CopySource=f'{bucket}/{object_key}',
Key=filename)
log_command_execution('S3 Copy objects result', response)
return filename
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Get object S3')
def get_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None):
filename = f'{ASSETS_DIR}/{uuid.uuid4()}'
try:
params = {'Bucket': bucket, 'Key': object_key}
if version_id:
params['VersionId'] = version_id
if isinstance(s3_client, AwsCliClient):
params['file_path'] = filename
response = s3_client.get_object(**params)
log_command_execution('S3 Get objects result', response)
if not isinstance(s3_client, AwsCliClient):
with open(f'{filename}', 'wb') as get_file:
chunk = response['Body'].read(1024)
while chunk:
get_file.write(chunk)
chunk = response['Body'].read(1024)
return filename
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Create multipart upload S3')
def create_multipart_upload_s3(s3_client, bucket_name: str, object_key: str) -> str:
try:
response = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_key)
log_command_execution('S3 Created multipart upload', response)
assert response.get('UploadId'), f'Expected UploadId in response:\n{response}'
return response.get('UploadId')
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('List multipart uploads S3')
def list_multipart_uploads_s3(s3_client, bucket_name: str) -> Optional[list[dict]]:
try:
response = s3_client.list_multipart_uploads(Bucket=bucket_name)
log_command_execution('S3 List multipart upload', response)
return response.get('Uploads')
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Abort multipart upload S3')
def abort_multipart_uploads_s3(s3_client, bucket_name: str, object_key: str, upload_id: str):
try:
response = s3_client.abort_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id)
log_command_execution('S3 Abort multipart upload', response)
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Upload part S3')
def upload_part_s3(s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, filepath: str) -> str:
if isinstance(s3_client, AwsCliClient):
file_content = filepath
else:
with open(filepath, 'rb') as put_file:
file_content = put_file.read()
try:
response = s3_client.upload_part(UploadId=upload_id, Bucket=bucket_name, Key=object_key, PartNumber=part_num,
Body=file_content)
log_command_execution('S3 Upload part', response)
assert response.get('ETag'), f'Expected ETag in response:\n{response}'
return response.get('ETag')
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('List parts S3')
def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> list[dict]:
try:
response = s3_client.list_parts(UploadId=upload_id, Bucket=bucket_name, Key=object_key)
log_command_execution('S3 List part', response)
assert response.get('Parts'), f'Expected Parts in response:\n{response}'
return response.get('Parts')
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Complete multipart upload S3')
def complete_multipart_upload_s3(s3_client, bucket_name: str, object_key: str, upload_id: str,
parts: list):
try:
parts = [{'ETag': etag, 'PartNumber': part_num} for part_num, etag in parts]
response = s3_client.complete_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id,
MultipartUpload={'Parts': parts})
log_command_execution('S3 Complete multipart upload', response)
except ClientError as err:
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
@allure.step('Put object tagging')
def put_object_tagging(s3_client, bucket_name: str, object_key: str, tags: list):
try:
tags = [{'Key': tag_key, 'Value': tag_value} for tag_key, tag_value in tags]
tagging = {'TagSet': tags}
s3_client.put_object_tagging(Bucket=bucket_name, Key=object_key, Tagging=tagging)
log_command_execution('S3 Put object tagging', str(tags))
except ClientError as err:
raise Exception(f'Got error during put object tagging: {err}') from err
@allure.step('Get object tagging')
def get_object_tagging(s3_client, bucket_name: str, object_key: str) -> list:
try:
response = s3_client.get_object_tagging(Bucket=bucket_name, Key=object_key)
log_command_execution('S3 Get object tagging', response)
return response.get('TagSet')
except ClientError as err:
raise Exception(f'Got error during get object tagging: {err}') from err
@allure.step('Delete object tagging')
def delete_object_tagging(s3_client, bucket_name: str, object_key: str):
try:
response = s3_client.delete_object_tagging(Bucket=bucket_name, Key=object_key)
log_command_execution('S3 Delete object tagging', response)
except ClientError as err:
raise Exception(f'Got error during delete object tagging: {err}') from err
@allure.step('Get object tagging')
def get_object_attributes(s3_client, bucket_name: str, object_key: str, *attributes: str, version_id: str = None,
max_parts: int = None, part_number: int = None, get_full_resp=True) -> dict:
try:
if not isinstance(s3_client, AwsCliClient):
logger.warn('Method get_object_attributes is not supported by boto3 client')
return {}
response = s3_client.get_object_attributes(bucket_name, object_key, *attributes, version_id=version_id,
max_parts=max_parts, part_number=part_number)
log_command_execution('S3 Get object attributes', response)
for attr in attributes:
assert attr in response, f'Expected attribute {attr} in {response}'
if get_full_resp:
return response
else:
return response.get(attributes[0])
except ClientError as err:
raise Exception(f'Got error during get object attributes: {err}') from err
def _make_objs_dict(key_names):
objs_list = []
for key in key_names:
obj_dict = {'Key': key}
objs_list.append(obj_dict)
objs_dict = {'Objects': objs_list}
return objs_dict