331 lines
13 KiB
Python
331 lines
13 KiB
Python
|
#!/usr/bin/python3
|
||
|
|
|
||
|
|
import os
|
||
|
|
import uuid
|
||
|
|
from enum import Enum
|
||
|
|
from typing import Optional, List
|
||
|
|
|
||
|
|
import urllib3
|
||
|
|
from botocore.exceptions import ClientError
|
||
|
|
from robot.api import logger
|
||
|
|
from robot.api.deco import keyword
|
||
|
|
|
||
|
|
from cli_helpers import log_command_execution
|
||
|
|
from python_keywords.aws_cli_client import AwsCliClient
|
||
|
|
|
||
|
|
##########################################################
|
||
|
|
# Disabling warnings on self-signed certificate which the
|
||
|
|
# boto library produces on requests to S3-gate in dev-env.
|
||
|
|
urllib3.disable_warnings()
|
||
|
|
##########################################################
|
||
|
|
|
||
|
|
ROBOT_AUTO_KEYWORDS = False
|
||
|
|
CREDENTIALS_CREATE_TIMEOUT = '30s'
|
||
|
|
|
||
|
|
ASSETS_DIR = os.getenv('ASSETS_DIR', 'TemporaryDir/')
|
||
|
|
|
||
|
|
|
||
|
|
class VersioningStatus(Enum):
|
||
|
|
ENABLED = 'Enabled'
|
||
|
|
SUSPENDED = 'Suspended'
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('List objects S3 v2')
|
||
|
|
def list_objects_s3_v2(s3_client, bucket: str) -> list:
|
||
|
|
try:
|
||
|
|
response = s3_client.list_objects_v2(Bucket=bucket)
|
||
|
|
content = response.get('Contents', [])
|
||
|
|
log_command_execution('S3 v2 List objects result', response)
|
||
|
|
obj_list = []
|
||
|
|
for obj in content:
|
||
|
|
obj_list.append(obj['Key'])
|
||
|
|
logger.info(f'Found s3 objects: {obj_list}')
|
||
|
|
return obj_list
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('List objects S3')
|
||
|
|
def list_objects_s3(s3_client, bucket: str) -> list:
|
||
|
|
try:
|
||
|
|
response = s3_client.list_objects(Bucket=bucket)
|
||
|
|
content = response.get('Contents', [])
|
||
|
|
log_command_execution('S3 List objects result', response)
|
||
|
|
obj_list = []
|
||
|
|
for obj in content:
|
||
|
|
obj_list.append(obj['Key'])
|
||
|
|
logger.info(f'Found s3 objects: {obj_list}')
|
||
|
|
return obj_list
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('List objects versions S3')
|
||
|
|
def list_objects_versions_s3(s3_client, bucket: str) -> list:
|
||
|
|
try:
|
||
|
|
response = s3_client.list_object_versions(Bucket=bucket)
|
||
|
|
versions = response.get('Versions', [])
|
||
|
|
log_command_execution('S3 List objects versions result', response)
|
||
|
|
return versions
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Put object S3')
|
||
|
|
def put_object_s3(s3_client, bucket: str, filepath: str):
|
||
|
|
filename = os.path.basename(filepath)
|
||
|
|
|
||
|
|
if isinstance(s3_client, AwsCliClient):
|
||
|
|
file_content = filepath
|
||
|
|
else:
|
||
|
|
with open(filepath, 'rb') as put_file:
|
||
|
|
file_content = put_file.read()
|
||
|
|
|
||
|
|
try:
|
||
|
|
response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename)
|
||
|
|
log_command_execution('S3 Put object result', response)
|
||
|
|
return response.get('VersionId')
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Head object S3')
|
||
|
|
def head_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None):
|
||
|
|
try:
|
||
|
|
params = {'Bucket': bucket, 'Key': object_key}
|
||
|
|
if version_id:
|
||
|
|
params['VersionId'] = version_id
|
||
|
|
response = s3_client.head_object(**params)
|
||
|
|
log_command_execution('S3 Head object result', response)
|
||
|
|
return response
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Delete object S3')
|
||
|
|
def delete_object_s3(s3_client, bucket, object_key, version_id: str = None):
|
||
|
|
try:
|
||
|
|
params = {'Bucket': bucket, 'Key': object_key}
|
||
|
|
if version_id:
|
||
|
|
params['VersionId'] = version_id
|
||
|
|
response = s3_client.delete_object(**params)
|
||
|
|
log_command_execution('S3 Delete object result', response)
|
||
|
|
return response
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Delete objects S3')
|
||
|
|
def delete_objects_s3(s3_client, bucket: str, object_keys: list):
|
||
|
|
try:
|
||
|
|
response = s3_client.delete_objects(Bucket=bucket, Delete=_make_objs_dict(object_keys))
|
||
|
|
log_command_execution('S3 Delete objects result', response)
|
||
|
|
return response
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Copy object S3')
|
||
|
|
def copy_object_s3(s3_client, bucket, object_key, bucket_dst=None):
|
||
|
|
filename = f'{os.getcwd()}/{uuid.uuid4()}'
|
||
|
|
try:
|
||
|
|
response = s3_client.copy_object(Bucket=bucket_dst or bucket,
|
||
|
|
CopySource=f'{bucket}/{object_key}',
|
||
|
|
Key=filename)
|
||
|
|
log_command_execution('S3 Copy objects result', response)
|
||
|
|
return filename
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Get object S3')
|
||
|
|
def get_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None):
|
||
|
|
filename = f'{ASSETS_DIR}/{uuid.uuid4()}'
|
||
|
|
try:
|
||
|
|
params = {'Bucket': bucket, 'Key': object_key}
|
||
|
|
if version_id:
|
||
|
|
params['VersionId'] = version_id
|
||
|
|
|
||
|
|
if isinstance(s3_client, AwsCliClient):
|
||
|
|
params['file_path'] = filename
|
||
|
|
|
||
|
|
response = s3_client.get_object(**params)
|
||
|
|
log_command_execution('S3 Get objects result', response)
|
||
|
|
|
||
|
|
if not isinstance(s3_client, AwsCliClient):
|
||
|
|
with open(f'{filename}', 'wb') as get_file:
|
||
|
|
chunk = response['Body'].read(1024)
|
||
|
|
while chunk:
|
||
|
|
get_file.write(chunk)
|
||
|
|
chunk = response['Body'].read(1024)
|
||
|
|
|
||
|
|
return filename
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Create multipart upload S3')
|
||
|
|
def create_multipart_upload_s3(s3_client, bucket_name: str, object_key: str) -> str:
|
||
|
|
try:
|
||
|
|
response = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_key)
|
||
|
|
log_command_execution('S3 Created multipart upload', response)
|
||
|
|
assert response.get('UploadId'), f'Expected UploadId in response:\n{response}'
|
||
|
|
|
||
|
|
return response.get('UploadId')
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('List multipart uploads S3')
|
||
|
|
def list_multipart_uploads_s3(s3_client, bucket_name: str) -> Optional[List[dict]]:
|
||
|
|
try:
|
||
|
|
response = s3_client.list_multipart_uploads(Bucket=bucket_name)
|
||
|
|
log_command_execution('S3 List multipart upload', response)
|
||
|
|
|
||
|
|
return response.get('Uploads')
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Abort multipart upload S3')
|
||
|
|
def abort_multipart_uploads_s3(s3_client, bucket_name: str, object_key: str, upload_id: str):
|
||
|
|
try:
|
||
|
|
response = s3_client.abort_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id)
|
||
|
|
log_command_execution('S3 Abort multipart upload', response)
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Upload part S3')
|
||
|
|
def upload_part_s3(s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, filepath: str) -> str:
|
||
|
|
if isinstance(s3_client, AwsCliClient):
|
||
|
|
file_content = filepath
|
||
|
|
else:
|
||
|
|
with open(filepath, 'rb') as put_file:
|
||
|
|
file_content = put_file.read()
|
||
|
|
|
||
|
|
try:
|
||
|
|
response = s3_client.upload_part(UploadId=upload_id, Bucket=bucket_name, Key=object_key, PartNumber=part_num,
|
||
|
|
Body=file_content)
|
||
|
|
log_command_execution('S3 Upload part', response)
|
||
|
|
assert response.get('ETag'), f'Expected ETag in response:\n{response}'
|
||
|
|
|
||
|
|
return response.get('ETag')
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('List parts S3')
|
||
|
|
def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> List[dict]:
|
||
|
|
try:
|
||
|
|
response = s3_client.list_parts(UploadId=upload_id, Bucket=bucket_name, Key=object_key)
|
||
|
|
log_command_execution('S3 List part', response)
|
||
|
|
assert response.get('Parts'), f'Expected Parts in response:\n{response}'
|
||
|
|
|
||
|
|
return response.get('Parts')
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Complete multipart upload S3')
|
||
|
|
def complete_multipart_upload_s3(s3_client, bucket_name: str, object_key: str, upload_id: str,
|
||
|
|
parts: list):
|
||
|
|
try:
|
||
|
|
parts = [{'ETag': etag, 'PartNumber': part_num} for part_num, etag in parts]
|
||
|
|
response = s3_client.complete_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id,
|
||
|
|
MultipartUpload={'Parts': parts})
|
||
|
|
log_command_execution('S3 Complete multipart upload', response)
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Put object tagging')
|
||
|
|
def put_object_tagging(s3_client, bucket_name: str, object_key: str, tags: list):
|
||
|
|
try:
|
||
|
|
tags = [{'Key': tag_key, 'Value': tag_value} for tag_key, tag_value in tags]
|
||
|
|
tagging = {'TagSet': tags}
|
||
|
|
s3_client.put_object_tagging(Bucket=bucket_name, Key=object_key, Tagging=tagging)
|
||
|
|
log_command_execution('S3 Put object tagging', str(tags))
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Got error during put object tagging: {err}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Get object tagging')
|
||
|
|
def get_object_tagging(s3_client, bucket_name: str, object_key: str) -> list:
|
||
|
|
try:
|
||
|
|
response = s3_client.get_object_tagging(Bucket=bucket_name, Key=object_key)
|
||
|
|
log_command_execution('S3 Get object tagging', response)
|
||
|
|
return response.get('TagSet')
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Got error during get object tagging: {err}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Delete object tagging')
|
||
|
|
def delete_object_tagging(s3_client, bucket_name: str, object_key: str):
|
||
|
|
try:
|
||
|
|
response = s3_client.delete_object_tagging(Bucket=bucket_name, Key=object_key)
|
||
|
|
log_command_execution('S3 Delete object tagging', response)
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Got error during delete object tagging: {err}') from err
|
||
|
|
|
||
|
|
|
||
|
|
@keyword('Get object tagging')
|
||
|
|
def get_object_attributes(s3_client, bucket_name: str, object_key: str, *attributes: str, version_id: str = None,
|
||
|
|
max_parts: int = None, part_number: int = None, get_full_resp=True) -> dict:
|
||
|
|
try:
|
||
|
|
if not isinstance(s3_client, AwsCliClient):
|
||
|
|
logger.warn('Method get_object_attributes is not supported by boto3 client')
|
||
|
|
return {}
|
||
|
|
response = s3_client.get_object_attributes(bucket_name, object_key, *attributes, version_id=version_id,
|
||
|
|
max_parts=max_parts, part_number=part_number)
|
||
|
|
log_command_execution('S3 Get object attributes', response)
|
||
|
|
for attr in attributes:
|
||
|
|
assert attr in response, f'Expected attribute {attr} in {response}'
|
||
|
|
|
||
|
|
if get_full_resp:
|
||
|
|
return response
|
||
|
|
else:
|
||
|
|
return response.get(attributes[0])
|
||
|
|
|
||
|
|
except ClientError as err:
|
||
|
|
raise Exception(f'Got error during get object attributes: {err}') from err
|
||
|
|
|
||
|
|
|
||
|
|
def _make_objs_dict(key_names):
|
||
|
|
objs_list = []
|
||
|
|
for key in key_names:
|
||
|
|
obj_dict = {'Key': key}
|
||
|
|
objs_list.append(obj_dict)
|
||
|
|
objs_dict = {'Objects': objs_list}
|
||
|
|
return objs_dict
|