forked from TrueCloudLab/frostfs-testcases
Add pytest tests.
Change AWS cli v1 to cli v2. Improve allure attachments. Add tests for S3 API. Signed-off-by: a.y.volkov <a.y.volkov@yadro.com>
This commit is contained in:
Anastasia Prasolova
parent
7abb2761c9
commit
6b1e1ab28d
24 changed files with 2210 additions and 247 deletions
330
robot/resources/lib/python_keywords/s3_gate_object.py
Normal file
330
robot/resources/lib/python_keywords/s3_gate_object.py
Normal file
|
|
@ -0,0 +1,330 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from enum import Enum
|
||||
from typing import Optional, List
|
||||
|
||||
import urllib3
|
||||
from botocore.exceptions import ClientError
|
||||
from robot.api import logger
|
||||
from robot.api.deco import keyword
|
||||
|
||||
from cli_helpers import log_command_execution
|
||||
from python_keywords.aws_cli_client import AwsCliClient
|
||||
|
||||
##########################################################
|
||||
# Disabling warnings on self-signed certificate which the
|
||||
# boto library produces on requests to S3-gate in dev-env.
|
||||
urllib3.disable_warnings()
|
||||
##########################################################
|
||||
|
||||
ROBOT_AUTO_KEYWORDS = False
|
||||
CREDENTIALS_CREATE_TIMEOUT = '30s'
|
||||
|
||||
ASSETS_DIR = os.getenv('ASSETS_DIR', 'TemporaryDir/')
|
||||
|
||||
|
||||
class VersioningStatus(Enum):
|
||||
ENABLED = 'Enabled'
|
||||
SUSPENDED = 'Suspended'
|
||||
|
||||
|
||||
@keyword('List objects S3 v2')
|
||||
def list_objects_s3_v2(s3_client, bucket: str) -> list:
|
||||
try:
|
||||
response = s3_client.list_objects_v2(Bucket=bucket)
|
||||
content = response.get('Contents', [])
|
||||
log_command_execution('S3 v2 List objects result', response)
|
||||
obj_list = []
|
||||
for obj in content:
|
||||
obj_list.append(obj['Key'])
|
||||
logger.info(f'Found s3 objects: {obj_list}')
|
||||
return obj_list
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('List objects S3')
|
||||
def list_objects_s3(s3_client, bucket: str) -> list:
|
||||
try:
|
||||
response = s3_client.list_objects(Bucket=bucket)
|
||||
content = response.get('Contents', [])
|
||||
log_command_execution('S3 List objects result', response)
|
||||
obj_list = []
|
||||
for obj in content:
|
||||
obj_list.append(obj['Key'])
|
||||
logger.info(f'Found s3 objects: {obj_list}')
|
||||
return obj_list
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('List objects versions S3')
|
||||
def list_objects_versions_s3(s3_client, bucket: str) -> list:
|
||||
try:
|
||||
response = s3_client.list_object_versions(Bucket=bucket)
|
||||
versions = response.get('Versions', [])
|
||||
log_command_execution('S3 List objects versions result', response)
|
||||
return versions
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Put object S3')
|
||||
def put_object_s3(s3_client, bucket: str, filepath: str):
|
||||
filename = os.path.basename(filepath)
|
||||
|
||||
if isinstance(s3_client, AwsCliClient):
|
||||
file_content = filepath
|
||||
else:
|
||||
with open(filepath, 'rb') as put_file:
|
||||
file_content = put_file.read()
|
||||
|
||||
try:
|
||||
response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename)
|
||||
log_command_execution('S3 Put object result', response)
|
||||
return response.get('VersionId')
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Head object S3')
|
||||
def head_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None):
|
||||
try:
|
||||
params = {'Bucket': bucket, 'Key': object_key}
|
||||
if version_id:
|
||||
params['VersionId'] = version_id
|
||||
response = s3_client.head_object(**params)
|
||||
log_command_execution('S3 Head object result', response)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Delete object S3')
|
||||
def delete_object_s3(s3_client, bucket, object_key, version_id: str = None):
|
||||
try:
|
||||
params = {'Bucket': bucket, 'Key': object_key}
|
||||
if version_id:
|
||||
params['VersionId'] = version_id
|
||||
response = s3_client.delete_object(**params)
|
||||
log_command_execution('S3 Delete object result', response)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Delete objects S3')
|
||||
def delete_objects_s3(s3_client, bucket: str, object_keys: list):
|
||||
try:
|
||||
response = s3_client.delete_objects(Bucket=bucket, Delete=_make_objs_dict(object_keys))
|
||||
log_command_execution('S3 Delete objects result', response)
|
||||
return response
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Copy object S3')
|
||||
def copy_object_s3(s3_client, bucket, object_key, bucket_dst=None):
|
||||
filename = f'{os.getcwd()}/{uuid.uuid4()}'
|
||||
try:
|
||||
response = s3_client.copy_object(Bucket=bucket_dst or bucket,
|
||||
CopySource=f'{bucket}/{object_key}',
|
||||
Key=filename)
|
||||
log_command_execution('S3 Copy objects result', response)
|
||||
return filename
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Get object S3')
|
||||
def get_object_s3(s3_client, bucket: str, object_key: str, version_id: str = None):
|
||||
filename = f'{ASSETS_DIR}/{uuid.uuid4()}'
|
||||
try:
|
||||
params = {'Bucket': bucket, 'Key': object_key}
|
||||
if version_id:
|
||||
params['VersionId'] = version_id
|
||||
|
||||
if isinstance(s3_client, AwsCliClient):
|
||||
params['file_path'] = filename
|
||||
|
||||
response = s3_client.get_object(**params)
|
||||
log_command_execution('S3 Get objects result', response)
|
||||
|
||||
if not isinstance(s3_client, AwsCliClient):
|
||||
with open(f'{filename}', 'wb') as get_file:
|
||||
chunk = response['Body'].read(1024)
|
||||
while chunk:
|
||||
get_file.write(chunk)
|
||||
chunk = response['Body'].read(1024)
|
||||
|
||||
return filename
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Create multipart upload S3')
|
||||
def create_multipart_upload_s3(s3_client, bucket_name: str, object_key: str) -> str:
|
||||
try:
|
||||
response = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution('S3 Created multipart upload', response)
|
||||
assert response.get('UploadId'), f'Expected UploadId in response:\n{response}'
|
||||
|
||||
return response.get('UploadId')
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('List multipart uploads S3')
|
||||
def list_multipart_uploads_s3(s3_client, bucket_name: str) -> Optional[List[dict]]:
|
||||
try:
|
||||
response = s3_client.list_multipart_uploads(Bucket=bucket_name)
|
||||
log_command_execution('S3 List multipart upload', response)
|
||||
|
||||
return response.get('Uploads')
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Abort multipart upload S3')
|
||||
def abort_multipart_uploads_s3(s3_client, bucket_name: str, object_key: str, upload_id: str):
|
||||
try:
|
||||
response = s3_client.abort_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id)
|
||||
log_command_execution('S3 Abort multipart upload', response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Upload part S3')
|
||||
def upload_part_s3(s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, filepath: str) -> str:
|
||||
if isinstance(s3_client, AwsCliClient):
|
||||
file_content = filepath
|
||||
else:
|
||||
with open(filepath, 'rb') as put_file:
|
||||
file_content = put_file.read()
|
||||
|
||||
try:
|
||||
response = s3_client.upload_part(UploadId=upload_id, Bucket=bucket_name, Key=object_key, PartNumber=part_num,
|
||||
Body=file_content)
|
||||
log_command_execution('S3 Upload part', response)
|
||||
assert response.get('ETag'), f'Expected ETag in response:\n{response}'
|
||||
|
||||
return response.get('ETag')
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('List parts S3')
|
||||
def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> List[dict]:
|
||||
try:
|
||||
response = s3_client.list_parts(UploadId=upload_id, Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution('S3 List part', response)
|
||||
assert response.get('Parts'), f'Expected Parts in response:\n{response}'
|
||||
|
||||
return response.get('Parts')
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Complete multipart upload S3')
|
||||
def complete_multipart_upload_s3(s3_client, bucket_name: str, object_key: str, upload_id: str,
|
||||
parts: list):
|
||||
try:
|
||||
parts = [{'ETag': etag, 'PartNumber': part_num} for part_num, etag in parts]
|
||||
response = s3_client.complete_multipart_upload(Bucket=bucket_name, Key=object_key, UploadId=upload_id,
|
||||
MultipartUpload={'Parts': parts})
|
||||
log_command_execution('S3 Complete multipart upload', response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Error Message: {err.response["Error"]["Message"]}\n'
|
||||
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}') from err
|
||||
|
||||
|
||||
@keyword('Put object tagging')
|
||||
def put_object_tagging(s3_client, bucket_name: str, object_key: str, tags: list):
|
||||
try:
|
||||
tags = [{'Key': tag_key, 'Value': tag_value} for tag_key, tag_value in tags]
|
||||
tagging = {'TagSet': tags}
|
||||
s3_client.put_object_tagging(Bucket=bucket_name, Key=object_key, Tagging=tagging)
|
||||
log_command_execution('S3 Put object tagging', str(tags))
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during put object tagging: {err}') from err
|
||||
|
||||
|
||||
@keyword('Get object tagging')
|
||||
def get_object_tagging(s3_client, bucket_name: str, object_key: str) -> list:
|
||||
try:
|
||||
response = s3_client.get_object_tagging(Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution('S3 Get object tagging', response)
|
||||
return response.get('TagSet')
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during get object tagging: {err}') from err
|
||||
|
||||
|
||||
@keyword('Delete object tagging')
|
||||
def delete_object_tagging(s3_client, bucket_name: str, object_key: str):
|
||||
try:
|
||||
response = s3_client.delete_object_tagging(Bucket=bucket_name, Key=object_key)
|
||||
log_command_execution('S3 Delete object tagging', response)
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during delete object tagging: {err}') from err
|
||||
|
||||
|
||||
@keyword('Get object tagging')
|
||||
def get_object_attributes(s3_client, bucket_name: str, object_key: str, *attributes: str, version_id: str = None,
|
||||
max_parts: int = None, part_number: int = None, get_full_resp=True) -> dict:
|
||||
try:
|
||||
if not isinstance(s3_client, AwsCliClient):
|
||||
logger.warn('Method get_object_attributes is not supported by boto3 client')
|
||||
return {}
|
||||
response = s3_client.get_object_attributes(bucket_name, object_key, *attributes, version_id=version_id,
|
||||
max_parts=max_parts, part_number=part_number)
|
||||
log_command_execution('S3 Get object attributes', response)
|
||||
for attr in attributes:
|
||||
assert attr in response, f'Expected attribute {attr} in {response}'
|
||||
|
||||
if get_full_resp:
|
||||
return response
|
||||
else:
|
||||
return response.get(attributes[0])
|
||||
|
||||
except ClientError as err:
|
||||
raise Exception(f'Got error during get object attributes: {err}') from err
|
||||
|
||||
|
||||
def _make_objs_dict(key_names):
|
||||
objs_list = []
|
||||
for key in key_names:
|
||||
obj_dict = {'Key': key}
|
||||
objs_list.append(obj_dict)
|
||||
objs_dict = {'Objects': objs_list}
|
||||
return objs_dict
|
||||
Loading…
Add table
Add a link
Reference in a new issue