anurindm/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Pull requests Activity

(#108) HTTP error logs for S3

Browse source 
Signed-off-by: Elizaveta Chichindaeva <elizaveta@nspcc.ru>
This commit is contained in:
Elizaveta Chichindaeva 2021-10-01 12:32:07 +03:00
parent 636eb65e5c
commit 36107fcc4c
1 changed files with 119 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
robot/resources/lib/gates.py
View file

@ -1,6 +1,8 @@
#!/usr/bin/python3.8
from logging import raiseExceptions
import boto3
import botocore
import os
import pexpect
import re
@ -58,6 +60,7 @@ def init_s3_credentials(wallet):
except subprocess.CalledProcessError as e:
raise Exception(f"Error: \nreturn code: {e.returncode}. \nOutput: {e.stderr}")
def _run_with_passwd(cmd):
p = pexpect.spawn(cmd)
p.expect(".*")
@ -66,8 +69,10 @@ def _run_with_passwd(cmd):
cmd = p.read()
return cmd.decode()
@keyword('Config S3 client')
def config_s3_client(access_key_id, secret_access_key):
try:
session = boto3.session.Session()
s3_client = session.client(
@ -79,8 +84,14 @@ def config_s3_client(access_key_id, secret_access_key):
return s3_client
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('List objects S3 v2')
def list_objects_s3_v2(s3_client, bucket):
try:
response = s3_client.list_objects_v2(Bucket=bucket)
logger.info("S3 v2 List objects result: %s" % response['Contents'])
obj_list = []
@ -89,9 +100,14 @@ def list_objects_s3_v2(s3_client, bucket):
logger.info("Found s3 objects: %s" % obj_list)
return obj_list
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('List objects S3')
def list_objects_s3(s3_client, bucket):
try:
response = s3_client.list_objects(Bucket=bucket)
logger.info("S3 List objects result: %s" % response['Contents'])
obj_list = []
@ -100,16 +116,29 @@ def list_objects_s3(s3_client, bucket):
logger.info("Found s3 objects: %s" % obj_list)
return obj_list
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Create bucket S3')
def create_bucket_s3(s3_client):
bucket_name = str(uuid.uuid4())
try:
s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
logger.info("Created S3 bucket: %s" % s3_bucket)
return bucket_name
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('List buckets S3')
def list_buckets_s3(s3_client):
found_buckets = []
try:
response = s3_client.list_buckets()
logger.info("S3 List buckets result: %s" % response)
@ -118,19 +147,35 @@ def list_buckets_s3(s3_client):
return found_buckets
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Delete bucket S3')
def delete_bucket_s3(s3_client, bucket):
try:
response = s3_client.delete_bucket(Bucket=bucket)
logger.info(f"S3 Delete bucket result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('HeadBucket S3')
def headbucket(bucket, s3_client):
try:
response = s3_client.head_bucket(Bucket=bucket)
logger.info(f"S3 HeadBucket result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Put object S3')
def put_object_s3(s3_client, bucket, filepath):
@ -139,40 +184,54 @@ def put_object_s3(s3_client, bucket, filepath):
with open(filepath, "rb") as f:
fileContent = f.read()
try:
response = s3_client.put_object(Body=fileContent, Bucket=bucket, Key=filename)
logger.info("S3 Put object result: %s" % response)
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Head object S3')
def head_object_s3(s3_client, bucket, object_key):
try:
response = s3_client.head_object(Bucket=bucket, Key=object_key)
logger.info("S3 Head object result: %s" % response)
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Delete object S3')
def delete_object_s3(s3_client, bucket, object_key):
try:
response = s3_client.delete_object(Bucket=bucket, Key=object_key)
logger.info("S3 Put object result: %s" % response)
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Copy object S3')
def copy_object_s3(s3_client, bucket, object_key, new_object):
try:
response = s3_client.copy_object(Bucket=bucket, CopySource=bucket+"/"+object_key, Key=new_object)
logger.info("S3 Copy object result: %s" % response)
except:
raise Exception(f"S3 object {new_object} hasn't been coppied")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Get object S3')
def get_object_s3(s3_client, bucket, object_key, target_file):
try:
response = s3_client.get_object(Bucket=bucket, Key=object_key)
with open(f"{target_file}", 'wb') as f:
@ -183,6 +242,10 @@ def get_object_s3(s3_client, bucket, object_key, target_file):
return target_file
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Get via HTTP Gate')
def get_via_http_gate(cid: str, oid: str):