anurindm/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Pull requests Activity

(#108) HTTP error logs for S3

Browse source 
Signed-off-by: Elizaveta Chichindaeva <elizaveta@nspcc.ru>
This commit is contained in:
Elizaveta Chichindaeva 2021-10-01 12:32:07 +03:00
parent 636eb65e5c
commit 36107fcc4c
1 changed files with 119 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

175
robot/resources/lib/gates.py
View file

@ -1,6 +1,8 @@
#!/usr/bin/python3.8 #!/usr/bin/python3.8
from logging import raiseExceptions
import boto3 import boto3
import botocore
import os import os
import pexpect import pexpect
import re import re
@ -58,6 +60,7 @@ def init_s3_credentials(wallet):
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
raise Exception(f"Error: \nreturn code: {e.returncode}. \nOutput: {e.stderr}") raise Exception(f"Error: \nreturn code: {e.returncode}. \nOutput: {e.stderr}")
def _run_with_passwd(cmd): def _run_with_passwd(cmd):
p = pexpect.spawn(cmd) p = pexpect.spawn(cmd)
p.expect(".*") p.expect(".*")
@ -66,70 +69,112 @@ def _run_with_passwd(cmd):
cmd = p.read() cmd = p.read()
return cmd.decode() return cmd.decode()
@keyword('Config S3 client') @keyword('Config S3 client')
def config_s3_client(access_key_id, secret_access_key): def config_s3_client(access_key_id, secret_access_key):
session = boto3.session.Session() try:
session = boto3.session.Session()
s3_client = session.client( s3_client = session.client(
service_name='s3', service_name='s3',
aws_access_key_id=access_key_id, aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key, aws_secret_access_key=secret_access_key,
endpoint_url=S3_GATE, verify=False endpoint_url=S3_GATE, verify=False
) )
return s3_client
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
return s3_client
@keyword('List objects S3 v2') @keyword('List objects S3 v2')
def list_objects_s3_v2(s3_client, bucket): def list_objects_s3_v2(s3_client, bucket):
response = s3_client.list_objects_v2(Bucket=bucket) try:
logger.info("S3 v2 List objects result: %s" % response['Contents']) response = s3_client.list_objects_v2(Bucket=bucket)
obj_list = [] logger.info("S3 v2 List objects result: %s" % response['Contents'])
for obj in response['Contents']: obj_list = []
obj_list.append(obj['Key']) for obj in response['Contents']:
logger.info("Found s3 objects: %s" % obj_list) obj_list.append(obj['Key'])
return obj_list logger.info("Found s3 objects: %s" % obj_list)
return obj_list
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('List objects S3') @keyword('List objects S3')
def list_objects_s3(s3_client, bucket): def list_objects_s3(s3_client, bucket):
response = s3_client.list_objects(Bucket=bucket) try:
logger.info("S3 List objects result: %s" % response['Contents']) response = s3_client.list_objects(Bucket=bucket)
obj_list = [] logger.info("S3 List objects result: %s" % response['Contents'])
for obj in response['Contents']: obj_list = []
obj_list.append(obj['Key']) for obj in response['Contents']:
logger.info("Found s3 objects: %s" % obj_list) obj_list.append(obj['Key'])
return obj_list logger.info("Found s3 objects: %s" % obj_list)
return obj_list
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Create bucket S3') @keyword('Create bucket S3')
def create_bucket_s3(s3_client): def create_bucket_s3(s3_client):
bucket_name = str(uuid.uuid4()) bucket_name = str(uuid.uuid4())
s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
logger.info("Created S3 bucket: %s" % s3_bucket) try:
return bucket_name s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
logger.info("Created S3 bucket: %s" % s3_bucket)
return bucket_name
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('List buckets S3') @keyword('List buckets S3')
def list_buckets_s3(s3_client): def list_buckets_s3(s3_client):
found_buckets = [] found_buckets = []
response = s3_client.list_buckets() try:
logger.info("S3 List buckets result: %s" % response) response = s3_client.list_buckets()
logger.info("S3 List buckets result: %s" % response)
for bucket in response['Buckets']: for bucket in response['Buckets']:
found_buckets.append(bucket['Name']) found_buckets.append(bucket['Name'])
return found_buckets
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
return found_buckets
@keyword('Delete bucket S3') @keyword('Delete bucket S3')
def delete_bucket_s3(s3_client, bucket): def delete_bucket_s3(s3_client, bucket):
response = s3_client.delete_bucket(Bucket=bucket) try:
logger.info(f"S3 Delete bucket result: {response}") response = s3_client.delete_bucket(Bucket=bucket)
return response logger.info(f"S3 Delete bucket result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('HeadBucket S3') @keyword('HeadBucket S3')
def headbucket(bucket, s3_client): def headbucket(bucket, s3_client):
response = s3_client.head_bucket(Bucket=bucket) try:
logger.info(f"S3 HeadBucket result: {response}") response = s3_client.head_bucket(Bucket=bucket)
return response logger.info(f"S3 HeadBucket result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Put object S3') @keyword('Put object S3')
@ -139,49 +184,67 @@ def put_object_s3(s3_client, bucket, filepath):
with open(filepath, "rb") as f: with open(filepath, "rb") as f:
fileContent = f.read() fileContent = f.read()
response = s3_client.put_object(Body=fileContent, Bucket=bucket, Key=filename) try:
logger.info("S3 Put object result: %s" % response) response = s3_client.put_object(Body=fileContent, Bucket=bucket, Key=filename)
return response logger.info("S3 Put object result: %s" % response)
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Head object S3') @keyword('Head object S3')
def head_object_s3(s3_client, bucket, object_key): def head_object_s3(s3_client, bucket, object_key):
response = s3_client.head_object(Bucket=bucket, Key=object_key)
logger.info("S3 Head object result: %s" % response) try:
return response response = s3_client.head_object(Bucket=bucket, Key=object_key)
logger.info("S3 Head object result: %s" % response)
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Delete object S3') @keyword('Delete object S3')
def delete_object_s3(s3_client, bucket, object_key): def delete_object_s3(s3_client, bucket, object_key):
try:
response = s3_client.delete_object(Bucket=bucket, Key=object_key)
logger.info("S3 Put object result: %s" % response)
return response
response = s3_client.delete_object(Bucket=bucket, Key=object_key) except botocore.exceptions.ClientError as err:
logger.info("S3 Put object result: %s" % response) raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
return response f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Copy object S3') @keyword('Copy object S3')
def copy_object_s3(s3_client, bucket, object_key, new_object): def copy_object_s3(s3_client, bucket, object_key, new_object):
try: try:
response = s3_client.copy_object(Bucket=bucket, CopySource=bucket+"/"+object_key, Key=new_object) response = s3_client.copy_object(Bucket=bucket, CopySource=bucket+"/"+object_key, Key=new_object)
logger.info("S3 Copy object result: %s" % response) logger.info("S3 Copy object result: %s" % response)
except: return response
raise Exception(f"S3 object {new_object} hasn't been coppied")
except botocore.exceptions.ClientError as err:
return response raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Get object S3') @keyword('Get object S3')
def get_object_s3(s3_client, bucket, object_key, target_file): def get_object_s3(s3_client, bucket, object_key, target_file):
response = s3_client.get_object(Bucket=bucket, Key=object_key) try:
response = s3_client.get_object(Bucket=bucket, Key=object_key)
with open(f"{target_file}", 'wb') as f: with open(f"{target_file}", 'wb') as f:
chunk = response['Body'].read(1024)
while chunk:
f.write(chunk)
chunk = response['Body'].read(1024) chunk = response['Body'].read(1024)
while chunk:
f.write(chunk)
chunk = response['Body'].read(1024)
return target_file return target_file
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Get via HTTP Gate') @keyword('Get via HTTP Gate')