frostfs-testcases/robot/resources/lib/python/s3_gate.py

236 lines
8.1 KiB
Python
Raw Normal View History

#!/usr/bin/python3
2020-11-30 10:33:05 +00:00
import json
2020-11-30 10:33:05 +00:00
import os
import uuid
import boto3
import botocore
from cli_helpers import _run_with_passwd
2020-11-30 10:33:05 +00:00
from common import GATE_PUB_KEY, NEOFS_ENDPOINT, S3_GATE
import urllib3
2020-11-30 10:33:05 +00:00
from robot.api.deco import keyword
from robot.api import logger
##########################################################
# Disabling warnings on self-signed certificate which the
# boto library produces on requests to S3-gate in dev-env.
urllib3.disable_warnings()
##########################################################
2020-11-30 10:33:05 +00:00
ROBOT_AUTO_KEYWORDS = False
NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate')
@keyword('Init S3 Credentials')
def init_s3_credentials(wallet):
bucket = str(uuid.uuid4())
s3_bearer_rules = "robot/resources/files/s3_bearer_rules.json"
cmd = (
f'{NEOFS_EXEC} --debug --with-log issue-secret --wallet {wallet} '
f'--gate-public-key={GATE_PUB_KEY} --peer {NEOFS_ENDPOINT} '
f'--container-friendly-name {bucket} '
f'--bearer-rules {s3_bearer_rules}'
)
logger.info(f"Executing command: {cmd}")
try:
output = _run_with_passwd(cmd)
logger.info(f"Command completed with output: {output}")
# first five string are log output, cutting them off and parse
# the rest of the output as JSON
output = '\n'.join(output.split('\n')[5:])
output_dict = json.loads(output)
return (output_dict['container_id'],
bucket,
output_dict['access_key_id'],
output_dict['secret_access_key'],
output_dict['owner_private_key'])
except Exception as exc:
raise RuntimeError("failed to init s3 credentials") from exc
@keyword('Config S3 client')
def config_s3_client(access_key_id, secret_access_key):
try:
session = boto3.session.Session()
s3_client = session.client(
service_name='s3',
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
endpoint_url=S3_GATE, verify=False
)
return s3_client
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('List objects S3 v2')
def list_objects_s3_v2(s3_client, bucket):
try:
response = s3_client.list_objects_v2(Bucket=bucket)
logger.info(f"S3 v2 List objects result: {response['Contents']}")
obj_list = []
for obj in response['Contents']:
obj_list.append(obj['Key'])
logger.info(f"Found s3 objects: {obj_list}")
return obj_list
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('List objects S3')
def list_objects_s3(s3_client, bucket):
try:
response = s3_client.list_objects(Bucket=bucket)
logger.info(f"S3 List objects result: {response['Contents']}")
obj_list = []
for obj in response['Contents']:
obj_list.append(obj['Key'])
logger.info(f"Found s3 objects: {obj_list}")
return obj_list
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Create bucket S3')
def create_bucket_s3(s3_client):
bucket_name = str(uuid.uuid4())
try:
s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
logger.info(f"Created S3 bucket: {s3_bucket}")
return bucket_name
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('List buckets S3')
def list_buckets_s3(s3_client):
found_buckets = []
try:
response = s3_client.list_buckets()
logger.info(f"S3 List buckets result: {response}")
for bucket in response['Buckets']:
found_buckets.append(bucket['Name'])
return found_buckets
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Delete bucket S3')
def delete_bucket_s3(s3_client, bucket):
try:
response = s3_client.delete_bucket(Bucket=bucket)
logger.info(f"S3 Delete bucket result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Head bucket S3')
def head_bucket(s3_client, bucket):
try:
response = s3_client.head_bucket(Bucket=bucket)
logger.info(f"S3 Head bucket result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Put object S3')
def put_object_s3(s3_client, bucket, filepath):
filename = os.path.basename(filepath)
with open(filepath, "rb") as put_file:
file_content = put_file.read()
try:
response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename)
logger.info(f"S3 Put object result: {response}")
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Head object S3')
def head_object_s3(s3_client, bucket, object_key):
try:
response = s3_client.head_object(Bucket=bucket, Key=object_key)
logger.info(f"S3 Head object result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Delete object S3')
def delete_object_s3(s3_client, bucket, object_key):
try:
response = s3_client.delete_object(Bucket=bucket, Key=object_key)
logger.info(f"S3 Put object result: {response}")
return response
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Copy object S3')
def copy_object_s3(s3_client, bucket, object_key):
filename = f"{os.getcwd()}/{uuid.uuid4()}"
try:
response = s3_client.copy_object(Bucket=bucket,
CopySource=f"{bucket}/{object_key}",
Key=filename)
logger.info(f"S3 Copy object result: {response}")
return filename
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
@keyword('Get object S3')
def get_object_s3(s3_client, bucket, object_key):
filename = f"{os.getcwd()}/{uuid.uuid4()}"
try:
response = s3_client.get_object(Bucket=bucket, Key=object_key)
with open(f"{filename}", 'wb') as get_file:
chunk = response['Body'].read(1024)
while chunk:
get_file.write(chunk)
chunk = response['Body'].read(1024)
return filename
except botocore.exceptions.ClientError as err:
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err