236 lines
8.3 KiB
Python
236 lines
8.3 KiB
Python
#!/usr/bin/python3
|
|
|
|
import json
|
|
import os
|
|
import uuid
|
|
|
|
import boto3
|
|
import botocore
|
|
import urllib3
|
|
from robot.api import logger
|
|
from robot.api.deco import keyword
|
|
|
|
from cli_helpers import _run_with_passwd
|
|
from common import GATE_PUB_KEY, NEOFS_ENDPOINT, S3_GATE
|
|
|
|
##########################################################
|
|
# Disabling warnings on self-signed certificate which the
|
|
# boto library produces on requests to S3-gate in dev-env.
|
|
urllib3.disable_warnings()
|
|
##########################################################
|
|
|
|
ROBOT_AUTO_KEYWORDS = False
|
|
CREDENTIALS_CREATE_TIMEOUT = '30s'
|
|
|
|
NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate')
|
|
|
|
|
|
@keyword('Init S3 Credentials')
|
|
def init_s3_credentials(wallet):
|
|
bucket = str(uuid.uuid4())
|
|
s3_bearer_rules = "robot/resources/files/s3_bearer_rules.json"
|
|
cmd = (
|
|
f'{NEOFS_EXEC} --debug --with-log --timeout {CREDENTIALS_CREATE_TIMEOUT} '
|
|
f'issue-secret --wallet {wallet} --gate-public-key={GATE_PUB_KEY} '
|
|
f'--peer {NEOFS_ENDPOINT} --container-friendly-name {bucket} '
|
|
f'--bearer-rules {s3_bearer_rules}'
|
|
)
|
|
logger.info(f"Executing command: {cmd}")
|
|
|
|
try:
|
|
output = _run_with_passwd(cmd)
|
|
logger.info(f"Command completed with output: {output}")
|
|
# first five string are log output, cutting them off and parse
|
|
# the rest of the output as JSON
|
|
output = '\n'.join(output.split('\n')[5:])
|
|
output_dict = json.loads(output)
|
|
|
|
return (output_dict['container_id'],
|
|
bucket,
|
|
output_dict['access_key_id'],
|
|
output_dict['secret_access_key'],
|
|
output_dict['owner_private_key'])
|
|
|
|
except Exception as exc:
|
|
raise RuntimeError("failed to init s3 credentials") from exc
|
|
|
|
|
|
@keyword('Config S3 client')
|
|
def config_s3_client(access_key_id, secret_access_key):
|
|
try:
|
|
|
|
session = boto3.session.Session()
|
|
|
|
s3_client = session.client(
|
|
service_name='s3',
|
|
aws_access_key_id=access_key_id,
|
|
aws_secret_access_key=secret_access_key,
|
|
endpoint_url=S3_GATE, verify=False
|
|
)
|
|
|
|
return s3_client
|
|
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
|
|
|
|
|
@keyword('List objects S3 v2')
|
|
def list_objects_s3_v2(s3_client, bucket):
|
|
try:
|
|
response = s3_client.list_objects_v2(Bucket=bucket)
|
|
logger.info(f"S3 v2 List objects result: {response['Contents']}")
|
|
obj_list = []
|
|
for obj in response['Contents']:
|
|
obj_list.append(obj['Key'])
|
|
logger.info(f"Found s3 objects: {obj_list}")
|
|
return obj_list
|
|
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
|
|
|
|
|
@keyword('List objects S3')
|
|
def list_objects_s3(s3_client, bucket):
|
|
try:
|
|
response = s3_client.list_objects(Bucket=bucket)
|
|
logger.info(f"S3 List objects result: {response['Contents']}")
|
|
obj_list = []
|
|
for obj in response['Contents']:
|
|
obj_list.append(obj['Key'])
|
|
logger.info(f"Found s3 objects: {obj_list}")
|
|
return obj_list
|
|
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
|
|
|
|
|
@keyword('Create bucket S3')
|
|
def create_bucket_s3(s3_client):
|
|
bucket_name = str(uuid.uuid4())
|
|
|
|
try:
|
|
s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
|
|
logger.info(f"Created S3 bucket: {s3_bucket}")
|
|
return bucket_name
|
|
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
|
|
|
|
|
@keyword('List buckets S3')
|
|
def list_buckets_s3(s3_client):
|
|
found_buckets = []
|
|
try:
|
|
response = s3_client.list_buckets()
|
|
logger.info(f"S3 List buckets result: {response}")
|
|
|
|
for bucket in response['Buckets']:
|
|
found_buckets.append(bucket['Name'])
|
|
|
|
return found_buckets
|
|
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
|
|
|
|
|
@keyword('Delete bucket S3')
|
|
def delete_bucket_s3(s3_client, bucket):
|
|
try:
|
|
response = s3_client.delete_bucket(Bucket=bucket)
|
|
logger.info(f"S3 Delete bucket result: {response}")
|
|
|
|
return response
|
|
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
|
|
|
|
|
@keyword('Head bucket S3')
|
|
def head_bucket(s3_client, bucket):
|
|
try:
|
|
response = s3_client.head_bucket(Bucket=bucket)
|
|
logger.info(f"S3 Head bucket result: {response}")
|
|
return response
|
|
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
|
|
|
|
|
@keyword('Put object S3')
|
|
def put_object_s3(s3_client, bucket, filepath):
|
|
filename = os.path.basename(filepath)
|
|
|
|
with open(filepath, "rb") as put_file:
|
|
file_content = put_file.read()
|
|
|
|
try:
|
|
response = s3_client.put_object(Body=file_content, Bucket=bucket, Key=filename)
|
|
logger.info(f"S3 Put object result: {response}")
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
|
|
|
|
|
@keyword('Head object S3')
|
|
def head_object_s3(s3_client, bucket, object_key):
|
|
try:
|
|
response = s3_client.head_object(Bucket=bucket, Key=object_key)
|
|
logger.info(f"S3 Head object result: {response}")
|
|
return response
|
|
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
|
|
|
|
|
@keyword('Delete object S3')
|
|
def delete_object_s3(s3_client, bucket, object_key):
|
|
try:
|
|
response = s3_client.delete_object(Bucket=bucket, Key=object_key)
|
|
logger.info(f"S3 Put object result: {response}")
|
|
return response
|
|
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
|
|
|
|
|
@keyword('Copy object S3')
|
|
def copy_object_s3(s3_client, bucket, object_key):
|
|
filename = f"{os.getcwd()}/{uuid.uuid4()}"
|
|
try:
|
|
response = s3_client.copy_object(Bucket=bucket,
|
|
CopySource=f"{bucket}/{object_key}",
|
|
Key=filename)
|
|
logger.info(f"S3 Copy object result: {response}")
|
|
return filename
|
|
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|
|
|
|
|
|
@keyword('Get object S3')
|
|
def get_object_s3(s3_client, bucket, object_key):
|
|
filename = f"{os.getcwd()}/{uuid.uuid4()}"
|
|
try:
|
|
response = s3_client.get_object(Bucket=bucket, Key=object_key)
|
|
|
|
with open(f"{filename}", 'wb') as get_file:
|
|
chunk = response['Body'].read(1024)
|
|
while chunk:
|
|
get_file.write(chunk)
|
|
chunk = response['Body'].read(1024)
|
|
|
|
return filename
|
|
|
|
except botocore.exceptions.ClientError as err:
|
|
raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
|
|
f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err
|