frostfs-testcases/robot/resources/lib/gates.py

#!/usr/bin/python3.8

import os
import re
import shutil
import subprocess
import uuid
import requests
import botocore
import boto3

from common import *
from robot.api.deco import keyword
from robot.api import logger
from cli_helpers import _run_with_passwd


ROBOT_AUTO_KEYWORDS = False

NEOFS_EXEC = os.getenv('NEOFS_EXEC', 'neofs-authmate')

@keyword('Init S3 Credentials')
def init_s3_credentials(wallet):
    bucket = str(uuid.uuid4())
    records = ' \' {"records":[{"operation":"PUT","action":"ALLOW","filters":[],"targets":[{"role":"OTHERS","keys":[]}]}, {"operation":"SEARCH","action":"ALLOW","filters":[],"targets":[{"role":"OTHERS","keys":[]}]}, {"operation":"GET","action":"ALLOW","filters":[],"targets":[{"role":"OTHERS","keys":[]}]}]} \' '
    Cmd = (
        f'{NEOFS_EXEC} --debug --with-log issue-secret --wallet {wallet} '
        f'--gate-public-key={GATE_PUB_KEY} --peer {NEOFS_ENDPOINT} '
        f'--container-friendly-name {bucket} --create-session-token '
        f'--bearer-rules {records}'
    )
    logger.info(f"Executing command: {Cmd}")

    try:
        output = _run_with_passwd(Cmd)
        logger.info(f"Command completed with output: {output}")

        m = re.search(r'"container_id":\s+"(\w+)"', output)
        cid = m.group(1)
        logger.info("cid: %s" % cid)

        m = re.search(r'"access_key_id":\s+"([\w\/]+)"', output)
        access_key_id = m.group(1)
        logger.info("access_key_id: %s" % access_key_id)

        m = re.search(r'"secret_access_key":\s+"(\w+)"', output)
        secret_access_key = m.group(1)
        logger.info("secret_access_key: %s" % secret_access_key)

        m = re.search(r'"owner_private_key":\s+"(\w+)"', output)
        owner_private_key = m.group(1)
        logger.info("owner_private_key: %s" % owner_private_key)

        return cid, bucket, access_key_id, secret_access_key, owner_private_key

    except subprocess.CalledProcessError as e:
        raise Exception(f"Error: \nreturn code: {e.returncode}. \nOutput: {e.stderr}")


@keyword('Config S3 client')
def config_s3_client(access_key_id, secret_access_key):
    try:
        session = boto3.session.Session()

        s3_client = session.client(
            service_name='s3',
            aws_access_key_id=access_key_id,
            aws_secret_access_key=secret_access_key,
            endpoint_url=S3_GATE, verify=False
        )

        return s3_client

    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err


@keyword('List objects S3 v2')
def list_objects_s3_v2(s3_client, bucket):
    try:
        response = s3_client.list_objects_v2(Bucket=bucket)
        logger.info("S3 v2 List objects result: %s" % response['Contents'])
        obj_list = []
        for obj in response['Contents']:
            obj_list.append(obj['Key'])
        logger.info("Found s3 objects: %s" % obj_list)
        return obj_list

    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err


@keyword('List objects S3')
def list_objects_s3(s3_client, bucket):
    try:
        response = s3_client.list_objects(Bucket=bucket)
        logger.info("S3 List objects result: %s" % response['Contents'])
        obj_list = []
        for obj in response['Contents']:
            obj_list.append(obj['Key'])
        logger.info("Found s3 objects: %s" % obj_list)
        return obj_list

    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err


@keyword('Create bucket S3')
def create_bucket_s3(s3_client):
    bucket_name = str(uuid.uuid4())

    try:
        s3_bucket = s3_client.create_bucket(Bucket=bucket_name)
        logger.info("Created S3 bucket: %s" % s3_bucket)
        return bucket_name

    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err 


@keyword('List buckets S3')
def list_buckets_s3(s3_client):
    found_buckets = []
    try:
        response = s3_client.list_buckets()
        logger.info("S3 List buckets result: %s" % response)

        for bucket in response['Buckets']:
            found_buckets.append(bucket['Name'])

        return found_buckets

    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err 


@keyword('Delete bucket S3')
def delete_bucket_s3(s3_client, bucket):
    try:
        response = s3_client.delete_bucket(Bucket=bucket)
        logger.info(f"S3 Delete bucket result: {response}")
        
        return response

    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err


@keyword('HeadBucket S3')
def headbucket(bucket, s3_client):
    try:
        response = s3_client.head_bucket(Bucket=bucket)
        logger.info(f"S3 HeadBucket result: {response}")
        return response

    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err


@keyword('Put object S3')
def put_object_s3(s3_client, bucket, filepath):
    filename = os.path.basename(filepath)

    with open(filepath, "rb") as f:
        fileContent = f.read()

    try:
        response = s3_client.put_object(Body=fileContent, Bucket=bucket, Key=filename)
        logger.info("S3 Put object result: %s" % response)
    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err


@keyword('Head object S3')
def head_object_s3(s3_client, bucket, object_key):

    try:
        response = s3_client.head_object(Bucket=bucket, Key=object_key)
        logger.info("S3 Head object result: %s" % response)
        return response

    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err   


@keyword('Delete object S3')
def delete_object_s3(s3_client, bucket, object_key):
    try:
        response = s3_client.delete_object(Bucket=bucket, Key=object_key)
        logger.info("S3 Put object result: %s" % response)
        return response

    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err


@keyword('Copy object S3')
def copy_object_s3(s3_client, bucket, object_key, new_object):
    try:
        response = s3_client.copy_object(Bucket=bucket, CopySource=bucket+"/"+object_key, Key=new_object)
        logger.info("S3 Copy object result: %s" % response)
        return response

    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err


@keyword('Get object S3')
def get_object_s3(s3_client, bucket, object_key, target_file):
    try:
        response = s3_client.get_object(Bucket=bucket, Key=object_key)

        with open(f"{target_file}", 'wb') as f:
            chunk = response['Body'].read(1024)
            while chunk:
                f.write(chunk)
                chunk = response['Body'].read(1024)

        return target_file

    except botocore.exceptions.ClientError as err:
        raise Exception(f"Error Message: {err.response['Error']['Message']}\n"
        f"Http status code: {err.response['ResponseMetadata']['HTTPStatusCode']}") from err


@keyword('Get via HTTP Gate')
def get_via_http_gate(cid: str, oid: str):
    """
    This function gets given object from HTTP gate
    :param cid:      CID to get object from
    :param oid:      object OID
    """
    request = f'{HTTP_GATE}/get/{cid}/{oid}'
    resp = requests.get(request, stream=True)

    if not resp.ok:
        raise Exception(f"""Failed to get object via HTTP gate:
                request: {resp.request.path_url},
                response: {resp.text},
                status code: {resp.status_code} {resp.reason}""")
        return

    logger.info(f'Request: {request}')
    filename = os.path.curdir + f"/{cid}_{oid}"
    with open(filename, "wb") as f:
        shutil.copyfileobj(resp.raw, f)
    del resp
    return filename