594 lines
21 KiB
Python
594 lines
21 KiB
Python
import logging
|
|
import os
|
|
import uuid
|
|
from time import sleep
|
|
from typing import Optional
|
|
|
|
import allure
|
|
import pytest
|
|
import urllib3
|
|
from aws_cli_client import AwsCliClient
|
|
from botocore.exceptions import ClientError
|
|
from cli_helpers import log_command_execution
|
|
from s3_gate_bucket import S3_SYNC_WAIT_TIME
|
|
|
|
##########################################################
|
|
# Disabling warnings on self-signed certificate which the
|
|
# boto library produces on requests to S3-gate in dev-env.
|
|
urllib3.disable_warnings()
|
|
##########################################################
|
|
logger = logging.getLogger("NeoLogger")
|
|
|
|
ACL_COPY = [
|
|
"private",
|
|
"public-read",
|
|
"public-read-write",
|
|
"authenticated-read",
|
|
"aws-exec-read",
|
|
"bucket-owner-read",
|
|
"bucket-owner-full-control",
|
|
]
|
|
|
|
ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/")
|
|
|
|
|
|
@allure.step("List objects S3 v2")
|
|
def list_objects_s3_v2(s3_client, bucket: str, full_output: bool = False) -> list:
|
|
try:
|
|
response = s3_client.list_objects_v2(Bucket=bucket)
|
|
content = response.get("Contents", [])
|
|
log_command_execution("S3 v2 List objects result", response)
|
|
obj_list = []
|
|
for obj in content:
|
|
obj_list.append(obj["Key"])
|
|
logger.info(f"Found s3 objects: {obj_list}")
|
|
return response if full_output else obj_list
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("List objects S3")
|
|
def list_objects_s3(s3_client, bucket: str, full_output: bool = False) -> list:
|
|
try:
|
|
response = s3_client.list_objects(Bucket=bucket)
|
|
content = response.get("Contents", [])
|
|
log_command_execution("S3 List objects result", response)
|
|
obj_list = []
|
|
for obj in content:
|
|
obj_list.append(obj["Key"])
|
|
logger.info(f"Found s3 objects: {obj_list}")
|
|
return response if full_output else obj_list
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("List objects versions S3")
|
|
def list_objects_versions_s3(s3_client, bucket: str, full_output: bool = False) -> list:
|
|
try:
|
|
response = s3_client.list_object_versions(Bucket=bucket)
|
|
versions = response.get("Versions", [])
|
|
log_command_execution("S3 List objects versions result", response)
|
|
return response if full_output else versions
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("List objects delete markers S3")
|
|
def list_objects_delete_markers_s3(s3_client, bucket: str, full_output: bool = False) -> list:
|
|
try:
|
|
response = s3_client.list_object_versions(Bucket=bucket)
|
|
delete_markers = response.get("DeleteMarkers", [])
|
|
log_command_execution("S3 List objects delete markers result", response)
|
|
return response if full_output else delete_markers
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Put object S3")
|
|
def put_object_s3(s3_client, bucket: str, filepath: str, **kwargs):
|
|
filename = os.path.basename(filepath)
|
|
|
|
if isinstance(s3_client, AwsCliClient):
|
|
file_content = filepath
|
|
else:
|
|
with open(filepath, "rb") as put_file:
|
|
file_content = put_file.read()
|
|
|
|
try:
|
|
params = {"Body": file_content, "Bucket": bucket, "Key": filename}
|
|
if kwargs:
|
|
params = {**params, **kwargs}
|
|
response = s3_client.put_object(**params)
|
|
log_command_execution("S3 Put object result", response)
|
|
return response.get("VersionId")
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Head object S3")
|
|
def head_object_s3(s3_client, bucket: str, object_key: str, version_id: Optional[str] = None):
|
|
try:
|
|
params = {"Bucket": bucket, "Key": object_key}
|
|
if version_id:
|
|
params["VersionId"] = version_id
|
|
response = s3_client.head_object(**params)
|
|
log_command_execution("S3 Head object result", response)
|
|
return response
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Delete object S3")
|
|
def delete_object_s3(
|
|
s3_client, bucket: str, object_key: str, version_id: Optional[str] = None
|
|
) -> dict:
|
|
try:
|
|
params = {"Bucket": bucket, "Key": object_key}
|
|
if version_id:
|
|
params["VersionId"] = version_id
|
|
response = s3_client.delete_object(**params)
|
|
log_command_execution("S3 Delete object result", response)
|
|
sleep(S3_SYNC_WAIT_TIME)
|
|
return response
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Delete objects S3")
|
|
def delete_objects_s3(s3_client, bucket: str, object_keys: list):
|
|
try:
|
|
response = s3_client.delete_objects(Bucket=bucket, Delete=_make_objs_dict(object_keys))
|
|
log_command_execution("S3 Delete objects result", response)
|
|
sleep(S3_SYNC_WAIT_TIME)
|
|
return response
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Delete object versions S3")
|
|
def delete_object_versions_s3(s3_client, bucket: str, object_versions: list):
|
|
try:
|
|
# Build deletion list in S3 format
|
|
delete_list = {
|
|
"Objects": [
|
|
{
|
|
"Key": object_version["Key"],
|
|
"VersionId": object_version["VersionId"],
|
|
}
|
|
for object_version in object_versions
|
|
]
|
|
}
|
|
response = s3_client.delete_objects(Bucket=bucket, Delete=delete_list)
|
|
log_command_execution("S3 Delete objects result", response)
|
|
return response
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Delete object versions S3 without delete markers")
|
|
def delete_object_versions_s3_without_dm(s3_client, bucket: str, object_versions: list):
|
|
try:
|
|
# Delete objects without creating delete markers
|
|
for object_version in object_versions:
|
|
params = {
|
|
"Bucket": bucket,
|
|
"Key": object_version["Key"],
|
|
"VersionId": object_version["VersionId"],
|
|
}
|
|
response = s3_client.delete_object(**params)
|
|
log_command_execution("S3 Delete object result", response)
|
|
return response
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Put object ACL")
|
|
def put_object_acl_s3(
|
|
s3_client,
|
|
bucket: str,
|
|
object_key: str,
|
|
acl: Optional[str] = None,
|
|
grant_write: Optional[str] = None,
|
|
grant_read: Optional[str] = None,
|
|
) -> list:
|
|
if not isinstance(s3_client, AwsCliClient):
|
|
pytest.skip("Method put_object_acl is not supported by boto3 client")
|
|
params = {"Bucket": bucket, "Key": object_key}
|
|
if acl:
|
|
params.update({"ACL": acl})
|
|
elif grant_write or grant_read:
|
|
if grant_write:
|
|
params.update({"GrantWrite": grant_write})
|
|
elif grant_read:
|
|
params.update({"GrantRead": grant_read})
|
|
try:
|
|
response = s3_client.put_object_acl(**params)
|
|
log_command_execution("S3 ACL objects result", response)
|
|
return response.get("Grants")
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Get object ACL")
|
|
def get_object_acl_s3(
|
|
s3_client, bucket: str, object_key: str, version_id: Optional[str] = None
|
|
) -> list:
|
|
params = {"Bucket": bucket, "Key": object_key}
|
|
try:
|
|
if version_id:
|
|
params.update({"VersionId": version_id})
|
|
response = s3_client.get_object_acl(**params)
|
|
log_command_execution("S3 ACL objects result", response)
|
|
return response.get("Grants")
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Copy object S3")
|
|
def copy_object_s3(
|
|
s3_client, bucket: str, object_key: str, bucket_dst: Optional[str] = None, **kwargs
|
|
) -> str:
|
|
filename = os.path.join(os.getcwd(), str(uuid.uuid4()))
|
|
try:
|
|
params = {
|
|
"Bucket": bucket_dst or bucket,
|
|
"CopySource": f"{bucket}/{object_key}",
|
|
"Key": filename,
|
|
}
|
|
if "ACL" in kwargs and kwargs["ACL"] in ACL_COPY:
|
|
params.update({"ACL": kwargs["ACL"]})
|
|
if "metadata_directive" in kwargs.keys():
|
|
params.update({"MetadataDirective": kwargs["metadata_directive"]})
|
|
if "metadata_directive" in kwargs.keys() and "metadata" in kwargs.keys():
|
|
params.update({"Metadata": kwargs["metadata"]})
|
|
if "tagging_directive" in kwargs.keys():
|
|
params.update({"TaggingDirective": kwargs["tagging_directive"]})
|
|
if "tagging_directive" in kwargs.keys() and "tagging" in kwargs.keys():
|
|
params.update({"Tagging": kwargs["tagging"]})
|
|
response = s3_client.copy_object(**params)
|
|
log_command_execution("S3 Copy objects result", response)
|
|
return filename
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Get object S3")
|
|
def get_object_s3(
|
|
s3_client,
|
|
bucket: str,
|
|
object_key: str,
|
|
version_id: Optional[str] = None,
|
|
range: Optional[list] = None,
|
|
full_output: bool = False,
|
|
):
|
|
filename = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))
|
|
try:
|
|
params = {"Bucket": bucket, "Key": object_key}
|
|
if version_id:
|
|
params["VersionId"] = version_id
|
|
|
|
if isinstance(s3_client, AwsCliClient):
|
|
params["file_path"] = filename
|
|
|
|
if range:
|
|
params["Range"] = f"bytes={range[0]}-{range[1]}"
|
|
|
|
response = s3_client.get_object(**params)
|
|
log_command_execution("S3 Get objects result", response)
|
|
|
|
if not isinstance(s3_client, AwsCliClient):
|
|
with open(f"{filename}", "wb") as get_file:
|
|
chunk = response["Body"].read(1024)
|
|
while chunk:
|
|
get_file.write(chunk)
|
|
chunk = response["Body"].read(1024)
|
|
return response if full_output else filename
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Create multipart upload S3")
|
|
def create_multipart_upload_s3(s3_client, bucket_name: str, object_key: str) -> str:
|
|
try:
|
|
response = s3_client.create_multipart_upload(Bucket=bucket_name, Key=object_key)
|
|
log_command_execution("S3 Created multipart upload", response)
|
|
assert response.get("UploadId"), f"Expected UploadId in response:\n{response}"
|
|
|
|
return response.get("UploadId")
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("List multipart uploads S3")
|
|
def list_multipart_uploads_s3(s3_client, bucket_name: str) -> Optional[list[dict]]:
|
|
try:
|
|
response = s3_client.list_multipart_uploads(Bucket=bucket_name)
|
|
log_command_execution("S3 List multipart upload", response)
|
|
|
|
return response.get("Uploads")
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Abort multipart upload S3")
|
|
def abort_multipart_uploads_s3(s3_client, bucket_name: str, object_key: str, upload_id: str):
|
|
try:
|
|
response = s3_client.abort_multipart_upload(
|
|
Bucket=bucket_name, Key=object_key, UploadId=upload_id
|
|
)
|
|
log_command_execution("S3 Abort multipart upload", response)
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Upload part S3")
|
|
def upload_part_s3(
|
|
s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, filepath: str
|
|
) -> str:
|
|
if isinstance(s3_client, AwsCliClient):
|
|
file_content = filepath
|
|
else:
|
|
with open(filepath, "rb") as put_file:
|
|
file_content = put_file.read()
|
|
|
|
try:
|
|
response = s3_client.upload_part(
|
|
UploadId=upload_id,
|
|
Bucket=bucket_name,
|
|
Key=object_key,
|
|
PartNumber=part_num,
|
|
Body=file_content,
|
|
)
|
|
log_command_execution("S3 Upload part", response)
|
|
assert response.get("ETag"), f"Expected ETag in response:\n{response}"
|
|
|
|
return response.get("ETag")
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Upload copy part S3")
|
|
def upload_part_copy_s3(
|
|
s3_client, bucket_name: str, object_key: str, upload_id: str, part_num: int, copy_source: str
|
|
) -> str:
|
|
|
|
try:
|
|
response = s3_client.upload_part_copy(
|
|
UploadId=upload_id,
|
|
Bucket=bucket_name,
|
|
Key=object_key,
|
|
PartNumber=part_num,
|
|
CopySource=copy_source,
|
|
)
|
|
log_command_execution("S3 Upload copy part", response)
|
|
assert response.get("CopyPartResult").get("ETag"), f"Expected ETag in response:\n{response}"
|
|
|
|
return response.get("CopyPartResult").get("ETag")
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("List parts S3")
|
|
def list_parts_s3(s3_client, bucket_name: str, object_key: str, upload_id: str) -> list[dict]:
|
|
try:
|
|
response = s3_client.list_parts(UploadId=upload_id, Bucket=bucket_name, Key=object_key)
|
|
log_command_execution("S3 List part", response)
|
|
assert response.get("Parts"), f"Expected Parts in response:\n{response}"
|
|
|
|
return response.get("Parts")
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Complete multipart upload S3")
|
|
def complete_multipart_upload_s3(
|
|
s3_client, bucket_name: str, object_key: str, upload_id: str, parts: list
|
|
):
|
|
try:
|
|
parts = [{"ETag": etag, "PartNumber": part_num} for part_num, etag in parts]
|
|
response = s3_client.complete_multipart_upload(
|
|
Bucket=bucket_name, Key=object_key, UploadId=upload_id, MultipartUpload={"Parts": parts}
|
|
)
|
|
log_command_execution("S3 Complete multipart upload", response)
|
|
|
|
except ClientError as err:
|
|
raise Exception(
|
|
f'Error Message: {err.response["Error"]["Message"]}\n'
|
|
f'Http status code: {err.response["ResponseMetadata"]["HTTPStatusCode"]}'
|
|
) from err
|
|
|
|
|
|
@allure.step("Put object retention")
|
|
def put_object_retention(
|
|
s3_client,
|
|
bucket_name: str,
|
|
object_key: str,
|
|
retention: dict,
|
|
version_id: Optional[str] = None,
|
|
bypass_governance_retention: Optional[bool] = None,
|
|
):
|
|
try:
|
|
params = {"Bucket": bucket_name, "Key": object_key, "Retention": retention}
|
|
if version_id:
|
|
params.update({"VersionId": version_id})
|
|
if not bypass_governance_retention is None:
|
|
params.update({"BypassGovernanceRetention": bypass_governance_retention})
|
|
s3_client.put_object_retention(**params)
|
|
log_command_execution("S3 Put object retention ", str(retention))
|
|
|
|
except ClientError as err:
|
|
raise Exception(f"Got error during put object tagging: {err}") from err
|
|
|
|
|
|
@allure.step("Put object legal hold")
|
|
def put_object_legal_hold(
|
|
s3_client, bucket_name: str, object_key: str, legal_hold: str, version_id: Optional[str] = None
|
|
):
|
|
try:
|
|
params = {"Bucket": bucket_name, "Key": object_key, "LegalHold": {"Status": legal_hold}}
|
|
if version_id:
|
|
params.update({"VersionId": version_id})
|
|
s3_client.put_object_legal_hold(**params)
|
|
log_command_execution("S3 Put object legal hold ", str(legal_hold))
|
|
|
|
except ClientError as err:
|
|
raise Exception(f"Got error during put object tagging: {err}") from err
|
|
|
|
|
|
@allure.step("Put object tagging")
|
|
def put_object_tagging(s3_client, bucket_name: str, object_key: str, tags: list):
|
|
try:
|
|
tags = [{"Key": tag_key, "Value": tag_value} for tag_key, tag_value in tags]
|
|
tagging = {"TagSet": tags}
|
|
s3_client.put_object_tagging(Bucket=bucket_name, Key=object_key, Tagging=tagging)
|
|
log_command_execution("S3 Put object tagging", str(tags))
|
|
|
|
except ClientError as err:
|
|
raise Exception(f"Got error during put object tagging: {err}") from err
|
|
|
|
|
|
@allure.step("Get object tagging")
|
|
def get_object_tagging(
|
|
s3_client, bucket_name: str, object_key: str, version_id: Optional[str] = None
|
|
) -> list:
|
|
try:
|
|
params = {"Bucket": bucket_name, "Key": object_key}
|
|
if version_id:
|
|
params.update({"VersionId": version_id})
|
|
response = s3_client.get_object_tagging(**params)
|
|
log_command_execution("S3 Get object tagging", response)
|
|
return response.get("TagSet")
|
|
|
|
except ClientError as err:
|
|
raise Exception(f"Got error during get object tagging: {err}") from err
|
|
|
|
|
|
@allure.step("Delete object tagging")
|
|
def delete_object_tagging(s3_client, bucket_name: str, object_key: str):
|
|
try:
|
|
response = s3_client.delete_object_tagging(Bucket=bucket_name, Key=object_key)
|
|
log_command_execution("S3 Delete object tagging", response)
|
|
|
|
except ClientError as err:
|
|
raise Exception(f"Got error during delete object tagging: {err}") from err
|
|
|
|
|
|
@allure.step("Get object attributes")
|
|
def get_object_attributes(
|
|
s3_client,
|
|
bucket_name: str,
|
|
object_key: str,
|
|
*attributes: str,
|
|
version_id: Optional[str] = None,
|
|
max_parts: Optional[int] = None,
|
|
part_number: Optional[int] = None,
|
|
get_full_resp: bool = True,
|
|
) -> dict:
|
|
try:
|
|
if not isinstance(s3_client, AwsCliClient):
|
|
logger.warning("Method get_object_attributes is not supported by boto3 client")
|
|
return {}
|
|
response = s3_client.get_object_attributes(
|
|
bucket_name,
|
|
object_key,
|
|
*attributes,
|
|
version_id=version_id,
|
|
max_parts=max_parts,
|
|
part_number=part_number,
|
|
)
|
|
log_command_execution("S3 Get object attributes", response)
|
|
for attr in attributes:
|
|
assert attr in response, f"Expected attribute {attr} in {response}"
|
|
|
|
if get_full_resp:
|
|
return response
|
|
else:
|
|
return response.get(attributes[0])
|
|
|
|
except ClientError as err:
|
|
raise Exception(f"Got error during get object attributes: {err}") from err
|
|
|
|
|
|
def _make_objs_dict(key_names):
|
|
objs_list = []
|
|
for key in key_names:
|
|
obj_dict = {"Key": key}
|
|
objs_list.append(obj_dict)
|
|
objs_dict = {"Objects": objs_list}
|
|
return objs_dict
|