[#133] Change reporter usage

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-11-29 15:27:17 +03:00
parent 39a17f3634
commit dc6b0e407f
37 changed files with 478 additions and 678 deletions

View file

@ -13,17 +13,11 @@ from botocore.config import Config
from botocore.exceptions import ClientError
from mypy_boto3_s3 import S3Client
from frostfs_testlib.reporter import get_reporter
from frostfs_testlib.resources.common import (
ASSETS_DIR,
MAX_REQUEST_ATTEMPTS,
RETRY_MODE,
S3_SYNC_WAIT_TIME,
)
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import ASSETS_DIR, MAX_REQUEST_ATTEMPTS, RETRY_MODE, S3_SYNC_WAIT_TIME
from frostfs_testlib.s3.interfaces import S3ClientWrapper, VersioningStatus, _make_objs_dict
from frostfs_testlib.utils.cli_utils import log_command_execution
reporter = get_reporter()
logger = logging.getLogger("NeoLogger")
# Disable warnings on self-signed certificate which the
@ -46,9 +40,11 @@ def report_error(func):
class Boto3ClientWrapper(S3ClientWrapper):
__repr_name__: str = "Boto3 client"
@reporter.step_deco("Configure S3 client (boto3)")
@reporter.step("Configure S3 client (boto3)")
@report_error
def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str='default') -> None:
def __init__(
self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str = "default"
) -> None:
self.boto3_client: S3Client = None
self.session = boto3.Session(profile_name=profile)
self.config = Config(
@ -62,7 +58,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
self.s3gate_endpoint: str = ""
self.set_endpoint(s3gate_endpoint)
@reporter.step_deco("Set endpoint S3 to {s3gate_endpoint}")
@reporter.step("Set endpoint S3 to {s3gate_endpoint}")
def set_endpoint(self, s3gate_endpoint: str):
if self.s3gate_endpoint == s3gate_endpoint:
return
@ -90,7 +86,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
return result
# BUCKET METHODS #
@reporter.step_deco("Create bucket S3")
@reporter.step("Create bucket S3")
@report_error
def create_bucket(
self,
@ -118,16 +114,14 @@ class Boto3ClientWrapper(S3ClientWrapper):
elif grant_full_control:
params.update({"GrantFullControl": grant_full_control})
if location_constraint:
params.update(
{"CreateBucketConfiguration": {"LocationConstraint": location_constraint}}
)
params.update({"CreateBucketConfiguration": {"LocationConstraint": location_constraint}})
s3_bucket = self.boto3_client.create_bucket(**params)
log_command_execution(f"Created S3 bucket {bucket}", s3_bucket)
sleep(S3_SYNC_WAIT_TIME)
return bucket
@reporter.step_deco("List buckets S3")
@reporter.step("List buckets S3")
@report_error
def list_buckets(self) -> list[str]:
found_buckets = []
@ -140,20 +134,20 @@ class Boto3ClientWrapper(S3ClientWrapper):
return found_buckets
@reporter.step_deco("Delete bucket S3")
@reporter.step("Delete bucket S3")
@report_error
def delete_bucket(self, bucket: str) -> None:
response = self.boto3_client.delete_bucket(Bucket=bucket)
log_command_execution("S3 Delete bucket result", response)
sleep(S3_SYNC_WAIT_TIME)
@reporter.step_deco("Head bucket S3")
@reporter.step("Head bucket S3")
@report_error
def head_bucket(self, bucket: str) -> None:
response = self.boto3_client.head_bucket(Bucket=bucket)
log_command_execution("S3 Head bucket result", response)
@reporter.step_deco("Put bucket versioning status")
@reporter.step("Put bucket versioning status")
@report_error
def put_bucket_versioning(self, bucket: str, status: VersioningStatus) -> None:
response = self.boto3_client.put_bucket_versioning(
@ -161,7 +155,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
)
log_command_execution("S3 Set bucket versioning to", response)
@reporter.step_deco("Get bucket versioning status")
@reporter.step("Get bucket versioning status")
@report_error
def get_bucket_versioning_status(self, bucket: str) -> Literal["Enabled", "Suspended"]:
response = self.boto3_client.get_bucket_versioning(Bucket=bucket)
@ -169,7 +163,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
log_command_execution("S3 Got bucket versioning status", response)
return status
@reporter.step_deco("Put bucket tagging")
@reporter.step("Put bucket tagging")
@report_error
def put_bucket_tagging(self, bucket: str, tags: list) -> None:
tags = [{"Key": tag_key, "Value": tag_value} for tag_key, tag_value in tags]
@ -177,27 +171,27 @@ class Boto3ClientWrapper(S3ClientWrapper):
response = self.boto3_client.put_bucket_tagging(Bucket=bucket, Tagging=tagging)
log_command_execution("S3 Put bucket tagging", response)
@reporter.step_deco("Get bucket tagging")
@reporter.step("Get bucket tagging")
@report_error
def get_bucket_tagging(self, bucket: str) -> list:
response = self.boto3_client.get_bucket_tagging(Bucket=bucket)
log_command_execution("S3 Get bucket tagging", response)
return response.get("TagSet")
@reporter.step_deco("Get bucket acl")
@reporter.step("Get bucket acl")
@report_error
def get_bucket_acl(self, bucket: str) -> list:
response = self.boto3_client.get_bucket_acl(Bucket=bucket)
log_command_execution("S3 Get bucket acl", response)
return response.get("Grants")
@reporter.step_deco("Delete bucket tagging")
@reporter.step("Delete bucket tagging")
@report_error
def delete_bucket_tagging(self, bucket: str) -> None:
response = self.boto3_client.delete_bucket_tagging(Bucket=bucket)
log_command_execution("S3 Delete bucket tagging", response)
@reporter.step_deco("Put bucket ACL")
@reporter.step("Put bucket ACL")
@report_error
def put_bucket_acl(
self,
@ -214,60 +208,56 @@ class Boto3ClientWrapper(S3ClientWrapper):
response = self.boto3_client.put_bucket_acl(**params)
log_command_execution("S3 ACL bucket result", response)
@reporter.step_deco("Put object lock configuration")
@reporter.step("Put object lock configuration")
@report_error
def put_object_lock_configuration(self, bucket: str, configuration: dict) -> dict:
response = self.boto3_client.put_object_lock_configuration(
Bucket=bucket, ObjectLockConfiguration=configuration
)
response = self.boto3_client.put_object_lock_configuration(Bucket=bucket, ObjectLockConfiguration=configuration)
log_command_execution("S3 put_object_lock_configuration result", response)
return response
@reporter.step_deco("Get object lock configuration")
@reporter.step("Get object lock configuration")
@report_error
def get_object_lock_configuration(self, bucket: str) -> dict:
response = self.boto3_client.get_object_lock_configuration(Bucket=bucket)
log_command_execution("S3 get_object_lock_configuration result", response)
return response.get("ObjectLockConfiguration")
@reporter.step_deco("Get bucket policy")
@reporter.step("Get bucket policy")
@report_error
def get_bucket_policy(self, bucket: str) -> str:
response = self.boto3_client.get_bucket_policy(Bucket=bucket)
log_command_execution("S3 get_bucket_policy result", response)
return response.get("Policy")
@reporter.step_deco("Put bucket policy")
@reporter.step("Put bucket policy")
@report_error
def put_bucket_policy(self, bucket: str, policy: dict) -> None:
response = self.boto3_client.put_bucket_policy(Bucket=bucket, Policy=json.dumps(policy))
log_command_execution("S3 put_bucket_policy result", response)
return response
@reporter.step_deco("Get bucket cors")
@reporter.step("Get bucket cors")
@report_error
def get_bucket_cors(self, bucket: str) -> dict:
response = self.boto3_client.get_bucket_cors(Bucket=bucket)
log_command_execution("S3 get_bucket_cors result", response)
return response.get("CORSRules")
@reporter.step_deco("Get bucket location")
@reporter.step("Get bucket location")
@report_error
def get_bucket_location(self, bucket: str) -> str:
response = self.boto3_client.get_bucket_location(Bucket=bucket)
log_command_execution("S3 get_bucket_location result", response)
return response.get("LocationConstraint")
@reporter.step_deco("Put bucket cors")
@reporter.step("Put bucket cors")
@report_error
def put_bucket_cors(self, bucket: str, cors_configuration: dict) -> None:
response = self.boto3_client.put_bucket_cors(
Bucket=bucket, CORSConfiguration=cors_configuration
)
response = self.boto3_client.put_bucket_cors(Bucket=bucket, CORSConfiguration=cors_configuration)
log_command_execution("S3 put_bucket_cors result", response)
return response
@reporter.step_deco("Delete bucket cors")
@reporter.step("Delete bucket cors")
@report_error
def delete_bucket_cors(self, bucket: str) -> None:
response = self.boto3_client.delete_bucket_cors(Bucket=bucket)
@ -276,7 +266,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
# END OF BUCKET METHODS #
# OBJECT METHODS #
@reporter.step_deco("List objects S3 v2")
@reporter.step("List objects S3 v2")
@report_error
def list_objects_v2(self, bucket: str, full_output: bool = False) -> Union[dict, list[str]]:
response = self.boto3_client.list_objects_v2(Bucket=bucket)
@ -287,7 +277,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
return response if full_output else obj_list
@reporter.step_deco("List objects S3")
@reporter.step("List objects S3")
@report_error
def list_objects(self, bucket: str, full_output: bool = False) -> Union[dict, list[str]]:
response = self.boto3_client.list_objects(Bucket=bucket)
@ -298,21 +288,21 @@ class Boto3ClientWrapper(S3ClientWrapper):
return response if full_output else obj_list
@reporter.step_deco("List objects versions S3")
@reporter.step("List objects versions S3")
@report_error
def list_objects_versions(self, bucket: str, full_output: bool = False) -> dict:
response = self.boto3_client.list_object_versions(Bucket=bucket)
log_command_execution("S3 List objects versions result", response)
return response if full_output else response.get("Versions", [])
@reporter.step_deco("List objects delete markers S3")
@reporter.step("List objects delete markers S3")
@report_error
def list_delete_markers(self, bucket: str, full_output: bool = False) -> list:
response = self.boto3_client.list_object_versions(Bucket=bucket)
log_command_execution("S3 List objects delete markers result", response)
return response if full_output else response.get("DeleteMarkers", [])
@reporter.step_deco("Put object S3")
@reporter.step("Put object S3")
@report_error
def put_object(
self,
@ -343,7 +333,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
log_command_execution("S3 Put object result", response)
return response.get("VersionId")
@reporter.step_deco("Head object S3")
@reporter.step("Head object S3")
@report_error
def head_object(self, bucket: str, key: str, version_id: Optional[str] = None) -> dict:
params = {
@ -355,7 +345,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
log_command_execution("S3 Head object result", response)
return response
@reporter.step_deco("Delete object S3")
@reporter.step("Delete object S3")
@report_error
def delete_object(self, bucket: str, key: str, version_id: Optional[str] = None) -> dict:
params = {
@ -368,7 +358,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
sleep(S3_SYNC_WAIT_TIME)
return response
@reporter.step_deco("Delete objects S3")
@reporter.step("Delete objects S3")
@report_error
def delete_objects(self, bucket: str, keys: list[str]) -> dict:
response = self.boto3_client.delete_objects(Bucket=bucket, Delete=_make_objs_dict(keys))
@ -379,7 +369,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
sleep(S3_SYNC_WAIT_TIME)
return response
@reporter.step_deco("Delete object versions S3")
@reporter.step("Delete object versions S3")
@report_error
def delete_object_versions(self, bucket: str, object_versions: list) -> dict:
# Build deletion list in S3 format
@ -396,7 +386,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
log_command_execution("S3 Delete objects result", response)
return response
@reporter.step_deco("Delete object versions S3 without delete markers")
@reporter.step("Delete object versions S3 without delete markers")
@report_error
def delete_object_versions_without_dm(self, bucket: str, object_versions: list) -> None:
# Delete objects without creating delete markers
@ -406,7 +396,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
)
log_command_execution("S3 Delete object result", response)
@reporter.step_deco("Put object ACL")
@reporter.step("Put object ACL")
@report_error
def put_object_acl(
self,
@ -419,7 +409,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
# pytest.skip("Method put_object_acl is not supported by boto3 client")
raise NotImplementedError("Unsupported for boto3 client")
@reporter.step_deco("Get object ACL")
@reporter.step("Get object ACL")
@report_error
def get_object_acl(self, bucket: str, key: str, version_id: Optional[str] = None) -> list:
params = {
@ -431,7 +421,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
log_command_execution("S3 ACL objects result", response)
return response.get("Grants")
@reporter.step_deco("Copy object S3")
@reporter.step("Copy object S3")
@report_error
def copy_object(
self,
@ -460,7 +450,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
log_command_execution("S3 Copy objects result", response)
return key
@reporter.step_deco("Get object S3")
@reporter.step("Get object S3")
@report_error
def get_object(
self,
@ -478,8 +468,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
params = {
self._to_s3_param(param): value
for param, value in {**locals(), **{"Range": range_str}}.items()
if param not in ["self", "object_range", "full_output", "range_str", "filename"]
and value is not None
if param not in ["self", "object_range", "full_output", "range_str", "filename"] and value is not None
}
response = self.boto3_client.get_object(**params)
log_command_execution("S3 Get objects result", response)
@ -491,7 +480,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
chunk = response["Body"].read(1024)
return response if full_output else filename
@reporter.step_deco("Create multipart upload S3")
@reporter.step("Create multipart upload S3")
@report_error
def create_multipart_upload(self, bucket: str, key: str) -> str:
response = self.boto3_client.create_multipart_upload(Bucket=bucket, Key=key)
@ -500,7 +489,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
return response["UploadId"]
@reporter.step_deco("List multipart uploads S3")
@reporter.step("List multipart uploads S3")
@report_error
def list_multipart_uploads(self, bucket: str) -> Optional[list[dict]]:
response = self.boto3_client.list_multipart_uploads(Bucket=bucket)
@ -508,19 +497,15 @@ class Boto3ClientWrapper(S3ClientWrapper):
return response.get("Uploads")
@reporter.step_deco("Abort multipart upload S3")
@reporter.step("Abort multipart upload S3")
@report_error
def abort_multipart_upload(self, bucket: str, key: str, upload_id: str) -> None:
response = self.boto3_client.abort_multipart_upload(
Bucket=bucket, Key=key, UploadId=upload_id
)
response = self.boto3_client.abort_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id)
log_command_execution("S3 Abort multipart upload", response)
@reporter.step_deco("Upload part S3")
@reporter.step("Upload part S3")
@report_error
def upload_part(
self, bucket: str, key: str, upload_id: str, part_num: int, filepath: str
) -> str:
def upload_part(self, bucket: str, key: str, upload_id: str, part_num: int, filepath: str) -> str:
with open(filepath, "rb") as put_file:
body = put_file.read()
@ -536,11 +521,9 @@ class Boto3ClientWrapper(S3ClientWrapper):
return response["ETag"]
@reporter.step_deco("Upload copy part S3")
@reporter.step("Upload copy part S3")
@report_error
def upload_part_copy(
self, bucket: str, key: str, upload_id: str, part_num: int, copy_source: str
) -> str:
def upload_part_copy(self, bucket: str, key: str, upload_id: str, part_num: int, copy_source: str) -> str:
response = self.boto3_client.upload_part_copy(
UploadId=upload_id,
Bucket=bucket,
@ -549,13 +532,11 @@ class Boto3ClientWrapper(S3ClientWrapper):
CopySource=copy_source,
)
log_command_execution("S3 Upload copy part", response)
assert response.get("CopyPartResult", []).get(
"ETag"
), f"Expected ETag in response:\n{response}"
assert response.get("CopyPartResult", []).get("ETag"), f"Expected ETag in response:\n{response}"
return response["CopyPartResult"]["ETag"]
@reporter.step_deco("List parts S3")
@reporter.step("List parts S3")
@report_error
def list_parts(self, bucket: str, key: str, upload_id: str) -> list[dict]:
response = self.boto3_client.list_parts(UploadId=upload_id, Bucket=bucket, Key=key)
@ -564,7 +545,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
return response["Parts"]
@reporter.step_deco("Complete multipart upload S3")
@reporter.step("Complete multipart upload S3")
@report_error
def complete_multipart_upload(self, bucket: str, key: str, upload_id: str, parts: list) -> None:
parts = [{"ETag": etag, "PartNumber": part_num} for part_num, etag in parts]
@ -573,7 +554,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
)
log_command_execution("S3 Complete multipart upload", response)
@reporter.step_deco("Put object retention")
@reporter.step("Put object retention")
@report_error
def put_object_retention(
self,
@ -591,7 +572,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
response = self.boto3_client.put_object_retention(**params)
log_command_execution("S3 Put object retention ", response)
@reporter.step_deco("Put object legal hold")
@reporter.step("Put object legal hold")
@report_error
def put_object_legal_hold(
self,
@ -609,7 +590,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
response = self.boto3_client.put_object_legal_hold(**params)
log_command_execution("S3 Put object legal hold ", response)
@reporter.step_deco("Put object tagging")
@reporter.step("Put object tagging")
@report_error
def put_object_tagging(self, bucket: str, key: str, tags: list) -> None:
tags = [{"Key": tag_key, "Value": tag_value} for tag_key, tag_value in tags]
@ -617,7 +598,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
response = self.boto3_client.put_object_tagging(Bucket=bucket, Key=key, Tagging=tagging)
log_command_execution("S3 Put object tagging", response)
@reporter.step_deco("Get object tagging")
@reporter.step("Get object tagging")
@report_error
def get_object_tagging(self, bucket: str, key: str, version_id: Optional[str] = None) -> list:
params = {
@ -629,13 +610,13 @@ class Boto3ClientWrapper(S3ClientWrapper):
log_command_execution("S3 Get object tagging", response)
return response.get("TagSet")
@reporter.step_deco("Delete object tagging")
@reporter.step("Delete object tagging")
@report_error
def delete_object_tagging(self, bucket: str, key: str) -> None:
response = self.boto3_client.delete_object_tagging(Bucket=bucket, Key=key)
log_command_execution("S3 Delete object tagging", response)
@reporter.step_deco("Get object attributes")
@reporter.step("Get object attributes")
@report_error
def get_object_attributes(
self,
@ -650,7 +631,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
logger.warning("Method get_object_attributes is not supported by boto3 client")
return {}
@reporter.step_deco("Sync directory S3")
@reporter.step("Sync directory S3")
@report_error
def sync(
self,
@ -661,7 +642,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
) -> dict:
raise NotImplementedError("Sync is not supported for boto3 client")
@reporter.step_deco("CP directory S3")
@reporter.step("CP directory S3")
@report_error
def cp(
self,