TrueCloudLab/frostfs-testlib
24
0
Fork 18
Code Issues Pull requests 2 Releases 1 Activity Actions 1

Support of AWS profiles

Browse source
This commit is contained in:
Yaroslava Lukoyanova 2023-11-24 16:32:26 +03:00 committed by ylukoyan
parent c17f0f6173
commit 47414eb866
3 changed files with 55 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

103
src/frostfs_testlib/s3/aws_cli_client.py
View file

@ -29,14 +29,15 @@ class AwsCliClient(S3ClientWrapper):
s3gate_endpoint: str s3gate_endpoint: str
@reporter.step_deco("Configure S3 client (aws cli)") @reporter.step_deco("Configure S3 client (aws cli)")
def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str) -> None: def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str='default') -> None:
self.s3gate_endpoint = s3gate_endpoint self.s3gate_endpoint = s3gate_endpoint
self.profile = profile
self.local_shell = LocalShell() self.local_shell = LocalShell()
try: try:
_configure_aws_cli("aws configure", access_key_id, secret_access_key) _configure_aws_cli(f"aws configure --profile {profile}", access_key_id, secret_access_key)
self.local_shell.exec(f"aws configure set max_attempts {MAX_REQUEST_ATTEMPTS}") self.local_shell.exec(f"aws configure set max_attempts {MAX_REQUEST_ATTEMPTS} --profile {profile}")
self.local_shell.exec( self.local_shell.exec(
f"aws configure set retry_mode {RETRY_MODE}", f"aws configure set retry_mode {RETRY_MODE} --profile {profile}",
) )
except Exception as err: except Exception as err:
raise RuntimeError("Error while configuring AwsCliClient") from err raise RuntimeError("Error while configuring AwsCliClient") from err
@ -67,7 +68,7 @@ class AwsCliClient(S3ClientWrapper):
object_lock = " --no-object-lock-enabled-for-bucket" object_lock = " --no-object-lock-enabled-for-bucket"
cmd = ( cmd = (
f"aws {self.common_flags} s3api create-bucket --bucket {bucket} " f"aws {self.common_flags} s3api create-bucket --bucket {bucket} "
f"{object_lock} --endpoint {self.s3gate_endpoint}" f"{object_lock} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
if acl: if acl:
cmd += f" --acl {acl}" cmd += f" --acl {acl}"
@ -86,20 +87,20 @@ class AwsCliClient(S3ClientWrapper):
@reporter.step_deco("List buckets S3") @reporter.step_deco("List buckets S3")
def list_buckets(self) -> list[str]: def list_buckets(self) -> list[str]:
cmd = f"aws {self.common_flags} s3api list-buckets --endpoint {self.s3gate_endpoint}" cmd = f"aws {self.common_flags} s3api list-buckets --endpoint {self.s3gate_endpoint} --profile {self.profile}"
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
buckets_json = self._to_json(output) buckets_json = self._to_json(output)
return [bucket["Name"] for bucket in buckets_json["Buckets"]] return [bucket["Name"] for bucket in buckets_json["Buckets"]]
@reporter.step_deco("Delete bucket S3") @reporter.step_deco("Delete bucket S3")
def delete_bucket(self, bucket: str) -> None: def delete_bucket(self, bucket: str) -> None:
cmd = f"aws {self.common_flags} s3api delete-bucket --bucket {bucket} --endpoint {self.s3gate_endpoint}" cmd = f"aws {self.common_flags} s3api delete-bucket --bucket {bucket} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
self.local_shell.exec(cmd, command_options) self.local_shell.exec(cmd, command_options)
sleep(S3_SYNC_WAIT_TIME) sleep(S3_SYNC_WAIT_TIME)
@reporter.step_deco("Head bucket S3") @reporter.step_deco("Head bucket S3")
def head_bucket(self, bucket: str) -> None: def head_bucket(self, bucket: str) -> None:
cmd = f"aws {self.common_flags} s3api head-bucket --bucket {bucket} --endpoint {self.s3gate_endpoint}" cmd = f"aws {self.common_flags} s3api head-bucket --bucket {bucket} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@reporter.step_deco("Put bucket versioning status") @reporter.step_deco("Put bucket versioning status")
@ -107,7 +108,7 @@ class AwsCliClient(S3ClientWrapper):
cmd = ( cmd = (
f"aws {self.common_flags} s3api put-bucket-versioning --bucket {bucket} " f"aws {self.common_flags} s3api put-bucket-versioning --bucket {bucket} "
f"--versioning-configuration Status={status.value} " f"--versioning-configuration Status={status.value} "
f"--endpoint {self.s3gate_endpoint}" f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@ -115,7 +116,7 @@ class AwsCliClient(S3ClientWrapper):
def get_bucket_versioning_status(self, bucket: str) -> Literal["Enabled", "Suspended"]: def get_bucket_versioning_status(self, bucket: str) -> Literal["Enabled", "Suspended"]:
cmd = ( cmd = (
f"aws {self.common_flags} s3api get-bucket-versioning --bucket {bucket} " f"aws {self.common_flags} s3api get-bucket-versioning --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}" f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -126,14 +127,14 @@ class AwsCliClient(S3ClientWrapper):
tags_json = {"TagSet": [{"Key": tag_key, "Value": tag_value} for tag_key, tag_value in tags]} tags_json = {"TagSet": [{"Key": tag_key, "Value": tag_value} for tag_key, tag_value in tags]}
cmd = ( cmd = (
f"aws {self.common_flags} s3api put-bucket-tagging --bucket {bucket} " f"aws {self.common_flags} s3api put-bucket-tagging --bucket {bucket} "
f"--tagging '{json.dumps(tags_json)}' --endpoint {self.s3gate_endpoint}" f"--tagging '{json.dumps(tags_json)}' --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@reporter.step_deco("Get bucket tagging") @reporter.step_deco("Get bucket tagging")
def get_bucket_tagging(self, bucket: str) -> list: def get_bucket_tagging(self, bucket: str) -> list:
cmd = ( cmd = (
f"aws {self.common_flags} s3api get-bucket-tagging --bucket {bucket} " f"--endpoint {self.s3gate_endpoint}" f"aws {self.common_flags} s3api get-bucket-tagging --bucket {bucket} " f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -141,7 +142,7 @@ class AwsCliClient(S3ClientWrapper):
@reporter.step_deco("Get bucket acl") @reporter.step_deco("Get bucket acl")
def get_bucket_acl(self, bucket: str) -> list: def get_bucket_acl(self, bucket: str) -> list:
cmd = f"aws {self.common_flags} s3api get-bucket-acl --bucket {bucket} " f"--endpoint {self.s3gate_endpoint}" cmd = f"aws {self.common_flags} s3api get-bucket-acl --bucket {bucket} " f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
return response.get("Grants") return response.get("Grants")
@ -149,7 +150,7 @@ class AwsCliClient(S3ClientWrapper):
@reporter.step_deco("Get bucket location") @reporter.step_deco("Get bucket location")
def get_bucket_location(self, bucket: str) -> dict: def get_bucket_location(self, bucket: str) -> dict:
cmd = ( cmd = (
f"aws {self.common_flags} s3api get-bucket-location --bucket {bucket} " f"--endpoint {self.s3gate_endpoint}" f"aws {self.common_flags} s3api get-bucket-location --bucket {bucket} " f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -157,7 +158,7 @@ class AwsCliClient(S3ClientWrapper):
@reporter.step_deco("List objects S3") @reporter.step_deco("List objects S3")
def list_objects(self, bucket: str, full_output: bool = False) -> Union[dict, list[str]]: def list_objects(self, bucket: str, full_output: bool = False) -> Union[dict, list[str]]:
cmd = f"aws {self.common_flags} s3api list-objects --bucket {bucket} " f"--endpoint {self.s3gate_endpoint}" cmd = f"aws {self.common_flags} s3api list-objects --bucket {bucket} " f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -168,7 +169,7 @@ class AwsCliClient(S3ClientWrapper):
@reporter.step_deco("List objects S3 v2") @reporter.step_deco("List objects S3 v2")
def list_objects_v2(self, bucket: str, full_output: bool = False) -> Union[dict, list[str]]: def list_objects_v2(self, bucket: str, full_output: bool = False) -> Union[dict, list[str]]:
cmd = f"aws {self.common_flags} s3api list-objects-v2 --bucket {bucket} " f"--endpoint {self.s3gate_endpoint}" cmd = f"aws {self.common_flags} s3api list-objects-v2 --bucket {bucket} " f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -181,7 +182,7 @@ class AwsCliClient(S3ClientWrapper):
def list_objects_versions(self, bucket: str, full_output: bool = False) -> dict: def list_objects_versions(self, bucket: str, full_output: bool = False) -> dict:
cmd = ( cmd = (
f"aws {self.common_flags} s3api list-object-versions --bucket {bucket} " f"aws {self.common_flags} s3api list-object-versions --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}" f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -191,7 +192,7 @@ class AwsCliClient(S3ClientWrapper):
def list_delete_markers(self, bucket: str, full_output: bool = False) -> list: def list_delete_markers(self, bucket: str, full_output: bool = False) -> list:
cmd = ( cmd = (
f"aws {self.common_flags} s3api list-object-versions --bucket {bucket} " f"aws {self.common_flags} s3api list-object-versions --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}" f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -218,7 +219,7 @@ class AwsCliClient(S3ClientWrapper):
cmd = ( cmd = (
f"aws {self.common_flags} s3api copy-object --copy-source {copy_source} " f"aws {self.common_flags} s3api copy-object --copy-source {copy_source} "
f"--bucket {bucket} --key {key} --endpoint {self.s3gate_endpoint}" f"--bucket {bucket} --key {key} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
if acl: if acl:
cmd += f" --acl {acl}" cmd += f" --acl {acl}"
@ -255,7 +256,7 @@ class AwsCliClient(S3ClientWrapper):
cmd = ( cmd = (
f"aws {self.common_flags} s3api put-object --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api put-object --bucket {bucket} --key {key} "
f"--body {filepath} --endpoint {self.s3gate_endpoint}" f"--body {filepath} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
if metadata: if metadata:
cmd += " --metadata" cmd += " --metadata"
@ -284,7 +285,7 @@ class AwsCliClient(S3ClientWrapper):
version = f" --version-id {version_id}" if version_id else "" version = f" --version-id {version_id}" if version_id else ""
cmd = ( cmd = (
f"aws {self.common_flags} s3api head-object --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api head-object --bucket {bucket} --key {key} "
f"{version} --endpoint {self.s3gate_endpoint}" f"{version} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -303,7 +304,7 @@ class AwsCliClient(S3ClientWrapper):
version = f" --version-id {version_id}" if version_id else "" version = f" --version-id {version_id}" if version_id else ""
cmd = ( cmd = (
f"aws {self.common_flags} s3api get-object --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api get-object --bucket {bucket} --key {key} "
f"{version} {file_path} --endpoint {self.s3gate_endpoint}" f"{version} {file_path} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
if object_range: if object_range:
cmd += f" --range bytes={object_range[0]}-{object_range[1]}" cmd += f" --range bytes={object_range[0]}-{object_range[1]}"
@ -316,7 +317,7 @@ class AwsCliClient(S3ClientWrapper):
version = f" --version-id {version_id}" if version_id else "" version = f" --version-id {version_id}" if version_id else ""
cmd = ( cmd = (
f"aws {self.common_flags} s3api get-object-acl --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api get-object-acl --bucket {bucket} --key {key} "
f"{version} --endpoint {self.s3gate_endpoint}" f"{version} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -333,7 +334,7 @@ class AwsCliClient(S3ClientWrapper):
) -> list: ) -> list:
cmd = ( cmd = (
f"aws {self.common_flags} s3api put-object-acl --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api put-object-acl --bucket {bucket} --key {key} "
f" --endpoint {self.s3gate_endpoint}" f" --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
if acl: if acl:
cmd += f" --acl {acl}" cmd += f" --acl {acl}"
@ -353,7 +354,7 @@ class AwsCliClient(S3ClientWrapper):
grant_write: Optional[str] = None, grant_write: Optional[str] = None,
grant_read: Optional[str] = None, grant_read: Optional[str] = None,
) -> None: ) -> None:
cmd = f"aws {self.common_flags} s3api put-bucket-acl --bucket {bucket} " f" --endpoint {self.s3gate_endpoint}" cmd = f"aws {self.common_flags} s3api put-bucket-acl --bucket {bucket} " f" --endpoint {self.s3gate_endpoint} --profile {self.profile}"
if acl: if acl:
cmd += f" --acl {acl}" cmd += f" --acl {acl}"
if grant_write: if grant_write:
@ -372,7 +373,7 @@ class AwsCliClient(S3ClientWrapper):
cmd = ( cmd = (
f"aws {self.common_flags} s3api delete-objects --bucket {bucket} " f"aws {self.common_flags} s3api delete-objects --bucket {bucket} "
f"--delete file://{file_path} --endpoint {self.s3gate_endpoint}" f"--delete file://{file_path} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd, command_options).stdout output = self.local_shell.exec(cmd, command_options).stdout
response = self._to_json(output) response = self._to_json(output)
@ -384,7 +385,7 @@ class AwsCliClient(S3ClientWrapper):
version = f" --version-id {version_id}" if version_id else "" version = f" --version-id {version_id}" if version_id else ""
cmd = ( cmd = (
f"aws {self.common_flags} s3api delete-object --bucket {bucket} " f"aws {self.common_flags} s3api delete-object --bucket {bucket} "
f"--key {key} {version} --endpoint {self.s3gate_endpoint}" f"--key {key} {version} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd, command_options).stdout output = self.local_shell.exec(cmd, command_options).stdout
sleep(S3_SYNC_WAIT_TIME) sleep(S3_SYNC_WAIT_TIME)
@ -411,7 +412,7 @@ class AwsCliClient(S3ClientWrapper):
cmd = ( cmd = (
f"aws {self.common_flags} s3api delete-objects --bucket {bucket} " f"aws {self.common_flags} s3api delete-objects --bucket {bucket} "
f"--delete file://{file_path} --endpoint {self.s3gate_endpoint}" f"--delete file://{file_path} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd, command_options).stdout output = self.local_shell.exec(cmd, command_options).stdout
sleep(S3_SYNC_WAIT_TIME) sleep(S3_SYNC_WAIT_TIME)
@ -442,7 +443,7 @@ class AwsCliClient(S3ClientWrapper):
cmd = ( cmd = (
f"aws {self.common_flags} s3api get-object-attributes --bucket {bucket} " f"aws {self.common_flags} s3api get-object-attributes --bucket {bucket} "
f"--key {key} {version} {parts} {part_number_str} --object-attributes {attrs} " f"--key {key} {version} {parts} {part_number_str} --object-attributes {attrs} "
f"--endpoint {self.s3gate_endpoint}" f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -457,7 +458,7 @@ class AwsCliClient(S3ClientWrapper):
@reporter.step_deco("Get bucket policy") @reporter.step_deco("Get bucket policy")
def get_bucket_policy(self, bucket: str) -> dict: def get_bucket_policy(self, bucket: str) -> dict:
cmd = f"aws {self.common_flags} s3api get-bucket-policy --bucket {bucket} " f"--endpoint {self.s3gate_endpoint}" cmd = f"aws {self.common_flags} s3api get-bucket-policy --bucket {bucket} " f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
return response.get("Policy") return response.get("Policy")
@ -473,13 +474,13 @@ class AwsCliClient(S3ClientWrapper):
dumped_policy = json.dumps(json.dumps(policy)) dumped_policy = json.dumps(json.dumps(policy))
cmd = ( cmd = (
f"aws {self.common_flags} s3api put-bucket-policy --bucket {bucket} " f"aws {self.common_flags} s3api put-bucket-policy --bucket {bucket} "
f"--policy {dumped_policy} --endpoint {self.s3gate_endpoint}" f"--policy {dumped_policy} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@reporter.step_deco("Get bucket cors") @reporter.step_deco("Get bucket cors")
def get_bucket_cors(self, bucket: str) -> dict: def get_bucket_cors(self, bucket: str) -> dict:
cmd = f"aws {self.common_flags} s3api get-bucket-cors --bucket {bucket} " f"--endpoint {self.s3gate_endpoint}" cmd = f"aws {self.common_flags} s3api get-bucket-cors --bucket {bucket} " f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
return response.get("CORSRules") return response.get("CORSRules")
@ -488,14 +489,14 @@ class AwsCliClient(S3ClientWrapper):
def put_bucket_cors(self, bucket: str, cors_configuration: dict) -> None: def put_bucket_cors(self, bucket: str, cors_configuration: dict) -> None:
cmd = ( cmd = (
f"aws {self.common_flags} s3api put-bucket-cors --bucket {bucket} " f"aws {self.common_flags} s3api put-bucket-cors --bucket {bucket} "
f"--cors-configuration '{json.dumps(cors_configuration)}' --endpoint {self.s3gate_endpoint}" f"--cors-configuration '{json.dumps(cors_configuration)}' --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@reporter.step_deco("Delete bucket cors") @reporter.step_deco("Delete bucket cors")
def delete_bucket_cors(self, bucket: str) -> None: def delete_bucket_cors(self, bucket: str) -> None:
cmd = ( cmd = (
f"aws {self.common_flags} s3api delete-bucket-cors --bucket {bucket} " f"--endpoint {self.s3gate_endpoint}" f"aws {self.common_flags} s3api delete-bucket-cors --bucket {bucket} " f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@ -503,7 +504,7 @@ class AwsCliClient(S3ClientWrapper):
def delete_bucket_tagging(self, bucket: str) -> None: def delete_bucket_tagging(self, bucket: str) -> None:
cmd = ( cmd = (
f"aws {self.common_flags} s3api delete-bucket-tagging --bucket {bucket} " f"aws {self.common_flags} s3api delete-bucket-tagging --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}" f"--endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@ -519,7 +520,7 @@ class AwsCliClient(S3ClientWrapper):
version = f" --version-id {version_id}" if version_id else "" version = f" --version-id {version_id}" if version_id else ""
cmd = ( cmd = (
f"aws {self.common_flags} s3api put-object-retention --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api put-object-retention --bucket {bucket} --key {key} "
f"{version} --retention '{json.dumps(retention, indent=4, sort_keys=True, default=str)}' --endpoint {self.s3gate_endpoint}" f"{version} --retention '{json.dumps(retention, indent=4, sort_keys=True, default=str)}' --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
if bypass_governance_retention is not None: if bypass_governance_retention is not None:
cmd += " --bypass-governance-retention" cmd += " --bypass-governance-retention"
@ -537,7 +538,7 @@ class AwsCliClient(S3ClientWrapper):
legal_hold = json.dumps({"Status": legal_hold_status}) legal_hold = json.dumps({"Status": legal_hold_status})
cmd = ( cmd = (
f"aws {self.common_flags} s3api put-object-legal-hold --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api put-object-legal-hold --bucket {bucket} --key {key} "
f"{version} --legal-hold '{legal_hold}' --endpoint {self.s3gate_endpoint}" f"{version} --legal-hold '{legal_hold}' --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@ -547,7 +548,7 @@ class AwsCliClient(S3ClientWrapper):
tagging = {"TagSet": tags} tagging = {"TagSet": tags}
cmd = ( cmd = (
f"aws {self.common_flags} s3api put-object-tagging --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api put-object-tagging --bucket {bucket} --key {key} "
f"--tagging '{json.dumps(tagging)}' --endpoint {self.s3gate_endpoint}" f"--tagging '{json.dumps(tagging)}' --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@ -556,7 +557,7 @@ class AwsCliClient(S3ClientWrapper):
version = f" --version-id {version_id}" if version_id else "" version = f" --version-id {version_id}" if version_id else ""
cmd = ( cmd = (
f"aws {self.common_flags} s3api get-object-tagging --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api get-object-tagging --bucket {bucket} --key {key} "
f"{version} --endpoint {self.s3gate_endpoint}" f"{version} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -566,7 +567,7 @@ class AwsCliClient(S3ClientWrapper):
def delete_object_tagging(self, bucket: str, key: str) -> None: def delete_object_tagging(self, bucket: str, key: str) -> None:
cmd = ( cmd = (
f"aws {self.common_flags} s3api delete-object-tagging --bucket {bucket} " f"aws {self.common_flags} s3api delete-object-tagging --bucket {bucket} "
f"--key {key} --endpoint {self.s3gate_endpoint}" f"--key {key} --endpoint {self.s3gate_endpoint} --profile {self.profile}"
) )
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@ -578,7 +579,7 @@ class AwsCliClient(S3ClientWrapper):
acl: Optional[str] = None, acl: Optional[str] = None,
metadata: Optional[dict] = None, metadata: Optional[dict] = None,
) -> dict: ) -> dict:
cmd = f"aws {self.common_flags} s3 sync {dir_path} s3://{bucket} " f"--endpoint-url {self.s3gate_endpoint}" cmd = f"aws {self.common_flags} s3 sync {dir_path} s3://{bucket} " f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
if metadata: if metadata:
cmd += " --metadata" cmd += " --metadata"
for key, value in metadata.items(): for key, value in metadata.items():
@ -598,7 +599,7 @@ class AwsCliClient(S3ClientWrapper):
) -> dict: ) -> dict:
cmd = ( cmd = (
f"aws {self.common_flags} s3 cp {dir_path} s3://{bucket} " f"aws {self.common_flags} s3 cp {dir_path} s3://{bucket} "
f"--endpoint-url {self.s3gate_endpoint} --recursive" f"--endpoint-url {self.s3gate_endpoint} --recursive --profile {self.profile}"
) )
if metadata: if metadata:
cmd += " --metadata" cmd += " --metadata"
@ -613,7 +614,7 @@ class AwsCliClient(S3ClientWrapper):
def create_multipart_upload(self, bucket: str, key: str) -> str: def create_multipart_upload(self, bucket: str, key: str) -> str:
cmd = ( cmd = (
f"aws {self.common_flags} s3api create-multipart-upload --bucket {bucket} " f"aws {self.common_flags} s3api create-multipart-upload --bucket {bucket} "
f"--key {key} --endpoint-url {self.s3gate_endpoint}" f"--key {key} --endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -626,7 +627,7 @@ class AwsCliClient(S3ClientWrapper):
def list_multipart_uploads(self, bucket: str) -> Optional[list[dict]]: def list_multipart_uploads(self, bucket: str) -> Optional[list[dict]]:
cmd = ( cmd = (
f"aws {self.common_flags} s3api list-multipart-uploads --bucket {bucket} " f"aws {self.common_flags} s3api list-multipart-uploads --bucket {bucket} "
f"--endpoint-url {self.s3gate_endpoint}" f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -636,7 +637,7 @@ class AwsCliClient(S3ClientWrapper):
def abort_multipart_upload(self, bucket: str, key: str, upload_id: str) -> None: def abort_multipart_upload(self, bucket: str, key: str, upload_id: str) -> None:
cmd = ( cmd = (
f"aws {self.common_flags} s3api abort-multipart-upload --bucket {bucket} " f"aws {self.common_flags} s3api abort-multipart-upload --bucket {bucket} "
f"--key {key} --upload-id {upload_id} --endpoint-url {self.s3gate_endpoint}" f"--key {key} --upload-id {upload_id} --endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
) )
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@ -645,7 +646,7 @@ class AwsCliClient(S3ClientWrapper):
cmd = ( cmd = (
f"aws {self.common_flags} s3api upload-part --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api upload-part --bucket {bucket} --key {key} "
f"--upload-id {upload_id} --part-number {part_num} --body {filepath} " f"--upload-id {upload_id} --part-number {part_num} --body {filepath} "
f"--endpoint-url {self.s3gate_endpoint}" f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd, command_options).stdout output = self.local_shell.exec(cmd, command_options).stdout
response = self._to_json(output) response = self._to_json(output)
@ -657,7 +658,7 @@ class AwsCliClient(S3ClientWrapper):
cmd = ( cmd = (
f"aws {self.common_flags} s3api upload-part-copy --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api upload-part-copy --bucket {bucket} --key {key} "
f"--upload-id {upload_id} --part-number {part_num} --copy-source {copy_source} " f"--upload-id {upload_id} --part-number {part_num} --copy-source {copy_source} "
f"--endpoint-url {self.s3gate_endpoint}" f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd, command_options).stdout output = self.local_shell.exec(cmd, command_options).stdout
response = self._to_json(output) response = self._to_json(output)
@ -669,7 +670,7 @@ class AwsCliClient(S3ClientWrapper):
def list_parts(self, bucket: str, key: str, upload_id: str) -> list[dict]: def list_parts(self, bucket: str, key: str, upload_id: str) -> list[dict]:
cmd = ( cmd = (
f"aws {self.common_flags} s3api list-parts --bucket {bucket} --key {key} " f"aws {self.common_flags} s3api list-parts --bucket {bucket} --key {key} "
f"--upload-id {upload_id} --endpoint-url {self.s3gate_endpoint}" f"--upload-id {upload_id} --endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)
@ -691,7 +692,7 @@ class AwsCliClient(S3ClientWrapper):
cmd = ( cmd = (
f"aws {self.common_flags} s3api complete-multipart-upload --bucket {bucket} " f"aws {self.common_flags} s3api complete-multipart-upload --bucket {bucket} "
f"--key {key} --upload-id {upload_id} --multipart-upload file://{file_path} " f"--key {key} --upload-id {upload_id} --multipart-upload file://{file_path} "
f"--endpoint-url {self.s3gate_endpoint}" f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
) )
self.local_shell.exec(cmd) self.local_shell.exec(cmd)
@ -699,7 +700,7 @@ class AwsCliClient(S3ClientWrapper):
def put_object_lock_configuration(self, bucket: str, configuration: dict) -> dict: def put_object_lock_configuration(self, bucket: str, configuration: dict) -> dict:
cmd = ( cmd = (
f"aws {self.common_flags} s3api put-object-lock-configuration --bucket {bucket} " f"aws {self.common_flags} s3api put-object-lock-configuration --bucket {bucket} "
f"--object-lock-configuration '{json.dumps(configuration)}' --endpoint-url {self.s3gate_endpoint}" f"--object-lock-configuration '{json.dumps(configuration)}' --endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
return self._to_json(output) return self._to_json(output)
@ -708,7 +709,7 @@ class AwsCliClient(S3ClientWrapper):
def get_object_lock_configuration(self, bucket: str): def get_object_lock_configuration(self, bucket: str):
cmd = ( cmd = (
f"aws {self.common_flags} s3api get-object-lock-configuration --bucket {bucket} " f"aws {self.common_flags} s3api get-object-lock-configuration --bucket {bucket} "
f"--endpoint-url {self.s3gate_endpoint}" f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
) )
output = self.local_shell.exec(cmd).stdout output = self.local_shell.exec(cmd).stdout
response = self._to_json(output) response = self._to_json(output)

4
src/frostfs_testlib/s3/boto3_client.py
View file

@ -48,9 +48,9 @@ class Boto3ClientWrapper(S3ClientWrapper):
@reporter.step_deco("Configure S3 client (boto3)") @reporter.step_deco("Configure S3 client (boto3)")
@report_error @report_error
def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str) -> None: def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str='default') -> None:
self.boto3_client: S3Client = None self.boto3_client: S3Client = None
self.session = boto3.Session() self.session = boto3.Session(profile_name=profile)
self.config = Config( self.config = Config(
retries={ retries={
"max_attempts": MAX_REQUEST_ATTEMPTS, "max_attempts": MAX_REQUEST_ATTEMPTS,

2
src/frostfs_testlib/s3/interfaces.py
View file

@ -33,7 +33,7 @@ ACL_COPY = [
class S3ClientWrapper(HumanReadableABC): class S3ClientWrapper(HumanReadableABC):
@abstractmethod @abstractmethod
def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str) -> None: def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str, profile: str) -> None:
pass pass
@abstractmethod @abstractmethod