abereziny/frostfs-testlib
1
0
Fork 0
forked from TrueCloudLab/frostfs-testlib
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add local shell and small fix

Browse source 
Signed-off-by: Dmitriy Zayakin <d.zayakin@yadro.com>
This commit is contained in:
Dmitriy Zayakin 2023-10-27 13:36:32 +03:00 • committed by Dmitriy Zayakin
parent f3c160f313
commit 137fd21561
4 changed files with 113 additions and 122 deletions
Download patch file Download diff file Expand all files Collapse all files

107
src/frostfs_testlib/s3/aws_cli_client.py
View file

@ -14,13 +14,15 @@ from frostfs_testlib.resources.common import (
S3_SYNC_WAIT_TIME,
)
from frostfs_testlib.s3.interfaces import S3ClientWrapper, VersioningStatus, _make_objs_dict
from frostfs_testlib.shell import CommandOptions
from frostfs_testlib.shell.local_shell import LocalShell
# TODO: Refactor this code to use shell instead of _cmd_run
from frostfs_testlib.utils.cli_utils import _cmd_run, _configure_aws_cli
from frostfs_testlib.utils.cli_utils import _configure_aws_cli
reporter = get_reporter()
logger = logging.getLogger("NeoLogger")
LONG_TIMEOUT = 240
command_options = CommandOptions(timeout=240)
class AwsCliClient(S3ClientWrapper):
@ -34,10 +36,13 @@ class AwsCliClient(S3ClientWrapper):
@reporter.step_deco("Configure S3 client (aws cli)")
def __init__(self, access_key_id: str, secret_access_key: str, s3gate_endpoint: str) -> None:
self.s3gate_endpoint = s3gate_endpoint
self.local_shell = LocalShell()
try:
_configure_aws_cli("aws configure", access_key_id, secret_access_key)
_cmd_run(f"aws configure set max_attempts {MAX_REQUEST_ATTEMPTS}")
_cmd_run(f"aws configure set retry_mode {RETRY_MODE}")
self.local_shell.exec(f"aws configure set max_attempts {MAX_REQUEST_ATTEMPTS}")
self.local_shell.exec(
f"aws configure set retry_mode {RETRY_MODE}",
)
except Exception as err:
raise RuntimeError("Error while configuring AwsCliClient") from err
@ -79,7 +84,7 @@ class AwsCliClient(S3ClientWrapper):
cmd += f" --grant-read {grant_read}"
if location_constraint:
cmd += f" --create-bucket-configuration LocationConstraint={location_constraint}"
_cmd_run(cmd)
self.local_shell.exec(cmd)
sleep(S3_SYNC_WAIT_TIME)
return bucket
@ -87,20 +92,20 @@ class AwsCliClient(S3ClientWrapper):
@reporter.step_deco("List buckets S3")
def list_buckets(self) -> list[str]:
cmd = f"aws {self.common_flags} s3api list-buckets --endpoint {self.s3gate_endpoint}"
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
buckets_json = self._to_json(output)
return [bucket["Name"] for bucket in buckets_json["Buckets"]]
@reporter.step_deco("Delete bucket S3")
def delete_bucket(self, bucket: str) -> None:
cmd = f"aws {self.common_flags} s3api delete-bucket --bucket {bucket} --endpoint {self.s3gate_endpoint}"
_cmd_run(cmd, LONG_TIMEOUT)
self.local_shell.exec(cmd, command_options)
sleep(S3_SYNC_WAIT_TIME)
@reporter.step_deco("Head bucket S3")
def head_bucket(self, bucket: str) -> None:
cmd = f"aws {self.common_flags} s3api head-bucket --bucket {bucket} --endpoint {self.s3gate_endpoint}"
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Put bucket versioning status")
def put_bucket_versioning(self, bucket: str, status: VersioningStatus) -> None:
@ -109,7 +114,7 @@ class AwsCliClient(S3ClientWrapper):
f"--versioning-configuration Status={status.value} "
f"--endpoint {self.s3gate_endpoint}"
)
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Get bucket versioning status")
def get_bucket_versioning_status(self, bucket: str) -> Literal["Enabled", "Suspended"]:
@ -117,7 +122,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api get-bucket-versioning --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response.get("Status")
@ -130,7 +135,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api put-bucket-tagging --bucket {bucket} "
f"--tagging '{json.dumps(tags_json)}' --endpoint {self.s3gate_endpoint}"
)
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Get bucket tagging")
def get_bucket_tagging(self, bucket: str) -> list:
@ -138,7 +143,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api get-bucket-tagging --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response.get("TagSet")
@ -148,7 +153,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api get-bucket-acl --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response.get("Grants")
@ -158,7 +163,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api get-bucket-location --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response.get("LocationConstraint")
@ -168,7 +173,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api list-objects --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
obj_list = [obj["Key"] for obj in response.get("Contents", [])]
@ -182,7 +187,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api list-objects-v2 --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
obj_list = [obj["Key"] for obj in response.get("Contents", [])]
@ -196,7 +201,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api list-object-versions --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response if full_output else response.get("Versions", [])
@ -206,7 +211,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api list-object-versions --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response if full_output else response.get("DeleteMarkers", [])
@ -245,7 +250,7 @@ class AwsCliClient(S3ClientWrapper):
cmd += f" --tagging-directive {tagging_directive}"
if tagging:
cmd += f" --tagging {tagging}"
_cmd_run(cmd, LONG_TIMEOUT)
self.local_shell.exec(cmd, command_options)
return key
@reporter.step_deco("Put object S3")
@ -288,7 +293,7 @@ class AwsCliClient(S3ClientWrapper):
cmd += f" --grant-full-control '{grant_full_control}'"
if grant_read:
cmd += f" --grant-read {grant_read}"
output = _cmd_run(cmd, LONG_TIMEOUT)
output = self.local_shell.exec(cmd, command_options).stdout
response = self._to_json(output)
return response.get("VersionId")
@ -299,7 +304,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api head-object --bucket {bucket} --key {key} "
f"{version} --endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response
@ -320,7 +325,7 @@ class AwsCliClient(S3ClientWrapper):
)
if object_range:
cmd += f" --range bytes={object_range[0]}-{object_range[1]}"
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response if full_output else file_path
@ -331,7 +336,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api get-object-acl --bucket {bucket} --key {key} "
f"{version} --endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response.get("Grants")
@ -354,7 +359,7 @@ class AwsCliClient(S3ClientWrapper):
cmd += f" --grant-write {grant_write}"
if grant_read:
cmd += f" --grant-read {grant_read}"
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response.get("Grants")
@ -376,7 +381,7 @@ class AwsCliClient(S3ClientWrapper):
cmd += f" --grant-write {grant_write}"
if grant_read:
cmd += f" --grant-read {grant_read}"
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Delete objects S3")
def delete_objects(self, bucket: str, keys: list[str]) -> dict:
@ -390,7 +395,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api delete-objects --bucket {bucket} "
f"--delete file://{file_path} --endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd, LONG_TIMEOUT)
output = self.local_shell.exec(cmd, command_options).stdout
response = self._to_json(output)
sleep(S3_SYNC_WAIT_TIME)
return response
@ -402,7 +407,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api delete-object --bucket {bucket} "
f"--key {key} {version} --endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd, LONG_TIMEOUT)
output = self.local_shell.exec(cmd, command_options).stdout
sleep(S3_SYNC_WAIT_TIME)
return self._to_json(output)
@ -429,7 +434,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api delete-objects --bucket {bucket} "
f"--delete file://{file_path} --endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd, LONG_TIMEOUT)
output = self.local_shell.exec(cmd, command_options).stdout
sleep(S3_SYNC_WAIT_TIME)
return self._to_json(output)
@ -462,7 +467,7 @@ class AwsCliClient(S3ClientWrapper):
f"--key {key} {version} {parts} {part_number_str} --object-attributes {attrs} "
f"--endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
for attr in attributes:
@ -479,7 +484,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api get-bucket-policy --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response.get("Policy")
@ -496,7 +501,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api put-bucket-policy --bucket {bucket} "
f"--policy {dumped_policy} --endpoint {self.s3gate_endpoint}"
)
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Get bucket cors")
def get_bucket_cors(self, bucket: str) -> dict:
@ -504,7 +509,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api get-bucket-cors --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response.get("CORSRules")
@ -514,7 +519,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api put-bucket-cors --bucket {bucket} "
f"--cors-configuration '{json.dumps(cors_configuration)}' --endpoint {self.s3gate_endpoint}"
)
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Delete bucket cors")
def delete_bucket_cors(self, bucket: str) -> None:
@ -522,7 +527,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api delete-bucket-cors --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Delete bucket tagging")
def delete_bucket_tagging(self, bucket: str) -> None:
@ -530,7 +535,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api delete-bucket-tagging --bucket {bucket} "
f"--endpoint {self.s3gate_endpoint}"
)
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Put object retention")
def put_object_retention(
@ -548,7 +553,7 @@ class AwsCliClient(S3ClientWrapper):
)
if bypass_governance_retention is not None:
cmd += " --bypass-governance-retention"
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Put object legal hold")
def put_object_legal_hold(
@ -564,7 +569,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api put-object-legal-hold --bucket {bucket} --key {key} "
f"{version} --legal-hold '{legal_hold}' --endpoint {self.s3gate_endpoint}"
)
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Put object tagging")
def put_object_tagging(self, bucket: str, key: str, tags: list) -> None:
@ -574,7 +579,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api put-object-tagging --bucket {bucket} --key {key} "
f"--tagging '{json.dumps(tagging)}' --endpoint {self.s3gate_endpoint}"
)
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Get object tagging")
def get_object_tagging(self, bucket: str, key: str, version_id: Optional[str] = None) -> list:
@ -583,7 +588,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api get-object-tagging --bucket {bucket} --key {key} "
f"{version} --endpoint {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response.get("TagSet")
@ -593,7 +598,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api delete-object-tagging --bucket {bucket} "
f"--key {key} --endpoint {self.s3gate_endpoint}"
)
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Sync directory S3")
def sync(
@ -613,7 +618,7 @@ class AwsCliClient(S3ClientWrapper):
cmd += f" {key}={value}"
if acl:
cmd += f" --acl {acl}"
output = _cmd_run(cmd, LONG_TIMEOUT)
output = self.local_shell.exec(cmd, command_options).stdout
return self._to_json(output)
@reporter.step_deco("CP directory S3")
@ -634,7 +639,7 @@ class AwsCliClient(S3ClientWrapper):
cmd += f" {key}={value}"
if acl:
cmd += f" --acl {acl}"
output = _cmd_run(cmd, LONG_TIMEOUT)
output = self.local_shell.exec(cmd, command_options).stdout
return self._to_json(output)
@reporter.step_deco("Create multipart upload S3")
@ -643,7 +648,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api create-multipart-upload --bucket {bucket} "
f"--key {key} --endpoint-url {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
assert response.get("UploadId"), f"Expected UploadId in response:\n{response}"
@ -656,7 +661,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api list-multipart-uploads --bucket {bucket} "
f"--endpoint-url {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response.get("Uploads")
@ -666,7 +671,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api abort-multipart-upload --bucket {bucket} "
f"--key {key} --upload-id {upload_id} --endpoint-url {self.s3gate_endpoint}"
)
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Upload part S3")
def upload_part(
@ -677,7 +682,7 @@ class AwsCliClient(S3ClientWrapper):
f"--upload-id {upload_id} --part-number {part_num} --body {filepath} "
f"--endpoint-url {self.s3gate_endpoint}"
)
output = _cmd_run(cmd, LONG_TIMEOUT)
output = self.local_shell.exec(cmd, command_options).stdout
response = self._to_json(output)
assert response.get("ETag"), f"Expected ETag in response:\n{response}"
return response["ETag"]
@ -691,7 +696,7 @@ class AwsCliClient(S3ClientWrapper):
f"--upload-id {upload_id} --part-number {part_num} --copy-source {copy_source} "
f"--endpoint-url {self.s3gate_endpoint}"
)
output = _cmd_run(cmd, LONG_TIMEOUT)
output = self.local_shell.exec(cmd, command_options).stdout
response = self._to_json(output)
assert response.get("CopyPartResult", []).get(
"ETag"
@ -705,7 +710,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api list-parts --bucket {bucket} --key {key} "
f"--upload-id {upload_id} --endpoint-url {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
assert response.get("Parts"), f"Expected Parts in response:\n{response}"
@ -727,7 +732,7 @@ class AwsCliClient(S3ClientWrapper):
f"--key {key} --upload-id {upload_id} --multipart-upload file://{file_path} "
f"--endpoint-url {self.s3gate_endpoint}"
)
_cmd_run(cmd)
self.local_shell.exec(cmd)
@reporter.step_deco("Put object lock configuration")
def put_object_lock_configuration(self, bucket: str, configuration: dict) -> dict:
@ -735,7 +740,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api put-object-lock-configuration --bucket {bucket} "
f"--object-lock-configuration '{json.dumps(configuration)}' --endpoint-url {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
return self._to_json(output)
@reporter.step_deco("Get object lock configuration")
@ -744,7 +749,7 @@ class AwsCliClient(S3ClientWrapper):
f"aws {self.common_flags} s3api get-object-lock-configuration --bucket {bucket} "
f"--endpoint-url {self.s3gate_endpoint}"
)
output = _cmd_run(cmd)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response.get("ObjectLockConfiguration")