From 8940fd17c776d00647cf2203078e5872e4415e32 Mon Sep 17 00:00:00 2001 From: "a.berezin" Date: Tue, 18 Jun 2024 12:37:25 +0300 Subject: [PATCH] [#246] Use TestFiles which automatically deletes itself Signed-off-by: a.berezin --- src/frostfs_testlib/s3/aws_cli_client.py | 199 +++++--------------- src/frostfs_testlib/s3/boto3_client.py | 123 +++--------- src/frostfs_testlib/s3/interfaces.py | 78 ++++---- src/frostfs_testlib/steps/cli/object.py | 17 +- src/frostfs_testlib/steps/http/http_gate.py | 43 ++--- src/frostfs_testlib/utils/file_utils.py | 79 +++++--- 6 files changed, 199 insertions(+), 340 deletions(-) diff --git a/src/frostfs_testlib/s3/aws_cli_client.py b/src/frostfs_testlib/s3/aws_cli_client.py index 69a097b..4d64429 100644 --- a/src/frostfs_testlib/s3/aws_cli_client.py +++ b/src/frostfs_testlib/s3/aws_cli_client.py @@ -14,6 +14,7 @@ from frostfs_testlib.shell.local_shell import LocalShell # TODO: Refactor this code to use shell instead of _cmd_run from frostfs_testlib.utils.cli_utils import _configure_aws_cli +from frostfs_testlib.utils.file_utils import TestFile logger = logging.getLogger("NeoLogger") command_options = CommandOptions(timeout=480) @@ -153,8 +154,7 @@ class AwsCliClient(S3ClientWrapper): @reporter.step("Get bucket acl") def get_bucket_acl(self, bucket: str) -> list: cmd = ( - f"aws {self.common_flags} s3api get-bucket-acl --bucket {bucket} " - f"--endpoint {self.s3gate_endpoint} --profile {self.profile}" + f"aws {self.common_flags} s3api get-bucket-acl --bucket {bucket} " f"--endpoint {self.s3gate_endpoint} --profile {self.profile}" ) output = self.local_shell.exec(cmd).stdout response = self._to_json(output) @@ -172,10 +172,7 @@ class AwsCliClient(S3ClientWrapper): @reporter.step("List objects S3") def list_objects(self, bucket: str, full_output: bool = False) -> Union[dict, list[str]]: - cmd = ( - f"aws {self.common_flags} s3api list-objects --bucket {bucket} " - f"--endpoint {self.s3gate_endpoint} --profile {self.profile}" - ) + cmd = f"aws {self.common_flags} s3api list-objects --bucket {bucket} " f"--endpoint {self.s3gate_endpoint} --profile {self.profile}" output = self.local_shell.exec(cmd).stdout response = self._to_json(output) @@ -319,18 +316,18 @@ class AwsCliClient(S3ClientWrapper): version_id: Optional[str] = None, object_range: Optional[tuple[int, int]] = None, full_output: bool = False, - ) -> Union[dict, str]: - file_path = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4())) + ) -> dict | TestFile: + test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))) version = f" --version-id {version_id}" if version_id else "" cmd = ( f"aws {self.common_flags} s3api get-object --bucket {bucket} --key {key} " - f"{version} {file_path} --endpoint {self.s3gate_endpoint} --profile {self.profile}" + f"{version} {test_file} --endpoint {self.s3gate_endpoint} --profile {self.profile}" ) if object_range: cmd += f" --range bytes={object_range[0]}-{object_range[1]}" output = self.local_shell.exec(cmd).stdout response = self._to_json(output) - return response if full_output else file_path + return response if full_output else test_file @reporter.step("Get object ACL") def get_object_acl(self, bucket: str, key: str, version_id: Optional[str] = None) -> list: @@ -583,7 +580,7 @@ class AwsCliClient(S3ClientWrapper): self.local_shell.exec(cmd) @reporter.step("Put object tagging") - def put_object_tagging(self, bucket: str, key: str, tags: list, version_id: Optional[str] = '') -> None: + def put_object_tagging(self, bucket: str, key: str, tags: list, version_id: Optional[str] = "") -> None: tags = [{"Key": tag_key, "Value": tag_value} for tag_key, tag_value in tags] tagging = {"TagSet": tags} version = f" --version-id {version_id}" if version_id else "" @@ -622,8 +619,7 @@ class AwsCliClient(S3ClientWrapper): metadata: Optional[dict] = None, ) -> dict: cmd = ( - f"aws {self.common_flags} s3 sync {dir_path} s3://{bucket} " - f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}" + f"aws {self.common_flags} s3 sync {dir_path} s3://{bucket} " f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}" ) if metadata: cmd += " --metadata" @@ -779,9 +775,7 @@ class AwsCliClient(S3ClientWrapper): @reporter.step("Adds the specified user to the specified group") def iam_add_user_to_group(self, user_name: str, group_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam add-user-to-group --user-name {user_name} --group-name {group_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam add-user-to-group --user-name {user_name} --group-name {group_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -789,12 +783,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Attaches the specified managed policy to the specified IAM group") def iam_attach_group_policy(self, group_name: str, policy_arn: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam attach-group-policy --group-name {group_name} --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam attach-group-policy --group-name {group_name} --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -803,12 +794,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Attaches the specified managed policy to the specified user") def iam_attach_user_policy(self, user_name: str, policy_arn: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam attach-user-policy --user-name {user_name} --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam attach-user-policy --user-name {user_name} --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -817,12 +805,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Creates a new AWS secret access key and access key ID for the specified user") def iam_create_access_key(self, user_name: Optional[str] = None) -> dict: - cmd = ( - f"aws {self.common_flags} iam create-access-key --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam create-access-key --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" if user_name: @@ -837,12 +822,9 @@ class AwsCliClient(S3ClientWrapper): return access_key_id, secret_access_key - @reporter.step("Creates a new group") def iam_create_group(self, group_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam create-group --group-name {group_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam create-group --group-name {group_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -853,7 +835,6 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Creates a new managed policy for your AWS account") def iam_create_policy(self, policy_name: str, policy_document: dict) -> dict: cmd = ( @@ -871,12 +852,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Creates a new IAM user for your AWS account") def iam_create_user(self, user_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam create-user --user-name {user_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam create-user --user-name {user_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -887,12 +865,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Deletes the access key pair associated with the specified IAM user") def iam_delete_access_key(self, access_key_id: str, user_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam delete-access-key --access-key-id {access_key_id} --user-name {user_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam delete-access-key --access-key-id {access_key_id} --user-name {user_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" @@ -901,12 +876,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Deletes the specified IAM group") def iam_delete_group(self, group_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam delete-group --group-name {group_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam delete-group --group-name {group_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -914,12 +886,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Deletes the specified inline policy that is embedded in the specified IAM group") def iam_delete_group_policy(self, group_name: str, policy_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam delete-group-policy --group-name {group_name} --policy-name {policy_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam delete-group-policy --group-name {group_name} --policy-name {policy_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -927,12 +896,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Deletes the specified managed policy") def iam_delete_policy(self, policy_arn: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam delete-policy --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam delete-policy --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -940,26 +906,19 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Deletes the specified IAM user") def iam_delete_user(self, user_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam delete-user --user-name {user_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam delete-user --user-name {user_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout response = self._to_json(output) - return response - @reporter.step("Deletes the specified inline policy that is embedded in the specified IAM user") def iam_delete_user_policy(self, user_name: str, policy_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam delete-user-policy --user-name {user_name} --policy-name {policy_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam delete-user-policy --user-name {user_name} --policy-name {policy_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -967,12 +926,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Removes the specified managed policy from the specified IAM group") def iam_detach_group_policy(self, group_name: str, policy_arn: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam detach-group-policy --group-name {group_name} --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam detach-group-policy --group-name {group_name} --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -981,12 +937,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Removes the specified managed policy from the specified user") def iam_detach_user_policy(self, user_name: str, policy_arn: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam detach-user-policy --user-name {user_name} --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam detach-user-policy --user-name {user_name} --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -995,12 +948,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Returns a list of IAM users that are in the specified IAM group") def iam_get_group(self, group_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam get-group --group-name {group_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam get-group --group-name {group_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1011,12 +961,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Retrieves the specified inline policy document that is embedded in the specified IAM group") def iam_get_group_policy(self, group_name: str, policy_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam get-group-policy --group-name {group_name} --policy-name {policy_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam get-group-policy --group-name {group_name} --policy-name {policy_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1024,12 +971,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Retrieves information about the specified managed policy") def iam_get_policy(self, policy_arn: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam get-policy --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam get-policy --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1040,12 +984,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Retrieves information about the specified version of the specified managed policy") def iam_get_policy_version(self, policy_arn: str, version_id: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam get-policy-version --policy-arn {policy_arn} --version-id {version_id} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam get-policy-version --policy-arn {policy_arn} --version-id {version_id} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1056,12 +997,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Retrieves information about the specified IAM user") def iam_get_user(self, user_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam get-user --user-name {user_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam get-user --user-name {user_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1072,12 +1010,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Retrieves the specified inline policy document that is embedded in the specified IAM user") def iam_get_user_policy(self, user_name: str, policy_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam get-user-policy --user-name {user_name} --policy-name {policy_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam get-user-policy --user-name {user_name} --policy-name {policy_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1087,12 +1022,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Returns information about the access key IDs associated with the specified IAM user") def iam_list_access_keys(self, user_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam list-access-keys --user-name {user_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam list-access-keys --user-name {user_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1100,12 +1032,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Lists all managed policies that are attached to the specified IAM group") def iam_list_attached_group_policies(self, group_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam list-attached-group-policies --group-name {group_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam list-attached-group-policies --group-name {group_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1115,12 +1044,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Lists all managed policies that are attached to the specified IAM user") def iam_list_attached_user_policies(self, user_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam list-attached-user-policies --user-name {user_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam list-attached-user-policies --user-name {user_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1130,12 +1056,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Lists all IAM users, groups, and roles that the specified managed policy is attached to") def iam_list_entities_for_policy(self, policy_arn: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam list-entities-for-policy --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam list-entities-for-policy --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1146,12 +1069,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Lists the names of the inline policies that are embedded in the specified IAM group") def iam_list_group_policies(self, group_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam list-group-policies --group-name {group_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam list-group-policies --group-name {group_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1161,12 +1081,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Lists the IAM groups") def iam_list_groups(self) -> dict: - cmd = ( - f"aws {self.common_flags} iam list-groups --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam list-groups --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1176,12 +1093,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Lists the IAM groups that the specified IAM user belongs to") def iam_list_groups_for_user(self, user_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam list-groups-for-user --user-name {user_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam list-groups-for-user --user-name {user_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1191,27 +1105,21 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Lists all the managed policies that are available in your AWS account") def iam_list_policies(self) -> dict: - cmd = ( - f"aws {self.common_flags} iam list-policies --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam list-policies --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout response = self._to_json(output) - assert 'Policies' in response.keys(), f"Expected Policies in response:\n{response}" + assert "Policies" in response.keys(), f"Expected Policies in response:\n{response}" return response - @reporter.step("Lists information about the versions of the specified managed policy") def iam_list_policy_versions(self, policy_arn: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam list-policy-versions --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam list-policy-versions --policy-arn {policy_arn} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1221,12 +1129,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Lists the names of the inline policies embedded in the specified IAM user") def iam_list_user_policies(self, user_name: str) -> dict: - cmd = ( - f"aws {self.common_flags} iam list-user-policies --user-name {user_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam list-user-policies --user-name {user_name} --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1236,12 +1141,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Lists the IAM users") def iam_list_users(self) -> dict: - cmd = ( - f"aws {self.common_flags} iam list-users --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam list-users --endpoint {self.iam_endpoint}" if self.profile: cmd += f" --profile {self.profile}" output = self.local_shell.exec(cmd).stdout @@ -1251,12 +1153,11 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Adds or updates an inline policy document that is embedded in the specified IAM group") def iam_put_group_policy(self, group_name: str, policy_name: str, policy_document: dict) -> dict: cmd = ( f"aws {self.common_flags} iam put-group-policy --endpoint {self.iam_endpoint}" - f" --group-name {group_name} --policy-name {policy_name} --policy-document \'{json.dumps(policy_document)}\'" + f" --group-name {group_name} --policy-name {policy_name} --policy-document '{json.dumps(policy_document)}'" ) if self.profile: cmd += f" --profile {self.profile}" @@ -1266,12 +1167,11 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Adds or updates an inline policy document that is embedded in the specified IAM user") def iam_put_user_policy(self, user_name: str, policy_name: str, policy_document: dict) -> dict: cmd = ( f"aws {self.common_flags} iam put-user-policy --endpoint {self.iam_endpoint}" - f" --user-name {user_name} --policy-name {policy_name} --policy-document \'{json.dumps(policy_document)}\'" + f" --user-name {user_name} --policy-name {policy_name} --policy-document '{json.dumps(policy_document)}'" ) if self.profile: cmd += f" --profile {self.profile}" @@ -1282,7 +1182,6 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Removes the specified user from the specified group") def iam_remove_user_from_group(self, group_name: str, user_name: str) -> dict: cmd = ( @@ -1296,12 +1195,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Updates the name and/or the path of the specified IAM group") def iam_update_group(self, group_name: str, new_name: Optional[str] = None, new_path: Optional[str] = None) -> dict: - cmd = ( - f"aws {self.common_flags} iam update-group --group-name {group_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam update-group --group-name {group_name} --endpoint {self.iam_endpoint}" if new_name: cmd += f" --new-group-name {new_name}" if new_path: @@ -1314,12 +1210,9 @@ class AwsCliClient(S3ClientWrapper): return response - @reporter.step("Updates the name and/or the path of the specified IAM user") def iam_update_user(self, user_name: str, new_name: Optional[str] = None, new_path: Optional[str] = None) -> dict: - cmd = ( - f"aws {self.common_flags} iam update-user --user-name {user_name} --endpoint {self.iam_endpoint}" - ) + cmd = f"aws {self.common_flags} iam update-user --user-name {user_name} --endpoint {self.iam_endpoint}" if new_name: cmd += f" --new-user-name {new_name}" if new_path: @@ -1331,5 +1224,3 @@ class AwsCliClient(S3ClientWrapper): response = self._to_json(output) return response - - diff --git a/src/frostfs_testlib/s3/boto3_client.py b/src/frostfs_testlib/s3/boto3_client.py index 59da55a..28e05c4 100644 --- a/src/frostfs_testlib/s3/boto3_client.py +++ b/src/frostfs_testlib/s3/boto3_client.py @@ -16,10 +16,10 @@ from mypy_boto3_s3 import S3Client from frostfs_testlib import reporter from frostfs_testlib.resources.common import ASSETS_DIR, MAX_REQUEST_ATTEMPTS, RETRY_MODE, S3_SYNC_WAIT_TIME from frostfs_testlib.s3.interfaces import S3ClientWrapper, VersioningStatus, _make_objs_dict -from frostfs_testlib.utils.cli_utils import log_command_execution # TODO: Refactor this code to use shell instead of _cmd_run -from frostfs_testlib.utils.cli_utils import _configure_aws_cli +from frostfs_testlib.utils.cli_utils import _configure_aws_cli, log_command_execution +from frostfs_testlib.utils.file_utils import TestFile logger = logging.getLogger("NeoLogger") @@ -80,7 +80,6 @@ class Boto3ClientWrapper(S3ClientWrapper): verify=False, ) - @reporter.step("Set endpoint IAM to {iam_endpoint}") def set_iam_endpoint(self, iam_endpoint: str): self.boto3_iam_client = self.session.client( @@ -88,8 +87,8 @@ class Boto3ClientWrapper(S3ClientWrapper): aws_access_key_id=self.access_key_id, aws_secret_access_key=self.secret_access_key, endpoint_url=iam_endpoint, - verify=False,) - + verify=False, + ) def _to_s3_param(self, param: str): replacement_map = { @@ -167,9 +166,7 @@ class Boto3ClientWrapper(S3ClientWrapper): @reporter.step("Put bucket versioning status") @report_error def put_bucket_versioning(self, bucket: str, status: VersioningStatus) -> None: - response = self.boto3_client.put_bucket_versioning( - Bucket=bucket, VersioningConfiguration={"Status": status.value} - ) + response = self.boto3_client.put_bucket_versioning(Bucket=bucket, VersioningConfiguration={"Status": status.value}) log_command_execution("S3 Set bucket versioning to", response) @reporter.step("Get bucket versioning status") @@ -217,11 +214,7 @@ class Boto3ClientWrapper(S3ClientWrapper): grant_write: Optional[str] = None, grant_read: Optional[str] = None, ) -> None: - params = { - self._to_s3_param(param): value - for param, value in locals().items() - if param not in ["self"] and value is not None - } + params = {self._to_s3_param(param): value for param, value in locals().items() if param not in ["self"] and value is not None} response = self.boto3_client.put_bucket_acl(**params) log_command_execution("S3 ACL bucket result", response) @@ -360,11 +353,7 @@ class Boto3ClientWrapper(S3ClientWrapper): @reporter.step("Head object S3") @report_error def head_object(self, bucket: str, key: str, version_id: Optional[str] = None) -> dict: - params = { - self._to_s3_param(param): value - for param, value in locals().items() - if param not in ["self"] and value is not None - } + params = {self._to_s3_param(param): value for param, value in locals().items() if param not in ["self"] and value is not None} response = self.boto3_client.head_object(**params) log_command_execution("S3 Head object result", response) return response @@ -372,11 +361,7 @@ class Boto3ClientWrapper(S3ClientWrapper): @reporter.step("Delete object S3") @report_error def delete_object(self, bucket: str, key: str, version_id: Optional[str] = None) -> dict: - params = { - self._to_s3_param(param): value - for param, value in locals().items() - if param not in ["self"] and value is not None - } + params = {self._to_s3_param(param): value for param, value in locals().items() if param not in ["self"] and value is not None} response = self.boto3_client.delete_object(**params) log_command_execution("S3 Delete object result", response) sleep(S3_SYNC_WAIT_TIME) @@ -415,9 +400,7 @@ class Boto3ClientWrapper(S3ClientWrapper): def delete_object_versions_without_dm(self, bucket: str, object_versions: list) -> None: # Delete objects without creating delete markers for object_version in object_versions: - response = self.boto3_client.delete_object( - Bucket=bucket, Key=object_version["Key"], VersionId=object_version["VersionId"] - ) + response = self.boto3_client.delete_object(Bucket=bucket, Key=object_version["Key"], VersionId=object_version["VersionId"]) log_command_execution("S3 Delete object result", response) @reporter.step("Put object ACL") @@ -436,11 +419,7 @@ class Boto3ClientWrapper(S3ClientWrapper): @reporter.step("Get object ACL") @report_error def get_object_acl(self, bucket: str, key: str, version_id: Optional[str] = None) -> list: - params = { - self._to_s3_param(param): value - for param, value in locals().items() - if param not in ["self"] and value is not None - } + params = {self._to_s3_param(param): value for param, value in locals().items() if param not in ["self"] and value is not None} response = self.boto3_client.get_object_acl(**params) log_command_execution("S3 ACL objects result", response) return response.get("Grants") @@ -483,8 +462,7 @@ class Boto3ClientWrapper(S3ClientWrapper): version_id: Optional[str] = None, object_range: Optional[tuple[int, int]] = None, full_output: bool = False, - ) -> Union[dict, str]: - filename = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4())) + ) -> dict | TestFile: range_str = None if object_range: range_str = f"bytes={object_range[0]}-{object_range[1]}" @@ -497,12 +475,16 @@ class Boto3ClientWrapper(S3ClientWrapper): response = self.boto3_client.get_object(**params) log_command_execution("S3 Get objects result", response) - with open(f"{filename}", "wb") as get_file: + if full_output: + return response + + test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))) + with open(test_file, "wb") as file: chunk = response["Body"].read(1024) while chunk: - get_file.write(chunk) + file.write(chunk) chunk = response["Body"].read(1024) - return response if full_output else filename + return test_file @reporter.step("Create multipart upload S3") @report_error @@ -573,9 +555,7 @@ class Boto3ClientWrapper(S3ClientWrapper): @report_error def complete_multipart_upload(self, bucket: str, key: str, upload_id: str, parts: list) -> None: parts = [{"ETag": etag, "PartNumber": part_num} for part_num, etag in parts] - response = self.boto3_client.complete_multipart_upload( - Bucket=bucket, Key=key, UploadId=upload_id, MultipartUpload={"Parts": parts} - ) + response = self.boto3_client.complete_multipart_upload(Bucket=bucket, Key=key, UploadId=upload_id, MultipartUpload={"Parts": parts}) log_command_execution("S3 Complete multipart upload", response) return response @@ -590,11 +570,7 @@ class Boto3ClientWrapper(S3ClientWrapper): version_id: Optional[str] = None, bypass_governance_retention: Optional[bool] = None, ) -> None: - params = { - self._to_s3_param(param): value - for param, value in locals().items() - if param not in ["self"] and value is not None - } + params = {self._to_s3_param(param): value for param, value in locals().items() if param not in ["self"] and value is not None} response = self.boto3_client.put_object_retention(**params) log_command_execution("S3 Put object retention ", response) @@ -618,7 +594,7 @@ class Boto3ClientWrapper(S3ClientWrapper): @reporter.step("Put object tagging") @report_error - def put_object_tagging(self, bucket: str, key: str, tags: list, version_id: Optional[str] = '') -> None: + def put_object_tagging(self, bucket: str, key: str, tags: list, version_id: Optional[str] = "") -> None: tags = [{"Key": tag_key, "Value": tag_value} for tag_key, tag_value in tags] tagging = {"TagSet": tags} response = self.boto3_client.put_object_tagging(Bucket=bucket, Key=key, Tagging=tagging, VersionId=version_id) @@ -627,11 +603,7 @@ class Boto3ClientWrapper(S3ClientWrapper): @reporter.step("Get object tagging") @report_error def get_object_tagging(self, bucket: str, key: str, version_id: Optional[str] = None) -> list: - params = { - self._to_s3_param(param): value - for param, value in locals().items() - if param not in ["self"] and value is not None - } + params = {self._to_s3_param(param): value for param, value in locals().items() if param not in ["self"] and value is not None} response = self.boto3_client.get_object_tagging(**params) log_command_execution("S3 Get object tagging", response) return response.get("TagSet") @@ -681,7 +653,6 @@ class Boto3ClientWrapper(S3ClientWrapper): # END OBJECT METHODS # - # IAM METHODS # # Some methods don't have checks because boto3 is silent in some cases (delete, attach, etc.) @@ -690,21 +661,18 @@ class Boto3ClientWrapper(S3ClientWrapper): response = self.boto3_iam_client.add_user_to_group(UserName=user_name, GroupName=group_name) return response - @reporter.step("Attaches the specified managed policy to the specified IAM group") def iam_attach_group_policy(self, group_name: str, policy_arn: str) -> dict: response = self.boto3_iam_client.attach_group_policy(GroupName=group_name, PolicyArn=policy_arn) sleep(S3_SYNC_WAIT_TIME * 10) return response - @reporter.step("Attaches the specified managed policy to the specified user") def iam_attach_user_policy(self, user_name: str, policy_arn: str) -> dict: response = self.boto3_iam_client.attach_user_policy(UserName=user_name, PolicyArn=policy_arn) sleep(S3_SYNC_WAIT_TIME * 10) return response - @reporter.step("Creates a new AWS secret access key and access key ID for the specified user") def iam_create_access_key(self, user_name: str) -> dict: response = self.boto3_iam_client.create_access_key(UserName=user_name) @@ -716,7 +684,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return access_key_id, secret_access_key - @reporter.step("Creates a new group") def iam_create_group(self, group_name: str) -> dict: response = self.boto3_iam_client.create_group(GroupName=group_name) @@ -725,7 +692,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Creates a new managed policy for your AWS account") def iam_create_policy(self, policy_name: str, policy_document: dict) -> dict: response = self.boto3_iam_client.create_policy(PolicyName=policy_name, PolicyDocument=json.dumps(policy_document)) @@ -734,7 +700,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Creates a new IAM user for your AWS account") def iam_create_user(self, user_name: str) -> dict: response = self.boto3_iam_client.create_user(UserName=user_name) @@ -743,57 +708,48 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Deletes the access key pair associated with the specified IAM user") def iam_delete_access_key(self, access_key_id: str, user_name: str) -> dict: response = self.boto3_iam_client.delete_access_key(AccessKeyId=access_key_id, UserName=user_name) return response - @reporter.step("Deletes the specified IAM group") def iam_delete_group(self, group_name: str) -> dict: response = self.boto3_iam_client.delete_group(GroupName=group_name) return response - @reporter.step("Deletes the specified inline policy that is embedded in the specified IAM group") def iam_delete_group_policy(self, group_name: str, policy_name: str) -> dict: response = self.boto3_iam_client.delete_group_policy(GroupName=group_name, PolicyName=policy_name) return response - @reporter.step("Deletes the specified managed policy") def iam_delete_policy(self, policy_arn: str) -> dict: response = self.boto3_iam_client.delete_policy(PolicyArn=policy_arn) return response - @reporter.step("Deletes the specified IAM user") def iam_delete_user(self, user_name: str) -> dict: response = self.boto3_iam_client.delete_user(UserName=user_name) return response - @reporter.step("Deletes the specified inline policy that is embedded in the specified IAM user") def iam_delete_user_policy(self, user_name: str, policy_name: str) -> dict: response = self.boto3_iam_client.delete_user_policy(UserName=user_name, PolicyName=policy_name) return response - @reporter.step("Removes the specified managed policy from the specified IAM group") def iam_detach_group_policy(self, group_name: str, policy_arn: str) -> dict: response = self.boto3_iam_client.detach_group_policy(GroupName=group_name, PolicyArn=policy_arn) sleep(S3_SYNC_WAIT_TIME * 10) return response - @reporter.step("Removes the specified managed policy from the specified user") def iam_detach_user_policy(self, user_name: str, policy_arn: str) -> dict: response = self.boto3_iam_client.detach_user_policy(UserName=user_name, PolicyArn=policy_arn) sleep(S3_SYNC_WAIT_TIME * 10) return response - @reporter.step("Returns a list of IAM users that are in the specified IAM group") def iam_get_group(self, group_name: str) -> dict: response = self.boto3_iam_client.get_group(GroupName=group_name) @@ -801,14 +757,12 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Retrieves the specified inline policy document that is embedded in the specified IAM group") def iam_get_group_policy(self, group_name: str, policy_name: str) -> dict: response = self.boto3_iam_client.get_group_policy(GroupName=group_name, PolicyName=policy_name) return response - @reporter.step("Retrieves information about the specified managed policy") def iam_get_policy(self, policy_arn: str) -> dict: response = self.boto3_iam_client.get_policy(PolicyArn=policy_arn) @@ -817,7 +771,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Retrieves information about the specified version of the specified managed policy") def iam_get_policy_version(self, policy_arn: str, version_id: str) -> dict: response = self.boto3_iam_client.get_policy_version(PolicyArn=policy_arn, VersionId=version_id) @@ -826,7 +779,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Retrieves information about the specified IAM user") def iam_get_user(self, user_name: str) -> dict: response = self.boto3_iam_client.get_user(UserName=user_name) @@ -835,7 +787,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Retrieves the specified inline policy document that is embedded in the specified IAM user") def iam_get_user_policy(self, user_name: str, policy_name: str) -> dict: response = self.boto3_iam_client.get_user_policy(UserName=user_name, PolicyName=policy_name) @@ -843,14 +794,12 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Returns information about the access key IDs associated with the specified IAM user") def iam_list_access_keys(self, user_name: str) -> dict: response = self.boto3_iam_client.list_access_keys(UserName=user_name) return response - @reporter.step("Lists all managed policies that are attached to the specified IAM group") def iam_list_attached_group_policies(self, group_name: str) -> dict: response = self.boto3_iam_client.list_attached_group_policies(GroupName=group_name) @@ -858,7 +807,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Lists all managed policies that are attached to the specified IAM user") def iam_list_attached_user_policies(self, user_name: str) -> dict: response = self.boto3_iam_client.list_attached_user_policies(UserName=user_name) @@ -866,7 +814,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Lists all IAM users, groups, and roles that the specified managed policy is attached to") def iam_list_entities_for_policy(self, policy_arn: str) -> dict: response = self.boto3_iam_client.list_entities_for_policy(PolicyArn=policy_arn) @@ -876,7 +823,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Lists the names of the inline policies that are embedded in the specified IAM group") def iam_list_group_policies(self, group_name: str) -> dict: response = self.boto3_iam_client.list_group_policies(GroupName=group_name) @@ -884,7 +830,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Lists the IAM groups") def iam_list_groups(self) -> dict: response = self.boto3_iam_client.list_groups() @@ -892,7 +837,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Lists the IAM groups that the specified IAM user belongs to") def iam_list_groups_for_user(self, user_name: str) -> dict: response = self.boto3_iam_client.list_groups_for_user(UserName=user_name) @@ -900,7 +844,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Lists all the managed policies that are available in your AWS account") def iam_list_policies(self) -> dict: response = self.boto3_iam_client.list_policies() @@ -908,7 +851,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Lists information about the versions of the specified managed policy") def iam_list_policy_versions(self, policy_arn: str) -> dict: response = self.boto3_iam_client.list_policy_versions(PolicyArn=policy_arn) @@ -916,7 +858,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Lists the names of the inline policies embedded in the specified IAM user") def iam_list_user_policies(self, user_name: str) -> dict: response = self.boto3_iam_client.list_user_policies(UserName=user_name) @@ -924,7 +865,6 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Lists the IAM users") def iam_list_users(self) -> dict: response = self.boto3_iam_client.list_users() @@ -932,35 +872,34 @@ class Boto3ClientWrapper(S3ClientWrapper): return response - @reporter.step("Adds or updates an inline policy document that is embedded in the specified IAM group") def iam_put_group_policy(self, group_name: str, policy_name: str, policy_document: dict) -> dict: - response = self.boto3_iam_client.put_group_policy(GroupName=group_name, PolicyName=policy_name, PolicyDocument=json.dumps(policy_document)) + response = self.boto3_iam_client.put_group_policy( + GroupName=group_name, PolicyName=policy_name, PolicyDocument=json.dumps(policy_document) + ) sleep(S3_SYNC_WAIT_TIME * 10) return response - @reporter.step("Adds or updates an inline policy document that is embedded in the specified IAM user") def iam_put_user_policy(self, user_name: str, policy_name: str, policy_document: dict) -> dict: - response = self.boto3_iam_client.put_user_policy(UserName=user_name, PolicyName=policy_name, PolicyDocument=json.dumps(policy_document)) + response = self.boto3_iam_client.put_user_policy( + UserName=user_name, PolicyName=policy_name, PolicyDocument=json.dumps(policy_document) + ) sleep(S3_SYNC_WAIT_TIME * 10) return response - @reporter.step("Removes the specified user from the specified group") def iam_remove_user_from_group(self, group_name: str, user_name: str) -> dict: response = self.boto3_iam_client.remove_user_from_group(GroupName=group_name, UserName=user_name) return response - @reporter.step("Updates the name and/or the path of the specified IAM group") def iam_update_group(self, group_name: str, new_name: str, new_path: Optional[str] = None) -> dict: - response = self.boto3_iam_client.update_group(GroupName=group_name, NewGroupName=new_name, NewPath='/') + response = self.boto3_iam_client.update_group(GroupName=group_name, NewGroupName=new_name, NewPath="/") return response - @reporter.step("Updates the name and/or the path of the specified IAM user") def iam_update_user(self, user_name: str, new_name: str, new_path: Optional[str] = None) -> dict: - response = self.boto3_iam_client.update_user(UserName=user_name, NewUserName=new_name, NewPath='/') - return response \ No newline at end of file + response = self.boto3_iam_client.update_user(UserName=user_name, NewUserName=new_name, NewPath="/") + return response diff --git a/src/frostfs_testlib/s3/interfaces.py b/src/frostfs_testlib/s3/interfaces.py index 8cfc2bb..271744e 100644 --- a/src/frostfs_testlib/s3/interfaces.py +++ b/src/frostfs_testlib/s3/interfaces.py @@ -4,6 +4,7 @@ from typing import Literal, Optional, Union from frostfs_testlib.storage.cluster import ClusterNode from frostfs_testlib.testing.readable import HumanReadableABC, HumanReadableEnum +from frostfs_testlib.utils.file_utils import TestFile def _make_objs_dict(key_names): @@ -289,7 +290,7 @@ class S3ClientWrapper(HumanReadableABC): version_id: Optional[str] = None, object_range: Optional[tuple[int, int]] = None, full_output: bool = False, - ) -> Union[dict, str]: + ) -> dict | TestFile: """Retrieves objects from S3.""" @abstractmethod @@ -400,153 +401,152 @@ class S3ClientWrapper(HumanReadableABC): # END OF OBJECT METHODS # - # IAM METHODS # @abstractmethod def iam_add_user_to_group(self, user_name: str, group_name: str) -> dict: - '''Adds the specified user to the specified group''' + """Adds the specified user to the specified group""" @abstractmethod def iam_attach_group_policy(self, group: str, policy_arn: str) -> dict: - '''Attaches the specified managed policy to the specified IAM group''' + """Attaches the specified managed policy to the specified IAM group""" @abstractmethod def iam_attach_user_policy(self, user_name: str, policy_arn: str) -> dict: - '''Attaches the specified managed policy to the specified user''' + """Attaches the specified managed policy to the specified user""" @abstractmethod def iam_create_access_key(self, user_name: str) -> dict: - '''Creates a new AWS secret access key and access key ID for the specified user''' + """Creates a new AWS secret access key and access key ID for the specified user""" @abstractmethod def iam_create_group(self, group_name: str) -> dict: - '''Creates a new group''' + """Creates a new group""" @abstractmethod def iam_create_policy(self, policy_name: str, policy_document: dict) -> dict: - '''Creates a new managed policy for your AWS account''' + """Creates a new managed policy for your AWS account""" @abstractmethod def iam_create_user(self, user_name: str) -> dict: - '''Creates a new IAM user for your AWS account''' + """Creates a new IAM user for your AWS account""" @abstractmethod def iam_delete_access_key(self, access_key_id: str, user_name: str) -> dict: - '''Deletes the access key pair associated with the specified IAM user''' + """Deletes the access key pair associated with the specified IAM user""" @abstractmethod def iam_delete_group(self, group_name: str) -> dict: - '''Deletes the specified IAM group''' + """Deletes the specified IAM group""" @abstractmethod def iam_delete_group_policy(self, group_name: str, policy_name: str) -> dict: - '''Deletes the specified inline policy that is embedded in the specified IAM group''' + """Deletes the specified inline policy that is embedded in the specified IAM group""" @abstractmethod def iam_delete_policy(self, policy_arn: str) -> dict: - '''Deletes the specified managed policy''' + """Deletes the specified managed policy""" @abstractmethod def iam_delete_user(self, user_name: str) -> dict: - '''Deletes the specified IAM user''' + """Deletes the specified IAM user""" @abstractmethod def iam_delete_user_policy(self, user_name: str, policy_name: str) -> dict: - '''Deletes the specified inline policy that is embedded in the specified IAM user''' + """Deletes the specified inline policy that is embedded in the specified IAM user""" @abstractmethod def iam_detach_group_policy(self, group_name: str, policy_arn: str) -> dict: - '''Removes the specified managed policy from the specified IAM group''' + """Removes the specified managed policy from the specified IAM group""" @abstractmethod def iam_detach_user_policy(self, user_name: str, policy_arn: str) -> dict: - '''Removes the specified managed policy from the specified user''' + """Removes the specified managed policy from the specified user""" @abstractmethod def iam_get_group(self, group_name: str) -> dict: - '''Returns a list of IAM users that are in the specified IAM group''' + """Returns a list of IAM users that are in the specified IAM group""" @abstractmethod def iam_get_group_policy(self, group_name: str, policy_name: str) -> dict: - '''Retrieves the specified inline policy document that is embedded in the specified IAM group''' + """Retrieves the specified inline policy document that is embedded in the specified IAM group""" @abstractmethod def iam_get_policy(self, policy_arn: str) -> dict: - '''Retrieves information about the specified managed policy''' + """Retrieves information about the specified managed policy""" @abstractmethod def iam_get_policy_version(self, policy_arn: str, version_id: str) -> dict: - '''Retrieves information about the specified version of the specified managed policy''' + """Retrieves information about the specified version of the specified managed policy""" @abstractmethod def iam_get_user(self, user_name: str) -> dict: - '''Retrieves information about the specified IAM user''' + """Retrieves information about the specified IAM user""" @abstractmethod def iam_get_user_policy(self, user_name: str, policy_name: str) -> dict: - '''Retrieves the specified inline policy document that is embedded in the specified IAM user''' + """Retrieves the specified inline policy document that is embedded in the specified IAM user""" @abstractmethod def iam_list_access_keys(self, user_name: str) -> dict: - '''Returns information about the access key IDs associated with the specified IAM user''' + """Returns information about the access key IDs associated with the specified IAM user""" @abstractmethod def iam_list_attached_group_policies(self, group_name: str) -> dict: - '''Lists all managed policies that are attached to the specified IAM group''' + """Lists all managed policies that are attached to the specified IAM group""" @abstractmethod def iam_list_attached_user_policies(self, user_name: str) -> dict: - '''Lists all managed policies that are attached to the specified IAM user''' + """Lists all managed policies that are attached to the specified IAM user""" @abstractmethod def iam_list_entities_for_policy(self, policy_arn: str) -> dict: - '''Lists all IAM users, groups, and roles that the specified managed policy is attached to''' + """Lists all IAM users, groups, and roles that the specified managed policy is attached to""" @abstractmethod def iam_list_group_policies(self, group_name: str) -> dict: - '''Lists the names of the inline policies that are embedded in the specified IAM group''' + """Lists the names of the inline policies that are embedded in the specified IAM group""" @abstractmethod def iam_list_groups(self) -> dict: - '''Lists the IAM groups''' + """Lists the IAM groups""" @abstractmethod def iam_list_groups_for_user(self, user_name: str) -> dict: - '''Lists the IAM groups that the specified IAM user belongs to''' + """Lists the IAM groups that the specified IAM user belongs to""" @abstractmethod def iam_list_policies(self) -> dict: - '''Lists all the managed policies that are available in your AWS account''' + """Lists all the managed policies that are available in your AWS account""" @abstractmethod def iam_list_policy_versions(self, policy_arn: str) -> dict: - '''Lists information about the versions of the specified managed policy''' + """Lists information about the versions of the specified managed policy""" @abstractmethod def iam_list_user_policies(self, user_name: str) -> dict: - '''Lists the names of the inline policies embedded in the specified IAM user''' + """Lists the names of the inline policies embedded in the specified IAM user""" @abstractmethod def iam_list_users(self) -> dict: - '''Lists the IAM users''' + """Lists the IAM users""" @abstractmethod def iam_put_group_policy(self, group_name: str, policy_name: str, policy_document: dict) -> dict: - '''Adds or updates an inline policy document that is embedded in the specified IAM group''' + """Adds or updates an inline policy document that is embedded in the specified IAM group""" @abstractmethod def iam_put_user_policy(self, user_name: str, policy_name: str, policy_document: dict) -> dict: - '''Adds or updates an inline policy document that is embedded in the specified IAM user''' + """Adds or updates an inline policy document that is embedded in the specified IAM user""" @abstractmethod def iam_remove_user_from_group(self, group_name: str, user_name: str) -> dict: - '''Removes the specified user from the specified group''' + """Removes the specified user from the specified group""" @abstractmethod def iam_update_group(self, group_name: str, new_name: Optional[str] = None, new_path: Optional[str] = None) -> dict: - '''Updates the name and/or the path of the specified IAM group''' + """Updates the name and/or the path of the specified IAM group""" @abstractmethod def iam_update_user(self, user_name: str, new_name: Optional[str] = None, new_path: Optional[str] = None) -> dict: - '''Updates the name and/or the path of the specified IAM user''' + """Updates the name and/or the path of the specified IAM user""" diff --git a/src/frostfs_testlib/steps/cli/object.py b/src/frostfs_testlib/steps/cli/object.py index cd58ec3..59e2696 100644 --- a/src/frostfs_testlib/steps/cli/object.py +++ b/src/frostfs_testlib/steps/cli/object.py @@ -16,6 +16,7 @@ from frostfs_testlib.storage.dataclasses.wallet import WalletInfo from frostfs_testlib.testing import wait_for_success from frostfs_testlib.utils import json_utils from frostfs_testlib.utils.cli_utils import parse_cmd_table, parse_netmap_output +from frostfs_testlib.utils.file_utils import TestFile logger = logging.getLogger("NeoLogger") @@ -81,7 +82,7 @@ def get_object( no_progress: bool = True, session: Optional[str] = None, timeout: Optional[str] = CLI_DEFAULT_TIMEOUT, -) -> str: +) -> TestFile: """ GET from FrostFS. @@ -103,14 +104,14 @@ def get_object( if not write_object: write_object = str(uuid.uuid4()) - file_path = os.path.join(ASSETS_DIR, write_object) + test_file = TestFile(os.path.join(ASSETS_DIR, write_object)) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path) cli.object.get( rpc_endpoint=endpoint, cid=cid, oid=oid, - file=file_path, + file=test_file, bearer=bearer, no_progress=no_progress, xhdr=xhdr, @@ -118,7 +119,7 @@ def get_object( timeout=timeout, ) - return file_path + return test_file @reporter.step("Get Range Hash from {endpoint}") @@ -357,7 +358,7 @@ def get_range( Returns: (str, bytes) - path to the file with range content and content of this file as bytes """ - range_file_path = os.path.join(ASSETS_DIR, str(uuid.uuid4())) + test_file = TestFile(os.path.join(ASSETS_DIR, str(uuid.uuid4()))) cli = FrostfsCli(shell, FROSTFS_CLI_EXEC, wallet.config_path) cli.object.range( @@ -365,16 +366,16 @@ def get_range( cid=cid, oid=oid, range=range_cut, - file=range_file_path, + file=test_file, bearer=bearer, xhdr=xhdr, session=session, timeout=timeout, ) - with open(range_file_path, "rb") as file: + with open(test_file, "rb") as file: content = file.read() - return range_file_path, content + return test_file, content @reporter.step("Lock Object") diff --git a/src/frostfs_testlib/steps/http/http_gate.py b/src/frostfs_testlib/steps/http/http_gate.py index 3f4d838..f2b50d7 100644 --- a/src/frostfs_testlib/steps/http/http_gate.py +++ b/src/frostfs_testlib/steps/http/http_gate.py @@ -12,7 +12,7 @@ import requests from frostfs_testlib import reporter from frostfs_testlib.cli import GenericCli -from frostfs_testlib.resources.common import SIMPLE_OBJECT_SIZE +from frostfs_testlib.resources.common import ASSETS_DIR, SIMPLE_OBJECT_SIZE from frostfs_testlib.s3.aws_cli_client import command_options from frostfs_testlib.shell import Shell from frostfs_testlib.shell.local_shell import LocalShell @@ -20,11 +20,10 @@ from frostfs_testlib.steps.cli.object import get_object from frostfs_testlib.steps.storage_policy import get_nodes_without_object from frostfs_testlib.storage.cluster import ClusterNode, StorageNode from frostfs_testlib.testing.test_control import retry -from frostfs_testlib.utils.file_utils import get_file_hash +from frostfs_testlib.utils.file_utils import TestFile, get_file_hash logger = logging.getLogger("NeoLogger") -ASSETS_DIR = os.getenv("ASSETS_DIR", "TemporaryDir/") local_shell = LocalShell() @@ -50,9 +49,7 @@ def get_via_http_gate( else: request = f"{node.http_gate.get_endpoint()}{request_path}" - resp = requests.get( - request, headers={"Host": node.storage_node.get_http_hostname()[0]}, stream=True, timeout=timeout, verify=False - ) + resp = requests.get(request, headers={"Host": node.storage_node.get_http_hostname()[0]}, stream=True, timeout=timeout, verify=False) if not resp.ok: raise Exception( @@ -66,10 +63,10 @@ def get_via_http_gate( logger.info(f"Request: {request}") _attach_allure_step(request, resp.status_code) - file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}") - with open(file_path, "wb") as file: + test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}")) + with open(test_file, "wb") as file: shutil.copyfileobj(resp.raw, file) - return file_path + return test_file @reporter.step("Get via Zip HTTP Gate") @@ -95,11 +92,11 @@ def get_via_zip_http_gate(cid: str, prefix: str, node: ClusterNode, timeout: Opt logger.info(f"Request: {request}") _attach_allure_step(request, resp.status_code) - file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_archive.zip") - with open(file_path, "wb") as file: + test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_archive.zip")) + with open(test_file, "wb") as file: shutil.copyfileobj(resp.raw, file) - with zipfile.ZipFile(file_path, "r") as zip_ref: + with zipfile.ZipFile(test_file, "r") as zip_ref: zip_ref.extractall(ASSETS_DIR) return os.path.join(os.getcwd(), ASSETS_DIR, prefix) @@ -129,9 +126,7 @@ def get_via_http_gate_by_attribute( else: request = f"{node.http_gate.get_endpoint()}{request_path}" - resp = requests.get( - request, stream=True, timeout=timeout, verify=False, headers={"Host": node.storage_node.get_http_hostname()[0]} - ) + resp = requests.get(request, stream=True, timeout=timeout, verify=False, headers={"Host": node.storage_node.get_http_hostname()[0]}) if not resp.ok: raise Exception( @@ -145,17 +140,15 @@ def get_via_http_gate_by_attribute( logger.info(f"Request: {request}") _attach_allure_step(request, resp.status_code) - file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{str(uuid.uuid4())}") - with open(file_path, "wb") as file: + test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{str(uuid.uuid4())}")) + with open(test_file, "wb") as file: shutil.copyfileobj(resp.raw, file) - return file_path + return test_file # TODO: pass http_hostname as a header @reporter.step("Upload via HTTP Gate") -def upload_via_http_gate( - cid: str, path: str, endpoint: str, headers: Optional[dict] = None, timeout: Optional[int] = 300 -) -> str: +def upload_via_http_gate(cid: str, path: str, endpoint: str, headers: Optional[dict] = None, timeout: Optional[int] = 300) -> str: """ This function upload given object through HTTP gate cid: CID to get object from @@ -248,7 +241,7 @@ def upload_via_http_gate_curl( @retry(max_attempts=3, sleep_interval=1) @reporter.step("Get via HTTP Gate using Curl") -def get_via_http_curl(cid: str, oid: str, node: ClusterNode) -> str: +def get_via_http_curl(cid: str, oid: str, node: ClusterNode) -> TestFile: """ This function gets given object from HTTP gate using curl utility. cid: CID to get object from @@ -256,12 +249,12 @@ def get_via_http_curl(cid: str, oid: str, node: ClusterNode) -> str: node: node for request """ request = f"{node.http_gate.get_endpoint()}/get/{cid}/{oid}" - file_path = os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}") + test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, f"{cid}_{oid}_{str(uuid.uuid4())}")) curl = GenericCli("curl", node.host) - curl(f'-k -H "Host: {node.storage_node.get_http_hostname()[0]}"', f"{request} > {file_path}", shell=local_shell) + curl(f'-k -H "Host: {node.storage_node.get_http_hostname()[0]}"', f"{request} > {test_file}", shell=local_shell) - return file_path + return test_file def _attach_allure_step(request: str, status_code: int, req_type="GET"): diff --git a/src/frostfs_testlib/utils/file_utils.py b/src/frostfs_testlib/utils/file_utils.py index d238106..c144891 100644 --- a/src/frostfs_testlib/utils/file_utils.py +++ b/src/frostfs_testlib/utils/file_utils.py @@ -10,7 +10,35 @@ from frostfs_testlib.resources.common import ASSETS_DIR logger = logging.getLogger("NeoLogger") -def generate_file(size: int) -> str: +class TestFile(os.PathLike): + def __init__(self, path: str): + self.path = path + + def __del__(self): + logger.debug(f"Removing file {self.path}") + if os.path.exists(self.path): + os.remove(self.path) + + def __str__(self): + return self.path + + def __fspath__(self): + return self.path + + +def ensure_directory(path): + directory = os.path.dirname(path) + + if not os.path.exists(directory): + os.makedirs(directory) + + +def ensure_directory_opener(path, flags): + ensure_directory(path) + return os.open(path, flags) + + +def generate_file(size: int) -> TestFile: """Generates a binary file with the specified size in bytes. Args: @@ -19,19 +47,19 @@ def generate_file(size: int) -> str: Returns: The path to the generated file. """ - file_path = os.path.join(ASSETS_DIR, str(uuid.uuid4())) - with open(file_path, "wb") as file: + test_file = TestFile(os.path.join(ASSETS_DIR, str(uuid.uuid4()))) + with open(test_file, "wb", opener=ensure_directory_opener) as file: file.write(os.urandom(size)) - logger.info(f"File with size {size} bytes has been generated: {file_path}") + logger.info(f"File with size {size} bytes has been generated: {test_file}") - return file_path + return test_file def generate_file_with_content( size: int, - file_path: Optional[str] = None, + file_path: Optional[str | TestFile] = None, content: Optional[str] = None, -) -> str: +) -> TestFile: """Creates a new file with specified content. Args: @@ -48,20 +76,22 @@ def generate_file_with_content( content = os.urandom(size) mode = "wb" + test_file = None if not file_path: - file_path = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4())) + test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))) + elif isinstance(file_path, TestFile): + test_file = file_path else: - if not os.path.exists(os.path.dirname(file_path)): - os.makedirs(os.path.dirname(file_path)) + test_file = TestFile(file_path) - with open(file_path, mode) as file: + with open(test_file, mode, opener=ensure_directory_opener) as file: file.write(content) - return file_path + return test_file @reporter.step("Get File Hash") -def get_file_hash(file_path: str, len: Optional[int] = None, offset: Optional[int] = None) -> str: +def get_file_hash(file_path: str | TestFile, len: Optional[int] = None, offset: Optional[int] = None) -> str: """Generates hash for the specified file. Args: @@ -88,7 +118,7 @@ def get_file_hash(file_path: str, len: Optional[int] = None, offset: Optional[in @reporter.step("Concatenation set of files to one file") -def concat_files(file_paths: list, resulting_file_path: Optional[str] = None) -> str: +def concat_files(file_paths: list[str | TestFile], resulting_file_path: Optional[str | TestFile] = None) -> TestFile: """Concatenates several files into a single file. Args: @@ -98,16 +128,23 @@ def concat_files(file_paths: list, resulting_file_path: Optional[str] = None) -> Returns: Path to the resulting file. """ + + test_file = None if not resulting_file_path: - resulting_file_path = os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4())) - with open(resulting_file_path, "wb") as f: + test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4()))) + elif isinstance(resulting_file_path, TestFile): + test_file = resulting_file_path + else: + test_file = TestFile(resulting_file_path) + + with open(test_file, "wb", opener=ensure_directory_opener) as f: for file in file_paths: with open(file, "rb") as part_file: f.write(part_file.read()) - return resulting_file_path + return test_file -def split_file(file_path: str, parts: int) -> list[str]: +def split_file(file_path: str | TestFile, parts: int) -> list[TestFile]: """Splits specified file into several specified number of parts. Each part is saved under name `{original_file}_part_{i}`. @@ -129,7 +166,7 @@ def split_file(file_path: str, parts: int) -> list[str]: part_file_paths = [] for content_offset in range(0, content_size + 1, chunk_size): part_file_name = f"{file_path}_part_{part_id}" - part_file_paths.append(part_file_name) + part_file_paths.append(TestFile(part_file_name)) with open(part_file_name, "wb") as out_file: out_file.write(content[content_offset : content_offset + chunk_size]) part_id += 1 @@ -137,9 +174,7 @@ def split_file(file_path: str, parts: int) -> list[str]: return part_file_paths -def get_file_content( - file_path: str, content_len: Optional[int] = None, mode: str = "r", offset: Optional[int] = None -) -> Any: +def get_file_content(file_path: str | TestFile, content_len: Optional[int] = None, mode: str = "r", offset: Optional[int] = None) -> Any: """Returns content of specified file. Args: