realloc/frostfs-testlib
1
0
Fork 0
forked from TrueCloudLab/frostfs-testlib
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#253] Update S3 clients and permission matrixes

Browse source 
Signed-off-by: a.berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2024-06-28 15:18:20 +03:00
parent c9e4c2c7bb
commit 3a4204f2e4
5 changed files with 33 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/frostfs_testlib/s3/aws_cli_client.py
View file

@ -1,7 +1,6 @@
import json
import logging
import os
import uuid
from datetime import datetime
from time import sleep
from typing import Literal, Optional, Union
@ -11,6 +10,7 @@ from frostfs_testlib.resources.common import ASSETS_DIR, MAX_REQUEST_ATTEMPTS, R
from frostfs_testlib.s3.interfaces import S3ClientWrapper, VersioningStatus, _make_objs_dict
from frostfs_testlib.shell import CommandOptions
from frostfs_testlib.shell.local_shell import LocalShell
from frostfs_testlib.utils import string_utils
# TODO: Refactor this code to use shell instead of _cmd_run
from frostfs_testlib.utils.cli_utils import _configure_aws_cli
@ -68,7 +68,7 @@ class AwsCliClient(S3ClientWrapper):
location_constraint: Optional[str] = None,
) -> str:
if bucket is None:
bucket = str(uuid.uuid4())
bucket = string_utils.unique_name("bucket-")
if object_lock_enabled_for_bucket is None:
object_lock = ""
@ -229,7 +229,7 @@ class AwsCliClient(S3ClientWrapper):
if bucket is None:
bucket = source_bucket
if key is None:
key = os.path.join(os.getcwd(), str(uuid.uuid4()))
key = string_utils.unique_name("copy-object-")
copy_source = f"{source_bucket}/{source_key}"
cmd = (
@ -315,7 +315,7 @@ class AwsCliClient(S3ClientWrapper):
object_range: Optional[tuple[int, int]] = None,
full_output: bool = False,
) -> dict | TestFile:
test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4())))
test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, string_utils.unique_name("dl-object-")))
version = f" --version-id {version_id}" if version_id else ""
cmd = (
f"aws {self.common_flags} s3api get-object --bucket {bucket} --key {key} "

8
src/frostfs_testlib/s3/boto3_client.py
View file

@ -1,7 +1,6 @@
import json
import logging
import os
import uuid
from datetime import datetime
from functools import wraps
from time import sleep
@ -16,6 +15,7 @@ from mypy_boto3_s3 import S3Client
from frostfs_testlib import reporter
from frostfs_testlib.resources.common import ASSETS_DIR, MAX_REQUEST_ATTEMPTS, RETRY_MODE, S3_SYNC_WAIT_TIME
from frostfs_testlib.s3.interfaces import S3ClientWrapper, VersioningStatus, _make_objs_dict
from frostfs_testlib.utils import string_utils
# TODO: Refactor this code to use shell instead of _cmd_run
from frostfs_testlib.utils.cli_utils import _configure_aws_cli, log_command_execution
@ -115,7 +115,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
location_constraint: Optional[str] = None,
) -> str:
if bucket is None:
bucket = str(uuid.uuid4())
bucket = string_utils.unique_name("bucket-")
params = {"Bucket": bucket}
if object_lock_enabled_for_bucket is not None:
@ -439,7 +439,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
if bucket is None:
bucket = source_bucket
if key is None:
key = os.path.join(os.getcwd(), str(uuid.uuid4()))
key = string_utils.unique_name("copy-object-")
copy_source = f"{source_bucket}/{source_key}"
params = {
@ -476,7 +476,7 @@ class Boto3ClientWrapper(S3ClientWrapper):
if full_output:
return response
test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, str(uuid.uuid4())))
test_file = TestFile(os.path.join(os.getcwd(), ASSETS_DIR, string_utils.unique_name("dl-object-")))
with open(test_file, "wb") as file:
chunk = response["Body"].read(1024)
while chunk: