[#354] Support of presigned url methods for S3
All checks were successful
DCO action / DCO (pull_request) Successful in 27s

Signed-off-by: Yaroslava Lukoyanova <y.lukoyanova@yadro.com>
This commit is contained in:
Yaroslava Lukoyanova 2025-02-03 12:44:21 +03:00
parent b44705eb2f
commit f27a73d4d3
4 changed files with 29 additions and 1 deletions

View file

@ -957,6 +957,15 @@ class AwsCliClient(S3ClientWrapper):
return json_output return json_output
@reporter.step("Create presign url for the object")
def create_presign_url(self, method: str, bucket: str, key: str, expires_in: Optional[int] = 3600):
# AWS CLI does not support method definition and world only in 'get_object' state by default
cmd = f"aws {self.common_flags} s3 presign s3://{bucket}/{key} " f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
if expires_in:
cmd += f" --expires-in {expires_in}"
response = self.local_shell.exec(cmd).stdout
return response.strip()
# IAM METHODS # # IAM METHODS #
# Some methods don't have checks because AWS is silent in some cases (delete, attach, etc.) # Some methods don't have checks because AWS is silent in some cases (delete, attach, etc.)

View file

@ -50,10 +50,11 @@ class Boto3ClientWrapper(S3ClientWrapper):
self.session = boto3.Session() self.session = boto3.Session()
self.config = Config( self.config = Config(
signature_version="s3v4",
retries={ retries={
"max_attempts": MAX_REQUEST_ATTEMPTS, "max_attempts": MAX_REQUEST_ATTEMPTS,
"mode": RETRY_MODE, "mode": RETRY_MODE,
} },
) )
self.set_endpoint(s3gate_endpoint) self.set_endpoint(s3gate_endpoint)
@ -816,6 +817,16 @@ class Boto3ClientWrapper(S3ClientWrapper):
) -> dict: ) -> dict:
raise NotImplementedError("Cp is not supported for boto3 client") raise NotImplementedError("Cp is not supported for boto3 client")
@reporter.step("Create presign url for the object")
def create_presign_url(self, method: str, bucket: str, key: str, expires_in: Optional[int] = 3600):
response = self._exec_request(
method=self.boto3_client.generate_presigned_url,
params={"ClientMethod": method, "Params": {"Bucket": bucket, "Key": key}, "ExpiresIn": expires_in},
endpoint=self.s3gate_endpoint,
profile=self.profile,
)
return response
# END OBJECT METHODS # # END OBJECT METHODS #
# IAM METHODS # # IAM METHODS #

View file

@ -417,6 +417,10 @@ class S3ClientWrapper(HumanReadableABC):
) -> dict: ) -> dict:
"""cp directory TODO: Add proper description""" """cp directory TODO: Add proper description"""
@abstractmethod
def create_presign_url(self, method: str, bucket: str, key: str, expires_in: Optional[int] = 3600):
"""Creates presign URL"""
# END OF OBJECT METHODS # # END OF OBJECT METHODS #
# IAM METHODS # # IAM METHODS #

View file

@ -33,6 +33,7 @@ def get_via_http_gate(
oid: str, oid: str,
node: ClusterNode, node: ClusterNode,
request_path: Optional[str] = None, request_path: Optional[str] = None,
presigned_url: Optional[str] = None,
timeout: Optional[int] = 300, timeout: Optional[int] = 300,
): ):
""" """
@ -47,6 +48,9 @@ def get_via_http_gate(
if request_path: if request_path:
request = f"{node.http_gate.get_endpoint()}{request_path}" request = f"{node.http_gate.get_endpoint()}{request_path}"
if presigned_url:
request = presigned_url
response = requests.get(request, stream=True, timeout=timeout, verify=False) response = requests.get(request, stream=True, timeout=timeout, verify=False)
if not response.ok: if not response.ok: