[#354] Support of presigned url methods for S3
All checks were successful
DCO action / DCO (pull_request) Successful in 27s
All checks were successful
DCO action / DCO (pull_request) Successful in 27s
Signed-off-by: Yaroslava Lukoyanova <y.lukoyanova@yadro.com>
This commit is contained in:
parent
b44705eb2f
commit
f27a73d4d3
4 changed files with 29 additions and 1 deletions
|
@ -957,6 +957,15 @@ class AwsCliClient(S3ClientWrapper):
|
||||||
|
|
||||||
return json_output
|
return json_output
|
||||||
|
|
||||||
|
@reporter.step("Create presign url for the object")
|
||||||
|
def create_presign_url(self, method: str, bucket: str, key: str, expires_in: Optional[int] = 3600):
|
||||||
|
# AWS CLI does not support method definition and world only in 'get_object' state by default
|
||||||
|
cmd = f"aws {self.common_flags} s3 presign s3://{bucket}/{key} " f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
|
||||||
|
if expires_in:
|
||||||
|
cmd += f" --expires-in {expires_in}"
|
||||||
|
response = self.local_shell.exec(cmd).stdout
|
||||||
|
return response.strip()
|
||||||
|
|
||||||
# IAM METHODS #
|
# IAM METHODS #
|
||||||
# Some methods don't have checks because AWS is silent in some cases (delete, attach, etc.)
|
# Some methods don't have checks because AWS is silent in some cases (delete, attach, etc.)
|
||||||
|
|
||||||
|
|
|
@ -50,10 +50,11 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
|
|
||||||
self.session = boto3.Session()
|
self.session = boto3.Session()
|
||||||
self.config = Config(
|
self.config = Config(
|
||||||
|
signature_version="s3v4",
|
||||||
retries={
|
retries={
|
||||||
"max_attempts": MAX_REQUEST_ATTEMPTS,
|
"max_attempts": MAX_REQUEST_ATTEMPTS,
|
||||||
"mode": RETRY_MODE,
|
"mode": RETRY_MODE,
|
||||||
}
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
self.set_endpoint(s3gate_endpoint)
|
self.set_endpoint(s3gate_endpoint)
|
||||||
|
@ -816,6 +817,16 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
) -> dict:
|
) -> dict:
|
||||||
raise NotImplementedError("Cp is not supported for boto3 client")
|
raise NotImplementedError("Cp is not supported for boto3 client")
|
||||||
|
|
||||||
|
@reporter.step("Create presign url for the object")
|
||||||
|
def create_presign_url(self, method: str, bucket: str, key: str, expires_in: Optional[int] = 3600):
|
||||||
|
response = self._exec_request(
|
||||||
|
method=self.boto3_client.generate_presigned_url,
|
||||||
|
params={"ClientMethod": method, "Params": {"Bucket": bucket, "Key": key}, "ExpiresIn": expires_in},
|
||||||
|
endpoint=self.s3gate_endpoint,
|
||||||
|
profile=self.profile,
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
# END OBJECT METHODS #
|
# END OBJECT METHODS #
|
||||||
|
|
||||||
# IAM METHODS #
|
# IAM METHODS #
|
||||||
|
|
|
@ -417,6 +417,10 @@ class S3ClientWrapper(HumanReadableABC):
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""cp directory TODO: Add proper description"""
|
"""cp directory TODO: Add proper description"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def create_presign_url(self, method: str, bucket: str, key: str, expires_in: Optional[int] = 3600):
|
||||||
|
"""Creates presign URL"""
|
||||||
|
|
||||||
# END OF OBJECT METHODS #
|
# END OF OBJECT METHODS #
|
||||||
|
|
||||||
# IAM METHODS #
|
# IAM METHODS #
|
||||||
|
|
|
@ -33,6 +33,7 @@ def get_via_http_gate(
|
||||||
oid: str,
|
oid: str,
|
||||||
node: ClusterNode,
|
node: ClusterNode,
|
||||||
request_path: Optional[str] = None,
|
request_path: Optional[str] = None,
|
||||||
|
presigned_url: Optional[str] = None,
|
||||||
timeout: Optional[int] = 300,
|
timeout: Optional[int] = 300,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
|
@ -47,6 +48,9 @@ def get_via_http_gate(
|
||||||
if request_path:
|
if request_path:
|
||||||
request = f"{node.http_gate.get_endpoint()}{request_path}"
|
request = f"{node.http_gate.get_endpoint()}{request_path}"
|
||||||
|
|
||||||
|
if presigned_url:
|
||||||
|
request = presigned_url
|
||||||
|
|
||||||
response = requests.get(request, stream=True, timeout=timeout, verify=False)
|
response = requests.get(request, stream=True, timeout=timeout, verify=False)
|
||||||
|
|
||||||
if not response.ok:
|
if not response.ok:
|
||||||
|
|
Loading…
Add table
Reference in a new issue