[#301] Added interfaces for put/get lifecycle configuration to s3 clients
This commit is contained in:
parent
a04eba8aec
commit
2a41f2b0f6
8 changed files with 80 additions and 22 deletions
|
@ -27,8 +27,8 @@ dependencies = [
|
|||
"testrail-api>=1.12.0",
|
||||
"pytest==7.1.2",
|
||||
"tenacity==8.0.1",
|
||||
"boto3==1.16.33",
|
||||
"boto3-stubs[essential]==1.16.33",
|
||||
"boto3==1.35.30",
|
||||
"boto3-stubs[essential]==1.35.30",
|
||||
]
|
||||
requires-python = ">=3.10"
|
||||
|
||||
|
|
|
@ -8,8 +8,8 @@ docstring_parser==0.15
|
|||
testrail-api==1.12.0
|
||||
tenacity==8.0.1
|
||||
pytest==7.1.2
|
||||
boto3==1.16.33
|
||||
boto3-stubs[essential]==1.16.33
|
||||
boto3==1.35.30
|
||||
boto3-stubs[essential]==1.35.30
|
||||
|
||||
# Dev dependencies
|
||||
black==22.8.0
|
||||
|
|
|
@ -69,9 +69,7 @@ class FrostfsAdmMorph(CliCommand):
|
|||
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
|
||||
)
|
||||
|
||||
def set_config(
|
||||
self, set_key_value: str, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None
|
||||
) -> CommandResult:
|
||||
def set_config(self, set_key_value: str, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None) -> CommandResult:
|
||||
"""Add/update global config value in the FrostFS network.
|
||||
|
||||
Args:
|
||||
|
@ -125,7 +123,7 @@ class FrostfsAdmMorph(CliCommand):
|
|||
)
|
||||
|
||||
def force_new_epoch(
|
||||
self, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None
|
||||
self, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None, delta: Optional[int] = None
|
||||
) -> CommandResult:
|
||||
"""Create new FrostFS epoch event in the side chain.
|
||||
|
||||
|
@ -344,11 +342,7 @@ class FrostfsAdmMorph(CliCommand):
|
|||
|
||||
return self._execute(
|
||||
f"morph remove-nodes {' '.join(node_netmap_keys)}",
|
||||
**{
|
||||
param: param_value
|
||||
for param, param_value in locals().items()
|
||||
if param not in ["self", "node_netmap_keys"]
|
||||
},
|
||||
**{param: param_value for param, param_value in locals().items() if param not in ["self", "node_netmap_keys"]},
|
||||
)
|
||||
|
||||
def add_rule(
|
||||
|
|
|
@ -754,6 +754,36 @@ class AwsCliClient(S3ClientWrapper):
|
|||
response = self._to_json(output)
|
||||
return response.get("ObjectLockConfiguration")
|
||||
|
||||
@reporter.step("Put bucket lifecycle configuration")
|
||||
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
|
||||
cmd = (
|
||||
f"aws {self.common_flags} s3api put-bucket-lifecycle-configuration --bucket {bucket} "
|
||||
f"--endpoint-url {self.s3gate_endpoint} --lifecycle-configuration file://{dumped_configuration} --profile {self.profile}"
|
||||
)
|
||||
output = self.local_shell.exec(cmd).stdout
|
||||
response = self._to_json(output)
|
||||
return response
|
||||
|
||||
@reporter.step("Get bucket lifecycle configuration")
|
||||
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
|
||||
cmd = (
|
||||
f"aws {self.common_flags} s3api get-bucket-lifecycle-configuration --bucket {bucket} "
|
||||
f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
|
||||
)
|
||||
output = self.local_shell.exec(cmd).stdout
|
||||
response = self._to_json(output)
|
||||
return response
|
||||
|
||||
@reporter.step("Delete bucket lifecycle configuration")
|
||||
def delete_bucket_lifecycle(self, bucket: str) -> dict:
|
||||
cmd = (
|
||||
f"aws {self.common_flags} s3api delete-bucket-lifecycle --bucket {bucket} "
|
||||
f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
|
||||
)
|
||||
output = self.local_shell.exec(cmd).stdout
|
||||
response = self._to_json(output)
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def _to_json(output: str) -> dict:
|
||||
json_output = {}
|
||||
|
|
|
@ -296,6 +296,27 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
|||
response = self.boto3_client.delete_bucket_cors(Bucket=bucket)
|
||||
log_command_execution(self.s3gate_endpoint, "S3 delete_bucket_cors result", response, {"Bucket": bucket})
|
||||
|
||||
@reporter.step("Put bucket lifecycle configuration")
|
||||
@report_error
|
||||
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
|
||||
response = self.boto3_client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle_configuration)
|
||||
log_command_execution(self.s3gate_endpoint, "S3 put_bucket_lifecycle_configuration result", response, {"Bucket": bucket})
|
||||
return response
|
||||
|
||||
@reporter.step("Get bucket lifecycle configuration")
|
||||
@report_error
|
||||
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
|
||||
response = self.boto3_client.get_bucket_lifecycle_configuration(Bucket=bucket)
|
||||
log_command_execution(self.s3gate_endpoint, "S3 get_bucket_lifecycle_configuration result", response, {"Bucket": bucket})
|
||||
return {"Rules": response.get("Rules")}
|
||||
|
||||
@reporter.step("Delete bucket lifecycle configuration")
|
||||
@report_error
|
||||
def delete_bucket_lifecycle(self, bucket: str) -> dict:
|
||||
response = self.boto3_client.delete_bucket_lifecycle(Bucket=bucket)
|
||||
log_command_execution(self.s3gate_endpoint, "S3 delete_bucket_lifecycle result", response, {"Bucket": bucket})
|
||||
return response
|
||||
|
||||
# END OF BUCKET METHODS #
|
||||
# OBJECT METHODS #
|
||||
|
||||
|
|
|
@ -366,6 +366,18 @@ class S3ClientWrapper(HumanReadableABC):
|
|||
def delete_object_tagging(self, bucket: str, key: str) -> None:
|
||||
"""Removes the entire tag set from the specified object."""
|
||||
|
||||
@abstractmethod
|
||||
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
|
||||
"""Adds or updates bucket lifecycle configuration"""
|
||||
|
||||
@abstractmethod
|
||||
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
|
||||
"""Gets bucket lifecycle configuration"""
|
||||
|
||||
@abstractmethod
|
||||
def delete_bucket_lifecycle(self, bucket: str) -> dict:
|
||||
"""Deletes bucket lifecycle"""
|
||||
|
||||
@abstractmethod
|
||||
def get_object_attributes(
|
||||
self,
|
||||
|
|
|
@ -69,7 +69,7 @@ def get_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
|||
|
||||
|
||||
@reporter.step("Tick Epoch")
|
||||
def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None):
|
||||
def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None, delta: Optional[int] = None):
|
||||
"""
|
||||
Tick epoch using frostfs-adm or NeoGo if frostfs-adm is not available (DevEnv)
|
||||
Args:
|
||||
|
@ -88,12 +88,17 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
|||
frostfs_adm_exec_path=FROSTFS_ADM_EXEC,
|
||||
config_file=FROSTFS_ADM_CONFIG_PATH,
|
||||
)
|
||||
frostfs_adm.morph.force_new_epoch()
|
||||
frostfs_adm.morph.force_new_epoch(delta=delta)
|
||||
return
|
||||
|
||||
# Otherwise we tick epoch using transaction
|
||||
cur_epoch = get_epoch(shell, cluster)
|
||||
|
||||
if delta:
|
||||
next_epoch = cur_epoch + delta
|
||||
else:
|
||||
next_epoch = cur_epoch + 1
|
||||
|
||||
# Use first node by default
|
||||
ir_node = cluster.services(InnerRing)[0]
|
||||
# In case if no local_wallet_path is provided, we use wallet_path
|
||||
|
@ -110,7 +115,7 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
|||
wallet_password=ir_wallet_pass,
|
||||
scripthash=get_contract_hash(morph_chain, "netmap.frostfs", shell=shell),
|
||||
method="newEpoch",
|
||||
arguments=f"int:{cur_epoch + 1}",
|
||||
arguments=f"int:{next_epoch}",
|
||||
multisig_hash=f"{ir_address}:Global",
|
||||
address=ir_address,
|
||||
rpc_endpoint=morph_endpoint,
|
||||
|
|
|
@ -25,12 +25,8 @@ class ClusterTestBase:
|
|||
for _ in range(epochs_to_tick):
|
||||
self.tick_epoch(alive_node, wait_block)
|
||||
|
||||
def tick_epoch(
|
||||
self,
|
||||
alive_node: Optional[StorageNode] = None,
|
||||
wait_block: int = None,
|
||||
):
|
||||
epoch.tick_epoch(self.shell, self.cluster, alive_node=alive_node)
|
||||
def tick_epoch(self, alive_node: Optional[StorageNode] = None, wait_block: int = None, delta: Optional[int] = None):
|
||||
epoch.tick_epoch(self.shell, self.cluster, alive_node=alive_node, delta=delta)
|
||||
if wait_block:
|
||||
self.wait_for_blocks(wait_block)
|
||||
|
||||
|
|
Loading…
Reference in a new issue