[#301] Added interfaces for put/get lifecycle configuration to s3 clients

This commit is contained in:
Yaroslava Lukoyanova 2024-09-23 17:54:40 +03:00 committed by Ekaterina Chernitsyna
parent 3219dd6389
commit 27359e9266
6 changed files with 76 additions and 18 deletions

View file

@ -69,9 +69,7 @@ class FrostfsAdmMorph(CliCommand):
**{param: param_value for param, param_value in locals().items() if param not in ["self"]}, **{param: param_value for param, param_value in locals().items() if param not in ["self"]},
) )
def set_config( def set_config(self, set_key_value: str, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None) -> CommandResult:
self, set_key_value: str, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None
) -> CommandResult:
"""Add/update global config value in the FrostFS network. """Add/update global config value in the FrostFS network.
Args: Args:
@ -125,7 +123,7 @@ class FrostfsAdmMorph(CliCommand):
) )
def force_new_epoch( def force_new_epoch(
self, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None self, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None, delta: Optional[int] = None
) -> CommandResult: ) -> CommandResult:
"""Create new FrostFS epoch event in the side chain. """Create new FrostFS epoch event in the side chain.
@ -344,11 +342,7 @@ class FrostfsAdmMorph(CliCommand):
return self._execute( return self._execute(
f"morph remove-nodes {' '.join(node_netmap_keys)}", f"morph remove-nodes {' '.join(node_netmap_keys)}",
**{ **{param: param_value for param, param_value in locals().items() if param not in ["self", "node_netmap_keys"]},
param: param_value
for param, param_value in locals().items()
if param not in ["self", "node_netmap_keys"]
},
) )
def add_rule( def add_rule(

View file

@ -754,6 +754,36 @@ class AwsCliClient(S3ClientWrapper):
response = self._to_json(output) response = self._to_json(output)
return response.get("ObjectLockConfiguration") return response.get("ObjectLockConfiguration")
@reporter.step("Put bucket lifecycle configuration")
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
cmd = (
f"aws {self.common_flags} s3api put-bucket-lifecycle-configuration --bucket {bucket} "
f"--endpoint-url {self.s3gate_endpoint} --lifecycle-configuration file://{dumped_configuration} --profile {self.profile}"
)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response
@reporter.step("Get bucket lifecycle configuration")
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
cmd = (
f"aws {self.common_flags} s3api get-bucket-lifecycle-configuration --bucket {bucket} "
f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response
@reporter.step("Delete bucket lifecycle configuration")
def delete_bucket_lifecycle(self, bucket: str) -> dict:
cmd = (
f"aws {self.common_flags} s3api delete-bucket-lifecycle --bucket {bucket} "
f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response
@staticmethod @staticmethod
def _to_json(output: str) -> dict: def _to_json(output: str) -> dict:
json_output = {} json_output = {}

View file

@ -297,6 +297,27 @@ class Boto3ClientWrapper(S3ClientWrapper):
response = self.boto3_client.delete_bucket_cors(Bucket=bucket) response = self.boto3_client.delete_bucket_cors(Bucket=bucket)
log_command_execution(self.s3gate_endpoint, "S3 delete_bucket_cors result", response, {"Bucket": bucket}) log_command_execution(self.s3gate_endpoint, "S3 delete_bucket_cors result", response, {"Bucket": bucket})
@reporter.step("Put bucket lifecycle configuration")
@report_error
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
response = self.boto3_client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle_configuration)
log_command_execution(self.s3gate_endpoint, "S3 put_bucket_lifecycle_configuration result", response, {"Bucket": bucket})
return response
@reporter.step("Get bucket lifecycle configuration")
@report_error
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
response = self.boto3_client.get_bucket_lifecycle_configuration(Bucket=bucket)
log_command_execution(self.s3gate_endpoint, "S3 get_bucket_lifecycle_configuration result", response, {"Bucket": bucket})
return {"Rules": response.get("Rules")}
@reporter.step("Delete bucket lifecycle configuration")
@report_error
def delete_bucket_lifecycle(self, bucket: str) -> dict:
response = self.boto3_client.delete_bucket_lifecycle(Bucket=bucket)
log_command_execution(self.s3gate_endpoint, "S3 delete_bucket_lifecycle result", response, {"Bucket": bucket})
return response
# END OF BUCKET METHODS # # END OF BUCKET METHODS #
# OBJECT METHODS # # OBJECT METHODS #

View file

@ -370,6 +370,18 @@ class S3ClientWrapper(HumanReadableABC):
def delete_object_tagging(self, bucket: str, key: str) -> None: def delete_object_tagging(self, bucket: str, key: str) -> None:
"""Removes the entire tag set from the specified object.""" """Removes the entire tag set from the specified object."""
@abstractmethod
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
"""Adds or updates bucket lifecycle configuration"""
@abstractmethod
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
"""Gets bucket lifecycle configuration"""
@abstractmethod
def delete_bucket_lifecycle(self, bucket: str) -> dict:
"""Deletes bucket lifecycle"""
@abstractmethod @abstractmethod
def get_object_attributes( def get_object_attributes(
self, self,

View file

@ -69,7 +69,7 @@ def get_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
@reporter.step("Tick Epoch") @reporter.step("Tick Epoch")
def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None): def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None, delta: Optional[int] = None):
""" """
Tick epoch using frostfs-adm or NeoGo if frostfs-adm is not available (DevEnv) Tick epoch using frostfs-adm or NeoGo if frostfs-adm is not available (DevEnv)
Args: Args:
@ -88,12 +88,17 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
frostfs_adm_exec_path=FROSTFS_ADM_EXEC, frostfs_adm_exec_path=FROSTFS_ADM_EXEC,
config_file=FROSTFS_ADM_CONFIG_PATH, config_file=FROSTFS_ADM_CONFIG_PATH,
) )
frostfs_adm.morph.force_new_epoch() frostfs_adm.morph.force_new_epoch(delta=delta)
return return
# Otherwise we tick epoch using transaction # Otherwise we tick epoch using transaction
cur_epoch = get_epoch(shell, cluster) cur_epoch = get_epoch(shell, cluster)
if delta:
next_epoch = cur_epoch + delta
else:
next_epoch = cur_epoch + 1
# Use first node by default # Use first node by default
ir_node = cluster.services(InnerRing)[0] ir_node = cluster.services(InnerRing)[0]
# In case if no local_wallet_path is provided, we use wallet_path # In case if no local_wallet_path is provided, we use wallet_path
@ -110,7 +115,7 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
wallet_password=ir_wallet_pass, wallet_password=ir_wallet_pass,
scripthash=get_contract_hash(morph_chain, "netmap.frostfs", shell=shell), scripthash=get_contract_hash(morph_chain, "netmap.frostfs", shell=shell),
method="newEpoch", method="newEpoch",
arguments=f"int:{cur_epoch + 1}", arguments=f"int:{next_epoch}",
multisig_hash=f"{ir_address}:Global", multisig_hash=f"{ir_address}:Global",
address=ir_address, address=ir_address,
rpc_endpoint=morph_endpoint, rpc_endpoint=morph_endpoint,

View file

@ -25,12 +25,8 @@ class ClusterTestBase:
for _ in range(epochs_to_tick): for _ in range(epochs_to_tick):
self.tick_epoch(alive_node, wait_block) self.tick_epoch(alive_node, wait_block)
def tick_epoch( def tick_epoch(self, alive_node: Optional[StorageNode] = None, wait_block: int = None, delta: Optional[int] = None):
self, epoch.tick_epoch(self.shell, self.cluster, alive_node=alive_node, delta=delta)
alive_node: Optional[StorageNode] = None,
wait_block: int = None,
):
epoch.tick_epoch(self.shell, self.cluster, alive_node=alive_node)
if wait_block: if wait_block:
self.wait_for_blocks(wait_block) self.wait_for_blocks(wait_block)