[#301] Added interfaces for put/get lifecycle configuration to s3 clients

This commit is contained in:
Yaroslava Lukoyanova 2024-09-23 17:54:40 +03:00 committed by ylukoyan
parent a04eba8aec
commit 2a41f2b0f6
8 changed files with 80 additions and 22 deletions

View file

@ -27,8 +27,8 @@ dependencies = [
"testrail-api>=1.12.0",
"pytest==7.1.2",
"tenacity==8.0.1",
"boto3==1.16.33",
"boto3-stubs[essential]==1.16.33",
"boto3==1.35.30",
"boto3-stubs[essential]==1.35.30",
]
requires-python = ">=3.10"

View file

@ -8,8 +8,8 @@ docstring_parser==0.15
testrail-api==1.12.0
tenacity==8.0.1
pytest==7.1.2
boto3==1.16.33
boto3-stubs[essential]==1.16.33
boto3==1.35.30
boto3-stubs[essential]==1.35.30
# Dev dependencies
black==22.8.0

View file

@ -69,9 +69,7 @@ class FrostfsAdmMorph(CliCommand):
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
)
def set_config(
self, set_key_value: str, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None
) -> CommandResult:
def set_config(self, set_key_value: str, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None) -> CommandResult:
"""Add/update global config value in the FrostFS network.
Args:
@ -125,7 +123,7 @@ class FrostfsAdmMorph(CliCommand):
)
def force_new_epoch(
self, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None
self, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None, delta: Optional[int] = None
) -> CommandResult:
"""Create new FrostFS epoch event in the side chain.
@ -344,11 +342,7 @@ class FrostfsAdmMorph(CliCommand):
return self._execute(
f"morph remove-nodes {' '.join(node_netmap_keys)}",
**{
param: param_value
for param, param_value in locals().items()
if param not in ["self", "node_netmap_keys"]
},
**{param: param_value for param, param_value in locals().items() if param not in ["self", "node_netmap_keys"]},
)
def add_rule(

View file

@ -754,6 +754,36 @@ class AwsCliClient(S3ClientWrapper):
response = self._to_json(output)
return response.get("ObjectLockConfiguration")
@reporter.step("Put bucket lifecycle configuration")
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
cmd = (
f"aws {self.common_flags} s3api put-bucket-lifecycle-configuration --bucket {bucket} "
f"--endpoint-url {self.s3gate_endpoint} --lifecycle-configuration file://{dumped_configuration} --profile {self.profile}"
)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response
@reporter.step("Get bucket lifecycle configuration")
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
cmd = (
f"aws {self.common_flags} s3api get-bucket-lifecycle-configuration --bucket {bucket} "
f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response
@reporter.step("Delete bucket lifecycle configuration")
def delete_bucket_lifecycle(self, bucket: str) -> dict:
cmd = (
f"aws {self.common_flags} s3api delete-bucket-lifecycle --bucket {bucket} "
f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
)
output = self.local_shell.exec(cmd).stdout
response = self._to_json(output)
return response
@staticmethod
def _to_json(output: str) -> dict:
json_output = {}

View file

@ -296,6 +296,27 @@ class Boto3ClientWrapper(S3ClientWrapper):
response = self.boto3_client.delete_bucket_cors(Bucket=bucket)
log_command_execution(self.s3gate_endpoint, "S3 delete_bucket_cors result", response, {"Bucket": bucket})
@reporter.step("Put bucket lifecycle configuration")
@report_error
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
response = self.boto3_client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle_configuration)
log_command_execution(self.s3gate_endpoint, "S3 put_bucket_lifecycle_configuration result", response, {"Bucket": bucket})
return response
@reporter.step("Get bucket lifecycle configuration")
@report_error
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
response = self.boto3_client.get_bucket_lifecycle_configuration(Bucket=bucket)
log_command_execution(self.s3gate_endpoint, "S3 get_bucket_lifecycle_configuration result", response, {"Bucket": bucket})
return {"Rules": response.get("Rules")}
@reporter.step("Delete bucket lifecycle configuration")
@report_error
def delete_bucket_lifecycle(self, bucket: str) -> dict:
response = self.boto3_client.delete_bucket_lifecycle(Bucket=bucket)
log_command_execution(self.s3gate_endpoint, "S3 delete_bucket_lifecycle result", response, {"Bucket": bucket})
return response
# END OF BUCKET METHODS #
# OBJECT METHODS #

View file

@ -366,6 +366,18 @@ class S3ClientWrapper(HumanReadableABC):
def delete_object_tagging(self, bucket: str, key: str) -> None:
"""Removes the entire tag set from the specified object."""
@abstractmethod
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
"""Adds or updates bucket lifecycle configuration"""
@abstractmethod
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
"""Gets bucket lifecycle configuration"""
@abstractmethod
def delete_bucket_lifecycle(self, bucket: str) -> dict:
"""Deletes bucket lifecycle"""
@abstractmethod
def get_object_attributes(
self,

View file

@ -69,7 +69,7 @@ def get_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
@reporter.step("Tick Epoch")
def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None):
def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None, delta: Optional[int] = None):
"""
Tick epoch using frostfs-adm or NeoGo if frostfs-adm is not available (DevEnv)
Args:
@ -88,12 +88,17 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
frostfs_adm_exec_path=FROSTFS_ADM_EXEC,
config_file=FROSTFS_ADM_CONFIG_PATH,
)
frostfs_adm.morph.force_new_epoch()
frostfs_adm.morph.force_new_epoch(delta=delta)
return
# Otherwise we tick epoch using transaction
cur_epoch = get_epoch(shell, cluster)
if delta:
next_epoch = cur_epoch + delta
else:
next_epoch = cur_epoch + 1
# Use first node by default
ir_node = cluster.services(InnerRing)[0]
# In case if no local_wallet_path is provided, we use wallet_path
@ -110,7 +115,7 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
wallet_password=ir_wallet_pass,
scripthash=get_contract_hash(morph_chain, "netmap.frostfs", shell=shell),
method="newEpoch",
arguments=f"int:{cur_epoch + 1}",
arguments=f"int:{next_epoch}",
multisig_hash=f"{ir_address}:Global",
address=ir_address,
rpc_endpoint=morph_endpoint,

View file

@ -25,12 +25,8 @@ class ClusterTestBase:
for _ in range(epochs_to_tick):
self.tick_epoch(alive_node, wait_block)
def tick_epoch(
self,
alive_node: Optional[StorageNode] = None,
wait_block: int = None,
):
epoch.tick_epoch(self.shell, self.cluster, alive_node=alive_node)
def tick_epoch(self, alive_node: Optional[StorageNode] = None, wait_block: int = None, delta: Optional[int] = None):
epoch.tick_epoch(self.shell, self.cluster, alive_node=alive_node, delta=delta)
if wait_block:
self.wait_for_blocks(wait_block)