[#301] Added interfaces for put/get lifecycle configuration to s3 clients #301
8 changed files with 80 additions and 22 deletions
|
@ -27,8 +27,8 @@ dependencies = [
|
||||||
"testrail-api>=1.12.0",
|
"testrail-api>=1.12.0",
|
||||||
"pytest==7.1.2",
|
"pytest==7.1.2",
|
||||||
"tenacity==8.0.1",
|
"tenacity==8.0.1",
|
||||||
"boto3==1.16.33",
|
"boto3==1.35.30",
|
||||||
"boto3-stubs[essential]==1.16.33",
|
"boto3-stubs[essential]==1.35.30",
|
||||||
]
|
]
|
||||||
requires-python = ">=3.10"
|
requires-python = ">=3.10"
|
||||||
|
|
||||||
|
|
|
@ -8,8 +8,8 @@ docstring_parser==0.15
|
||||||
testrail-api==1.12.0
|
testrail-api==1.12.0
|
||||||
tenacity==8.0.1
|
tenacity==8.0.1
|
||||||
pytest==7.1.2
|
pytest==7.1.2
|
||||||
boto3==1.16.33
|
boto3==1.35.30
|
||||||
boto3-stubs[essential]==1.16.33
|
boto3-stubs[essential]==1.35.30
|
||||||
|
|
||||||
# Dev dependencies
|
# Dev dependencies
|
||||||
black==22.8.0
|
black==22.8.0
|
||||||
|
|
|
@ -69,9 +69,7 @@ class FrostfsAdmMorph(CliCommand):
|
||||||
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
|
**{param: param_value for param, param_value in locals().items() if param not in ["self"]},
|
||||||
)
|
)
|
||||||
|
|
||||||
def set_config(
|
def set_config(self, set_key_value: str, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None) -> CommandResult:
|
||||||
self, set_key_value: str, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None
|
|
||||||
) -> CommandResult:
|
|
||||||
"""Add/update global config value in the FrostFS network.
|
"""Add/update global config value in the FrostFS network.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
@ -125,7 +123,7 @@ class FrostfsAdmMorph(CliCommand):
|
||||||
)
|
)
|
||||||
|
|
||||||
def force_new_epoch(
|
def force_new_epoch(
|
||||||
self, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None
|
self, rpc_endpoint: Optional[str] = None, alphabet_wallets: Optional[str] = None, delta: Optional[int] = None
|
||||||
) -> CommandResult:
|
) -> CommandResult:
|
||||||
"""Create new FrostFS epoch event in the side chain.
|
"""Create new FrostFS epoch event in the side chain.
|
||||||
|
|
||||||
|
@ -344,9 +342,5 @@ class FrostfsAdmMorph(CliCommand):
|
||||||
|
|
||||||
return self._execute(
|
return self._execute(
|
||||||
f"morph remove-nodes {' '.join(node_netmap_keys)}",
|
f"morph remove-nodes {' '.join(node_netmap_keys)}",
|
||||||
**{
|
**{param: param_value for param, param_value in locals().items() if param not in ["self", "node_netmap_keys"]},
|
||||||
param: param_value
|
|
||||||
for param, param_value in locals().items()
|
|
||||||
if param not in ["self", "node_netmap_keys"]
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
|
|
|
@ -754,6 +754,36 @@ class AwsCliClient(S3ClientWrapper):
|
||||||
response = self._to_json(output)
|
response = self._to_json(output)
|
||||||
return response.get("ObjectLockConfiguration")
|
return response.get("ObjectLockConfiguration")
|
||||||
|
|
||||||
|
@reporter.step("Put bucket lifecycle configuration")
|
||||||
|
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
|
||||||
|
cmd = (
|
||||||
|
f"aws {self.common_flags} s3api put-bucket-lifecycle-configuration --bucket {bucket} "
|
||||||
|
f"--endpoint-url {self.s3gate_endpoint} --lifecycle-configuration file://{dumped_configuration} --profile {self.profile}"
|
||||||
|
)
|
||||||
|
output = self.local_shell.exec(cmd).stdout
|
||||||
|
response = self._to_json(output)
|
||||||
|
return response
|
||||||
|
|
||||||
|
@reporter.step("Get bucket lifecycle configuration")
|
||||||
|
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
|
||||||
|
cmd = (
|
||||||
|
f"aws {self.common_flags} s3api get-bucket-lifecycle-configuration --bucket {bucket} "
|
||||||
|
f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
|
||||||
|
)
|
||||||
|
output = self.local_shell.exec(cmd).stdout
|
||||||
|
response = self._to_json(output)
|
||||||
|
return response
|
||||||
|
|
||||||
|
@reporter.step("Delete bucket lifecycle configuration")
|
||||||
|
def delete_bucket_lifecycle(self, bucket: str) -> dict:
|
||||||
|
cmd = (
|
||||||
|
f"aws {self.common_flags} s3api delete-bucket-lifecycle --bucket {bucket} "
|
||||||
|
f"--endpoint-url {self.s3gate_endpoint} --profile {self.profile}"
|
||||||
|
)
|
||||||
|
output = self.local_shell.exec(cmd).stdout
|
||||||
|
response = self._to_json(output)
|
||||||
|
return response
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _to_json(output: str) -> dict:
|
def _to_json(output: str) -> dict:
|
||||||
json_output = {}
|
json_output = {}
|
||||||
|
|
|
@ -296,6 +296,27 @@ class Boto3ClientWrapper(S3ClientWrapper):
|
||||||
response = self.boto3_client.delete_bucket_cors(Bucket=bucket)
|
response = self.boto3_client.delete_bucket_cors(Bucket=bucket)
|
||||||
log_command_execution(self.s3gate_endpoint, "S3 delete_bucket_cors result", response, {"Bucket": bucket})
|
log_command_execution(self.s3gate_endpoint, "S3 delete_bucket_cors result", response, {"Bucket": bucket})
|
||||||
|
|
||||||
|
@reporter.step("Put bucket lifecycle configuration")
|
||||||
|
@report_error
|
||||||
|
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
|
||||||
|
response = self.boto3_client.put_bucket_lifecycle_configuration(Bucket=bucket, LifecycleConfiguration=lifecycle_configuration)
|
||||||
|
log_command_execution(self.s3gate_endpoint, "S3 put_bucket_lifecycle_configuration result", response, {"Bucket": bucket})
|
||||||
|
return response
|
||||||
|
|
||||||
|
@reporter.step("Get bucket lifecycle configuration")
|
||||||
|
@report_error
|
||||||
|
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
|
||||||
|
response = self.boto3_client.get_bucket_lifecycle_configuration(Bucket=bucket)
|
||||||
|
log_command_execution(self.s3gate_endpoint, "S3 get_bucket_lifecycle_configuration result", response, {"Bucket": bucket})
|
||||||
|
return {"Rules": response.get("Rules")}
|
||||||
|
|
||||||
|
@reporter.step("Delete bucket lifecycle configuration")
|
||||||
|
@report_error
|
||||||
|
def delete_bucket_lifecycle(self, bucket: str) -> dict:
|
||||||
|
response = self.boto3_client.delete_bucket_lifecycle(Bucket=bucket)
|
||||||
|
log_command_execution(self.s3gate_endpoint, "S3 delete_bucket_lifecycle result", response, {"Bucket": bucket})
|
||||||
|
return response
|
||||||
|
|
||||||
# END OF BUCKET METHODS #
|
# END OF BUCKET METHODS #
|
||||||
# OBJECT METHODS #
|
# OBJECT METHODS #
|
||||||
|
|
||||||
|
|
|
@ -366,6 +366,18 @@ class S3ClientWrapper(HumanReadableABC):
|
||||||
def delete_object_tagging(self, bucket: str, key: str) -> None:
|
def delete_object_tagging(self, bucket: str, key: str) -> None:
|
||||||
"""Removes the entire tag set from the specified object."""
|
"""Removes the entire tag set from the specified object."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def put_bucket_lifecycle_configuration(self, bucket: str, lifecycle_configuration: dict, dumped_configuration: str) -> dict:
|
||||||
|
"""Adds or updates bucket lifecycle configuration"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_bucket_lifecycle_configuration(self, bucket: str) -> dict:
|
||||||
|
"""Gets bucket lifecycle configuration"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def delete_bucket_lifecycle(self, bucket: str) -> dict:
|
||||||
|
"""Deletes bucket lifecycle"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_object_attributes(
|
def get_object_attributes(
|
||||||
self,
|
self,
|
||||||
|
|
|
@ -69,7 +69,7 @@ def get_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
||||||
|
|
||||||
|
|
||||||
@reporter.step("Tick Epoch")
|
@reporter.step("Tick Epoch")
|
||||||
def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None):
|
def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode] = None, delta: Optional[int] = None):
|
||||||
"""
|
"""
|
||||||
Tick epoch using frostfs-adm or NeoGo if frostfs-adm is not available (DevEnv)
|
Tick epoch using frostfs-adm or NeoGo if frostfs-adm is not available (DevEnv)
|
||||||
Args:
|
Args:
|
||||||
|
@ -88,12 +88,17 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
||||||
frostfs_adm_exec_path=FROSTFS_ADM_EXEC,
|
frostfs_adm_exec_path=FROSTFS_ADM_EXEC,
|
||||||
config_file=FROSTFS_ADM_CONFIG_PATH,
|
config_file=FROSTFS_ADM_CONFIG_PATH,
|
||||||
)
|
)
|
||||||
frostfs_adm.morph.force_new_epoch()
|
frostfs_adm.morph.force_new_epoch(delta=delta)
|
||||||
return
|
return
|
||||||
|
|
||||||
# Otherwise we tick epoch using transaction
|
# Otherwise we tick epoch using transaction
|
||||||
cur_epoch = get_epoch(shell, cluster)
|
cur_epoch = get_epoch(shell, cluster)
|
||||||
|
|
||||||
|
if delta:
|
||||||
|
next_epoch = cur_epoch + delta
|
||||||
|
else:
|
||||||
|
next_epoch = cur_epoch + 1
|
||||||
|
|
||||||
# Use first node by default
|
# Use first node by default
|
||||||
ir_node = cluster.services(InnerRing)[0]
|
ir_node = cluster.services(InnerRing)[0]
|
||||||
# In case if no local_wallet_path is provided, we use wallet_path
|
# In case if no local_wallet_path is provided, we use wallet_path
|
||||||
|
@ -110,7 +115,7 @@ def tick_epoch(shell: Shell, cluster: Cluster, alive_node: Optional[StorageNode]
|
||||||
wallet_password=ir_wallet_pass,
|
wallet_password=ir_wallet_pass,
|
||||||
scripthash=get_contract_hash(morph_chain, "netmap.frostfs", shell=shell),
|
scripthash=get_contract_hash(morph_chain, "netmap.frostfs", shell=shell),
|
||||||
method="newEpoch",
|
method="newEpoch",
|
||||||
arguments=f"int:{cur_epoch + 1}",
|
arguments=f"int:{next_epoch}",
|
||||||
multisig_hash=f"{ir_address}:Global",
|
multisig_hash=f"{ir_address}:Global",
|
||||||
address=ir_address,
|
address=ir_address,
|
||||||
rpc_endpoint=morph_endpoint,
|
rpc_endpoint=morph_endpoint,
|
||||||
|
|
|
@ -25,12 +25,8 @@ class ClusterTestBase:
|
||||||
for _ in range(epochs_to_tick):
|
for _ in range(epochs_to_tick):
|
||||||
self.tick_epoch(alive_node, wait_block)
|
self.tick_epoch(alive_node, wait_block)
|
||||||
|
|
||||||
def tick_epoch(
|
def tick_epoch(self, alive_node: Optional[StorageNode] = None, wait_block: int = None, delta: Optional[int] = None):
|
||||||
self,
|
epoch.tick_epoch(self.shell, self.cluster, alive_node=alive_node, delta=delta)
|
||||||
alive_node: Optional[StorageNode] = None,
|
|
||||||
wait_block: int = None,
|
|
||||||
):
|
|
||||||
epoch.tick_epoch(self.shell, self.cluster, alive_node=alive_node)
|
|
||||||
if wait_block:
|
if wait_block:
|
||||||
self.wait_for_blocks(wait_block)
|
self.wait_for_blocks(wait_block)
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue