s.makhov/frostfs-testlib
1
0
Fork 0
forked from TrueCloudLab/frostfs-testlib
Code Pull requests Activity

[#91] Failover enhancements

Browse source 
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-10-03 15:18:29 +03:00
parent 9feb8135e3
commit 98ccd4c382
16 changed files with 200 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

64
src/frostfs_testlib/load/load_verifiers.py
View file

@ -12,7 +12,7 @@ class LoadVerifier:
def __init__(self, load_params: LoadParams) -> None:
self.load_params = load_params
def verify_load_results(self, load_summaries: dict[str, dict]):
def collect_load_issues(self, load_summaries: dict[str, dict]) -> list[str]:
write_operations = 0
write_errors = 0
@ -41,38 +41,58 @@ class LoadVerifier:
delete_operations += metrics.delete_total_iterations
delete_errors += metrics.delete_failed_iterations
exceptions = []
issues = []
if writers and not write_operations:
exceptions.append(f"No any write operation was performed")
issues.append(f"No any write operation was performed")
if readers and not read_operations:
exceptions.append(f"No any read operation was performed")
issues.append(f"No any read operation was performed")
if deleters and not delete_operations:
exceptions.append(f"No any delete operation was performed")
issues.append(f"No any delete operation was performed")
if write_operations and writers and write_errors / write_operations * 100 > self.load_params.error_threshold:
exceptions.append(
if (
write_operations
and writers
and write_errors / write_operations * 100 > self.load_params.error_threshold
):
issues.append(
f"Write error rate is greater than threshold: {write_errors / write_operations * 100} > {self.load_params.error_threshold}"
)
if read_operations and readers and read_errors / read_operations * 100 > self.load_params.error_threshold:
exceptions.append(
if (
read_operations
and readers
and read_errors / read_operations * 100 > self.load_params.error_threshold
):
issues.append(
f"Read error rate is greater than threshold: {read_errors / read_operations * 100} > {self.load_params.error_threshold}"
)
if delete_operations and deleters and delete_errors / delete_operations * 100 > self.load_params.error_threshold:
exceptions.append(
if (
delete_operations
and deleters
and delete_errors / delete_operations * 100 > self.load_params.error_threshold
):
issues.append(
f"Delete error rate is greater than threshold: {delete_errors / delete_operations * 100} > {self.load_params.error_threshold}"
)
assert not exceptions, "\n".join(exceptions)
return issues
def check_verify_results(self, load_summaries, verification_summaries) -> None:
for node_or_endpoint in load_summaries:
with reporter.step(f"Check verify scenario results for {node_or_endpoint}"):
self._check_verify_result(
load_summaries[node_or_endpoint], verification_summaries[node_or_endpoint]
def collect_verify_issues(self, load_summaries, verification_summaries) -> list[str]:
verify_issues: list[str] = []
for k6_process_label in load_summaries:
with reporter.step(f"Check verify scenario results for {k6_process_label}"):
verify_issues.extend(
self._collect_verify_issues_on_process(
k6_process_label,
load_summaries[k6_process_label],
verification_summaries[k6_process_label],
)
)
return verify_issues
def _check_verify_result(self, load_summary, verification_summary) -> None:
exceptions = []
def _collect_verify_issues_on_process(
self, label, load_summary, verification_summary
) -> list[str]:
issues = []
load_metrics = get_metrics_object(self.load_params.scenario, load_summary)
@ -92,8 +112,8 @@ class LoadVerifier:
# Due to interruptions we may see total verified objects to be less than written on writers count
if abs(total_left_objects - verified_objects) > writers:
exceptions.append(
f"Verified objects mismatch. Total: {total_left_objects}, Verified: {verified_objects}. Writers: {writers}."
issues.append(
f"Verified objects mismatch for {label}. Total: {total_left_objects}, Verified: {verified_objects}. Writers: {writers}."
)
assert not exceptions, "\n".join(exceptions)
return issues