Add error_threshold parameter, add error check after load

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>

src/frostfs_testlib/load/load_config.py

@@ -139,6 +139,11 @@ class LoadParams:
     verify: Optional[bool] = None
     # Just id for load so distinct it between runs. Filled automatically.
     load_id: Optional[str] = None
+    # Acceptable number of load errors in %
+    # 100 means 100% errors allowed
+    # 1.5 means 1.5% errors allowed
+    # 0 means no errors allowed
+    error_threshold: Optional[float] = None
     # Working directory
     working_dir: Optional[str] = None
     # Preset for the k6 run

src/frostfs_testlib/load/load_report.py

@@ -154,6 +154,7 @@ class LoadReport:
         <tr><th colspan="2" bgcolor="gainsboro">Errors</th></tr>
         {per_node_errors_html}
         {self._row("Total", f"{total_errors} ({total_errors/total_operations*100.0:.2f}%)")}
+        {self._row("Threshold", f"{self.load_params.error_threshold:.2f}%")}
         </tbody></table><br><hr>
         """
 

src/frostfs_testlib/load/load_verifiers.py

@@ -2,7 +2,9 @@ import logging
 
 from frostfs_testlib.load.load_config import LoadParams, LoadScenario
 from frostfs_testlib.load.load_metrics import get_metrics_object
+from frostfs_testlib.reporter import get_reporter
 
+reporter = get_reporter()
 logger = logging.getLogger("NeoLogger")
 
 
@@ -10,54 +12,88 @@ class LoadVerifier:
     def __init__(self, load_params: LoadParams) -> None:
         self.load_params = load_params
 
-    def verify_summaries(self, load_summary, verification_summary) -> None:
-        exceptions = []
+    def verify_load_results(self, load_summaries: dict[str, dict]):
+        write_operations = 0
+        write_errors = 0
 
-        if not verification_summary or not load_summary:
-            logger.info("Can't check load results due to missing summary")
+        read_operations = 0
+        read_errors = 0
 
-        load_metrics = get_metrics_object(self.load_params.scenario, load_summary)
+        delete_operations = 0
+        delete_errors = 0
 
         writers = self.load_params.writers or self.load_params.preallocated_writers or 0
         readers = self.load_params.readers or self.load_params.preallocated_readers or 0
         deleters = self.load_params.deleters or self.load_params.preallocated_deleters or 0
 
-        objects_count = load_metrics.write_success_iterations
-        fails_count = load_metrics.write_failed_iterations
+        for load_summary in load_summaries.values():
+            metrics = get_metrics_object(self.load_params.scenario, load_summary)
 
-        if writers > 0:
-            if objects_count < 1:
-                exceptions.append("Total put objects should be greater than 0")
-            if fails_count > 0:
-                exceptions.append(f"There were {fails_count} failed write operations")
+            if writers:
+                write_operations += metrics.write_total_iterations
+                write_errors += metrics.write_failed_iterations
 
-        if readers > 0:
-            read_count = load_metrics.read_success_iterations
-            read_fails_count = load_metrics.read_failed_iterations
-            if read_count < 1:
-                exceptions.append("Total read operations should be greater than 0")
-            if read_fails_count > 0:
-                exceptions.append(f"There were {read_fails_count} failed read operations")
+            if readers:
+                read_operations += metrics.read_total_iterations
+                read_errors += metrics.read_failed_iterations
+
+            if deleters:
+                delete_operations += metrics.delete_total_iterations
+                delete_errors += metrics.delete_failed_iterations
+
+        exceptions = []
+        if writers and not write_operations:
+            exceptions.append(f"No any write operation was performed")
+        if readers and not read_operations:
+            exceptions.append(f"No any read operation was performed")
+        if deleters and not delete_operations:
+            exceptions.append(f"No any delete operation was performed")
+
+        if writers and write_errors / write_operations * 100 > self.load_params.error_threshold:
+            exceptions.append(
+                f"Write error rate is greater than threshold: {write_errors / write_operations * 100} > {self.load_params.error_threshold}"
+            )
+        if readers and read_errors / read_operations * 100 > self.load_params.error_threshold:
+            exceptions.append(
+                f"Read error rate is greater than threshold: {read_errors / read_operations * 100} > {self.load_params.error_threshold}"
+            )
+        if deleters and delete_errors / delete_operations * 100 > self.load_params.error_threshold:
+            exceptions.append(
+                f"Delete error rate is greater than threshold: {delete_errors / delete_operations * 100} > {self.load_params.error_threshold}"
+            )
+
+        assert not exceptions, "\n".join(exceptions)
+
+    def check_verify_results(self, load_summaries, verification_summaries) -> None:
+        for node_or_endpoint in load_summaries:
+            with reporter.step(f"Check verify scenario results for {node_or_endpoint}"):
+                self._check_verify_result(
+                    load_summaries[node_or_endpoint], verification_summaries[node_or_endpoint]
+                )
+
+    def _check_verify_result(self, load_summary, verification_summary) -> None:
+        exceptions = []
+
+        load_metrics = get_metrics_object(self.load_params.scenario, load_summary)
+
+        writers = self.load_params.writers or self.load_params.preallocated_writers or 0
+        deleters = self.load_params.deleters or self.load_params.preallocated_deleters or 0
+
+        delete_success = 0
 
         if deleters > 0:
-            delete_count = load_metrics.delete_success_iterations
-            delete_fails_count = load_metrics.delete_failed_iterations
-            if delete_count < 1:
-                exceptions.append("Total delete operations should be greater than 0")
-            if delete_fails_count > 0:
-                exceptions.append(f"There were {delete_fails_count} failed delete operations")
+            delete_success = load_metrics.delete_success_iterations
 
         if verification_summary:
             verify_metrics = get_metrics_object(LoadScenario.VERIFY, verification_summary)
             verified_objects = verify_metrics.read_success_iterations
             invalid_objects = verify_metrics.read_failed_iterations
+            total_left_objects = load_metrics.write_success_iterations - delete_success
 
-            if invalid_objects > 0:
-                exceptions.append(f"There were {invalid_objects} verification fails")
             # Due to interruptions we may see total verified objects to be less than written on writers count
-            if abs(objects_count - verified_objects) > writers:
+            if abs(total_left_objects - verified_objects) > writers:
                 exceptions.append(
-                    f"Verified objects mismatch. Total: {objects_count}, Verified: {verified_objects}. Writers: {writers}."
+                    f"Verified objects mismatch. Total: {total_left_objects}, Verified: {verified_objects}. Writers: {writers}."
                 )
 
         assert not exceptions, "\n".join(exceptions)