Changes required to run multiple loads during one test

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>

src/frostfs_testlib/storage/controllers/background_load_controller.py

@@ -1,3 +1,4 @@
+import copy
 import time
 
 import frostfs_testlib.resources.optionals as optionals
@@ -9,7 +10,9 @@ from frostfs_testlib.load.load_config import (
     LoadScenario,
     LoadType,
 )
+from frostfs_testlib.load.load_report import LoadReport
 from frostfs_testlib.load.load_steps import init_s3_client, prepare_k6_instances
+from frostfs_testlib.load.load_verifiers import LoadVerifier
 from frostfs_testlib.reporter import get_reporter
 from frostfs_testlib.resources.load_params import (
     K6_TEARDOWN_PERIOD,
@@ -33,11 +36,14 @@ class BackgroundLoadController:
     k6_instances: list[K6]
     k6_dir: str
     load_params: LoadParams
+    original_load_params: LoadParams
     load_nodes: list[str]
     verification_params: LoadParams
     nodes_under_load: list[ClusterNode]
+    load_counter: int
     ssh_credentials: SshCredentials
     loaders_wallet: WalletInfo
+    load_summaries: dict
     endpoints: list[str]
 
     def __init__(
@@ -48,8 +54,10 @@ class BackgroundLoadController:
         nodes_under_load: list[ClusterNode],
     ) -> None:
         self.k6_dir = k6_dir
-        self.load_params = load_params
+        self.original_load_params = load_params
+        self.load_params = copy.deepcopy(self.original_load_params)
         self.nodes_under_load = nodes_under_load
+        self.load_counter = 1
         self.load_nodes = LOAD_NODES
         self.loaders_wallet = loaders_wallet
 
@@ -59,17 +67,7 @@ class BackgroundLoadController:
         self.endpoints = self._get_endpoints(
             load_params.load_type, load_params.endpoint_selection_strategy
         )
-        self.verification_params = LoadParams(
-            verify_clients=load_params.verify_clients,
-            scenario=LoadScenario.VERIFY,
-            registry_file=load_params.registry_file,
-            verify_time=load_params.verify_time,
-            load_type=load_params.load_type,
-            load_id=load_params.load_id,
-            working_dir=load_params.working_dir,
-            endpoint_selection_strategy=load_params.endpoint_selection_strategy,
-            k6_process_allocation_strategy=load_params.k6_process_allocation_strategy,
-        )
+
         self.ssh_credentials = SshCredentials(
             LOAD_NODE_SSH_USER,
             LOAD_NODE_SSH_PASSWORD,
@@ -179,6 +177,66 @@ class BackgroundLoadController:
 
         return True
 
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
+    @reporter.step_deco("Reset background load")
+    def _reset_for_consequent_load(self):
+        """This method is required if we want to run multiple loads during test run.
+        Raise load counter by 1 and append it to load_id
+        """
+        self.load_counter += 1
+        self.load_params = copy.deepcopy(self.original_load_params)
+        self.load_params.set_id(f"{self.load_params.load_id}_{self.load_counter}")
+
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
+    @reporter.step_deco("Startup background load")
+    def startup(self):
+        self.prepare()
+        self.start()
+
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
+    @reporter.step_deco("Stop and get results of background load")
+    def teardown(self, load_report: LoadReport = None):
+        if not self.k6_instances:
+            return
+
+        self.stop()
+        self.load_summaries = self.get_results()
+        self.k6_instances = []
+        if load_report:
+            load_report.add_summaries(self.load_summaries)
+
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
+    @reporter.step_deco("Verify results of background load")
+    def verify(self):
+        try:
+            if self.load_params.verify:
+                self.verification_params = LoadParams(
+                    verify_clients=self.load_params.verify_clients,
+                    scenario=LoadScenario.VERIFY,
+                    registry_file=self.load_params.registry_file,
+                    verify_time=self.load_params.verify_time,
+                    load_type=self.load_params.load_type,
+                    load_id=self.load_params.load_id,
+                    working_dir=self.load_params.working_dir,
+                    endpoint_selection_strategy=self.load_params.endpoint_selection_strategy,
+                    k6_process_allocation_strategy=self.load_params.k6_process_allocation_strategy,
+                )
+                self._run_verify_scenario()
+                verification_summaries = self.get_results()
+                self.verify_summaries(self.load_summaries, verification_summaries)
+        finally:
+            self._reset_for_consequent_load()
+
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
+    @reporter.step_deco("Verify summaries from k6")
+    def verify_summaries(self, load_summaries: dict, verification_summaries: dict):
+        verifier = LoadVerifier(self.load_params)
+        for node_or_endpoint in load_summaries:
+            with reporter.step(f"Verify load summaries for {node_or_endpoint}"):
+                verifier.verify_summaries(
+                    load_summaries[node_or_endpoint], verification_summaries[node_or_endpoint]
+                )
+
     @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
     def wait_until_finish(self):
         if self.load_params.load_time is None:
@@ -188,7 +246,8 @@ class BackgroundLoadController:
             k6_instance.wait_until_finished(self.load_params.load_time + int(K6_TEARDOWN_PERIOD))
 
     @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
-    def verify(self):
+    @reporter.step_deco("Run verify scenario for background load")
+    def _run_verify_scenario(self):
         if self.verification_params.verify_time is None:
             raise RuntimeError("verify_time should not be none")