forked from TrueCloudLab/frostfs-testlib
Changes required to run multiple loads during one test
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
parent
123b5425a8
commit
cc35b2e6da
6 changed files with 217 additions and 138 deletions
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import time
|
||||
|
||||
import frostfs_testlib.resources.optionals as optionals
|
||||
|
|
@ -9,7 +10,9 @@ from frostfs_testlib.load.load_config import (
|
|||
LoadScenario,
|
||||
LoadType,
|
||||
)
|
||||
from frostfs_testlib.load.load_report import LoadReport
|
||||
from frostfs_testlib.load.load_steps import init_s3_client, prepare_k6_instances
|
||||
from frostfs_testlib.load.load_verifiers import LoadVerifier
|
||||
from frostfs_testlib.reporter import get_reporter
|
||||
from frostfs_testlib.resources.load_params import (
|
||||
K6_TEARDOWN_PERIOD,
|
||||
|
|
@ -33,11 +36,14 @@ class BackgroundLoadController:
|
|||
k6_instances: list[K6]
|
||||
k6_dir: str
|
||||
load_params: LoadParams
|
||||
original_load_params: LoadParams
|
||||
load_nodes: list[str]
|
||||
verification_params: LoadParams
|
||||
nodes_under_load: list[ClusterNode]
|
||||
load_counter: int
|
||||
ssh_credentials: SshCredentials
|
||||
loaders_wallet: WalletInfo
|
||||
load_summaries: dict
|
||||
endpoints: list[str]
|
||||
|
||||
def __init__(
|
||||
|
|
@ -48,8 +54,10 @@ class BackgroundLoadController:
|
|||
nodes_under_load: list[ClusterNode],
|
||||
) -> None:
|
||||
self.k6_dir = k6_dir
|
||||
self.load_params = load_params
|
||||
self.original_load_params = load_params
|
||||
self.load_params = copy.deepcopy(self.original_load_params)
|
||||
self.nodes_under_load = nodes_under_load
|
||||
self.load_counter = 1
|
||||
self.load_nodes = LOAD_NODES
|
||||
self.loaders_wallet = loaders_wallet
|
||||
|
||||
|
|
@ -59,17 +67,7 @@ class BackgroundLoadController:
|
|||
self.endpoints = self._get_endpoints(
|
||||
load_params.load_type, load_params.endpoint_selection_strategy
|
||||
)
|
||||
self.verification_params = LoadParams(
|
||||
verify_clients=load_params.verify_clients,
|
||||
scenario=LoadScenario.VERIFY,
|
||||
registry_file=load_params.registry_file,
|
||||
verify_time=load_params.verify_time,
|
||||
load_type=load_params.load_type,
|
||||
load_id=load_params.load_id,
|
||||
working_dir=load_params.working_dir,
|
||||
endpoint_selection_strategy=load_params.endpoint_selection_strategy,
|
||||
k6_process_allocation_strategy=load_params.k6_process_allocation_strategy,
|
||||
)
|
||||
|
||||
self.ssh_credentials = SshCredentials(
|
||||
LOAD_NODE_SSH_USER,
|
||||
LOAD_NODE_SSH_PASSWORD,
|
||||
|
|
@ -179,6 +177,66 @@ class BackgroundLoadController:
|
|||
|
||||
return True
|
||||
|
||||
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
|
||||
@reporter.step_deco("Reset background load")
|
||||
def _reset_for_consequent_load(self):
|
||||
"""This method is required if we want to run multiple loads during test run.
|
||||
Raise load counter by 1 and append it to load_id
|
||||
"""
|
||||
self.load_counter += 1
|
||||
self.load_params = copy.deepcopy(self.original_load_params)
|
||||
self.load_params.set_id(f"{self.load_params.load_id}_{self.load_counter}")
|
||||
|
||||
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
|
||||
@reporter.step_deco("Startup background load")
|
||||
def startup(self):
|
||||
self.prepare()
|
||||
self.start()
|
||||
|
||||
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
|
||||
@reporter.step_deco("Stop and get results of background load")
|
||||
def teardown(self, load_report: LoadReport = None):
|
||||
if not self.k6_instances:
|
||||
return
|
||||
|
||||
self.stop()
|
||||
self.load_summaries = self.get_results()
|
||||
self.k6_instances = []
|
||||
if load_report:
|
||||
load_report.add_summaries(self.load_summaries)
|
||||
|
||||
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
|
||||
@reporter.step_deco("Verify results of background load")
|
||||
def verify(self):
|
||||
try:
|
||||
if self.load_params.verify:
|
||||
self.verification_params = LoadParams(
|
||||
verify_clients=self.load_params.verify_clients,
|
||||
scenario=LoadScenario.VERIFY,
|
||||
registry_file=self.load_params.registry_file,
|
||||
verify_time=self.load_params.verify_time,
|
||||
load_type=self.load_params.load_type,
|
||||
load_id=self.load_params.load_id,
|
||||
working_dir=self.load_params.working_dir,
|
||||
endpoint_selection_strategy=self.load_params.endpoint_selection_strategy,
|
||||
k6_process_allocation_strategy=self.load_params.k6_process_allocation_strategy,
|
||||
)
|
||||
self._run_verify_scenario()
|
||||
verification_summaries = self.get_results()
|
||||
self.verify_summaries(self.load_summaries, verification_summaries)
|
||||
finally:
|
||||
self._reset_for_consequent_load()
|
||||
|
||||
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
|
||||
@reporter.step_deco("Verify summaries from k6")
|
||||
def verify_summaries(self, load_summaries: dict, verification_summaries: dict):
|
||||
verifier = LoadVerifier(self.load_params)
|
||||
for node_or_endpoint in load_summaries:
|
||||
with reporter.step(f"Verify load summaries for {node_or_endpoint}"):
|
||||
verifier.verify_summaries(
|
||||
load_summaries[node_or_endpoint], verification_summaries[node_or_endpoint]
|
||||
)
|
||||
|
||||
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
|
||||
def wait_until_finish(self):
|
||||
if self.load_params.load_time is None:
|
||||
|
|
@ -188,7 +246,8 @@ class BackgroundLoadController:
|
|||
k6_instance.wait_until_finished(self.load_params.load_time + int(K6_TEARDOWN_PERIOD))
|
||||
|
||||
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
|
||||
def verify(self):
|
||||
@reporter.step_deco("Run verify scenario for background load")
|
||||
def _run_verify_scenario(self):
|
||||
if self.verification_params.verify_time is None:
|
||||
raise RuntimeError("verify_time should not be none")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
import time
|
||||
|
||||
import allure
|
||||
|
||||
import frostfs_testlib.resources.optionals as optionals
|
||||
from frostfs_testlib.reporter import get_reporter
|
||||
from frostfs_testlib.shell import CommandOptions, Shell
|
||||
|
|
@ -30,15 +28,29 @@ class ClusterStateController:
|
|||
@run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
|
||||
@reporter.step_deco("Stop host of node {node}")
|
||||
def stop_node_host(self, node: ClusterNode, mode: str):
|
||||
with allure.step(f"Stop host {node.host.config.address}"):
|
||||
with reporter.step(f"Stop host {node.host.config.address}"):
|
||||
node.host.stop_host(mode=mode)
|
||||
wait_for_host_offline(self.shell, node.storage_node)
|
||||
self.stopped_nodes.append(node)
|
||||
|
||||
@run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
|
||||
@reporter.step_deco("Shutdown whole cluster")
|
||||
def shutdown_cluster(self, mode: str, reversed_order: bool = False):
|
||||
nodes = (
|
||||
reversed(self.cluster.cluster_nodes) if reversed_order else self.cluster.cluster_nodes
|
||||
)
|
||||
for node in nodes:
|
||||
with reporter.step(f"Stop host {node.host.config.address}"):
|
||||
self.stopped_nodes.append(node)
|
||||
node.host.stop_host(mode=mode)
|
||||
|
||||
for node in nodes:
|
||||
wait_for_host_offline(self.shell, node.storage_node)
|
||||
|
||||
@run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
|
||||
@reporter.step_deco("Start host of node {node}")
|
||||
def start_node_host(self, node: ClusterNode):
|
||||
with allure.step(f"Start host {node.host.config.address}"):
|
||||
with reporter.step(f"Start host {node.host.config.address}"):
|
||||
node.host.start_host()
|
||||
wait_for_host_online(self.shell, node.storage_node)
|
||||
wait_for_node_online(node.storage_node)
|
||||
|
|
@ -46,9 +58,11 @@ class ClusterStateController:
|
|||
|
||||
@run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
|
||||
@reporter.step_deco("Start stopped hosts")
|
||||
def start_stopped_hosts(self):
|
||||
for node in self.stopped_nodes:
|
||||
node.host.start_host()
|
||||
def start_stopped_hosts(self, reversed_order: bool = False):
|
||||
nodes = reversed(self.stopped_nodes) if reversed_order else self.stopped_nodes
|
||||
for node in nodes:
|
||||
with reporter.step(f"Start host {node.host.config.address}"):
|
||||
node.host.start_host()
|
||||
self.stopped_nodes = []
|
||||
wait_all_storage_nodes_returned(self.shell, self.cluster)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue