forked from TrueCloudLab/frostfs-testlib
Changes required to run multiple loads during one test
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
parent
123b5425a8
commit
cc35b2e6da
6 changed files with 217 additions and 138 deletions
|
|
@ -153,10 +153,6 @@ class K6:
|
|||
|
||||
@reporter.step_deco("Start K6 on initiator")
|
||||
def start(self) -> None:
|
||||
# Make working_dir directory
|
||||
self.shell.exec(f"sudo mkdir -p {self.load_params.working_dir}")
|
||||
self.shell.exec(f"sudo chown {LOAD_NODE_SSH_USER} {self.load_params.working_dir}")
|
||||
|
||||
command = (
|
||||
f"{self._k6_dir}/k6 run {self._generate_env_variables()} "
|
||||
f"{self._k6_dir}/scenarios/{self.scenario.value}.js"
|
||||
|
|
@ -170,13 +166,12 @@ class K6:
|
|||
assert "No k6 instances were executed"
|
||||
if k6_should_be_running:
|
||||
assert self._k6_process.running(), "k6 should be running."
|
||||
while timeout >= 0:
|
||||
while timeout > 0:
|
||||
if not self._k6_process.running():
|
||||
return
|
||||
logger.info(f"K6 is running. Waiting {wait_interval} seconds...")
|
||||
if timeout > 0:
|
||||
sleep(wait_interval)
|
||||
timeout -= wait_interval
|
||||
sleep(wait_interval)
|
||||
timeout -= wait_interval
|
||||
self.stop()
|
||||
raise TimeoutError(f"Expected K6 finished in {timeout} sec.")
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ from frostfs_testlib.load.load_metrics import get_metrics_object
|
|||
class LoadReport:
|
||||
def __init__(self, load_test) -> None:
|
||||
self.load_test = load_test
|
||||
self.load_summaries: Optional[dict] = None
|
||||
# List of load summaries dict
|
||||
self.load_summaries_list: Optional[list[dict]] = []
|
||||
self.load_params: Optional[LoadParams] = None
|
||||
self.start_time: Optional[datetime] = None
|
||||
self.end_time: Optional[datetime] = None
|
||||
|
|
@ -21,8 +22,8 @@ class LoadReport:
|
|||
def set_end_time(self):
|
||||
self.end_time = datetime.utcnow()
|
||||
|
||||
def set_summaries(self, load_summaries: dict):
|
||||
self.load_summaries = load_summaries
|
||||
def add_summaries(self, load_summaries: dict):
|
||||
self.load_summaries_list.append(load_summaries)
|
||||
|
||||
def set_load_params(self, load_params: LoadParams):
|
||||
self.load_params = load_params
|
||||
|
|
@ -30,7 +31,7 @@ class LoadReport:
|
|||
def get_report_html(self):
|
||||
report_sections = [
|
||||
[self.load_test, self._get_load_params_section_html],
|
||||
[self.load_summaries, self._get_totals_section_html],
|
||||
[self.load_summaries_list, self._get_totals_section_html],
|
||||
[self.end_time, self._get_test_time_html],
|
||||
]
|
||||
|
||||
|
|
@ -156,110 +157,113 @@ class LoadReport:
|
|||
return html
|
||||
|
||||
def _get_totals_section_html(self):
|
||||
html = ""
|
||||
for i, load_summaries in enumerate(self.load_summaries_list, 1):
|
||||
html += f"<h3>Load Results for load #{i}</h3>"
|
||||
|
||||
html = "<h3>Load Results</h3>"
|
||||
|
||||
write_operations = 0
|
||||
write_op_sec = 0
|
||||
write_throughput = 0
|
||||
write_errors = {}
|
||||
requested_write_rate = self.load_params.write_rate
|
||||
requested_write_rate_str = f"{requested_write_rate}op/sec" if requested_write_rate else ""
|
||||
|
||||
read_operations = 0
|
||||
read_op_sec = 0
|
||||
read_throughput = 0
|
||||
read_errors = {}
|
||||
requested_read_rate = self.load_params.read_rate
|
||||
requested_read_rate_str = f"{requested_read_rate}op/sec" if requested_read_rate else ""
|
||||
|
||||
delete_operations = 0
|
||||
delete_op_sec = 0
|
||||
delete_errors = {}
|
||||
requested_delete_rate = self.load_params.delete_rate
|
||||
requested_delete_rate_str = (
|
||||
f"{requested_delete_rate}op/sec" if requested_delete_rate else ""
|
||||
)
|
||||
|
||||
if self.load_params.scenario in [LoadScenario.gRPC_CAR, LoadScenario.S3_CAR]:
|
||||
delete_vus = max(
|
||||
self.load_params.preallocated_deleters or 0, self.load_params.max_deleters or 0
|
||||
)
|
||||
write_vus = max(
|
||||
self.load_params.preallocated_writers or 0, self.load_params.max_writers or 0
|
||||
)
|
||||
read_vus = max(
|
||||
self.load_params.preallocated_readers or 0, self.load_params.max_readers or 0
|
||||
)
|
||||
else:
|
||||
write_vus = self.load_params.writers
|
||||
read_vus = self.load_params.readers
|
||||
delete_vus = self.load_params.deleters
|
||||
|
||||
write_vus_str = f"{write_vus}th"
|
||||
read_vus_str = f"{read_vus}th"
|
||||
delete_vus_str = f"{delete_vus}th"
|
||||
|
||||
write_section_required = False
|
||||
read_section_required = False
|
||||
delete_section_required = False
|
||||
|
||||
for node_key, load_summary in self.load_summaries.items():
|
||||
metrics = get_metrics_object(self.load_params.scenario, load_summary)
|
||||
write_operations += metrics.write_total_iterations
|
||||
if write_operations:
|
||||
write_section_required = True
|
||||
write_op_sec += metrics.write_rate
|
||||
write_throughput += metrics.write_throughput
|
||||
if metrics.write_failed_iterations:
|
||||
write_errors[node_key] = metrics.write_failed_iterations
|
||||
|
||||
read_operations += metrics.read_total_iterations
|
||||
if read_operations:
|
||||
read_section_required = True
|
||||
read_op_sec += metrics.read_rate
|
||||
read_throughput += metrics.read_throughput
|
||||
if metrics.read_failed_iterations:
|
||||
read_errors[node_key] = metrics.read_failed_iterations
|
||||
|
||||
delete_operations += metrics.delete_total_iterations
|
||||
if delete_operations:
|
||||
delete_section_required = True
|
||||
delete_op_sec += metrics.delete_rate
|
||||
if metrics.delete_failed_iterations:
|
||||
delete_errors[node_key] = metrics.delete_failed_iterations
|
||||
|
||||
if write_section_required:
|
||||
html += self._get_oprations_sub_section_html(
|
||||
"Write",
|
||||
write_operations,
|
||||
requested_write_rate_str,
|
||||
write_vus_str,
|
||||
write_op_sec,
|
||||
write_throughput,
|
||||
write_errors,
|
||||
write_operations = 0
|
||||
write_op_sec = 0
|
||||
write_throughput = 0
|
||||
write_errors = {}
|
||||
requested_write_rate = self.load_params.write_rate
|
||||
requested_write_rate_str = (
|
||||
f"{requested_write_rate}op/sec" if requested_write_rate else ""
|
||||
)
|
||||
|
||||
if read_section_required:
|
||||
html += self._get_oprations_sub_section_html(
|
||||
"Read",
|
||||
read_operations,
|
||||
requested_read_rate_str,
|
||||
read_vus_str,
|
||||
read_op_sec,
|
||||
read_throughput,
|
||||
read_errors,
|
||||
read_operations = 0
|
||||
read_op_sec = 0
|
||||
read_throughput = 0
|
||||
read_errors = {}
|
||||
requested_read_rate = self.load_params.read_rate
|
||||
requested_read_rate_str = f"{requested_read_rate}op/sec" if requested_read_rate else ""
|
||||
|
||||
delete_operations = 0
|
||||
delete_op_sec = 0
|
||||
delete_errors = {}
|
||||
requested_delete_rate = self.load_params.delete_rate
|
||||
requested_delete_rate_str = (
|
||||
f"{requested_delete_rate}op/sec" if requested_delete_rate else ""
|
||||
)
|
||||
|
||||
if delete_section_required:
|
||||
html += self._get_oprations_sub_section_html(
|
||||
"Delete",
|
||||
delete_operations,
|
||||
requested_delete_rate_str,
|
||||
delete_vus_str,
|
||||
delete_op_sec,
|
||||
0,
|
||||
delete_errors,
|
||||
)
|
||||
if self.load_params.scenario in [LoadScenario.gRPC_CAR, LoadScenario.S3_CAR]:
|
||||
delete_vus = max(
|
||||
self.load_params.preallocated_deleters or 0, self.load_params.max_deleters or 0
|
||||
)
|
||||
write_vus = max(
|
||||
self.load_params.preallocated_writers or 0, self.load_params.max_writers or 0
|
||||
)
|
||||
read_vus = max(
|
||||
self.load_params.preallocated_readers or 0, self.load_params.max_readers or 0
|
||||
)
|
||||
else:
|
||||
write_vus = self.load_params.writers
|
||||
read_vus = self.load_params.readers
|
||||
delete_vus = self.load_params.deleters
|
||||
|
||||
write_vus_str = f"{write_vus}th"
|
||||
read_vus_str = f"{read_vus}th"
|
||||
delete_vus_str = f"{delete_vus}th"
|
||||
|
||||
write_section_required = False
|
||||
read_section_required = False
|
||||
delete_section_required = False
|
||||
|
||||
for node_key, load_summary in load_summaries.items():
|
||||
metrics = get_metrics_object(self.load_params.scenario, load_summary)
|
||||
write_operations += metrics.write_total_iterations
|
||||
if write_operations:
|
||||
write_section_required = True
|
||||
write_op_sec += metrics.write_rate
|
||||
write_throughput += metrics.write_throughput
|
||||
if metrics.write_failed_iterations:
|
||||
write_errors[node_key] = metrics.write_failed_iterations
|
||||
|
||||
read_operations += metrics.read_total_iterations
|
||||
if read_operations:
|
||||
read_section_required = True
|
||||
read_op_sec += metrics.read_rate
|
||||
read_throughput += metrics.read_throughput
|
||||
if metrics.read_failed_iterations:
|
||||
read_errors[node_key] = metrics.read_failed_iterations
|
||||
|
||||
delete_operations += metrics.delete_total_iterations
|
||||
if delete_operations:
|
||||
delete_section_required = True
|
||||
delete_op_sec += metrics.delete_rate
|
||||
if metrics.delete_failed_iterations:
|
||||
delete_errors[node_key] = metrics.delete_failed_iterations
|
||||
|
||||
if write_section_required:
|
||||
html += self._get_oprations_sub_section_html(
|
||||
"Write",
|
||||
write_operations,
|
||||
requested_write_rate_str,
|
||||
write_vus_str,
|
||||
write_op_sec,
|
||||
write_throughput,
|
||||
write_errors,
|
||||
)
|
||||
|
||||
if read_section_required:
|
||||
html += self._get_oprations_sub_section_html(
|
||||
"Read",
|
||||
read_operations,
|
||||
requested_read_rate_str,
|
||||
read_vus_str,
|
||||
read_op_sec,
|
||||
read_throughput,
|
||||
read_errors,
|
||||
)
|
||||
|
||||
if delete_section_required:
|
||||
html += self._get_oprations_sub_section_html(
|
||||
"Delete",
|
||||
delete_operations,
|
||||
requested_delete_rate_str,
|
||||
delete_vus_str,
|
||||
delete_op_sec,
|
||||
0,
|
||||
delete_errors,
|
||||
)
|
||||
|
||||
return html
|
||||
|
|
|
|||
|
|
@ -9,7 +9,10 @@ from frostfs_testlib.load.k6 import K6
|
|||
from frostfs_testlib.load.load_config import K6ProcessAllocationStrategy, LoadParams
|
||||
from frostfs_testlib.reporter import get_reporter
|
||||
from frostfs_testlib.resources.cli import FROSTFS_AUTHMATE_EXEC
|
||||
from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_VUS_COUNT_DIVISOR
|
||||
from frostfs_testlib.resources.load_params import (
|
||||
BACKGROUND_LOAD_VUS_COUNT_DIVISOR,
|
||||
LOAD_NODE_SSH_USER,
|
||||
)
|
||||
from frostfs_testlib.shell import CommandOptions, SSHShell
|
||||
from frostfs_testlib.shell.interfaces import InteractiveInput, SshCredentials
|
||||
from frostfs_testlib.storage.cluster import ClusterNode
|
||||
|
|
@ -35,7 +38,7 @@ def init_s3_client(
|
|||
grpc_peer = storage_node.get_rpc_endpoint()
|
||||
|
||||
for load_node in load_nodes:
|
||||
ssh_client = _get_ssh_client(ssh_credentials, load_node)
|
||||
ssh_client = _get_shell(ssh_credentials, load_node)
|
||||
frostfs_authmate_exec: FrostfsAuthmate = FrostfsAuthmate(ssh_client, FROSTFS_AUTHMATE_EXEC)
|
||||
issue_secret_output = frostfs_authmate_exec.secret.issue(
|
||||
wallet=wallet.path,
|
||||
|
|
@ -99,12 +102,16 @@ def prepare_k6_instances(
|
|||
|
||||
for distributed_load_params in distributed_load_params_list:
|
||||
load_node = next(nodes)
|
||||
ssh_client = _get_ssh_client(ssh_credentials, load_node)
|
||||
shell = _get_shell(ssh_credentials, load_node)
|
||||
# Make working_dir directory
|
||||
shell.exec(f"sudo mkdir -p {distributed_load_params.working_dir}")
|
||||
shell.exec(f"sudo chown {LOAD_NODE_SSH_USER} {distributed_load_params.working_dir}")
|
||||
|
||||
k6_load_object = K6(
|
||||
distributed_load_params,
|
||||
next(endpoints_gen),
|
||||
k6_dir,
|
||||
ssh_client,
|
||||
shell,
|
||||
load_node,
|
||||
loaders_wallet,
|
||||
)
|
||||
|
|
@ -115,7 +122,7 @@ def prepare_k6_instances(
|
|||
return k6_load_objects
|
||||
|
||||
|
||||
def _get_ssh_client(ssh_credentials: SshCredentials, load_node: str):
|
||||
def _get_shell(ssh_credentials: SshCredentials, load_node: str) -> SSHShell:
|
||||
ssh_client = SSHShell(
|
||||
host=load_node,
|
||||
login=ssh_credentials.ssh_login,
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ class LoadVerifier:
|
|||
# Due to interruptions we may see total verified objects to be less than written on writers count
|
||||
if abs(objects_count - verified_objects) > writers:
|
||||
exceptions.append(
|
||||
f"Verified objects is less than total objects. Total: {objects_count}, Verified: {verified_objects}. Writers: {writers}."
|
||||
f"Verified objects mismatch. Total: {objects_count}, Verified: {verified_objects}. Writers: {writers}."
|
||||
)
|
||||
|
||||
assert not exceptions, "\n".join(exceptions)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue