forked from TrueCloudLab/frostfs-testlib
Move shared code to testlib
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
parent
d97a02d1d3
commit
997e768e92
69 changed files with 9213 additions and 64 deletions
237
src/frostfs_testlib/load/k6.py
Normal file
237
src/frostfs_testlib/load/k6.py
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, fields
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
from frostfs_testlib.load.load_config import (
|
||||
K6ProcessAllocationStrategy,
|
||||
LoadParams,
|
||||
LoadScenario,
|
||||
LoadType,
|
||||
)
|
||||
from frostfs_testlib.processes.remote_process import RemoteProcess
|
||||
from frostfs_testlib.reporter import get_reporter
|
||||
from frostfs_testlib.shell import Shell
|
||||
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
|
||||
|
||||
EXIT_RESULT_CODE = 0
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
reporter = get_reporter()
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadResults:
|
||||
data_sent: float = 0.0
|
||||
data_received: float = 0.0
|
||||
read_ops: float = 0.0
|
||||
write_ops: float = 0.0
|
||||
total_ops: float = 0.0
|
||||
|
||||
|
||||
class K6:
|
||||
_k6_process: RemoteProcess
|
||||
_k6_stop_attempts: int = 5
|
||||
_k6_stop_check_interval: int = 15
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
load_params: LoadParams,
|
||||
endpoints: list[str],
|
||||
k6_dir: str,
|
||||
shell: Shell,
|
||||
load_node: str,
|
||||
wallet: WalletInfo,
|
||||
):
|
||||
if load_params.scenario is None:
|
||||
raise RuntimeError("Scenario should not be none")
|
||||
|
||||
self.load_params: LoadParams = load_params
|
||||
self.endpoints = endpoints
|
||||
self.load_node: str = load_node
|
||||
self.shell: Shell = shell
|
||||
self.wallet = wallet
|
||||
self.scenario: LoadScenario = load_params.scenario
|
||||
self.summary_json: str = os.path.join(
|
||||
self.load_params.working_dir,
|
||||
f"{self.load_params.load_id}_{self.scenario.value}_summary.json",
|
||||
)
|
||||
|
||||
self._k6_dir: str = k6_dir
|
||||
|
||||
@property
|
||||
def process_dir(self) -> str:
|
||||
return self._k6_process.process_dir
|
||||
|
||||
@reporter.step_deco("Preset containers and objects")
|
||||
def preset(self) -> str:
|
||||
preset_grpc = f"{self._k6_dir}/scenarios/preset/preset_grpc.py"
|
||||
preset_s3 = f"{self._k6_dir}/scenarios/preset/preset_s3.py"
|
||||
preset_map = {
|
||||
LoadType.gRPC: preset_grpc,
|
||||
LoadType.S3: preset_s3,
|
||||
LoadType.HTTP: preset_grpc,
|
||||
}
|
||||
|
||||
base_args = {
|
||||
preset_grpc: [
|
||||
preset_grpc,
|
||||
f"--endpoint {self.endpoints[0]}",
|
||||
f"--wallet {self.wallet.path} ",
|
||||
f"--config {self.wallet.config_path} ",
|
||||
],
|
||||
preset_s3: [
|
||||
preset_s3,
|
||||
f"--endpoint {self.endpoints[0]}",
|
||||
],
|
||||
}
|
||||
|
||||
preset_scenario = preset_map[self.load_params.load_type]
|
||||
command_args = base_args[preset_scenario].copy()
|
||||
|
||||
command_args += [
|
||||
f"--{field.metadata['preset_argument']} '{getattr(self.load_params, field.name)}'"
|
||||
for field in fields(self.load_params)
|
||||
if field.metadata
|
||||
and self.scenario in field.metadata["applicable_scenarios"]
|
||||
and field.metadata["preset_argument"]
|
||||
and getattr(self.load_params, field.name) is not None
|
||||
]
|
||||
|
||||
if self.load_params.preset:
|
||||
command_args += [
|
||||
f"--{field.metadata['preset_argument']} '{getattr(self.load_params.preset, field.name)}'"
|
||||
for field in fields(self.load_params.preset)
|
||||
if field.metadata
|
||||
and self.scenario in field.metadata["applicable_scenarios"]
|
||||
and field.metadata["preset_argument"]
|
||||
and getattr(self.load_params.preset, field.name) is not None
|
||||
]
|
||||
|
||||
command = " ".join(command_args)
|
||||
result = self.shell.exec(command)
|
||||
|
||||
assert (
|
||||
result.return_code == EXIT_RESULT_CODE
|
||||
), f"Return code of preset is not zero: {result.stdout}"
|
||||
return result.stdout.strip("\n")
|
||||
|
||||
@reporter.step_deco("Generate K6 command")
|
||||
def _generate_env_variables(self) -> str:
|
||||
env_vars = {
|
||||
field.metadata["env_variable"]: getattr(self.load_params, field.name)
|
||||
for field in fields(self.load_params)
|
||||
if field.metadata
|
||||
and self.scenario in field.metadata["applicable_scenarios"]
|
||||
and field.metadata["env_variable"]
|
||||
and getattr(self.load_params, field.name) is not None
|
||||
}
|
||||
|
||||
if self.load_params.preset:
|
||||
env_vars.update(
|
||||
{
|
||||
field.metadata["env_variable"]: getattr(self.load_params.preset, field.name)
|
||||
for field in fields(self.load_params.preset)
|
||||
if field.metadata
|
||||
and self.scenario in field.metadata["applicable_scenarios"]
|
||||
and field.metadata["env_variable"]
|
||||
and getattr(self.load_params.preset, field.name) is not None
|
||||
}
|
||||
)
|
||||
|
||||
env_vars[f"{self.load_params.load_type.value.upper()}_ENDPOINTS"] = ",".join(self.endpoints)
|
||||
env_vars["SUMMARY_JSON"] = self.summary_json
|
||||
|
||||
reporter.attach(
|
||||
"\n".join(f"{param}: {value}" for param, value in env_vars.items()), "K6 ENV variables"
|
||||
)
|
||||
return " ".join(
|
||||
[f"-e {param}='{value}'" for param, value in env_vars.items() if value is not None]
|
||||
)
|
||||
|
||||
@reporter.step_deco("Start K6 on initiator")
|
||||
def start(self) -> None:
|
||||
command = (
|
||||
f"{self._k6_dir}/k6 run {self._generate_env_variables()} "
|
||||
f"{self._k6_dir}/scenarios/{self.scenario.value}.js"
|
||||
)
|
||||
self._k6_process = RemoteProcess.create(command, self.shell, self.load_params.working_dir)
|
||||
|
||||
@reporter.step_deco("Wait until K6 is finished")
|
||||
def wait_until_finished(self, timeout: int = 0, k6_should_be_running: bool = False) -> None:
|
||||
wait_interval = 10
|
||||
if self._k6_process is None:
|
||||
assert "No k6 instances were executed"
|
||||
if k6_should_be_running:
|
||||
assert self._k6_process.running(), "k6 should be running."
|
||||
while timeout >= 0:
|
||||
if not self._k6_process.running():
|
||||
return
|
||||
logger.info(f"K6 is running. Waiting {wait_interval} seconds...")
|
||||
if timeout > 0:
|
||||
sleep(wait_interval)
|
||||
timeout -= wait_interval
|
||||
self._stop()
|
||||
raise TimeoutError(f"Expected K6 finished in {timeout} sec.")
|
||||
|
||||
def get_results(self) -> Any:
|
||||
with reporter.step(f"K6 results from {self.load_node}"):
|
||||
self.__log_output()
|
||||
|
||||
if not self.summary_json:
|
||||
return None
|
||||
|
||||
summary_text = self.shell.exec(f"cat {self.summary_json}").stdout
|
||||
summary_json = json.loads(summary_text)
|
||||
|
||||
allure_filenames = {
|
||||
K6ProcessAllocationStrategy.PER_LOAD_NODE: f"{self.load_node}_{self.scenario.value}_summary.json",
|
||||
K6ProcessAllocationStrategy.PER_ENDPOINT: f"{self.load_node}_{self.scenario.value}_{self.endpoints[0]}_summary.json",
|
||||
}
|
||||
allure_filename = allure_filenames[self.load_params.k6_process_allocation_strategy]
|
||||
|
||||
reporter.attach(summary_text, allure_filename)
|
||||
return summary_json
|
||||
|
||||
@reporter.step_deco("Assert K6 should be finished")
|
||||
def _k6_should_be_finished(self) -> None:
|
||||
k6_rc = self._k6_process.rc()
|
||||
assert k6_rc == 0, f"K6 unexpectedly finished with RC {k6_rc}"
|
||||
|
||||
@reporter.step_deco("Terminate K6 on initiator")
|
||||
def stop(self) -> None:
|
||||
if not self.is_running:
|
||||
self.get_results()
|
||||
raise AssertionError("K6 unexpectedly finished")
|
||||
|
||||
self._stop()
|
||||
|
||||
k6_rc = self._k6_process.rc()
|
||||
assert k6_rc == EXIT_RESULT_CODE, f"Return code of K6 job should be 0, but {k6_rc}"
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
if self._k6_process:
|
||||
return self._k6_process.running()
|
||||
return False
|
||||
|
||||
@reporter.step_deco("Try to stop K6 with SIGTERM")
|
||||
def _stop(self) -> None:
|
||||
self._k6_process.stop()
|
||||
with reporter.step("Wait until process end"):
|
||||
for _ in range(self._k6_stop_attempts):
|
||||
if not self._k6_process.running():
|
||||
break
|
||||
|
||||
sleep(self._k6_stop_check_interval)
|
||||
else:
|
||||
raise AssertionError("Can not stop K6 process within timeout")
|
||||
|
||||
def _kill(self) -> None:
|
||||
self._k6_process.kill()
|
||||
|
||||
def __log_output(self) -> None:
|
||||
reporter.attach(self._k6_process.stdout(full=True), "K6 stdout")
|
||||
reporter.attach(self._k6_process.stderr(full=True), "K6 stderr")
|
||||
211
src/frostfs_testlib/load/load_config.py
Normal file
211
src/frostfs_testlib/load/load_config.py
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class LoadType(Enum):
|
||||
gRPC = "grpc"
|
||||
S3 = "s3"
|
||||
HTTP = "http"
|
||||
|
||||
|
||||
class LoadScenario(Enum):
|
||||
gRPC = "grpc"
|
||||
gRPC_CAR = "grpc_car"
|
||||
S3 = "s3"
|
||||
S3_CAR = "s3_car"
|
||||
HTTP = "http"
|
||||
VERIFY = "verify"
|
||||
|
||||
|
||||
all_load_scenarios = [
|
||||
LoadScenario.gRPC,
|
||||
LoadScenario.S3,
|
||||
LoadScenario.HTTP,
|
||||
LoadScenario.S3_CAR,
|
||||
LoadScenario.gRPC_CAR,
|
||||
]
|
||||
all_scenarios = all_load_scenarios.copy() + [LoadScenario.VERIFY]
|
||||
|
||||
constant_vus_scenarios = [LoadScenario.gRPC, LoadScenario.S3, LoadScenario.HTTP]
|
||||
constant_arrival_rate_scenarios = [LoadScenario.gRPC_CAR, LoadScenario.S3_CAR]
|
||||
|
||||
grpc_preset_scenarios = [LoadScenario.gRPC, LoadScenario.HTTP, LoadScenario.gRPC_CAR]
|
||||
s3_preset_scenarios = [LoadScenario.S3, LoadScenario.S3_CAR]
|
||||
|
||||
|
||||
def metadata_field(
|
||||
applicable_scenarios: list[LoadScenario],
|
||||
preset_param: Optional[str] = None,
|
||||
scenario_variable: Optional[str] = None,
|
||||
distributed: Optional[bool] = False,
|
||||
):
|
||||
return field(
|
||||
default=None,
|
||||
metadata={
|
||||
"applicable_scenarios": applicable_scenarios,
|
||||
"preset_argument": preset_param,
|
||||
"env_variable": scenario_variable,
|
||||
"distributed": distributed,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class NodesSelectionStrategy(Enum):
|
||||
# Select ONE random node from cluster nodes.
|
||||
RANDOM_SINGLE = "RANDOM_SINGLE"
|
||||
# Select All nodes.
|
||||
ALL = "ALL"
|
||||
# Select All nodes except node under test (useful for failover). This is DEFAULT one
|
||||
ALL_EXCEPT_UNDER_TEST = "ALL_EXCEPT_UNDER_TEST"
|
||||
# Select ONE random node except under test (useful for failover).
|
||||
RANDOM_SINGLE_EXCEPT_UNDER_TEST = "RANDOM_SINGLE_EXCEPT_UNDER_TEST"
|
||||
|
||||
|
||||
class EndpointSelectionStrategy(Enum):
|
||||
"""Enum which defines which endpoint to select from each storage node"""
|
||||
|
||||
# Select All endpoints.
|
||||
ALL = "ALL"
|
||||
# Select first endpoint from node
|
||||
FIRST = "FIRST"
|
||||
|
||||
|
||||
class K6ProcessAllocationStrategy(Enum):
|
||||
"""Enum which defines how K6 processes should be allocated"""
|
||||
|
||||
# Each load node will get one k6 process with all endpoints (Default)
|
||||
PER_LOAD_NODE = "PER_LOAD_NODE"
|
||||
# Each endpoint will get it's own k6 process regardless of number of load nodes.
|
||||
# If there is not enough load nodes, some nodes may have multiple k6 processes
|
||||
PER_ENDPOINT = "PER_ENDPOINT"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Preset:
|
||||
# ------ COMMON ------
|
||||
# Amount of objects which should be created
|
||||
objects_count: Optional[int] = metadata_field(all_load_scenarios, "preload_obj", None)
|
||||
# Preset json. Filled automatically.
|
||||
pregen_json: Optional[str] = metadata_field(all_load_scenarios, "out", "PREGEN_JSON")
|
||||
# Workers count for preset
|
||||
workers: Optional[int] = metadata_field(all_load_scenarios, "workers", None)
|
||||
|
||||
# ------ GRPC ------
|
||||
# Amount of containers which should be created
|
||||
containers_count: Optional[int] = metadata_field(grpc_preset_scenarios, "containers", None)
|
||||
# Container placement policy for containers for gRPC
|
||||
container_placement_policy: Optional[str] = metadata_field(
|
||||
grpc_preset_scenarios, "policy", None
|
||||
)
|
||||
|
||||
# ------ S3 ------
|
||||
# Amount of buckets which should be created
|
||||
buckets_count: Optional[int] = metadata_field(s3_preset_scenarios, "buckets", None)
|
||||
# S3 region (AKA placement policy for S3 buckets)
|
||||
s3_location: Optional[str] = metadata_field(s3_preset_scenarios, "location", None)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadParams:
|
||||
# ------- CONTROL PARAMS -------
|
||||
# Load type can be gRPC, HTTP, S3.
|
||||
load_type: LoadType
|
||||
# Load scenario from k6 scenarios
|
||||
scenario: Optional[LoadScenario] = None
|
||||
# Strategy to select nodes under load. See NodesSelectionStrategy class for more details.
|
||||
# default is ALL_EXCEPT_UNDER_TEST
|
||||
nodes_selection_strategy: Optional[NodesSelectionStrategy] = None
|
||||
# Strategy which defines which endpoint to select from each storage node
|
||||
endpoint_selection_strategy: Optional[EndpointSelectionStrategy] = None
|
||||
# Strategy which defines how K6 processes should be allocated
|
||||
k6_process_allocation_strategy: Optional[K6ProcessAllocationStrategy] = None
|
||||
# Set to true in order to verify uploaded objects after K6 load finish. Default is True.
|
||||
verify: Optional[bool] = None
|
||||
# Just id for load so distinct it between runs. Filled automatically.
|
||||
load_id: Optional[str] = None
|
||||
# Working directory
|
||||
working_dir: Optional[str] = None
|
||||
# Preset for the k6 run
|
||||
preset: Optional[Preset] = None
|
||||
|
||||
# ------- COMMON SCENARIO PARAMS -------
|
||||
# Load time is the maximum duration for k6 to give load. Default is the BACKGROUND_LOAD_DEFAULT_TIME value.
|
||||
load_time: Optional[int] = metadata_field(all_load_scenarios, None, "DURATION")
|
||||
# Object size in KB for load and preset.
|
||||
object_size: Optional[int] = metadata_field(all_load_scenarios, "size", "WRITE_OBJ_SIZE")
|
||||
# Output registry K6 file. Filled automatically.
|
||||
registry_file: Optional[str] = metadata_field(all_scenarios, None, "REGISTRY_FILE")
|
||||
# Specifies the minimum duration of every single execution (i.e. iteration).
|
||||
# Any iterations that are shorter than this value will cause that VU to
|
||||
# sleep for the remainder of the time until the specified minimum duration is reached.
|
||||
min_iteration_duration: Optional[str] = metadata_field(
|
||||
all_load_scenarios, None, "K6_MIN_ITERATION_DURATION"
|
||||
)
|
||||
|
||||
# ------- CONSTANT VUS SCENARIO PARAMS -------
|
||||
# Amount of Writers VU.
|
||||
writers: Optional[int] = metadata_field(constant_vus_scenarios, None, "WRITERS", True)
|
||||
# Amount of Readers VU.
|
||||
readers: Optional[int] = metadata_field(constant_vus_scenarios, None, "READERS", True)
|
||||
# Amount of Deleters VU.
|
||||
deleters: Optional[int] = metadata_field(constant_vus_scenarios, None, "DELETERS", True)
|
||||
|
||||
# ------- CONSTANT ARRIVAL RATE SCENARIO PARAMS -------
|
||||
# Number of iterations to start during each timeUnit period for write.
|
||||
write_rate: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "WRITE_RATE", True
|
||||
)
|
||||
|
||||
# Number of iterations to start during each timeUnit period for read.
|
||||
read_rate: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "READ_RATE", True
|
||||
)
|
||||
|
||||
# Number of iterations to start during each timeUnit period for delete.
|
||||
delete_rate: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "DELETE_RATE", True
|
||||
)
|
||||
|
||||
# Amount of preAllocatedVUs for write operations.
|
||||
preallocated_writers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "PRE_ALLOC_WRITERS", True
|
||||
)
|
||||
# Amount of maxVUs for write operations.
|
||||
max_writers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "MAX_WRITERS", True
|
||||
)
|
||||
|
||||
# Amount of preAllocatedVUs for read operations.
|
||||
preallocated_readers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "PRE_ALLOC_READERS", True
|
||||
)
|
||||
# Amount of maxVUs for read operations.
|
||||
max_readers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "MAX_READERS", True
|
||||
)
|
||||
|
||||
# Amount of preAllocatedVUs for read operations.
|
||||
preallocated_deleters: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "PRE_ALLOC_DELETERS", True
|
||||
)
|
||||
# Amount of maxVUs for delete operations.
|
||||
max_deleters: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "MAX_DELETERS", True
|
||||
)
|
||||
|
||||
# Period of time to apply the rate value.
|
||||
time_unit: Optional[str] = metadata_field(constant_arrival_rate_scenarios, None, "TIME_UNIT")
|
||||
|
||||
# ------- VERIFY SCENARIO PARAMS -------
|
||||
# Maximum verification time for k6 to verify objects. Default is BACKGROUND_LOAD_MAX_VERIFY_TIME (3600).
|
||||
verify_time: Optional[int] = metadata_field([LoadScenario.VERIFY], None, "TIME_LIMIT")
|
||||
# Amount of Verification VU.
|
||||
clients: Optional[int] = metadata_field([LoadScenario.VERIFY], None, "CLIENTS")
|
||||
|
||||
def set_id(self, load_id):
|
||||
self.load_id = load_id
|
||||
self.registry_file = os.path.join(self.working_dir, f"{load_id}_registry.bolt")
|
||||
if self.preset:
|
||||
self.preset.pregen_json = os.path.join(self.working_dir, f"{load_id}_prepare.json")
|
||||
162
src/frostfs_testlib/load/load_metrics.py
Normal file
162
src/frostfs_testlib/load/load_metrics.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
from abc import ABC
|
||||
from typing import Any
|
||||
|
||||
from frostfs_testlib.load.load_config import LoadScenario
|
||||
|
||||
|
||||
class MetricsBase(ABC):
|
||||
_WRITE_SUCCESS = ""
|
||||
_WRITE_ERRORS = ""
|
||||
_WRITE_THROUGHPUT = "data_sent"
|
||||
|
||||
_READ_SUCCESS = ""
|
||||
_READ_ERRORS = ""
|
||||
_READ_THROUGHPUT = "data_received"
|
||||
|
||||
_DELETE_SUCCESS = ""
|
||||
_DELETE_ERRORS = ""
|
||||
|
||||
def __init__(self, summary) -> None:
|
||||
self.summary = summary
|
||||
self.metrics = summary["metrics"]
|
||||
|
||||
@property
|
||||
def write_total_iterations(self) -> int:
|
||||
return self._get_metric(self._WRITE_SUCCESS) + self._get_metric(self._WRITE_ERRORS)
|
||||
|
||||
@property
|
||||
def write_success_iterations(self) -> int:
|
||||
return self._get_metric(self._WRITE_SUCCESS)
|
||||
|
||||
@property
|
||||
def write_rate(self) -> float:
|
||||
return self._get_metric_rate(self._WRITE_SUCCESS)
|
||||
|
||||
@property
|
||||
def write_failed_iterations(self) -> int:
|
||||
return self._get_metric(self._WRITE_ERRORS)
|
||||
|
||||
@property
|
||||
def write_throughput(self) -> float:
|
||||
return self._get_metric_rate(self._WRITE_THROUGHPUT)
|
||||
|
||||
@property
|
||||
def read_total_iterations(self) -> int:
|
||||
return self._get_metric(self._READ_SUCCESS) + self._get_metric(self._READ_ERRORS)
|
||||
|
||||
@property
|
||||
def read_success_iterations(self) -> int:
|
||||
return self._get_metric(self._READ_SUCCESS)
|
||||
|
||||
@property
|
||||
def read_rate(self) -> int:
|
||||
return self._get_metric_rate(self._READ_SUCCESS)
|
||||
|
||||
@property
|
||||
def read_failed_iterations(self) -> int:
|
||||
return self._get_metric(self._READ_ERRORS)
|
||||
|
||||
@property
|
||||
def read_throughput(self) -> float:
|
||||
return self._get_metric_rate(self._READ_THROUGHPUT)
|
||||
|
||||
@property
|
||||
def delete_total_iterations(self) -> int:
|
||||
return self._get_metric(self._DELETE_SUCCESS) + self._get_metric(self._DELETE_ERRORS)
|
||||
|
||||
@property
|
||||
def delete_success_iterations(self) -> int:
|
||||
return self._get_metric(self._DELETE_SUCCESS)
|
||||
|
||||
@property
|
||||
def delete_failed_iterations(self) -> int:
|
||||
return self._get_metric(self._DELETE_ERRORS)
|
||||
|
||||
@property
|
||||
def delete_rate(self) -> int:
|
||||
return self._get_metric_rate(self._DELETE_SUCCESS)
|
||||
|
||||
def _get_metric(self, metric: str) -> int:
|
||||
metrics_method_map = {"counter": self._get_counter_metric, "gauge": self._get_gauge_metric}
|
||||
|
||||
if metric not in self.metrics:
|
||||
return 0
|
||||
|
||||
metric = self.metrics[metric]
|
||||
metric_type = metric["type"]
|
||||
if metric_type not in metrics_method_map:
|
||||
raise Exception(
|
||||
f"Unsupported metric type: {metric_type}, supported: {metrics_method_map.keys()}"
|
||||
)
|
||||
|
||||
return metrics_method_map[metric_type](metric)
|
||||
|
||||
def _get_metric_rate(self, metric: str) -> int:
|
||||
metrics_method_map = {"counter": self._get_counter_metric_rate}
|
||||
|
||||
if metric not in self.metrics:
|
||||
return 0
|
||||
|
||||
metric = self.metrics[metric]
|
||||
metric_type = metric["type"]
|
||||
if metric_type not in metrics_method_map:
|
||||
raise Exception(
|
||||
f"Unsupported rate metric type: {metric_type}, supported: {metrics_method_map.keys()}"
|
||||
)
|
||||
|
||||
return metrics_method_map[metric_type](metric)
|
||||
|
||||
def _get_counter_metric_rate(self, metric: str) -> int:
|
||||
return metric["values"]["rate"]
|
||||
|
||||
def _get_counter_metric(self, metric: str) -> int:
|
||||
return metric["values"]["count"]
|
||||
|
||||
def _get_gauge_metric(self, metric: str) -> int:
|
||||
return metric["values"]["value"]
|
||||
|
||||
|
||||
class GrpcMetrics(MetricsBase):
|
||||
_WRITE_SUCCESS = "frostfs_obj_put_total"
|
||||
_WRITE_ERRORS = "frostfs_obj_put_fails"
|
||||
|
||||
_READ_SUCCESS = "frostfs_obj_get_total"
|
||||
_READ_ERRORS = "frostfs_obj_get_fails"
|
||||
|
||||
_DELETE_SUCCESS = "frostfs_obj_delete_total"
|
||||
_DELETE_ERRORS = "frostfs_obj_delete_fails"
|
||||
|
||||
|
||||
class S3Metrics(MetricsBase):
|
||||
_WRITE_SUCCESS = "aws_obj_put_total"
|
||||
_WRITE_ERRORS = "aws_obj_put_fails"
|
||||
|
||||
_READ_SUCCESS = "aws_obj_get_total"
|
||||
_READ_ERRORS = "aws_obj_get_fails"
|
||||
|
||||
_DELETE_SUCCESS = "aws_obj_delete_total"
|
||||
_DELETE_ERRORS = "aws_obj_delete_fails"
|
||||
|
||||
|
||||
class VerifyMetrics(MetricsBase):
|
||||
_WRITE_SUCCESS = "N/A"
|
||||
_WRITE_ERRORS = "N/A"
|
||||
|
||||
_READ_SUCCESS = "verified_obj"
|
||||
_READ_ERRORS = "invalid_obj"
|
||||
|
||||
_DELETE_SUCCESS = "N/A"
|
||||
_DELETE_ERRORS = "N/A"
|
||||
|
||||
|
||||
def get_metrics_object(load_type: LoadScenario, summary: dict[str, Any]) -> MetricsBase:
|
||||
class_map = {
|
||||
LoadScenario.gRPC: GrpcMetrics,
|
||||
LoadScenario.gRPC_CAR: GrpcMetrics,
|
||||
LoadScenario.HTTP: GrpcMetrics,
|
||||
LoadScenario.S3: S3Metrics,
|
||||
LoadScenario.S3_CAR: S3Metrics,
|
||||
LoadScenario.VERIFY: VerifyMetrics,
|
||||
}
|
||||
|
||||
return class_map[load_type](summary)
|
||||
265
src/frostfs_testlib/load/load_report.py
Normal file
265
src/frostfs_testlib/load/load_report.py
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
from datetime import datetime
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import yaml
|
||||
|
||||
from frostfs_testlib.load.load_config import K6ProcessAllocationStrategy, LoadParams, LoadScenario
|
||||
from frostfs_testlib.load.load_metrics import get_metrics_object
|
||||
|
||||
|
||||
class LoadReport:
|
||||
def __init__(self, load_test) -> None:
|
||||
self.load_test = load_test
|
||||
self.load_summaries: Optional[dict] = None
|
||||
self.load_params: Optional[LoadParams] = None
|
||||
self.start_time: Optional[datetime] = None
|
||||
self.end_time: Optional[datetime] = None
|
||||
|
||||
def set_start_time(self):
|
||||
self.start_time = datetime.utcnow()
|
||||
|
||||
def set_end_time(self):
|
||||
self.end_time = datetime.utcnow()
|
||||
|
||||
def set_summaries(self, load_summaries: dict):
|
||||
self.load_summaries = load_summaries
|
||||
|
||||
def set_load_params(self, load_params: LoadParams):
|
||||
self.load_params = load_params
|
||||
|
||||
def get_report_html(self):
|
||||
report_sections = [
|
||||
[self.load_test, self._get_load_params_section_html],
|
||||
[self.load_summaries, self._get_totals_section_html],
|
||||
[self.end_time, self._get_test_time_html],
|
||||
]
|
||||
|
||||
html = ""
|
||||
for section in report_sections:
|
||||
if section[0] is not None:
|
||||
html += section[1]()
|
||||
|
||||
return html
|
||||
|
||||
def _get_load_params_section_html(self) -> str:
|
||||
params: str = yaml.safe_dump(self.load_test, sort_keys=False)
|
||||
params = params.replace("\n", "<br>")
|
||||
section_html = f"""<h3>Scenario params</h3>
|
||||
|
||||
<pre>{params}</pre>
|
||||
<hr>"""
|
||||
|
||||
return section_html
|
||||
|
||||
def _get_test_time_html(self) -> str:
|
||||
html = f"""<h3>Scenario duration in UTC time (from agent)</h3>
|
||||
{self.start_time} - {self.end_time}<br>
|
||||
<hr>
|
||||
"""
|
||||
|
||||
return html
|
||||
|
||||
def _calc_unit(self, value: float, skip_units: int = 0) -> Tuple[float, str]:
|
||||
units = ["B", "KB", "MB", "GB", "TB"]
|
||||
|
||||
for unit in units[skip_units:]:
|
||||
if value < 1024:
|
||||
return value, unit
|
||||
|
||||
value = value / 1024.0
|
||||
|
||||
return value, unit
|
||||
|
||||
def _seconds_to_formatted_duration(self, seconds: int) -> str:
|
||||
"""Converts N number of seconds to formatted output ignoring zeroes.
|
||||
Examples:
|
||||
186399 -> "2d3h46m39s"
|
||||
86399 -> "23h59m59s"
|
||||
86399 -> "23h59m59s"
|
||||
3605 -> "1h5s"
|
||||
123 -> "2m3s"
|
||||
"""
|
||||
units = {"d": 86400, "h": 3600, "m": 60, "s": 1}
|
||||
parts = []
|
||||
remaining = seconds
|
||||
for divisor in units.values():
|
||||
part = remaining // divisor
|
||||
remaining -= divisor * part
|
||||
parts.append(part)
|
||||
|
||||
return "".join([f"{val}{unit}" for unit, val in zip(units, parts) if val > 0])
|
||||
|
||||
def _row(self, caption: str, value: str) -> str:
|
||||
return f"<tr><th>{caption}</th><td>{value}</td></tr>"
|
||||
|
||||
def _get_model_string(self):
|
||||
if self.load_params.min_iteration_duration is not None:
|
||||
return f"min_iteration_duration={self.load_params.min_iteration_duration}"
|
||||
|
||||
model_map = {
|
||||
LoadScenario.gRPC: "closed model",
|
||||
LoadScenario.S3: "closed model",
|
||||
LoadScenario.HTTP: "closed model",
|
||||
LoadScenario.gRPC_CAR: "open model",
|
||||
LoadScenario.S3_CAR: "open model",
|
||||
}
|
||||
|
||||
return model_map[self.load_params.scenario]
|
||||
|
||||
def _get_oprations_sub_section_html(
|
||||
self,
|
||||
operation_type: str,
|
||||
total_operations: int,
|
||||
requested_rate_str: str,
|
||||
vus_str: str,
|
||||
total_rate: float,
|
||||
throughput: float,
|
||||
errors: dict[str, int],
|
||||
):
|
||||
throughput_html = ""
|
||||
if throughput > 0:
|
||||
throughput, unit = self._calc_unit(throughput)
|
||||
throughput_html = self._row("Throughput", f"{throughput:.2f} {unit}/sec")
|
||||
|
||||
per_node_errors_html = ""
|
||||
total_errors = 0
|
||||
if errors:
|
||||
total_errors: int = 0
|
||||
for node_key, errors in errors.items():
|
||||
total_errors += errors
|
||||
if (
|
||||
self.load_params.k6_process_allocation_strategy
|
||||
== K6ProcessAllocationStrategy.PER_ENDPOINT
|
||||
):
|
||||
per_node_errors_html += self._row(f"At {node_key}", errors)
|
||||
|
||||
object_size, object_size_unit = self._calc_unit(self.load_params.object_size, 1)
|
||||
duration = self._seconds_to_formatted_duration(self.load_params.load_time)
|
||||
model = self._get_model_string()
|
||||
# write 8KB 15h49m 50op/sec 50th open model/closed model/min_iteration duration=1s - 1.636MB/s 199.57451/s
|
||||
short_summary = f"{operation_type} {object_size}{object_size_unit} {duration} {requested_rate_str} {vus_str} {model} - {throughput:.2f}{unit} {total_rate:.2f}/s"
|
||||
|
||||
html = f"""
|
||||
<table border="1" cellpadding="5px"><tbody>
|
||||
<th colspan="2" bgcolor="gainsboro">{short_summary}</th>
|
||||
<th colspan="2" bgcolor="gainsboro">Metrics</th>
|
||||
{self._row("Total operations", total_operations)}
|
||||
{self._row("OP/sec", f"{total_rate:.2f}")}
|
||||
{throughput_html}
|
||||
|
||||
<th colspan="2" bgcolor="gainsboro">Errors</th>
|
||||
{per_node_errors_html}
|
||||
{self._row("Total", f"{total_errors} ({total_errors/total_operations*100.0:.2f}%)")}
|
||||
</tbody></table><br><hr>
|
||||
"""
|
||||
|
||||
return html
|
||||
|
||||
def _get_totals_section_html(self):
|
||||
|
||||
html = "<h3>Load Results</h3>"
|
||||
|
||||
write_operations = 0
|
||||
write_op_sec = 0
|
||||
write_throughput = 0
|
||||
write_errors = {}
|
||||
requested_write_rate = self.load_params.write_rate
|
||||
requested_write_rate_str = f"{requested_write_rate}op/sec" if requested_write_rate else ""
|
||||
|
||||
read_operations = 0
|
||||
read_op_sec = 0
|
||||
read_throughput = 0
|
||||
read_errors = {}
|
||||
requested_read_rate = self.load_params.read_rate
|
||||
requested_read_rate_str = f"{requested_read_rate}op/sec" if requested_read_rate else ""
|
||||
|
||||
delete_operations = 0
|
||||
delete_op_sec = 0
|
||||
delete_errors = {}
|
||||
requested_delete_rate = self.load_params.delete_rate
|
||||
requested_delete_rate_str = (
|
||||
f"{requested_delete_rate}op/sec" if requested_delete_rate else ""
|
||||
)
|
||||
|
||||
if self.load_params.scenario in [LoadScenario.gRPC_CAR, LoadScenario.S3_CAR]:
|
||||
delete_vus = max(
|
||||
self.load_params.preallocated_deleters or 0, self.load_params.max_deleters or 0
|
||||
)
|
||||
write_vus = max(
|
||||
self.load_params.preallocated_writers or 0, self.load_params.max_writers or 0
|
||||
)
|
||||
read_vus = max(
|
||||
self.load_params.preallocated_readers or 0, self.load_params.max_readers or 0
|
||||
)
|
||||
else:
|
||||
write_vus = self.load_params.writers
|
||||
read_vus = self.load_params.readers
|
||||
delete_vus = self.load_params.deleters
|
||||
|
||||
write_vus_str = f"{write_vus}th"
|
||||
read_vus_str = f"{read_vus}th"
|
||||
delete_vus_str = f"{delete_vus}th"
|
||||
|
||||
write_section_required = False
|
||||
read_section_required = False
|
||||
delete_section_required = False
|
||||
|
||||
for node_key, load_summary in self.load_summaries.items():
|
||||
metrics = get_metrics_object(self.load_params.scenario, load_summary)
|
||||
write_operations += metrics.write_total_iterations
|
||||
if write_operations:
|
||||
write_section_required = True
|
||||
write_op_sec += metrics.write_rate
|
||||
write_throughput += metrics.write_throughput
|
||||
if metrics.write_failed_iterations:
|
||||
write_errors[node_key] = metrics.write_failed_iterations
|
||||
|
||||
read_operations += metrics.read_total_iterations
|
||||
if read_operations:
|
||||
read_section_required = True
|
||||
read_op_sec += metrics.read_rate
|
||||
read_throughput += metrics.read_throughput
|
||||
if metrics.read_failed_iterations:
|
||||
read_errors[node_key] = metrics.read_failed_iterations
|
||||
|
||||
delete_operations += metrics.delete_total_iterations
|
||||
if delete_operations:
|
||||
delete_section_required = True
|
||||
delete_op_sec += metrics.delete_rate
|
||||
if metrics.delete_failed_iterations:
|
||||
delete_errors[node_key] = metrics.delete_failed_iterations
|
||||
|
||||
if write_section_required:
|
||||
html += self._get_oprations_sub_section_html(
|
||||
"Write",
|
||||
write_operations,
|
||||
requested_write_rate_str,
|
||||
write_vus_str,
|
||||
write_op_sec,
|
||||
write_throughput,
|
||||
write_errors,
|
||||
)
|
||||
|
||||
if read_section_required:
|
||||
html += self._get_oprations_sub_section_html(
|
||||
"Read",
|
||||
read_operations,
|
||||
requested_read_rate_str,
|
||||
read_vus_str,
|
||||
read_op_sec,
|
||||
read_throughput,
|
||||
read_errors,
|
||||
)
|
||||
|
||||
if delete_section_required:
|
||||
html += self._get_oprations_sub_section_html(
|
||||
"Delete",
|
||||
delete_operations,
|
||||
requested_delete_rate_str,
|
||||
delete_vus_str,
|
||||
delete_op_sec,
|
||||
0,
|
||||
delete_errors,
|
||||
)
|
||||
|
||||
return html
|
||||
184
src/frostfs_testlib/load/load_steps.py
Normal file
184
src/frostfs_testlib/load/load_steps.py
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
import copy
|
||||
import itertools
|
||||
import math
|
||||
import re
|
||||
from dataclasses import fields
|
||||
|
||||
from frostfs_testlib.cli import FrostfsAuthmate
|
||||
from frostfs_testlib.load.k6 import K6
|
||||
from frostfs_testlib.load.load_config import K6ProcessAllocationStrategy, LoadParams
|
||||
from frostfs_testlib.reporter import get_reporter
|
||||
from frostfs_testlib.resources.cli import FROSTFS_AUTHMATE_EXEC
|
||||
from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_VUS_COUNT_DIVISOR
|
||||
from frostfs_testlib.shell import CommandOptions, SSHShell
|
||||
from frostfs_testlib.shell.interfaces import InteractiveInput, SshCredentials
|
||||
from frostfs_testlib.storage.cluster import ClusterNode
|
||||
from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate, StorageNode
|
||||
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
|
||||
|
||||
reporter = get_reporter()
|
||||
|
||||
STOPPED_HOSTS = []
|
||||
|
||||
|
||||
@reporter.step_deco("Init s3 client on load nodes")
|
||||
def init_s3_client(
|
||||
load_nodes: list[str],
|
||||
load_params: LoadParams,
|
||||
k6_directory: str,
|
||||
ssh_credentials: SshCredentials,
|
||||
nodes_under_load: list[ClusterNode],
|
||||
wallet: WalletInfo,
|
||||
):
|
||||
storage_node = nodes_under_load[0].service(StorageNode)
|
||||
s3_public_keys = [node.service(S3Gate).get_wallet_public_key() for node in nodes_under_load]
|
||||
grpc_peer = storage_node.get_rpc_endpoint()
|
||||
|
||||
for load_node in load_nodes:
|
||||
ssh_client = _get_ssh_client(ssh_credentials, load_node)
|
||||
frostfs_authmate_exec: FrostfsAuthmate = FrostfsAuthmate(ssh_client, FROSTFS_AUTHMATE_EXEC)
|
||||
issue_secret_output = frostfs_authmate_exec.secret.issue(
|
||||
wallet=wallet.path,
|
||||
peer=grpc_peer,
|
||||
bearer_rules=f"{k6_directory}/scenarios/files/rules.json",
|
||||
gate_public_key=s3_public_keys,
|
||||
container_placement_policy=load_params.preset.container_placement_policy,
|
||||
container_policy=f"{k6_directory}/scenarios/files/policy.json",
|
||||
wallet_password=wallet.password,
|
||||
).stdout
|
||||
aws_access_key_id = str(
|
||||
re.search(r"access_key_id.*:\s.(?P<aws_access_key_id>\w*)", issue_secret_output).group(
|
||||
"aws_access_key_id"
|
||||
)
|
||||
)
|
||||
aws_secret_access_key = str(
|
||||
re.search(
|
||||
r"secret_access_key.*:\s.(?P<aws_secret_access_key>\w*)", issue_secret_output
|
||||
).group("aws_secret_access_key")
|
||||
)
|
||||
# prompt_pattern doesn't work at the moment
|
||||
configure_input = [
|
||||
InteractiveInput(prompt_pattern=r"AWS Access Key ID.*", input=aws_access_key_id),
|
||||
InteractiveInput(
|
||||
prompt_pattern=r"AWS Secret Access Key.*", input=aws_secret_access_key
|
||||
),
|
||||
InteractiveInput(prompt_pattern=r".*", input=""),
|
||||
InteractiveInput(prompt_pattern=r".*", input=""),
|
||||
]
|
||||
ssh_client.exec("aws configure", CommandOptions(interactive_inputs=configure_input))
|
||||
|
||||
|
||||
@reporter.step_deco("Prepare K6 instances and objects")
|
||||
def prepare_k6_instances(
|
||||
load_nodes: list[str],
|
||||
ssh_credentials: SshCredentials,
|
||||
k6_dir: str,
|
||||
load_params: LoadParams,
|
||||
endpoints: list[str],
|
||||
loaders_wallet: WalletInfo,
|
||||
) -> list[K6]:
|
||||
k6_load_objects: list[K6] = []
|
||||
nodes = itertools.cycle(load_nodes)
|
||||
|
||||
k6_distribution_count = {
|
||||
K6ProcessAllocationStrategy.PER_LOAD_NODE: len(load_nodes),
|
||||
K6ProcessAllocationStrategy.PER_ENDPOINT: len(endpoints),
|
||||
}
|
||||
endpoints_generators = {
|
||||
K6ProcessAllocationStrategy.PER_LOAD_NODE: itertools.cycle([endpoints]),
|
||||
K6ProcessAllocationStrategy.PER_ENDPOINT: itertools.cycle(
|
||||
[[endpoint] for endpoint in endpoints]
|
||||
),
|
||||
}
|
||||
k6_processes_count = k6_distribution_count[load_params.k6_process_allocation_strategy]
|
||||
endpoints_gen = endpoints_generators[load_params.k6_process_allocation_strategy]
|
||||
|
||||
distributed_load_params_list = _get_distributed_load_params_list(
|
||||
load_params, k6_processes_count
|
||||
)
|
||||
|
||||
for distributed_load_params in distributed_load_params_list:
|
||||
load_node = next(nodes)
|
||||
ssh_client = _get_ssh_client(ssh_credentials, load_node)
|
||||
k6_load_object = K6(
|
||||
distributed_load_params,
|
||||
next(endpoints_gen),
|
||||
k6_dir,
|
||||
ssh_client,
|
||||
load_node,
|
||||
loaders_wallet,
|
||||
)
|
||||
k6_load_objects.append(k6_load_object)
|
||||
if load_params.preset:
|
||||
k6_load_object.preset()
|
||||
|
||||
return k6_load_objects
|
||||
|
||||
|
||||
def _get_ssh_client(ssh_credentials: SshCredentials, load_node: str):
|
||||
ssh_client = SSHShell(
|
||||
host=load_node,
|
||||
login=ssh_credentials.ssh_login,
|
||||
password=ssh_credentials.ssh_password,
|
||||
private_key_path=ssh_credentials.ssh_key_path,
|
||||
private_key_passphrase=ssh_credentials.ssh_key_passphrase,
|
||||
)
|
||||
|
||||
return ssh_client
|
||||
|
||||
|
||||
def _get_distributed_load_params_list(
|
||||
original_load_params: LoadParams, workers_count: int
|
||||
) -> list[LoadParams]:
|
||||
divisor = int(BACKGROUND_LOAD_VUS_COUNT_DIVISOR)
|
||||
distributed_load_params: list[LoadParams] = []
|
||||
|
||||
for i in range(workers_count):
|
||||
load_params = copy.deepcopy(original_load_params)
|
||||
# Append #i here in case if multiple k6 processes goes into same load node
|
||||
load_params.set_id(f"{load_params.load_id}_{i}")
|
||||
distributed_load_params.append(load_params)
|
||||
|
||||
load_fields = fields(original_load_params)
|
||||
|
||||
for field in load_fields:
|
||||
if (
|
||||
field.metadata
|
||||
and original_load_params.scenario in field.metadata["applicable_scenarios"]
|
||||
and field.metadata["distributed"]
|
||||
and getattr(original_load_params, field.name) is not None
|
||||
):
|
||||
original_value = getattr(original_load_params, field.name)
|
||||
distribution = _get_distribution(math.ceil(original_value / divisor), workers_count)
|
||||
for i in range(workers_count):
|
||||
setattr(distributed_load_params[i], field.name, distribution[i])
|
||||
|
||||
return distributed_load_params
|
||||
|
||||
|
||||
def _get_distribution(clients_count: int, workers_count: int) -> list[int]:
|
||||
"""
|
||||
This function will distribute evenly as possible X clients to Y workers.
|
||||
For example if we have 150 readers (clients) and we want to spread it over 4 load nodes (workers)
|
||||
this will return [38, 38, 37, 37].
|
||||
|
||||
Args:
|
||||
clients_count: amount of things needs to be distributed.
|
||||
workers_count: amount of workers.
|
||||
|
||||
Returns:
|
||||
list of distribution.
|
||||
"""
|
||||
if workers_count < 1:
|
||||
raise Exception("Workers cannot be less then 1")
|
||||
|
||||
# Amount of guaranteed payload on one worker
|
||||
clients_per_worker = clients_count // workers_count
|
||||
# Remainder of clients left to be distributed
|
||||
remainder = clients_count - clients_per_worker * workers_count
|
||||
|
||||
distribution = [
|
||||
clients_per_worker + 1 if i < remainder else clients_per_worker
|
||||
for i in range(workers_count)
|
||||
]
|
||||
return distribution
|
||||
36
src/frostfs_testlib/load/load_verifiers.py
Normal file
36
src/frostfs_testlib/load/load_verifiers.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import logging
|
||||
|
||||
from frostfs_testlib.load.load_config import LoadParams, LoadScenario
|
||||
from frostfs_testlib.load.load_metrics import get_metrics_object
|
||||
|
||||
logger = logging.getLogger("NeoLogger")
|
||||
|
||||
|
||||
class LoadVerifier:
|
||||
def __init__(self, load_params: LoadParams) -> None:
|
||||
self.load_params = load_params
|
||||
|
||||
def verify_summaries(self, load_summary, verification_summary) -> None:
|
||||
if not verification_summary or not load_summary:
|
||||
logger.info("Can't check load results due to missing summary")
|
||||
|
||||
load_metrics = get_metrics_object(self.load_params.scenario, load_summary)
|
||||
writers = self.load_params.writers or 0
|
||||
|
||||
objects_count = load_metrics.write_success_iterations
|
||||
fails_count = load_metrics.write_failed_iterations
|
||||
|
||||
if writers > 0:
|
||||
assert objects_count > 0, "Total put objects should be greater than 0"
|
||||
assert fails_count == 0, f"There were {fails_count} failed put objects operations"
|
||||
|
||||
if verification_summary:
|
||||
verify_metrics = get_metrics_object(LoadScenario.VERIFY, verification_summary)
|
||||
verified_objects = verify_metrics.read_success_iterations
|
||||
invalid_objects = verify_metrics.read_failed_iterations
|
||||
|
||||
assert invalid_objects == 0, f"There were {invalid_objects} verification fails"
|
||||
# Due to interruptions we may see total verified objects to be less than written on writers count
|
||||
assert (
|
||||
abs(objects_count - verified_objects) <= writers
|
||||
), f"Verified objects is less than total objects. Total: {objects_count}, Verified: {verified_objects}. Writers: {writers}."
|
||||
Loading…
Add table
Add a link
Reference in a new issue