forked from TrueCloudLab/frostfs-testlib
[#139] Use readers for init time calculation
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>master
parent
81dfc723da
commit
ae566b413b
|
|
@ -40,11 +40,18 @@ all_load_scenarios = [
|
|||
LoadScenario.gRPC_CAR,
|
||||
LoadScenario.LOCAL,
|
||||
LoadScenario.S3_MULTIPART,
|
||||
LoadScenario.S3_LOCAL
|
||||
LoadScenario.S3_LOCAL,
|
||||
]
|
||||
all_scenarios = all_load_scenarios.copy() + [LoadScenario.VERIFY]
|
||||
|
||||
constant_vus_scenarios = [LoadScenario.gRPC, LoadScenario.S3, LoadScenario.HTTP, LoadScenario.LOCAL, LoadScenario.S3_MULTIPART, LoadScenario.S3_LOCAL]
|
||||
constant_vus_scenarios = [
|
||||
LoadScenario.gRPC,
|
||||
LoadScenario.S3,
|
||||
LoadScenario.HTTP,
|
||||
LoadScenario.LOCAL,
|
||||
LoadScenario.S3_MULTIPART,
|
||||
LoadScenario.S3_LOCAL,
|
||||
]
|
||||
constant_arrival_rate_scenarios = [LoadScenario.gRPC_CAR, LoadScenario.S3_CAR]
|
||||
|
||||
grpc_preset_scenarios = [
|
||||
|
|
@ -124,13 +131,9 @@ class Preset:
|
|||
|
||||
# ------ GRPC ------
|
||||
# Amount of containers which should be created
|
||||
containers_count: Optional[int] = metadata_field(
|
||||
grpc_preset_scenarios, "containers", None, False
|
||||
)
|
||||
containers_count: Optional[int] = metadata_field(grpc_preset_scenarios, "containers", None, False)
|
||||
# Container placement policy for containers for gRPC
|
||||
container_placement_policy: Optional[str] = metadata_field(
|
||||
grpc_preset_scenarios, "policy", None, False
|
||||
)
|
||||
container_placement_policy: Optional[str] = metadata_field(grpc_preset_scenarios, "policy", None, False)
|
||||
|
||||
# ------ S3 ------
|
||||
# Amount of buckets which should be created
|
||||
|
|
@ -180,7 +183,14 @@ class LoadParams:
|
|||
awscli_url: Optional[str] = None
|
||||
# No ssl verification flag
|
||||
no_verify_ssl: Optional[bool] = metadata_field(
|
||||
[LoadScenario.S3, LoadScenario.S3_CAR, LoadScenario.S3_MULTIPART, LoadScenario.S3_LOCAL, LoadScenario.VERIFY, LoadScenario.HTTP],
|
||||
[
|
||||
LoadScenario.S3,
|
||||
LoadScenario.S3_CAR,
|
||||
LoadScenario.S3_MULTIPART,
|
||||
LoadScenario.S3_LOCAL,
|
||||
LoadScenario.VERIFY,
|
||||
LoadScenario.HTTP,
|
||||
],
|
||||
"no-verify-ssl",
|
||||
"NO_VERIFY_SSL",
|
||||
False,
|
||||
|
|
@ -198,9 +208,7 @@ class LoadParams:
|
|||
# Specifies the minimum duration of every single execution (i.e. iteration).
|
||||
# Any iterations that are shorter than this value will cause that VU to
|
||||
# sleep for the remainder of the time until the specified minimum duration is reached.
|
||||
min_iteration_duration: Optional[str] = metadata_field(
|
||||
all_load_scenarios, None, "K6_MIN_ITERATION_DURATION", False
|
||||
)
|
||||
min_iteration_duration: Optional[str] = metadata_field(all_load_scenarios, None, "K6_MIN_ITERATION_DURATION", False)
|
||||
# Prepare/cut objects locally on client before sending
|
||||
prepare_locally: Optional[bool] = metadata_field(
|
||||
[LoadScenario.gRPC, LoadScenario.gRPC_CAR], None, "PREPARE_LOCALLY", False
|
||||
|
|
@ -225,46 +233,34 @@ class LoadParams:
|
|||
|
||||
# ------- CONSTANT ARRIVAL RATE SCENARIO PARAMS -------
|
||||
# Number of iterations to start during each timeUnit period for write.
|
||||
write_rate: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "WRITE_RATE", True, True
|
||||
)
|
||||
write_rate: Optional[int] = metadata_field(constant_arrival_rate_scenarios, None, "WRITE_RATE", True, True)
|
||||
|
||||
# Number of iterations to start during each timeUnit period for read.
|
||||
read_rate: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "READ_RATE", True, True
|
||||
)
|
||||
read_rate: Optional[int] = metadata_field(constant_arrival_rate_scenarios, None, "READ_RATE", True, True)
|
||||
|
||||
# Number of iterations to start during each timeUnit period for delete.
|
||||
delete_rate: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "DELETE_RATE", True, True
|
||||
)
|
||||
delete_rate: Optional[int] = metadata_field(constant_arrival_rate_scenarios, None, "DELETE_RATE", True, True)
|
||||
|
||||
# Amount of preAllocatedVUs for write operations.
|
||||
preallocated_writers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "PRE_ALLOC_WRITERS", True, True
|
||||
)
|
||||
# Amount of maxVUs for write operations.
|
||||
max_writers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "MAX_WRITERS", False, True
|
||||
)
|
||||
max_writers: Optional[int] = metadata_field(constant_arrival_rate_scenarios, None, "MAX_WRITERS", False, True)
|
||||
|
||||
# Amount of preAllocatedVUs for read operations.
|
||||
preallocated_readers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "PRE_ALLOC_READERS", True, True
|
||||
)
|
||||
# Amount of maxVUs for read operations.
|
||||
max_readers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "MAX_READERS", False, True
|
||||
)
|
||||
max_readers: Optional[int] = metadata_field(constant_arrival_rate_scenarios, None, "MAX_READERS", False, True)
|
||||
|
||||
# Amount of preAllocatedVUs for read operations.
|
||||
preallocated_deleters: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "PRE_ALLOC_DELETERS", True, True
|
||||
)
|
||||
# Amount of maxVUs for delete operations.
|
||||
max_deleters: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "MAX_DELETERS", False, True
|
||||
)
|
||||
max_deleters: Optional[int] = metadata_field(constant_arrival_rate_scenarios, None, "MAX_DELETERS", False, True)
|
||||
|
||||
# Multipart
|
||||
# Number of parts to upload in parallel
|
||||
|
|
@ -272,20 +268,18 @@ class LoadParams:
|
|||
[LoadScenario.S3_MULTIPART], None, "WRITERS_MULTIPART", False, True
|
||||
)
|
||||
# part size must be greater than (5 MB)
|
||||
write_object_part_size: Optional[int] = metadata_field([LoadScenario.S3_MULTIPART], None, "WRITE_OBJ_PART_SIZE", False)
|
||||
write_object_part_size: Optional[int] = metadata_field(
|
||||
[LoadScenario.S3_MULTIPART], None, "WRITE_OBJ_PART_SIZE", False
|
||||
)
|
||||
|
||||
# Period of time to apply the rate value.
|
||||
time_unit: Optional[str] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "TIME_UNIT", False
|
||||
)
|
||||
time_unit: Optional[str] = metadata_field(constant_arrival_rate_scenarios, None, "TIME_UNIT", False)
|
||||
|
||||
# ------- VERIFY SCENARIO PARAMS -------
|
||||
# Maximum verification time for k6 to verify objects. Default is BACKGROUND_LOAD_MAX_VERIFY_TIME (3600).
|
||||
verify_time: Optional[int] = metadata_field([LoadScenario.VERIFY], None, "TIME_LIMIT", False)
|
||||
# Amount of Verification VU.
|
||||
verify_clients: Optional[int] = metadata_field(
|
||||
[LoadScenario.VERIFY], None, "CLIENTS", True, False
|
||||
)
|
||||
verify_clients: Optional[int] = metadata_field([LoadScenario.VERIFY], None, "CLIENTS", True, False)
|
||||
|
||||
# ------- LOCAL SCENARIO PARAMS -------
|
||||
# Config file location (filled automatically)
|
||||
|
|
@ -341,10 +335,8 @@ class LoadParams:
|
|||
return math.ceil(self._get_total_vus() * self.vu_init_time)
|
||||
|
||||
def _get_total_vus(self) -> int:
|
||||
vu_fields = ["writers", "preallocated_writers"]
|
||||
data_fields = [
|
||||
getattr(self, field.name) or 0 for field in fields(self) if field.name in vu_fields
|
||||
]
|
||||
vu_fields = ["writers", "preallocated_writers", "readers", "preallocated_readers"]
|
||||
data_fields = [getattr(self, field.name) or 0 for field in fields(self) if field.name in vu_fields]
|
||||
return sum(data_fields)
|
||||
|
||||
def _get_applicable_fields(self):
|
||||
|
|
@ -375,9 +367,7 @@ class LoadParams:
|
|||
]
|
||||
|
||||
for field in data_fields:
|
||||
actual_field_type = (
|
||||
get_args(field.type)[0] if len(get_args(field.type)) else get_args(field.type)
|
||||
)
|
||||
actual_field_type = get_args(field.type)[0] if len(get_args(field.type)) else get_args(field.type)
|
||||
if is_dataclass(actual_field_type) and getattr(instance, field.name):
|
||||
fields_with_data += LoadParams._get_meta_fields(getattr(instance, field.name))
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from frostfs_testlib.load.load_config import (
|
|||
ReadFrom,
|
||||
)
|
||||
from frostfs_testlib.load.runners import DefaultRunner
|
||||
from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_DEFAULT_VU_INIT_TIME
|
||||
from frostfs_testlib.storage.cluster import ClusterNode
|
||||
from frostfs_testlib.storage.controllers.background_load_controller import BackgroundLoadController
|
||||
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
|
||||
|
|
@ -53,6 +54,25 @@ class TestLoadConfig:
|
|||
assert repr(load_params) == expected
|
||||
assert f"{load_params}" == expected
|
||||
|
||||
def test_load_params_init_time(self):
|
||||
load_params = LoadParams(load_type=LoadType.S3)
|
||||
vus = 100
|
||||
|
||||
load_params.vu_init_time = BACKGROUND_LOAD_DEFAULT_VU_INIT_TIME
|
||||
# Used in time calculations
|
||||
load_params.readers = vus
|
||||
load_params.writers = vus
|
||||
load_params.preallocated_readers = vus
|
||||
load_params.preallocated_writers = vus
|
||||
|
||||
# Not used in time calculations
|
||||
load_params.deleters = vus
|
||||
load_params.preallocated_deleters = vus
|
||||
|
||||
expected = vus * 4 * BACKGROUND_LOAD_DEFAULT_VU_INIT_TIME
|
||||
actual = load_params.get_init_time()
|
||||
assert actual == expected, "Incorrect time for get_init_time()"
|
||||
|
||||
def test_load_params_initially_have_all_values_none(self):
|
||||
load_params = LoadParams(load_type=LoadType.S3)
|
||||
self._check_all_values_none(load_params, ["load_type", "scenario"])
|
||||
|
|
@ -285,9 +305,7 @@ class TestLoadConfig:
|
|||
self._check_preset_params(load_params, expected_preset_args)
|
||||
self._check_env_vars(load_params, expected_env_vars)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"load_params, load_type", [(LoadScenario.VERIFY, LoadType.S3)], indirect=True
|
||||
)
|
||||
@pytest.mark.parametrize("load_params, load_type", [(LoadScenario.VERIFY, LoadType.S3)], indirect=True)
|
||||
def test_argument_parsing_for_s3_verify_scenario(self, load_params: LoadParams):
|
||||
expected_env_vars = {
|
||||
"CLIENTS": 14,
|
||||
|
|
@ -299,9 +317,7 @@ class TestLoadConfig:
|
|||
|
||||
self._check_env_vars(load_params, expected_env_vars)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"load_params, load_type", [(LoadScenario.VERIFY, LoadType.gRPC)], indirect=True
|
||||
)
|
||||
@pytest.mark.parametrize("load_params, load_type", [(LoadScenario.VERIFY, LoadType.gRPC)], indirect=True)
|
||||
def test_argument_parsing_for_grpc_verify_scenario(self, load_params: LoadParams):
|
||||
expected_env_vars = {
|
||||
"CLIENTS": 14,
|
||||
|
|
@ -339,9 +355,7 @@ class TestLoadConfig:
|
|||
self._check_preset_params(load_params, expected_preset_args)
|
||||
self._check_env_vars(load_params, expected_env_vars)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"load_params, set_empty", [(LoadScenario.gRPC_CAR, True)], indirect=True
|
||||
)
|
||||
@pytest.mark.parametrize("load_params, set_empty", [(LoadScenario.gRPC_CAR, True)], indirect=True)
|
||||
def test_empty_argument_parsing_for_grpc_car_scenario(self, load_params: LoadParams):
|
||||
expected_preset_args = [
|
||||
"--size '0'",
|
||||
|
|
|
|||
Loading…
Reference in New Issue