[#156] load_time in the format of days, hours and minutes; new params

Signed-off-by: Liza <e.chichindaeva@yadro.com>
This commit is contained in:
Elizaveta Chichindaeva 2023-12-29 01:25:13 +03:00
parent d6a2cf92a2
commit df8d99d83c
2 changed files with 131 additions and 2 deletions

View file

@ -3,11 +3,28 @@ import os
from dataclasses import dataclass, field, fields, is_dataclass from dataclasses import dataclass, field, fields, is_dataclass
from enum import Enum from enum import Enum
from types import MappingProxyType from types import MappingProxyType
from typing import Any, Optional, get_args from typing import Any, Callable, Optional, get_args
from frostfs_testlib.utils.converting_utils import calc_unit from frostfs_testlib.utils.converting_utils import calc_unit
def convert_time_to_seconds(time: int | str | None) -> int:
if time is None:
return None
if str(time).isdigit():
seconds = int(time)
else:
days, hours, minutes = 0, 0, 0
if "d" in time:
days, time = time.split("d")
if "h" in time:
hours, time = time.split("h")
if "min" in time:
minutes = time.replace("min", "")
seconds = int(days) * 86400 + int(hours) * 3600 + int(minutes) * 60
return seconds
class LoadType(Enum): class LoadType(Enum):
gRPC = "grpc" gRPC = "grpc"
S3 = "s3" S3 = "s3"
@ -76,6 +93,7 @@ def metadata_field(
scenario_variable: Optional[str] = None, scenario_variable: Optional[str] = None,
string_repr: Optional[bool] = True, string_repr: Optional[bool] = True,
distributed: Optional[bool] = False, distributed: Optional[bool] = False,
formatter: Optional[Callable] = None,
): ):
return field( return field(
default=None, default=None,
@ -85,6 +103,7 @@ def metadata_field(
"env_variable": scenario_variable, "env_variable": scenario_variable,
"string_repr": string_repr, "string_repr": string_repr,
"distributed": distributed, "distributed": distributed,
"formatter": formatter,
}, },
) )
@ -200,7 +219,9 @@ class LoadParams:
# ------- COMMON SCENARIO PARAMS ------- # ------- COMMON SCENARIO PARAMS -------
# Load time is the maximum duration for k6 to give load. Default is the BACKGROUND_LOAD_DEFAULT_TIME value. # Load time is the maximum duration for k6 to give load. Default is the BACKGROUND_LOAD_DEFAULT_TIME value.
load_time: Optional[int] = metadata_field(all_load_scenarios, None, "DURATION", False) load_time: Optional[int] = metadata_field(
all_load_scenarios, None, "DURATION", False, formatter=convert_time_to_seconds
)
# Object size in KB for load and preset. # Object size in KB for load and preset.
object_size: Optional[int] = metadata_field(all_load_scenarios, "size", "WRITE_OBJ_SIZE", False) object_size: Optional[int] = metadata_field(all_load_scenarios, "size", "WRITE_OBJ_SIZE", False)
# For read operations, controls from which set get objects to read # For read operations, controls from which set get objects to read
@ -384,6 +405,25 @@ class LoadParams:
return fields_with_data or [] return fields_with_data or []
def _get_field_formatter(self, field_name: str) -> Callable | None:
data_fields = fields(self)
formatters = [
field.metadata["formatter"]
for field in data_fields
if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None
]
if formatters:
return formatters[0]
return None
def __setattr__(self, field_name, value):
formatter = self._get_field_formatter(field_name)
if formatter:
value = formatter(value)
super().__setattr__(field_name, value)
def __str__(self) -> str: def __str__(self) -> str:
load_type_str = self.scenario.value if self.scenario else self.load_type.value load_type_str = self.scenario.value if self.scenario else self.load_type.value
# TODO: migrate load_params defaults to testlib # TODO: migrate load_params defaults to testlib

View file

@ -141,6 +141,8 @@ class TestLoadConfig:
"--workers '7'", "--workers '7'",
"--containers '16'", "--containers '16'",
"--policy 'container_placement_policy'", "--policy 'container_placement_policy'",
"--ignore-errors",
"--sleep '19'",
] ]
expected_env_vars = { expected_env_vars = {
"DURATION": 9, "DURATION": 9,
@ -151,6 +153,7 @@ class TestLoadConfig:
"WRITERS": 7, "WRITERS": 7,
"READERS": 7, "READERS": 7,
"DELETERS": 8, "DELETERS": 8,
"READ_AGE": 8,
"PREGEN_JSON": "pregen_json", "PREGEN_JSON": "pregen_json",
"PREPARE_LOCALLY": True, "PREPARE_LOCALLY": True,
} }
@ -167,6 +170,8 @@ class TestLoadConfig:
"--workers '7'", "--workers '7'",
"--containers '16'", "--containers '16'",
"--policy 'container_placement_policy'", "--policy 'container_placement_policy'",
"--ignore-errors",
"--sleep '19'",
] ]
expected_env_vars = { expected_env_vars = {
"DURATION": 9, "DURATION": 9,
@ -184,6 +189,7 @@ class TestLoadConfig:
"TIME_UNIT": "time_unit", "TIME_UNIT": "time_unit",
"WRITE_RATE": 10, "WRITE_RATE": 10,
"READ_RATE": 9, "READ_RATE": 9,
"READ_AGE": 8,
"DELETE_RATE": 11, "DELETE_RATE": 11,
"PREPARE_LOCALLY": True, "PREPARE_LOCALLY": True,
} }
@ -201,6 +207,8 @@ class TestLoadConfig:
"--workers '7'", "--workers '7'",
"--buckets '13'", "--buckets '13'",
"--location 's3_location'", "--location 's3_location'",
"--ignore-errors",
"--sleep '19'",
] ]
expected_env_vars = { expected_env_vars = {
"DURATION": 9, "DURATION": 9,
@ -211,6 +219,7 @@ class TestLoadConfig:
"WRITERS": 7, "WRITERS": 7,
"READERS": 7, "READERS": 7,
"DELETERS": 8, "DELETERS": 8,
"READ_AGE": 8,
"NO_VERIFY_SSL": True, "NO_VERIFY_SSL": True,
"PREGEN_JSON": "pregen_json", "PREGEN_JSON": "pregen_json",
} }
@ -218,6 +227,44 @@ class TestLoadConfig:
self._check_preset_params(load_params, expected_preset_args) self._check_preset_params(load_params, expected_preset_args)
self._check_env_vars(load_params, expected_env_vars) self._check_env_vars(load_params, expected_env_vars)
@pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True)
def test_argument_parsing_for_s3_car_scenario_with_stringed_time(self, load_params: LoadParams):
load_params.load_time = "2d3h5min"
expected_preset_args = [
"--size '11'",
"--preload_obj '13'",
"--no-verify-ssl",
"--out 'pregen_json'",
"--workers '7'",
"--buckets '13'",
"--location 's3_location'",
"--ignore-errors",
"--sleep '19'",
]
expected_env_vars = {
"DURATION": 183900,
"WRITE_OBJ_SIZE": 11,
"REGISTRY_FILE": "registry_file",
"K6_MIN_ITERATION_DURATION": "min_iteration_duration",
"K6_SETUP_TIMEOUT": "setup_timeout",
"NO_VERIFY_SSL": True,
"MAX_WRITERS": 11,
"MAX_READERS": 11,
"MAX_DELETERS": 12,
"PRE_ALLOC_DELETERS": 21,
"PRE_ALLOC_READERS": 20,
"PRE_ALLOC_WRITERS": 20,
"PREGEN_JSON": "pregen_json",
"TIME_UNIT": "time_unit",
"WRITE_RATE": 10,
"READ_RATE": 9,
"READ_AGE": 8,
"DELETE_RATE": 11,
}
self._check_preset_params(load_params, expected_preset_args)
self._check_env_vars(load_params, expected_env_vars)
@pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True) @pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True)
def test_argument_parsing_for_s3_car_scenario(self, load_params: LoadParams): def test_argument_parsing_for_s3_car_scenario(self, load_params: LoadParams):
expected_preset_args = [ expected_preset_args = [
@ -228,6 +275,8 @@ class TestLoadConfig:
"--workers '7'", "--workers '7'",
"--buckets '13'", "--buckets '13'",
"--location 's3_location'", "--location 's3_location'",
"--ignore-errors",
"--sleep '19'",
] ]
expected_env_vars = { expected_env_vars = {
"DURATION": 9, "DURATION": 9,
@ -246,6 +295,7 @@ class TestLoadConfig:
"TIME_UNIT": "time_unit", "TIME_UNIT": "time_unit",
"WRITE_RATE": 10, "WRITE_RATE": 10,
"READ_RATE": 9, "READ_RATE": 9,
"READ_AGE": 8,
"DELETE_RATE": 11, "DELETE_RATE": 11,
} }
@ -262,6 +312,8 @@ class TestLoadConfig:
"--workers '7'", "--workers '7'",
"--containers '16'", "--containers '16'",
"--policy 'container_placement_policy'", "--policy 'container_placement_policy'",
"--ignore-errors",
"--sleep '19'",
] ]
expected_env_vars = { expected_env_vars = {
"DURATION": 9, "DURATION": 9,
@ -273,6 +325,7 @@ class TestLoadConfig:
"WRITERS": 7, "WRITERS": 7,
"READERS": 7, "READERS": 7,
"DELETERS": 8, "DELETERS": 8,
"READ_AGE": 8,
"PREGEN_JSON": "pregen_json", "PREGEN_JSON": "pregen_json",
} }
@ -288,6 +341,8 @@ class TestLoadConfig:
"--workers '7'", "--workers '7'",
"--containers '16'", "--containers '16'",
"--policy 'container_placement_policy'", "--policy 'container_placement_policy'",
"--ignore-errors",
"--sleep '19'",
] ]
expected_env_vars = { expected_env_vars = {
"CONFIG_FILE": "config_file", "CONFIG_FILE": "config_file",
@ -299,6 +354,7 @@ class TestLoadConfig:
"WRITERS": 7, "WRITERS": 7,
"READERS": 7, "READERS": 7,
"DELETERS": 8, "DELETERS": 8,
"READ_AGE": 8,
"PREGEN_JSON": "pregen_json", "PREGEN_JSON": "pregen_json",
} }
@ -338,6 +394,7 @@ class TestLoadConfig:
"--workers '0'", "--workers '0'",
"--containers '0'", "--containers '0'",
"--policy ''", "--policy ''",
"--sleep '0'",
] ]
expected_env_vars = { expected_env_vars = {
"DURATION": 0, "DURATION": 0,
@ -348,6 +405,7 @@ class TestLoadConfig:
"WRITERS": 0, "WRITERS": 0,
"READERS": 0, "READERS": 0,
"DELETERS": 0, "DELETERS": 0,
"READ_AGE": 0,
"PREGEN_JSON": "", "PREGEN_JSON": "",
"PREPARE_LOCALLY": False, "PREPARE_LOCALLY": False,
} }
@ -364,6 +422,7 @@ class TestLoadConfig:
"--workers '0'", "--workers '0'",
"--containers '0'", "--containers '0'",
"--policy ''", "--policy ''",
"--sleep '0'",
] ]
expected_env_vars = { expected_env_vars = {
"DURATION": 0, "DURATION": 0,
@ -382,6 +441,7 @@ class TestLoadConfig:
"WRITE_RATE": 0, "WRITE_RATE": 0,
"READ_RATE": 0, "READ_RATE": 0,
"DELETE_RATE": 0, "DELETE_RATE": 0,
"READ_AGE": 0,
"PREPARE_LOCALLY": False, "PREPARE_LOCALLY": False,
} }
@ -397,6 +457,7 @@ class TestLoadConfig:
"--workers '0'", "--workers '0'",
"--buckets '0'", "--buckets '0'",
"--location ''", "--location ''",
"--sleep '0'",
] ]
expected_env_vars = { expected_env_vars = {
"DURATION": 0, "DURATION": 0,
@ -407,6 +468,7 @@ class TestLoadConfig:
"WRITERS": 0, "WRITERS": 0,
"READERS": 0, "READERS": 0,
"DELETERS": 0, "DELETERS": 0,
"READ_AGE": 0,
"NO_VERIFY_SSL": False, "NO_VERIFY_SSL": False,
"PREGEN_JSON": "", "PREGEN_JSON": "",
} }
@ -423,6 +485,7 @@ class TestLoadConfig:
"--workers '0'", "--workers '0'",
"--buckets '0'", "--buckets '0'",
"--location ''", "--location ''",
"--sleep '0'",
] ]
expected_env_vars = { expected_env_vars = {
"DURATION": 0, "DURATION": 0,
@ -442,6 +505,7 @@ class TestLoadConfig:
"WRITE_RATE": 0, "WRITE_RATE": 0,
"READ_RATE": 0, "READ_RATE": 0,
"DELETE_RATE": 0, "DELETE_RATE": 0,
"READ_AGE": 0,
} }
self._check_preset_params(load_params, expected_preset_args) self._check_preset_params(load_params, expected_preset_args)
@ -456,6 +520,7 @@ class TestLoadConfig:
"--workers '0'", "--workers '0'",
"--containers '0'", "--containers '0'",
"--policy ''", "--policy ''",
"--sleep '0'",
] ]
expected_env_vars = { expected_env_vars = {
"DURATION": 0, "DURATION": 0,
@ -467,6 +532,7 @@ class TestLoadConfig:
"WRITERS": 0, "WRITERS": 0,
"READERS": 0, "READERS": 0,
"DELETERS": 0, "DELETERS": 0,
"READ_AGE": 0,
"PREGEN_JSON": "", "PREGEN_JSON": "",
} }
@ -482,6 +548,7 @@ class TestLoadConfig:
"--workers '0'", "--workers '0'",
"--containers '0'", "--containers '0'",
"--policy ''", "--policy ''",
"--sleep '0'",
] ]
expected_env_vars = { expected_env_vars = {
"CONFIG_FILE": "", "CONFIG_FILE": "",
@ -493,6 +560,7 @@ class TestLoadConfig:
"WRITERS": 0, "WRITERS": 0,
"READERS": 0, "READERS": 0,
"DELETERS": 0, "DELETERS": 0,
"READ_AGE": 0,
"PREGEN_JSON": "", "PREGEN_JSON": "",
} }
@ -531,6 +599,27 @@ class TestLoadConfig:
self._check_env_vars(load_params, expected_env_vars) self._check_env_vars(load_params, expected_env_vars)
@pytest.mark.parametrize(
"load_params, load_type",
[(LoadScenario.gRPC, LoadType.gRPC)],
indirect=True,
)
@pytest.mark.parametrize(
"load_time, expected_seconds",
[
(300, 300),
("2d3h45min", 186300),
("1d6h", 108000),
("1d", 86400),
("1d1min", 86460),
("2h", 7200),
("2h2min", 7320),
],
)
def test_convert_time_to_seconds(self, load_params: LoadParams, load_time: str | int, expected_seconds: int):
load_params.load_time = load_time
assert load_params.load_time == expected_seconds
def _check_preset_params(self, load_params: LoadParams, expected_preset_args: list[str]): def _check_preset_params(self, load_params: LoadParams, expected_preset_args: list[str]):
preset_parameters = load_params.get_preset_arguments() preset_parameters = load_params.get_preset_arguments()
assert sorted(preset_parameters) == sorted(expected_preset_args) assert sorted(preset_parameters) == sorted(expected_preset_args)