Compare commits
2 commits
master
...
bereza/upd
Author | SHA1 | Date | |
---|---|---|---|
9f2c8adfb5 | |||
ee1f3ab0c1 |
2 changed files with 131 additions and 2 deletions
|
@ -3,11 +3,28 @@ import os
|
||||||
from dataclasses import dataclass, field, fields, is_dataclass
|
from dataclasses import dataclass, field, fields, is_dataclass
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from types import MappingProxyType
|
from types import MappingProxyType
|
||||||
from typing import Any, Optional, get_args
|
from typing import Any, Callable, Optional, get_args
|
||||||
|
|
||||||
from frostfs_testlib.utils.converting_utils import calc_unit
|
from frostfs_testlib.utils.converting_utils import calc_unit
|
||||||
|
|
||||||
|
|
||||||
|
def convert_time_to_seconds(time: int | str | None) -> int:
|
||||||
|
if time is None:
|
||||||
|
return None
|
||||||
|
if str(time).isdigit():
|
||||||
|
seconds = int(time)
|
||||||
|
else:
|
||||||
|
days, hours, minutes = 0, 0, 0
|
||||||
|
if "d" in time:
|
||||||
|
days, time = time.split("d")
|
||||||
|
if "h" in time:
|
||||||
|
hours, time = time.split("h")
|
||||||
|
if "min" in time:
|
||||||
|
minutes = time.replace("min", "")
|
||||||
|
seconds = int(days) * 86400 + int(hours) * 3600 + int(minutes) * 60
|
||||||
|
return seconds
|
||||||
|
|
||||||
|
|
||||||
class LoadType(Enum):
|
class LoadType(Enum):
|
||||||
gRPC = "grpc"
|
gRPC = "grpc"
|
||||||
S3 = "s3"
|
S3 = "s3"
|
||||||
|
@ -76,6 +93,7 @@ def metadata_field(
|
||||||
scenario_variable: Optional[str] = None,
|
scenario_variable: Optional[str] = None,
|
||||||
string_repr: Optional[bool] = True,
|
string_repr: Optional[bool] = True,
|
||||||
distributed: Optional[bool] = False,
|
distributed: Optional[bool] = False,
|
||||||
|
formatter: Optional[Callable] = None,
|
||||||
):
|
):
|
||||||
return field(
|
return field(
|
||||||
default=None,
|
default=None,
|
||||||
|
@ -85,6 +103,7 @@ def metadata_field(
|
||||||
"env_variable": scenario_variable,
|
"env_variable": scenario_variable,
|
||||||
"string_repr": string_repr,
|
"string_repr": string_repr,
|
||||||
"distributed": distributed,
|
"distributed": distributed,
|
||||||
|
"formatter": formatter,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -200,7 +219,9 @@ class LoadParams:
|
||||||
|
|
||||||
# ------- COMMON SCENARIO PARAMS -------
|
# ------- COMMON SCENARIO PARAMS -------
|
||||||
# Load time is the maximum duration for k6 to give load. Default is the BACKGROUND_LOAD_DEFAULT_TIME value.
|
# Load time is the maximum duration for k6 to give load. Default is the BACKGROUND_LOAD_DEFAULT_TIME value.
|
||||||
load_time: Optional[int] = metadata_field(all_load_scenarios, None, "DURATION", False)
|
load_time: Optional[int] = metadata_field(
|
||||||
|
all_load_scenarios, None, "DURATION", False, formatter=convert_time_to_seconds
|
||||||
|
)
|
||||||
# Object size in KB for load and preset.
|
# Object size in KB for load and preset.
|
||||||
object_size: Optional[int] = metadata_field(all_load_scenarios, "size", "WRITE_OBJ_SIZE", False)
|
object_size: Optional[int] = metadata_field(all_load_scenarios, "size", "WRITE_OBJ_SIZE", False)
|
||||||
# For read operations, controls from which set get objects to read
|
# For read operations, controls from which set get objects to read
|
||||||
|
@ -384,6 +405,25 @@ class LoadParams:
|
||||||
|
|
||||||
return fields_with_data or []
|
return fields_with_data or []
|
||||||
|
|
||||||
|
def _get_field_formatter(self, field_name: str) -> Callable | None:
|
||||||
|
data_fields = fields(self)
|
||||||
|
formatters = [
|
||||||
|
field.metadata["formatter"]
|
||||||
|
for field in data_fields
|
||||||
|
if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None
|
||||||
|
]
|
||||||
|
if formatters:
|
||||||
|
return formatters[0]
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def __setattr__(self, field_name, value):
|
||||||
|
formatter = self._get_field_formatter(field_name)
|
||||||
|
if formatter:
|
||||||
|
value = formatter(value)
|
||||||
|
|
||||||
|
super().__setattr__(field_name, value)
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
load_type_str = self.scenario.value if self.scenario else self.load_type.value
|
load_type_str = self.scenario.value if self.scenario else self.load_type.value
|
||||||
# TODO: migrate load_params defaults to testlib
|
# TODO: migrate load_params defaults to testlib
|
||||||
|
|
|
@ -141,6 +141,8 @@ class TestLoadConfig:
|
||||||
"--workers '7'",
|
"--workers '7'",
|
||||||
"--containers '16'",
|
"--containers '16'",
|
||||||
"--policy 'container_placement_policy'",
|
"--policy 'container_placement_policy'",
|
||||||
|
"--ignore-errors",
|
||||||
|
"--sleep '19'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"DURATION": 9,
|
"DURATION": 9,
|
||||||
|
@ -151,6 +153,7 @@ class TestLoadConfig:
|
||||||
"WRITERS": 7,
|
"WRITERS": 7,
|
||||||
"READERS": 7,
|
"READERS": 7,
|
||||||
"DELETERS": 8,
|
"DELETERS": 8,
|
||||||
|
"READ_AGE": 8,
|
||||||
"PREGEN_JSON": "pregen_json",
|
"PREGEN_JSON": "pregen_json",
|
||||||
"PREPARE_LOCALLY": True,
|
"PREPARE_LOCALLY": True,
|
||||||
}
|
}
|
||||||
|
@ -167,6 +170,8 @@ class TestLoadConfig:
|
||||||
"--workers '7'",
|
"--workers '7'",
|
||||||
"--containers '16'",
|
"--containers '16'",
|
||||||
"--policy 'container_placement_policy'",
|
"--policy 'container_placement_policy'",
|
||||||
|
"--ignore-errors",
|
||||||
|
"--sleep '19'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"DURATION": 9,
|
"DURATION": 9,
|
||||||
|
@ -184,6 +189,7 @@ class TestLoadConfig:
|
||||||
"TIME_UNIT": "time_unit",
|
"TIME_UNIT": "time_unit",
|
||||||
"WRITE_RATE": 10,
|
"WRITE_RATE": 10,
|
||||||
"READ_RATE": 9,
|
"READ_RATE": 9,
|
||||||
|
"READ_AGE": 8,
|
||||||
"DELETE_RATE": 11,
|
"DELETE_RATE": 11,
|
||||||
"PREPARE_LOCALLY": True,
|
"PREPARE_LOCALLY": True,
|
||||||
}
|
}
|
||||||
|
@ -201,6 +207,8 @@ class TestLoadConfig:
|
||||||
"--workers '7'",
|
"--workers '7'",
|
||||||
"--buckets '13'",
|
"--buckets '13'",
|
||||||
"--location 's3_location'",
|
"--location 's3_location'",
|
||||||
|
"--ignore-errors",
|
||||||
|
"--sleep '19'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"DURATION": 9,
|
"DURATION": 9,
|
||||||
|
@ -211,6 +219,7 @@ class TestLoadConfig:
|
||||||
"WRITERS": 7,
|
"WRITERS": 7,
|
||||||
"READERS": 7,
|
"READERS": 7,
|
||||||
"DELETERS": 8,
|
"DELETERS": 8,
|
||||||
|
"READ_AGE": 8,
|
||||||
"NO_VERIFY_SSL": True,
|
"NO_VERIFY_SSL": True,
|
||||||
"PREGEN_JSON": "pregen_json",
|
"PREGEN_JSON": "pregen_json",
|
||||||
}
|
}
|
||||||
|
@ -218,6 +227,44 @@ class TestLoadConfig:
|
||||||
self._check_preset_params(load_params, expected_preset_args)
|
self._check_preset_params(load_params, expected_preset_args)
|
||||||
self._check_env_vars(load_params, expected_env_vars)
|
self._check_env_vars(load_params, expected_env_vars)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True)
|
||||||
|
def test_argument_parsing_for_s3_car_scenario_with_stringed_time(self, load_params: LoadParams):
|
||||||
|
load_params.load_time = "2d3h5min"
|
||||||
|
expected_preset_args = [
|
||||||
|
"--size '11'",
|
||||||
|
"--preload_obj '13'",
|
||||||
|
"--no-verify-ssl",
|
||||||
|
"--out 'pregen_json'",
|
||||||
|
"--workers '7'",
|
||||||
|
"--buckets '13'",
|
||||||
|
"--location 's3_location'",
|
||||||
|
"--ignore-errors",
|
||||||
|
"--sleep '19'",
|
||||||
|
]
|
||||||
|
expected_env_vars = {
|
||||||
|
"DURATION": 183900,
|
||||||
|
"WRITE_OBJ_SIZE": 11,
|
||||||
|
"REGISTRY_FILE": "registry_file",
|
||||||
|
"K6_MIN_ITERATION_DURATION": "min_iteration_duration",
|
||||||
|
"K6_SETUP_TIMEOUT": "setup_timeout",
|
||||||
|
"NO_VERIFY_SSL": True,
|
||||||
|
"MAX_WRITERS": 11,
|
||||||
|
"MAX_READERS": 11,
|
||||||
|
"MAX_DELETERS": 12,
|
||||||
|
"PRE_ALLOC_DELETERS": 21,
|
||||||
|
"PRE_ALLOC_READERS": 20,
|
||||||
|
"PRE_ALLOC_WRITERS": 20,
|
||||||
|
"PREGEN_JSON": "pregen_json",
|
||||||
|
"TIME_UNIT": "time_unit",
|
||||||
|
"WRITE_RATE": 10,
|
||||||
|
"READ_RATE": 9,
|
||||||
|
"READ_AGE": 8,
|
||||||
|
"DELETE_RATE": 11,
|
||||||
|
}
|
||||||
|
|
||||||
|
self._check_preset_params(load_params, expected_preset_args)
|
||||||
|
self._check_env_vars(load_params, expected_env_vars)
|
||||||
|
|
||||||
@pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True)
|
@pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True)
|
||||||
def test_argument_parsing_for_s3_car_scenario(self, load_params: LoadParams):
|
def test_argument_parsing_for_s3_car_scenario(self, load_params: LoadParams):
|
||||||
expected_preset_args = [
|
expected_preset_args = [
|
||||||
|
@ -228,6 +275,8 @@ class TestLoadConfig:
|
||||||
"--workers '7'",
|
"--workers '7'",
|
||||||
"--buckets '13'",
|
"--buckets '13'",
|
||||||
"--location 's3_location'",
|
"--location 's3_location'",
|
||||||
|
"--ignore-errors",
|
||||||
|
"--sleep '19'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"DURATION": 9,
|
"DURATION": 9,
|
||||||
|
@ -246,6 +295,7 @@ class TestLoadConfig:
|
||||||
"TIME_UNIT": "time_unit",
|
"TIME_UNIT": "time_unit",
|
||||||
"WRITE_RATE": 10,
|
"WRITE_RATE": 10,
|
||||||
"READ_RATE": 9,
|
"READ_RATE": 9,
|
||||||
|
"READ_AGE": 8,
|
||||||
"DELETE_RATE": 11,
|
"DELETE_RATE": 11,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -262,6 +312,8 @@ class TestLoadConfig:
|
||||||
"--workers '7'",
|
"--workers '7'",
|
||||||
"--containers '16'",
|
"--containers '16'",
|
||||||
"--policy 'container_placement_policy'",
|
"--policy 'container_placement_policy'",
|
||||||
|
"--ignore-errors",
|
||||||
|
"--sleep '19'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"DURATION": 9,
|
"DURATION": 9,
|
||||||
|
@ -273,6 +325,7 @@ class TestLoadConfig:
|
||||||
"WRITERS": 7,
|
"WRITERS": 7,
|
||||||
"READERS": 7,
|
"READERS": 7,
|
||||||
"DELETERS": 8,
|
"DELETERS": 8,
|
||||||
|
"READ_AGE": 8,
|
||||||
"PREGEN_JSON": "pregen_json",
|
"PREGEN_JSON": "pregen_json",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -288,6 +341,8 @@ class TestLoadConfig:
|
||||||
"--workers '7'",
|
"--workers '7'",
|
||||||
"--containers '16'",
|
"--containers '16'",
|
||||||
"--policy 'container_placement_policy'",
|
"--policy 'container_placement_policy'",
|
||||||
|
"--ignore-errors",
|
||||||
|
"--sleep '19'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"CONFIG_FILE": "config_file",
|
"CONFIG_FILE": "config_file",
|
||||||
|
@ -299,6 +354,7 @@ class TestLoadConfig:
|
||||||
"WRITERS": 7,
|
"WRITERS": 7,
|
||||||
"READERS": 7,
|
"READERS": 7,
|
||||||
"DELETERS": 8,
|
"DELETERS": 8,
|
||||||
|
"READ_AGE": 8,
|
||||||
"PREGEN_JSON": "pregen_json",
|
"PREGEN_JSON": "pregen_json",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -338,6 +394,7 @@ class TestLoadConfig:
|
||||||
"--workers '0'",
|
"--workers '0'",
|
||||||
"--containers '0'",
|
"--containers '0'",
|
||||||
"--policy ''",
|
"--policy ''",
|
||||||
|
"--sleep '0'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"DURATION": 0,
|
"DURATION": 0,
|
||||||
|
@ -348,6 +405,7 @@ class TestLoadConfig:
|
||||||
"WRITERS": 0,
|
"WRITERS": 0,
|
||||||
"READERS": 0,
|
"READERS": 0,
|
||||||
"DELETERS": 0,
|
"DELETERS": 0,
|
||||||
|
"READ_AGE": 0,
|
||||||
"PREGEN_JSON": "",
|
"PREGEN_JSON": "",
|
||||||
"PREPARE_LOCALLY": False,
|
"PREPARE_LOCALLY": False,
|
||||||
}
|
}
|
||||||
|
@ -364,6 +422,7 @@ class TestLoadConfig:
|
||||||
"--workers '0'",
|
"--workers '0'",
|
||||||
"--containers '0'",
|
"--containers '0'",
|
||||||
"--policy ''",
|
"--policy ''",
|
||||||
|
"--sleep '0'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"DURATION": 0,
|
"DURATION": 0,
|
||||||
|
@ -382,6 +441,7 @@ class TestLoadConfig:
|
||||||
"WRITE_RATE": 0,
|
"WRITE_RATE": 0,
|
||||||
"READ_RATE": 0,
|
"READ_RATE": 0,
|
||||||
"DELETE_RATE": 0,
|
"DELETE_RATE": 0,
|
||||||
|
"READ_AGE": 0,
|
||||||
"PREPARE_LOCALLY": False,
|
"PREPARE_LOCALLY": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -397,6 +457,7 @@ class TestLoadConfig:
|
||||||
"--workers '0'",
|
"--workers '0'",
|
||||||
"--buckets '0'",
|
"--buckets '0'",
|
||||||
"--location ''",
|
"--location ''",
|
||||||
|
"--sleep '0'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"DURATION": 0,
|
"DURATION": 0,
|
||||||
|
@ -407,6 +468,7 @@ class TestLoadConfig:
|
||||||
"WRITERS": 0,
|
"WRITERS": 0,
|
||||||
"READERS": 0,
|
"READERS": 0,
|
||||||
"DELETERS": 0,
|
"DELETERS": 0,
|
||||||
|
"READ_AGE": 0,
|
||||||
"NO_VERIFY_SSL": False,
|
"NO_VERIFY_SSL": False,
|
||||||
"PREGEN_JSON": "",
|
"PREGEN_JSON": "",
|
||||||
}
|
}
|
||||||
|
@ -423,6 +485,7 @@ class TestLoadConfig:
|
||||||
"--workers '0'",
|
"--workers '0'",
|
||||||
"--buckets '0'",
|
"--buckets '0'",
|
||||||
"--location ''",
|
"--location ''",
|
||||||
|
"--sleep '0'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"DURATION": 0,
|
"DURATION": 0,
|
||||||
|
@ -442,6 +505,7 @@ class TestLoadConfig:
|
||||||
"WRITE_RATE": 0,
|
"WRITE_RATE": 0,
|
||||||
"READ_RATE": 0,
|
"READ_RATE": 0,
|
||||||
"DELETE_RATE": 0,
|
"DELETE_RATE": 0,
|
||||||
|
"READ_AGE": 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
self._check_preset_params(load_params, expected_preset_args)
|
self._check_preset_params(load_params, expected_preset_args)
|
||||||
|
@ -456,6 +520,7 @@ class TestLoadConfig:
|
||||||
"--workers '0'",
|
"--workers '0'",
|
||||||
"--containers '0'",
|
"--containers '0'",
|
||||||
"--policy ''",
|
"--policy ''",
|
||||||
|
"--sleep '0'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"DURATION": 0,
|
"DURATION": 0,
|
||||||
|
@ -467,6 +532,7 @@ class TestLoadConfig:
|
||||||
"WRITERS": 0,
|
"WRITERS": 0,
|
||||||
"READERS": 0,
|
"READERS": 0,
|
||||||
"DELETERS": 0,
|
"DELETERS": 0,
|
||||||
|
"READ_AGE": 0,
|
||||||
"PREGEN_JSON": "",
|
"PREGEN_JSON": "",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -482,6 +548,7 @@ class TestLoadConfig:
|
||||||
"--workers '0'",
|
"--workers '0'",
|
||||||
"--containers '0'",
|
"--containers '0'",
|
||||||
"--policy ''",
|
"--policy ''",
|
||||||
|
"--sleep '0'",
|
||||||
]
|
]
|
||||||
expected_env_vars = {
|
expected_env_vars = {
|
||||||
"CONFIG_FILE": "",
|
"CONFIG_FILE": "",
|
||||||
|
@ -493,6 +560,7 @@ class TestLoadConfig:
|
||||||
"WRITERS": 0,
|
"WRITERS": 0,
|
||||||
"READERS": 0,
|
"READERS": 0,
|
||||||
"DELETERS": 0,
|
"DELETERS": 0,
|
||||||
|
"READ_AGE": 0,
|
||||||
"PREGEN_JSON": "",
|
"PREGEN_JSON": "",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -531,6 +599,27 @@ class TestLoadConfig:
|
||||||
|
|
||||||
self._check_env_vars(load_params, expected_env_vars)
|
self._check_env_vars(load_params, expected_env_vars)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"load_params, load_type",
|
||||||
|
[(LoadScenario.gRPC, LoadType.gRPC)],
|
||||||
|
indirect=True,
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"load_time, expected_seconds",
|
||||||
|
[
|
||||||
|
(300, 300),
|
||||||
|
("2d3h45min", 186300),
|
||||||
|
("1d6h", 108000),
|
||||||
|
("1d", 86400),
|
||||||
|
("1d1min", 86460),
|
||||||
|
("2h", 7200),
|
||||||
|
("2h2min", 7320),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_convert_time_to_seconds(self, load_params: LoadParams, load_time: str | int, expected_seconds: int):
|
||||||
|
load_params.load_time = load_time
|
||||||
|
assert load_params.load_time == expected_seconds
|
||||||
|
|
||||||
def _check_preset_params(self, load_params: LoadParams, expected_preset_args: list[str]):
|
def _check_preset_params(self, load_params: LoadParams, expected_preset_args: list[str]):
|
||||||
preset_parameters = load_params.get_preset_arguments()
|
preset_parameters = load_params.get_preset_arguments()
|
||||||
assert sorted(preset_parameters) == sorted(expected_preset_args)
|
assert sorted(preset_parameters) == sorted(expected_preset_args)
|
||||||
|
|
Loading…
Reference in a new issue