EliChin/frostfs-testlib
1
0
Fork 0
forked from TrueCloudLab/frostfs-testlib
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#156] load_time in the format of days, hours and minutes; new params

Browse source 
Signed-off-by: Liza <e.chichindaeva@yadro.com>
This commit is contained in:
Elizaveta Chichindaeva 2023-12-29 01:25:13 +03:00
parent d6a2cf92a2
commit df8d99d83c
2 changed files with 131 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
src/frostfs_testlib/load/load_config.py
View file

@ -3,11 +3,28 @@ import os
from dataclasses import dataclass, field, fields, is_dataclass
from enum import Enum
from types import MappingProxyType
from typing import Any, Optional, get_args
from typing import Any, Callable, Optional, get_args
from frostfs_testlib.utils.converting_utils import calc_unit
def convert_time_to_seconds(time: int | str | None) -> int:
if time is None:
return None
if str(time).isdigit():
seconds = int(time)
else:
days, hours, minutes = 0, 0, 0
if "d" in time:
days, time = time.split("d")
if "h" in time:
hours, time = time.split("h")
if "min" in time:
minutes = time.replace("min", "")
seconds = int(days) * 86400 + int(hours) * 3600 + int(minutes) * 60
return seconds
class LoadType(Enum):
gRPC = "grpc"
S3 = "s3"
@ -76,6 +93,7 @@ def metadata_field(
scenario_variable: Optional[str] = None,
string_repr: Optional[bool] = True,
distributed: Optional[bool] = False,
formatter: Optional[Callable] = None,
):
return field(
default=None,
@ -85,6 +103,7 @@ def metadata_field(
"env_variable": scenario_variable,
"string_repr": string_repr,
"distributed": distributed,
"formatter": formatter,
},
)
@ -200,7 +219,9 @@ class LoadParams:
# ------- COMMON SCENARIO PARAMS -------
# Load time is the maximum duration for k6 to give load. Default is the BACKGROUND_LOAD_DEFAULT_TIME value.
load_time: Optional[int] = metadata_field(all_load_scenarios, None, "DURATION", False)
load_time: Optional[int] = metadata_field(
all_load_scenarios, None, "DURATION", False, formatter=convert_time_to_seconds
)
# Object size in KB for load and preset.
object_size: Optional[int] = metadata_field(all_load_scenarios, "size", "WRITE_OBJ_SIZE", False)
# For read operations, controls from which set get objects to read
@ -384,6 +405,25 @@ class LoadParams:
return fields_with_data or []
def _get_field_formatter(self, field_name: str) -> Callable | None:
data_fields = fields(self)
formatters = [
field.metadata["formatter"]
for field in data_fields
if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None
]
if formatters:
return formatters[0]
return None
def __setattr__(self, field_name, value):
formatter = self._get_field_formatter(field_name)
if formatter:
value = formatter(value)
super().__setattr__(field_name, value)
def __str__(self) -> str:
load_type_str = self.scenario.value if self.scenario else self.load_type.value
# TODO: migrate load_params defaults to testlib

89
tests/test_load_config.py
View file

@ -141,6 +141,8 @@ class TestLoadConfig:
"--workers '7'",
"--containers '16'",
"--policy 'container_placement_policy'",
"--ignore-errors",
"--sleep '19'",
]
expected_env_vars = {
"DURATION": 9,
@ -151,6 +153,7 @@ class TestLoadConfig:
"WRITERS": 7,
"READERS": 7,
"DELETERS": 8,
"READ_AGE": 8,
"PREGEN_JSON": "pregen_json",
"PREPARE_LOCALLY": True,
}
@ -167,6 +170,8 @@ class TestLoadConfig:
"--workers '7'",
"--containers '16'",
"--policy 'container_placement_policy'",
"--ignore-errors",
"--sleep '19'",
]
expected_env_vars = {
"DURATION": 9,
@ -184,6 +189,7 @@ class TestLoadConfig:
"TIME_UNIT": "time_unit",
"WRITE_RATE": 10,
"READ_RATE": 9,
"READ_AGE": 8,
"DELETE_RATE": 11,
"PREPARE_LOCALLY": True,
}
@ -201,6 +207,8 @@ class TestLoadConfig:
"--workers '7'",
"--buckets '13'",
"--location 's3_location'",
"--ignore-errors",
"--sleep '19'",
]
expected_env_vars = {
"DURATION": 9,
@ -211,6 +219,7 @@ class TestLoadConfig:
"WRITERS": 7,
"READERS": 7,
"DELETERS": 8,
"READ_AGE": 8,
"NO_VERIFY_SSL": True,
"PREGEN_JSON": "pregen_json",
}
@ -218,6 +227,44 @@ class TestLoadConfig:
self._check_preset_params(load_params, expected_preset_args)
self._check_env_vars(load_params, expected_env_vars)
@pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True)
def test_argument_parsing_for_s3_car_scenario_with_stringed_time(self, load_params: LoadParams):
load_params.load_time = "2d3h5min"
expected_preset_args = [
"--size '11'",
"--preload_obj '13'",
"--no-verify-ssl",
"--out 'pregen_json'",
"--workers '7'",
"--buckets '13'",
"--location 's3_location'",
"--ignore-errors",
"--sleep '19'",
]
expected_env_vars = {
"DURATION": 183900,
"WRITE_OBJ_SIZE": 11,
"REGISTRY_FILE": "registry_file",
"K6_MIN_ITERATION_DURATION": "min_iteration_duration",
"K6_SETUP_TIMEOUT": "setup_timeout",
"NO_VERIFY_SSL": True,
"MAX_WRITERS": 11,
"MAX_READERS": 11,
"MAX_DELETERS": 12,
"PRE_ALLOC_DELETERS": 21,
"PRE_ALLOC_READERS": 20,
"PRE_ALLOC_WRITERS": 20,
"PREGEN_JSON": "pregen_json",
"TIME_UNIT": "time_unit",
"WRITE_RATE": 10,
"READ_RATE": 9,
"READ_AGE": 8,
"DELETE_RATE": 11,
}
self._check_preset_params(load_params, expected_preset_args)
self._check_env_vars(load_params, expected_env_vars)
@pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True)
def test_argument_parsing_for_s3_car_scenario(self, load_params: LoadParams):
expected_preset_args = [
@ -228,6 +275,8 @@ class TestLoadConfig:
"--workers '7'",
"--buckets '13'",
"--location 's3_location'",
"--ignore-errors",
"--sleep '19'",
]
expected_env_vars = {
"DURATION": 9,
@ -246,6 +295,7 @@ class TestLoadConfig:
"TIME_UNIT": "time_unit",
"WRITE_RATE": 10,
"READ_RATE": 9,
"READ_AGE": 8,
"DELETE_RATE": 11,
}
@ -262,6 +312,8 @@ class TestLoadConfig:
"--workers '7'",
"--containers '16'",
"--policy 'container_placement_policy'",
"--ignore-errors",
"--sleep '19'",
]
expected_env_vars = {
"DURATION": 9,
@ -273,6 +325,7 @@ class TestLoadConfig:
"WRITERS": 7,
"READERS": 7,
"DELETERS": 8,
"READ_AGE": 8,
"PREGEN_JSON": "pregen_json",
}
@ -288,6 +341,8 @@ class TestLoadConfig:
"--workers '7'",
"--containers '16'",
"--policy 'container_placement_policy'",
"--ignore-errors",
"--sleep '19'",
]
expected_env_vars = {
"CONFIG_FILE": "config_file",
@ -299,6 +354,7 @@ class TestLoadConfig:
"WRITERS": 7,
"READERS": 7,
"DELETERS": 8,
"READ_AGE": 8,
"PREGEN_JSON": "pregen_json",
}
@ -338,6 +394,7 @@ class TestLoadConfig:
"--workers '0'",
"--containers '0'",
"--policy ''",
"--sleep '0'",
]
expected_env_vars = {
"DURATION": 0,
@ -348,6 +405,7 @@ class TestLoadConfig:
"WRITERS": 0,
"READERS": 0,
"DELETERS": 0,
"READ_AGE": 0,
"PREGEN_JSON": "",
"PREPARE_LOCALLY": False,
}
@ -364,6 +422,7 @@ class TestLoadConfig:
"--workers '0'",
"--containers '0'",
"--policy ''",
"--sleep '0'",
]
expected_env_vars = {
"DURATION": 0,
@ -382,6 +441,7 @@ class TestLoadConfig:
"WRITE_RATE": 0,
"READ_RATE": 0,
"DELETE_RATE": 0,
"READ_AGE": 0,
"PREPARE_LOCALLY": False,
}
@ -397,6 +457,7 @@ class TestLoadConfig:
"--workers '0'",
"--buckets '0'",
"--location ''",
"--sleep '0'",
]
expected_env_vars = {
"DURATION": 0,
@ -407,6 +468,7 @@ class TestLoadConfig:
"WRITERS": 0,
"READERS": 0,
"DELETERS": 0,
"READ_AGE": 0,
"NO_VERIFY_SSL": False,
"PREGEN_JSON": "",
}
@ -423,6 +485,7 @@ class TestLoadConfig:
"--workers '0'",
"--buckets '0'",
"--location ''",
"--sleep '0'",
]
expected_env_vars = {
"DURATION": 0,
@ -442,6 +505,7 @@ class TestLoadConfig:
"WRITE_RATE": 0,
"READ_RATE": 0,
"DELETE_RATE": 0,
"READ_AGE": 0,
}
self._check_preset_params(load_params, expected_preset_args)
@ -456,6 +520,7 @@ class TestLoadConfig:
"--workers '0'",
"--containers '0'",
"--policy ''",
"--sleep '0'",
]
expected_env_vars = {
"DURATION": 0,
@ -467,6 +532,7 @@ class TestLoadConfig:
"WRITERS": 0,
"READERS": 0,
"DELETERS": 0,
"READ_AGE": 0,
"PREGEN_JSON": "",
}
@ -482,6 +548,7 @@ class TestLoadConfig:
"--workers '0'",
"--containers '0'",
"--policy ''",
"--sleep '0'",
]
expected_env_vars = {
"CONFIG_FILE": "",
@ -493,6 +560,7 @@ class TestLoadConfig:
"WRITERS": 0,
"READERS": 0,
"DELETERS": 0,
"READ_AGE": 0,
"PREGEN_JSON": "",
}
@ -531,6 +599,27 @@ class TestLoadConfig:
self._check_env_vars(load_params, expected_env_vars)
@pytest.mark.parametrize(
"load_params, load_type",
[(LoadScenario.gRPC, LoadType.gRPC)],
indirect=True,
)
@pytest.mark.parametrize(
"load_time, expected_seconds",
[
(300, 300),
("2d3h45min", 186300),
("1d6h", 108000),
("1d", 86400),
("1d1min", 86460),
("2h", 7200),
("2h2min", 7320),
],
)
def test_convert_time_to_seconds(self, load_params: LoadParams, load_time: str | int, expected_seconds: int):
load_params.load_time = load_time
assert load_params.load_time == expected_seconds
def _check_preset_params(self, load_params: LoadParams, expected_preset_args: list[str]):
preset_parameters = load_params.get_preset_arguments()
assert sorted(preset_parameters) == sorted(expected_preset_args)