From df8d99d83cf4efd3ceb8b5c629e1f6083e92f153 Mon Sep 17 00:00:00 2001 From: Liza Date: Fri, 29 Dec 2023 01:25:13 +0300 Subject: [PATCH] [#156] load_time in the format of days, hours and minutes; new params Signed-off-by: Liza --- src/frostfs_testlib/load/load_config.py | 44 +++++++++++- tests/test_load_config.py | 89 +++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 2 deletions(-) diff --git a/src/frostfs_testlib/load/load_config.py b/src/frostfs_testlib/load/load_config.py index 3ea66b8..6f355fc 100644 --- a/src/frostfs_testlib/load/load_config.py +++ b/src/frostfs_testlib/load/load_config.py @@ -3,11 +3,28 @@ import os from dataclasses import dataclass, field, fields, is_dataclass from enum import Enum from types import MappingProxyType -from typing import Any, Optional, get_args +from typing import Any, Callable, Optional, get_args from frostfs_testlib.utils.converting_utils import calc_unit +def convert_time_to_seconds(time: int | str | None) -> int: + if time is None: + return None + if str(time).isdigit(): + seconds = int(time) + else: + days, hours, minutes = 0, 0, 0 + if "d" in time: + days, time = time.split("d") + if "h" in time: + hours, time = time.split("h") + if "min" in time: + minutes = time.replace("min", "") + seconds = int(days) * 86400 + int(hours) * 3600 + int(minutes) * 60 + return seconds + + class LoadType(Enum): gRPC = "grpc" S3 = "s3" @@ -76,6 +93,7 @@ def metadata_field( scenario_variable: Optional[str] = None, string_repr: Optional[bool] = True, distributed: Optional[bool] = False, + formatter: Optional[Callable] = None, ): return field( default=None, @@ -85,6 +103,7 @@ def metadata_field( "env_variable": scenario_variable, "string_repr": string_repr, "distributed": distributed, + "formatter": formatter, }, ) @@ -200,7 +219,9 @@ class LoadParams: # ------- COMMON SCENARIO PARAMS ------- # Load time is the maximum duration for k6 to give load. Default is the BACKGROUND_LOAD_DEFAULT_TIME value. - load_time: Optional[int] = metadata_field(all_load_scenarios, None, "DURATION", False) + load_time: Optional[int] = metadata_field( + all_load_scenarios, None, "DURATION", False, formatter=convert_time_to_seconds + ) # Object size in KB for load and preset. object_size: Optional[int] = metadata_field(all_load_scenarios, "size", "WRITE_OBJ_SIZE", False) # For read operations, controls from which set get objects to read @@ -384,6 +405,25 @@ class LoadParams: return fields_with_data or [] + def _get_field_formatter(self, field_name: str) -> Callable | None: + data_fields = fields(self) + formatters = [ + field.metadata["formatter"] + for field in data_fields + if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None + ] + if formatters: + return formatters[0] + + return None + + def __setattr__(self, field_name, value): + formatter = self._get_field_formatter(field_name) + if formatter: + value = formatter(value) + + super().__setattr__(field_name, value) + def __str__(self) -> str: load_type_str = self.scenario.value if self.scenario else self.load_type.value # TODO: migrate load_params defaults to testlib diff --git a/tests/test_load_config.py b/tests/test_load_config.py index 926399b..f4fa022 100644 --- a/tests/test_load_config.py +++ b/tests/test_load_config.py @@ -141,6 +141,8 @@ class TestLoadConfig: "--workers '7'", "--containers '16'", "--policy 'container_placement_policy'", + "--ignore-errors", + "--sleep '19'", ] expected_env_vars = { "DURATION": 9, @@ -151,6 +153,7 @@ class TestLoadConfig: "WRITERS": 7, "READERS": 7, "DELETERS": 8, + "READ_AGE": 8, "PREGEN_JSON": "pregen_json", "PREPARE_LOCALLY": True, } @@ -167,6 +170,8 @@ class TestLoadConfig: "--workers '7'", "--containers '16'", "--policy 'container_placement_policy'", + "--ignore-errors", + "--sleep '19'", ] expected_env_vars = { "DURATION": 9, @@ -184,6 +189,7 @@ class TestLoadConfig: "TIME_UNIT": "time_unit", "WRITE_RATE": 10, "READ_RATE": 9, + "READ_AGE": 8, "DELETE_RATE": 11, "PREPARE_LOCALLY": True, } @@ -201,6 +207,8 @@ class TestLoadConfig: "--workers '7'", "--buckets '13'", "--location 's3_location'", + "--ignore-errors", + "--sleep '19'", ] expected_env_vars = { "DURATION": 9, @@ -211,6 +219,7 @@ class TestLoadConfig: "WRITERS": 7, "READERS": 7, "DELETERS": 8, + "READ_AGE": 8, "NO_VERIFY_SSL": True, "PREGEN_JSON": "pregen_json", } @@ -218,6 +227,44 @@ class TestLoadConfig: self._check_preset_params(load_params, expected_preset_args) self._check_env_vars(load_params, expected_env_vars) + @pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True) + def test_argument_parsing_for_s3_car_scenario_with_stringed_time(self, load_params: LoadParams): + load_params.load_time = "2d3h5min" + expected_preset_args = [ + "--size '11'", + "--preload_obj '13'", + "--no-verify-ssl", + "--out 'pregen_json'", + "--workers '7'", + "--buckets '13'", + "--location 's3_location'", + "--ignore-errors", + "--sleep '19'", + ] + expected_env_vars = { + "DURATION": 183900, + "WRITE_OBJ_SIZE": 11, + "REGISTRY_FILE": "registry_file", + "K6_MIN_ITERATION_DURATION": "min_iteration_duration", + "K6_SETUP_TIMEOUT": "setup_timeout", + "NO_VERIFY_SSL": True, + "MAX_WRITERS": 11, + "MAX_READERS": 11, + "MAX_DELETERS": 12, + "PRE_ALLOC_DELETERS": 21, + "PRE_ALLOC_READERS": 20, + "PRE_ALLOC_WRITERS": 20, + "PREGEN_JSON": "pregen_json", + "TIME_UNIT": "time_unit", + "WRITE_RATE": 10, + "READ_RATE": 9, + "READ_AGE": 8, + "DELETE_RATE": 11, + } + + self._check_preset_params(load_params, expected_preset_args) + self._check_env_vars(load_params, expected_env_vars) + @pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True) def test_argument_parsing_for_s3_car_scenario(self, load_params: LoadParams): expected_preset_args = [ @@ -228,6 +275,8 @@ class TestLoadConfig: "--workers '7'", "--buckets '13'", "--location 's3_location'", + "--ignore-errors", + "--sleep '19'", ] expected_env_vars = { "DURATION": 9, @@ -246,6 +295,7 @@ class TestLoadConfig: "TIME_UNIT": "time_unit", "WRITE_RATE": 10, "READ_RATE": 9, + "READ_AGE": 8, "DELETE_RATE": 11, } @@ -262,6 +312,8 @@ class TestLoadConfig: "--workers '7'", "--containers '16'", "--policy 'container_placement_policy'", + "--ignore-errors", + "--sleep '19'", ] expected_env_vars = { "DURATION": 9, @@ -273,6 +325,7 @@ class TestLoadConfig: "WRITERS": 7, "READERS": 7, "DELETERS": 8, + "READ_AGE": 8, "PREGEN_JSON": "pregen_json", } @@ -288,6 +341,8 @@ class TestLoadConfig: "--workers '7'", "--containers '16'", "--policy 'container_placement_policy'", + "--ignore-errors", + "--sleep '19'", ] expected_env_vars = { "CONFIG_FILE": "config_file", @@ -299,6 +354,7 @@ class TestLoadConfig: "WRITERS": 7, "READERS": 7, "DELETERS": 8, + "READ_AGE": 8, "PREGEN_JSON": "pregen_json", } @@ -338,6 +394,7 @@ class TestLoadConfig: "--workers '0'", "--containers '0'", "--policy ''", + "--sleep '0'", ] expected_env_vars = { "DURATION": 0, @@ -348,6 +405,7 @@ class TestLoadConfig: "WRITERS": 0, "READERS": 0, "DELETERS": 0, + "READ_AGE": 0, "PREGEN_JSON": "", "PREPARE_LOCALLY": False, } @@ -364,6 +422,7 @@ class TestLoadConfig: "--workers '0'", "--containers '0'", "--policy ''", + "--sleep '0'", ] expected_env_vars = { "DURATION": 0, @@ -382,6 +441,7 @@ class TestLoadConfig: "WRITE_RATE": 0, "READ_RATE": 0, "DELETE_RATE": 0, + "READ_AGE": 0, "PREPARE_LOCALLY": False, } @@ -397,6 +457,7 @@ class TestLoadConfig: "--workers '0'", "--buckets '0'", "--location ''", + "--sleep '0'", ] expected_env_vars = { "DURATION": 0, @@ -407,6 +468,7 @@ class TestLoadConfig: "WRITERS": 0, "READERS": 0, "DELETERS": 0, + "READ_AGE": 0, "NO_VERIFY_SSL": False, "PREGEN_JSON": "", } @@ -423,6 +485,7 @@ class TestLoadConfig: "--workers '0'", "--buckets '0'", "--location ''", + "--sleep '0'", ] expected_env_vars = { "DURATION": 0, @@ -442,6 +505,7 @@ class TestLoadConfig: "WRITE_RATE": 0, "READ_RATE": 0, "DELETE_RATE": 0, + "READ_AGE": 0, } self._check_preset_params(load_params, expected_preset_args) @@ -456,6 +520,7 @@ class TestLoadConfig: "--workers '0'", "--containers '0'", "--policy ''", + "--sleep '0'", ] expected_env_vars = { "DURATION": 0, @@ -467,6 +532,7 @@ class TestLoadConfig: "WRITERS": 0, "READERS": 0, "DELETERS": 0, + "READ_AGE": 0, "PREGEN_JSON": "", } @@ -482,6 +548,7 @@ class TestLoadConfig: "--workers '0'", "--containers '0'", "--policy ''", + "--sleep '0'", ] expected_env_vars = { "CONFIG_FILE": "", @@ -493,6 +560,7 @@ class TestLoadConfig: "WRITERS": 0, "READERS": 0, "DELETERS": 0, + "READ_AGE": 0, "PREGEN_JSON": "", } @@ -531,6 +599,27 @@ class TestLoadConfig: self._check_env_vars(load_params, expected_env_vars) + @pytest.mark.parametrize( + "load_params, load_type", + [(LoadScenario.gRPC, LoadType.gRPC)], + indirect=True, + ) + @pytest.mark.parametrize( + "load_time, expected_seconds", + [ + (300, 300), + ("2d3h45min", 186300), + ("1d6h", 108000), + ("1d", 86400), + ("1d1min", 86460), + ("2h", 7200), + ("2h2min", 7320), + ], + ) + def test_convert_time_to_seconds(self, load_params: LoadParams, load_time: str | int, expected_seconds: int): + load_params.load_time = load_time + assert load_params.load_time == expected_seconds + def _check_preset_params(self, load_params: LoadParams, expected_preset_args: list[str]): preset_parameters = load_params.get_preset_arguments() assert sorted(preset_parameters) == sorted(expected_preset_args)