From 429698944e94cc7bcc40e603aa868e9ba3d12481 Mon Sep 17 00:00:00 2001 From: "a.berezin" Date: Wed, 3 Jul 2024 12:02:40 +0300 Subject: [PATCH] [#256] Allow to set mix of policies for containers and buckets Signed-off-by: a.berezin --- src/frostfs_testlib/load/load_config.py | 65 ++++++++++++------- tests/test_load_config.py | 84 ++++++++++++++++--------- 2 files changed, 95 insertions(+), 54 deletions(-) diff --git a/src/frostfs_testlib/load/load_config.py b/src/frostfs_testlib/load/load_config.py index 1128096..767e9f2 100644 --- a/src/frostfs_testlib/load/load_config.py +++ b/src/frostfs_testlib/load/load_config.py @@ -25,6 +25,16 @@ def convert_time_to_seconds(time: int | str | None) -> int: return seconds +def force_list(input: str | list[str]): + if input is None: + return None + + if isinstance(input, list): + return list(map(str.strip, input)) + + return [input.strip()] + + class LoadType(Enum): gRPC = "grpc" S3 = "s3" @@ -142,8 +152,29 @@ class K6ProcessAllocationStrategy(Enum): PER_ENDPOINT = "PER_ENDPOINT" +class MetaConfig: + def _get_field_formatter(self, field_name: str) -> Callable | None: + data_fields = fields(self) + formatters = [ + field.metadata["formatter"] + for field in data_fields + if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None + ] + if formatters: + return formatters[0] + + return None + + def __setattr__(self, field_name, value): + formatter = self._get_field_formatter(field_name) + if formatter: + value = formatter(value) + + super().__setattr__(field_name, value) + + @dataclass -class Preset: +class Preset(MetaConfig): # ------ COMMON ------ # Amount of objects which should be created objects_count: Optional[int] = metadata_field(all_load_scenarios, "preload_obj", None, False) @@ -158,13 +189,13 @@ class Preset: # Amount of containers which should be created containers_count: Optional[int] = metadata_field(grpc_preset_scenarios, "containers", None, False) # Container placement policy for containers for gRPC - container_placement_policy: Optional[str] = metadata_field(grpc_preset_scenarios, "policy", None, False) + container_placement_policy: Optional[list[str]] = metadata_field(grpc_preset_scenarios, "policy", None, False, formatter=force_list) # ------ S3 ------ # Amount of buckets which should be created buckets_count: Optional[int] = metadata_field(s3_preset_scenarios, "buckets", None, False) # S3 region (AKA placement policy for S3 buckets) - s3_location: Optional[str] = metadata_field(s3_preset_scenarios, "location", None, False) + s3_location: Optional[list[str]] = metadata_field(s3_preset_scenarios, "location", None, False, formatter=force_list) # Delay between containers creation and object upload for preset object_upload_delay: Optional[int] = metadata_field(all_load_scenarios, "sleep", None, False) @@ -177,7 +208,7 @@ class Preset: @dataclass -class PrometheusParams: +class PrometheusParams(MetaConfig): # Prometheus server URL server_url: Optional[str] = metadata_field(all_load_scenarios, env_variable="K6_PROMETHEUS_RW_SERVER_URL", string_repr=False) # Prometheus trend stats @@ -187,7 +218,7 @@ class PrometheusParams: @dataclass -class LoadParams: +class LoadParams(MetaConfig): # ------- CONTROL PARAMS ------- # Load type can be gRPC, HTTP, S3. load_type: LoadType @@ -412,6 +443,11 @@ class LoadParams: # For preset calls, bool values are passed with just -- if the value is True return f"--{meta_field.metadata['preset_argument']}" if meta_field.value else "" + if isinstance(meta_field.value, list): + return ( + " ".join(f"--{meta_field.metadata['preset_argument']} '{value}'" for value in meta_field.value) if meta_field.value else "" + ) + return f"--{meta_field.metadata['preset_argument']} '{meta_field.value}'" @staticmethod @@ -431,25 +467,6 @@ class LoadParams: return fields_with_data or [] - def _get_field_formatter(self, field_name: str) -> Callable | None: - data_fields = fields(self) - formatters = [ - field.metadata["formatter"] - for field in data_fields - if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None - ] - if formatters: - return formatters[0] - - return None - - def __setattr__(self, field_name, value): - formatter = self._get_field_formatter(field_name) - if formatter: - value = formatter(value) - - super().__setattr__(field_name, value) - def __str__(self) -> str: load_type_str = self.scenario.value if self.scenario else self.load_type.value # TODO: migrate load_params defaults to testlib diff --git a/tests/test_load_config.py b/tests/test_load_config.py index 62339f6..883b1f2 100644 --- a/tests/test_load_config.py +++ b/tests/test_load_config.py @@ -3,14 +3,7 @@ from typing import Any, get_args import pytest -from frostfs_testlib.load.load_config import ( - EndpointSelectionStrategy, - LoadParams, - LoadScenario, - LoadType, - Preset, - ReadFrom, -) +from frostfs_testlib.load.load_config import EndpointSelectionStrategy, LoadParams, LoadScenario, LoadType, Preset, ReadFrom from frostfs_testlib.load.runners import DefaultRunner from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_DEFAULT_VU_INIT_TIME from frostfs_testlib.storage.cluster import ClusterNode @@ -99,9 +92,7 @@ class TestLoadConfig: def test_load_controller_string_representation(self, load_params: LoadParams): load_params.endpoint_selection_strategy = EndpointSelectionStrategy.ALL load_params.object_size = 512 - background_load_controller = BackgroundLoadController( - "tmp", load_params, "wallet", None, None, DefaultRunner(None) - ) + background_load_controller = BackgroundLoadController("tmp", load_params, None, None, DefaultRunner(None)) expected = "grpc 512 KiB, writers=7, readers=7, deleters=8" assert f"{background_load_controller}" == expected assert repr(background_load_controller) == expected @@ -141,7 +132,7 @@ class TestLoadConfig: "--out 'pregen_json'", "--workers '7'", "--containers '16'", - "--policy 'container_placement_policy'", + "--policy 'container_placement_policy' --policy 'container_placement_policy_2'", "--ignore-errors", "--sleep '19'", "--local", @@ -173,7 +164,7 @@ class TestLoadConfig: "--out 'pregen_json'", "--workers '7'", "--containers '16'", - "--policy 'container_placement_policy'", + "--policy 'container_placement_policy' --policy 'container_placement_policy_2'", "--ignore-errors", "--sleep '19'", "--local", @@ -214,7 +205,7 @@ class TestLoadConfig: "--out 'pregen_json'", "--workers '7'", "--buckets '13'", - "--location 's3_location'", + "--location 's3_location' --location 's3_location_2'", "--ignore-errors", "--sleep '19'", "--acl 'acl'", @@ -248,7 +239,7 @@ class TestLoadConfig: "--out 'pregen_json'", "--workers '7'", "--buckets '13'", - "--location 's3_location'", + "--location 's3_location' --location 's3_location_2'", "--ignore-errors", "--sleep '19'", "--acl 'acl'", @@ -288,7 +279,7 @@ class TestLoadConfig: "--out 'pregen_json'", "--workers '7'", "--buckets '13'", - "--location 's3_location'", + "--location 's3_location' --location 's3_location_2'", "--ignore-errors", "--sleep '19'", "--acl 'acl'", @@ -329,7 +320,7 @@ class TestLoadConfig: "--out 'pregen_json'", "--workers '7'", "--containers '16'", - "--policy 'container_placement_policy'", + "--policy 'container_placement_policy' --policy 'container_placement_policy_2'", "--ignore-errors", "--sleep '19'", "--acl 'acl'", @@ -362,12 +353,13 @@ class TestLoadConfig: "--out 'pregen_json'", "--workers '7'", "--containers '16'", - "--policy 'container_placement_policy'", + "--policy 'container_placement_policy' --policy 'container_placement_policy_2'", "--ignore-errors", "--sleep '19'", "--acl 'acl'", ] expected_env_vars = { + "CONFIG_DIR": "config_dir", "CONFIG_FILE": "config_file", "DURATION": 9, "WRITE_OBJ_SIZE": 11, @@ -380,12 +372,49 @@ class TestLoadConfig: "DELETERS": 8, "READ_AGE": 8, "STREAMING": 9, + "MAX_TOTAL_SIZE_GB": 17, "PREGEN_JSON": "pregen_json", } self._check_preset_params(load_params, expected_preset_args) self._check_env_vars(load_params, expected_env_vars) + @pytest.mark.parametrize( + "input, value, params", + [ + (["A C ", " B"], ["A C", "B"], [f"--policy 'A C' --policy 'B'"]), + (" A ", ["A"], ["--policy 'A'"]), + (" A , B ", ["A , B"], ["--policy 'A , B'"]), + ([" A", "B "], ["A", "B"], ["--policy 'A' --policy 'B'"]), + (None, None, []), + ], + ) + def test_grpc_list_parsing_formatter(self, input, value, params): + load_params = LoadParams(LoadType.gRPC) + load_params.preset = Preset() + load_params.preset.container_placement_policy = input + assert load_params.preset.container_placement_policy == value + + self._check_preset_params(load_params, params) + + @pytest.mark.parametrize( + "input, value, params", + [ + (["A C ", " B"], ["A C", "B"], [f"--location 'A C' --location 'B'"]), + (" A ", ["A"], ["--location 'A'"]), + (" A , B ", ["A , B"], ["--location 'A , B'"]), + ([" A", "B "], ["A", "B"], ["--location 'A' --location 'B'"]), + (None, None, []), + ], + ) + def test_s3_list_parsing_formatter(self, input, value, params): + load_params = LoadParams(LoadType.S3) + load_params.preset = Preset() + load_params.preset.s3_location = input + assert load_params.preset.s3_location == value + + self._check_preset_params(load_params, params) + @pytest.mark.parametrize("load_params, load_type", [(LoadScenario.VERIFY, LoadType.S3)], indirect=True) def test_argument_parsing_for_s3_verify_scenario(self, load_params: LoadParams): expected_env_vars = { @@ -592,6 +621,7 @@ class TestLoadConfig: "--acl ''", ] expected_env_vars = { + "CONFIG_DIR": "", "CONFIG_FILE": "", "DURATION": 0, "WRITE_OBJ_SIZE": 0, @@ -599,6 +629,7 @@ class TestLoadConfig: "K6_OUT": "", "K6_MIN_ITERATION_DURATION": "", "K6_SETUP_TIMEOUT": "", + "MAX_TOTAL_SIZE_GB": 0, "WRITERS": 0, "READERS": 0, "DELETERS": 0, @@ -689,9 +720,7 @@ class TestLoadConfig: value = getattr(dataclass, field.name) assert value is not None, f"{field.name} is not None" - def _get_filled_load_params( - self, load_type: LoadType, load_scenario: LoadScenario, set_emtpy: bool = False - ) -> LoadParams: + def _get_filled_load_params(self, load_type: LoadType, load_scenario: LoadScenario, set_emtpy: bool = False) -> LoadParams: load_type_map = { LoadScenario.S3: LoadType.S3, LoadScenario.S3_CAR: LoadType.S3, @@ -708,13 +737,12 @@ class TestLoadConfig: meta_fields = self._get_meta_fields(load_params) for field in meta_fields: - if ( - getattr(field.instance, field.field.name) is None - and load_params.scenario in field.field.metadata["applicable_scenarios"] - ): + if getattr(field.instance, field.field.name) is None and load_params.scenario in field.field.metadata["applicable_scenarios"]: value_to_set_map = { int: 0 if set_emtpy else len(field.field.name), + float: 0 if set_emtpy else len(field.field.name), str: "" if set_emtpy else field.field.name, + list[str]: "" if set_emtpy else [field.field.name, f"{field.field.name}_2"], bool: False if set_emtpy else True, } value_to_set = value_to_set_map[field.field_type] @@ -727,11 +755,7 @@ class TestLoadConfig: def _get_meta_fields(self, instance): data_fields = fields(instance) - fields_with_data = [ - MetaTestField(field, self._get_actual_field_type(field), instance) - for field in data_fields - if field.metadata - ] + fields_with_data = [MetaTestField(field, self._get_actual_field_type(field), instance) for field in data_fields if field.metadata] for field in data_fields: actual_field_type = self._get_actual_field_type(field)