[#256] Allow to set mix of policies for containers and buckets

Signed-off-by: a.berezin <a.berezin@yadro.com>

src/frostfs_testlib/load/load_config.py

@@ -25,6 +25,16 @@ def convert_time_to_seconds(time: int | str | None) -> int:
     return seconds
 
 
+def force_list(input: str | list[str]):
+    if input is None:
+        return None
+
+    if isinstance(input, list):
+        return list(map(str.strip, input))
+
+    return [input.strip()]
+
+
 class LoadType(Enum):
     gRPC = "grpc"
     S3 = "s3"
@@ -142,8 +152,29 @@ class K6ProcessAllocationStrategy(Enum):
     PER_ENDPOINT = "PER_ENDPOINT"
 
 
+class MetaConfig:
+    def _get_field_formatter(self, field_name: str) -> Callable | None:
+        data_fields = fields(self)
+        formatters = [
+            field.metadata["formatter"]
+            for field in data_fields
+            if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None
+        ]
+        if formatters:
+            return formatters[0]
+
+        return None
+
+    def __setattr__(self, field_name, value):
+        formatter = self._get_field_formatter(field_name)
+        if formatter:
+            value = formatter(value)
+
+        super().__setattr__(field_name, value)
+
+
 @dataclass
-class Preset:
+class Preset(MetaConfig):
     # ------ COMMON ------
     # Amount of objects which should be created
     objects_count: Optional[int] = metadata_field(all_load_scenarios, "preload_obj", None, False)
@@ -158,13 +189,13 @@ class Preset:
     # Amount of containers which should be created
     containers_count: Optional[int] = metadata_field(grpc_preset_scenarios, "containers", None, False)
     # Container placement policy for containers for gRPC
-    container_placement_policy: Optional[str] = metadata_field(grpc_preset_scenarios, "policy", None, False)
+    container_placement_policy: Optional[list[str]] = metadata_field(grpc_preset_scenarios, "policy", None, False, formatter=force_list)
 
     # ------ S3 ------
     # Amount of buckets which should be created
     buckets_count: Optional[int] = metadata_field(s3_preset_scenarios, "buckets", None, False)
     # S3 region (AKA placement policy for S3 buckets)
-    s3_location: Optional[str] = metadata_field(s3_preset_scenarios, "location", None, False)
+    s3_location: Optional[list[str]] = metadata_field(s3_preset_scenarios, "location", None, False, formatter=force_list)
 
     # Delay between containers creation and object upload for preset
     object_upload_delay: Optional[int] = metadata_field(all_load_scenarios, "sleep", None, False)
@@ -177,7 +208,7 @@ class Preset:
 
 
 @dataclass
-class PrometheusParams:
+class PrometheusParams(MetaConfig):
     # Prometheus server URL
     server_url: Optional[str] = metadata_field(all_load_scenarios, env_variable="K6_PROMETHEUS_RW_SERVER_URL", string_repr=False)
     # Prometheus trend stats
@@ -187,7 +218,7 @@ class PrometheusParams:
 
 
 @dataclass
-class LoadParams:
+class LoadParams(MetaConfig):
     # ------- CONTROL PARAMS -------
     # Load type can be gRPC, HTTP, S3.
     load_type: LoadType
@@ -412,6 +443,11 @@ class LoadParams:
             # For preset calls, bool values are passed with just --<argument_name> if the value is True
             return f"--{meta_field.metadata['preset_argument']}" if meta_field.value else ""
 
+        if isinstance(meta_field.value, list):
+            return (
+                " ".join(f"--{meta_field.metadata['preset_argument']} '{value}'" for value in meta_field.value) if meta_field.value else ""
+            )
+
         return f"--{meta_field.metadata['preset_argument']} '{meta_field.value}'"
 
     @staticmethod
@@ -431,25 +467,6 @@ class LoadParams:
 
         return fields_with_data or []
 
-    def _get_field_formatter(self, field_name: str) -> Callable | None:
-        data_fields = fields(self)
-        formatters = [
-            field.metadata["formatter"]
-            for field in data_fields
-            if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None
-        ]
-        if formatters:
-            return formatters[0]
-
-        return None
-
-    def __setattr__(self, field_name, value):
-        formatter = self._get_field_formatter(field_name)
-        if formatter:
-            value = formatter(value)
-
-        super().__setattr__(field_name, value)
-
     def __str__(self) -> str:
         load_type_str = self.scenario.value if self.scenario else self.load_type.value
         # TODO: migrate load_params defaults to testlib

tests/test_load_config.py

@@ -3,14 +3,7 @@ from typing import Any, get_args
 
 import pytest
 
-from frostfs_testlib.load.load_config import (
+from frostfs_testlib.load.load_config import EndpointSelectionStrategy, LoadParams, LoadScenario, LoadType, Preset, ReadFrom
-    EndpointSelectionStrategy,
-    LoadParams,
-    LoadScenario,
-    LoadType,
-    Preset,
-    ReadFrom,
-)
 from frostfs_testlib.load.runners import DefaultRunner
 from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_DEFAULT_VU_INIT_TIME
 from frostfs_testlib.storage.cluster import ClusterNode
@@ -99,9 +92,7 @@ class TestLoadConfig:
     def test_load_controller_string_representation(self, load_params: LoadParams):
         load_params.endpoint_selection_strategy = EndpointSelectionStrategy.ALL
         load_params.object_size = 512
-        background_load_controller = BackgroundLoadController(
+        background_load_controller = BackgroundLoadController("tmp", load_params, None, None, DefaultRunner(None))
-            "tmp", load_params, "wallet", None, None, DefaultRunner(None)
-        )
         expected = "grpc 512 KiB, writers=7, readers=7, deleters=8"
         assert f"{background_load_controller}" == expected
         assert repr(background_load_controller) == expected
@@ -141,7 +132,7 @@ class TestLoadConfig:
             "--out 'pregen_json'",
             "--workers '7'",
             "--containers '16'",
-            "--policy 'container_placement_policy'",
+            "--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
             "--ignore-errors",
             "--sleep '19'",
             "--local",
@@ -173,7 +164,7 @@ class TestLoadConfig:
             "--out 'pregen_json'",
             "--workers '7'",
             "--containers '16'",
-            "--policy 'container_placement_policy'",
+            "--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
             "--ignore-errors",
             "--sleep '19'",
             "--local",
@@ -214,7 +205,7 @@ class TestLoadConfig:
             "--out 'pregen_json'",
             "--workers '7'",
             "--buckets '13'",
-            "--location 's3_location'",
+            "--location 's3_location' --location 's3_location_2'",
             "--ignore-errors",
             "--sleep '19'",
             "--acl 'acl'",
@@ -248,7 +239,7 @@ class TestLoadConfig:
             "--out 'pregen_json'",
             "--workers '7'",
             "--buckets '13'",
-            "--location 's3_location'",
+            "--location 's3_location' --location 's3_location_2'",
             "--ignore-errors",
             "--sleep '19'",
             "--acl 'acl'",
@@ -288,7 +279,7 @@ class TestLoadConfig:
             "--out 'pregen_json'",
             "--workers '7'",
             "--buckets '13'",
-            "--location 's3_location'",
+            "--location 's3_location' --location 's3_location_2'",
             "--ignore-errors",
             "--sleep '19'",
             "--acl 'acl'",
@@ -329,7 +320,7 @@ class TestLoadConfig:
             "--out 'pregen_json'",
             "--workers '7'",
             "--containers '16'",
-            "--policy 'container_placement_policy'",
+            "--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
             "--ignore-errors",
             "--sleep '19'",
             "--acl 'acl'",
@@ -362,12 +353,13 @@ class TestLoadConfig:
             "--out 'pregen_json'",
             "--workers '7'",
             "--containers '16'",
-            "--policy 'container_placement_policy'",
+            "--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
             "--ignore-errors",
             "--sleep '19'",
             "--acl 'acl'",
         ]
         expected_env_vars = {
+            "CONFIG_DIR": "config_dir",
             "CONFIG_FILE": "config_file",
             "DURATION": 9,
             "WRITE_OBJ_SIZE": 11,
@@ -380,12 +372,49 @@ class TestLoadConfig:
             "DELETERS": 8,
             "READ_AGE": 8,
             "STREAMING": 9,
+            "MAX_TOTAL_SIZE_GB": 17,
             "PREGEN_JSON": "pregen_json",
         }
 
         self._check_preset_params(load_params, expected_preset_args)
         self._check_env_vars(load_params, expected_env_vars)
 
+    @pytest.mark.parametrize(
+        "input, value, params",
+        [
+            (["A C ", " B"], ["A C", "B"], [f"--policy 'A C' --policy 'B'"]),
+            (" A ", ["A"], ["--policy 'A'"]),
+            (" A , B ", ["A , B"], ["--policy 'A , B'"]),
+            ([" A", "B "], ["A", "B"], ["--policy 'A' --policy 'B'"]),
+            (None, None, []),
+        ],
+    )
+    def test_grpc_list_parsing_formatter(self, input, value, params):
+        load_params = LoadParams(LoadType.gRPC)
+        load_params.preset = Preset()
+        load_params.preset.container_placement_policy = input
+        assert load_params.preset.container_placement_policy == value
+
+        self._check_preset_params(load_params, params)
+
+    @pytest.mark.parametrize(
+        "input, value, params",
+        [
+            (["A C ", " B"], ["A C", "B"], [f"--location 'A C' --location 'B'"]),
+            (" A ", ["A"], ["--location 'A'"]),
+            (" A , B ", ["A , B"], ["--location 'A , B'"]),
+            ([" A", "B "], ["A", "B"], ["--location 'A' --location 'B'"]),
+            (None, None, []),
+        ],
+    )
+    def test_s3_list_parsing_formatter(self, input, value, params):
+        load_params = LoadParams(LoadType.S3)
+        load_params.preset = Preset()
+        load_params.preset.s3_location = input
+        assert load_params.preset.s3_location == value
+
+        self._check_preset_params(load_params, params)
+
     @pytest.mark.parametrize("load_params, load_type", [(LoadScenario.VERIFY, LoadType.S3)], indirect=True)
     def test_argument_parsing_for_s3_verify_scenario(self, load_params: LoadParams):
         expected_env_vars = {
@@ -592,6 +621,7 @@ class TestLoadConfig:
             "--acl ''",
         ]
         expected_env_vars = {
+            "CONFIG_DIR": "",
             "CONFIG_FILE": "",
             "DURATION": 0,
             "WRITE_OBJ_SIZE": 0,
@@ -599,6 +629,7 @@ class TestLoadConfig:
             "K6_OUT": "",
             "K6_MIN_ITERATION_DURATION": "",
             "K6_SETUP_TIMEOUT": "",
+            "MAX_TOTAL_SIZE_GB": 0,
             "WRITERS": 0,
             "READERS": 0,
             "DELETERS": 0,
@@ -689,9 +720,7 @@ class TestLoadConfig:
             value = getattr(dataclass, field.name)
             assert value is not None, f"{field.name} is not None"
 
-    def _get_filled_load_params(
+    def _get_filled_load_params(self, load_type: LoadType, load_scenario: LoadScenario, set_emtpy: bool = False) -> LoadParams:
-        self, load_type: LoadType, load_scenario: LoadScenario, set_emtpy: bool = False
-    ) -> LoadParams:
         load_type_map = {
             LoadScenario.S3: LoadType.S3,
             LoadScenario.S3_CAR: LoadType.S3,
@@ -708,13 +737,12 @@ class TestLoadConfig:
 
         meta_fields = self._get_meta_fields(load_params)
         for field in meta_fields:
-            if (
+            if getattr(field.instance, field.field.name) is None and load_params.scenario in field.field.metadata["applicable_scenarios"]:
-                getattr(field.instance, field.field.name) is None
-                and load_params.scenario in field.field.metadata["applicable_scenarios"]
-            ):
                 value_to_set_map = {
                     int: 0 if set_emtpy else len(field.field.name),
+                    float: 0 if set_emtpy else len(field.field.name),
                     str: "" if set_emtpy else field.field.name,
+                    list[str]: "" if set_emtpy else [field.field.name, f"{field.field.name}_2"],
                     bool: False if set_emtpy else True,
                 }
                 value_to_set = value_to_set_map[field.field_type]
@@ -727,11 +755,7 @@ class TestLoadConfig:
 
     def _get_meta_fields(self, instance):
         data_fields = fields(instance)
-        fields_with_data = [
+        fields_with_data = [MetaTestField(field, self._get_actual_field_type(field), instance) for field in data_fields if field.metadata]
-            MetaTestField(field, self._get_actual_field_type(field), instance)
-            for field in data_fields
-            if field.metadata
-        ]
 
         for field in data_fields:
             actual_field_type = self._get_actual_field_type(field)