[#256] Allow to set mix of policies for containers and buckets
Some checks reported warnings
DCO action / DCO (pull_request) Has been cancelled
Some checks reported warnings
DCO action / DCO (pull_request) Has been cancelled
Signed-off-by: a.berezin <a.berezin@yadro.com>
This commit is contained in:
parent
376499a7e8
commit
429698944e
2 changed files with 95 additions and 54 deletions
|
|
@ -25,6 +25,16 @@ def convert_time_to_seconds(time: int | str | None) -> int:
|
|||
return seconds
|
||||
|
||||
|
||||
def force_list(input: str | list[str]):
|
||||
if input is None:
|
||||
return None
|
||||
|
||||
if isinstance(input, list):
|
||||
return list(map(str.strip, input))
|
||||
|
||||
return [input.strip()]
|
||||
|
||||
|
||||
class LoadType(Enum):
|
||||
gRPC = "grpc"
|
||||
S3 = "s3"
|
||||
|
|
@ -142,8 +152,29 @@ class K6ProcessAllocationStrategy(Enum):
|
|||
PER_ENDPOINT = "PER_ENDPOINT"
|
||||
|
||||
|
||||
class MetaConfig:
|
||||
def _get_field_formatter(self, field_name: str) -> Callable | None:
|
||||
data_fields = fields(self)
|
||||
formatters = [
|
||||
field.metadata["formatter"]
|
||||
for field in data_fields
|
||||
if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None
|
||||
]
|
||||
if formatters:
|
||||
return formatters[0]
|
||||
|
||||
return None
|
||||
|
||||
def __setattr__(self, field_name, value):
|
||||
formatter = self._get_field_formatter(field_name)
|
||||
if formatter:
|
||||
value = formatter(value)
|
||||
|
||||
super().__setattr__(field_name, value)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Preset:
|
||||
class Preset(MetaConfig):
|
||||
# ------ COMMON ------
|
||||
# Amount of objects which should be created
|
||||
objects_count: Optional[int] = metadata_field(all_load_scenarios, "preload_obj", None, False)
|
||||
|
|
@ -158,13 +189,13 @@ class Preset:
|
|||
# Amount of containers which should be created
|
||||
containers_count: Optional[int] = metadata_field(grpc_preset_scenarios, "containers", None, False)
|
||||
# Container placement policy for containers for gRPC
|
||||
container_placement_policy: Optional[str] = metadata_field(grpc_preset_scenarios, "policy", None, False)
|
||||
container_placement_policy: Optional[list[str]] = metadata_field(grpc_preset_scenarios, "policy", None, False, formatter=force_list)
|
||||
|
||||
# ------ S3 ------
|
||||
# Amount of buckets which should be created
|
||||
buckets_count: Optional[int] = metadata_field(s3_preset_scenarios, "buckets", None, False)
|
||||
# S3 region (AKA placement policy for S3 buckets)
|
||||
s3_location: Optional[str] = metadata_field(s3_preset_scenarios, "location", None, False)
|
||||
s3_location: Optional[list[str]] = metadata_field(s3_preset_scenarios, "location", None, False, formatter=force_list)
|
||||
|
||||
# Delay between containers creation and object upload for preset
|
||||
object_upload_delay: Optional[int] = metadata_field(all_load_scenarios, "sleep", None, False)
|
||||
|
|
@ -177,7 +208,7 @@ class Preset:
|
|||
|
||||
|
||||
@dataclass
|
||||
class PrometheusParams:
|
||||
class PrometheusParams(MetaConfig):
|
||||
# Prometheus server URL
|
||||
server_url: Optional[str] = metadata_field(all_load_scenarios, env_variable="K6_PROMETHEUS_RW_SERVER_URL", string_repr=False)
|
||||
# Prometheus trend stats
|
||||
|
|
@ -187,7 +218,7 @@ class PrometheusParams:
|
|||
|
||||
|
||||
@dataclass
|
||||
class LoadParams:
|
||||
class LoadParams(MetaConfig):
|
||||
# ------- CONTROL PARAMS -------
|
||||
# Load type can be gRPC, HTTP, S3.
|
||||
load_type: LoadType
|
||||
|
|
@ -412,6 +443,11 @@ class LoadParams:
|
|||
# For preset calls, bool values are passed with just --<argument_name> if the value is True
|
||||
return f"--{meta_field.metadata['preset_argument']}" if meta_field.value else ""
|
||||
|
||||
if isinstance(meta_field.value, list):
|
||||
return (
|
||||
" ".join(f"--{meta_field.metadata['preset_argument']} '{value}'" for value in meta_field.value) if meta_field.value else ""
|
||||
)
|
||||
|
||||
return f"--{meta_field.metadata['preset_argument']} '{meta_field.value}'"
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -431,25 +467,6 @@ class LoadParams:
|
|||
|
||||
return fields_with_data or []
|
||||
|
||||
def _get_field_formatter(self, field_name: str) -> Callable | None:
|
||||
data_fields = fields(self)
|
||||
formatters = [
|
||||
field.metadata["formatter"]
|
||||
for field in data_fields
|
||||
if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None
|
||||
]
|
||||
if formatters:
|
||||
return formatters[0]
|
||||
|
||||
return None
|
||||
|
||||
def __setattr__(self, field_name, value):
|
||||
formatter = self._get_field_formatter(field_name)
|
||||
if formatter:
|
||||
value = formatter(value)
|
||||
|
||||
super().__setattr__(field_name, value)
|
||||
|
||||
def __str__(self) -> str:
|
||||
load_type_str = self.scenario.value if self.scenario else self.load_type.value
|
||||
# TODO: migrate load_params defaults to testlib
|
||||
|
|
|
|||
|
|
@ -3,14 +3,7 @@ from typing import Any, get_args
|
|||
|
||||
import pytest
|
||||
|
||||
from frostfs_testlib.load.load_config import (
|
||||
EndpointSelectionStrategy,
|
||||
LoadParams,
|
||||
LoadScenario,
|
||||
LoadType,
|
||||
Preset,
|
||||
ReadFrom,
|
||||
)
|
||||
from frostfs_testlib.load.load_config import EndpointSelectionStrategy, LoadParams, LoadScenario, LoadType, Preset, ReadFrom
|
||||
from frostfs_testlib.load.runners import DefaultRunner
|
||||
from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_DEFAULT_VU_INIT_TIME
|
||||
from frostfs_testlib.storage.cluster import ClusterNode
|
||||
|
|
@ -99,9 +92,7 @@ class TestLoadConfig:
|
|||
def test_load_controller_string_representation(self, load_params: LoadParams):
|
||||
load_params.endpoint_selection_strategy = EndpointSelectionStrategy.ALL
|
||||
load_params.object_size = 512
|
||||
background_load_controller = BackgroundLoadController(
|
||||
"tmp", load_params, "wallet", None, None, DefaultRunner(None)
|
||||
)
|
||||
background_load_controller = BackgroundLoadController("tmp", load_params, None, None, DefaultRunner(None))
|
||||
expected = "grpc 512 KiB, writers=7, readers=7, deleters=8"
|
||||
assert f"{background_load_controller}" == expected
|
||||
assert repr(background_load_controller) == expected
|
||||
|
|
@ -141,7 +132,7 @@ class TestLoadConfig:
|
|||
"--out 'pregen_json'",
|
||||
"--workers '7'",
|
||||
"--containers '16'",
|
||||
"--policy 'container_placement_policy'",
|
||||
"--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
|
||||
"--ignore-errors",
|
||||
"--sleep '19'",
|
||||
"--local",
|
||||
|
|
@ -173,7 +164,7 @@ class TestLoadConfig:
|
|||
"--out 'pregen_json'",
|
||||
"--workers '7'",
|
||||
"--containers '16'",
|
||||
"--policy 'container_placement_policy'",
|
||||
"--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
|
||||
"--ignore-errors",
|
||||
"--sleep '19'",
|
||||
"--local",
|
||||
|
|
@ -214,7 +205,7 @@ class TestLoadConfig:
|
|||
"--out 'pregen_json'",
|
||||
"--workers '7'",
|
||||
"--buckets '13'",
|
||||
"--location 's3_location'",
|
||||
"--location 's3_location' --location 's3_location_2'",
|
||||
"--ignore-errors",
|
||||
"--sleep '19'",
|
||||
"--acl 'acl'",
|
||||
|
|
@ -248,7 +239,7 @@ class TestLoadConfig:
|
|||
"--out 'pregen_json'",
|
||||
"--workers '7'",
|
||||
"--buckets '13'",
|
||||
"--location 's3_location'",
|
||||
"--location 's3_location' --location 's3_location_2'",
|
||||
"--ignore-errors",
|
||||
"--sleep '19'",
|
||||
"--acl 'acl'",
|
||||
|
|
@ -288,7 +279,7 @@ class TestLoadConfig:
|
|||
"--out 'pregen_json'",
|
||||
"--workers '7'",
|
||||
"--buckets '13'",
|
||||
"--location 's3_location'",
|
||||
"--location 's3_location' --location 's3_location_2'",
|
||||
"--ignore-errors",
|
||||
"--sleep '19'",
|
||||
"--acl 'acl'",
|
||||
|
|
@ -329,7 +320,7 @@ class TestLoadConfig:
|
|||
"--out 'pregen_json'",
|
||||
"--workers '7'",
|
||||
"--containers '16'",
|
||||
"--policy 'container_placement_policy'",
|
||||
"--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
|
||||
"--ignore-errors",
|
||||
"--sleep '19'",
|
||||
"--acl 'acl'",
|
||||
|
|
@ -362,12 +353,13 @@ class TestLoadConfig:
|
|||
"--out 'pregen_json'",
|
||||
"--workers '7'",
|
||||
"--containers '16'",
|
||||
"--policy 'container_placement_policy'",
|
||||
"--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
|
||||
"--ignore-errors",
|
||||
"--sleep '19'",
|
||||
"--acl 'acl'",
|
||||
]
|
||||
expected_env_vars = {
|
||||
"CONFIG_DIR": "config_dir",
|
||||
"CONFIG_FILE": "config_file",
|
||||
"DURATION": 9,
|
||||
"WRITE_OBJ_SIZE": 11,
|
||||
|
|
@ -380,12 +372,49 @@ class TestLoadConfig:
|
|||
"DELETERS": 8,
|
||||
"READ_AGE": 8,
|
||||
"STREAMING": 9,
|
||||
"MAX_TOTAL_SIZE_GB": 17,
|
||||
"PREGEN_JSON": "pregen_json",
|
||||
}
|
||||
|
||||
self._check_preset_params(load_params, expected_preset_args)
|
||||
self._check_env_vars(load_params, expected_env_vars)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input, value, params",
|
||||
[
|
||||
(["A C ", " B"], ["A C", "B"], [f"--policy 'A C' --policy 'B'"]),
|
||||
(" A ", ["A"], ["--policy 'A'"]),
|
||||
(" A , B ", ["A , B"], ["--policy 'A , B'"]),
|
||||
([" A", "B "], ["A", "B"], ["--policy 'A' --policy 'B'"]),
|
||||
(None, None, []),
|
||||
],
|
||||
)
|
||||
def test_grpc_list_parsing_formatter(self, input, value, params):
|
||||
load_params = LoadParams(LoadType.gRPC)
|
||||
load_params.preset = Preset()
|
||||
load_params.preset.container_placement_policy = input
|
||||
assert load_params.preset.container_placement_policy == value
|
||||
|
||||
self._check_preset_params(load_params, params)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input, value, params",
|
||||
[
|
||||
(["A C ", " B"], ["A C", "B"], [f"--location 'A C' --location 'B'"]),
|
||||
(" A ", ["A"], ["--location 'A'"]),
|
||||
(" A , B ", ["A , B"], ["--location 'A , B'"]),
|
||||
([" A", "B "], ["A", "B"], ["--location 'A' --location 'B'"]),
|
||||
(None, None, []),
|
||||
],
|
||||
)
|
||||
def test_s3_list_parsing_formatter(self, input, value, params):
|
||||
load_params = LoadParams(LoadType.S3)
|
||||
load_params.preset = Preset()
|
||||
load_params.preset.s3_location = input
|
||||
assert load_params.preset.s3_location == value
|
||||
|
||||
self._check_preset_params(load_params, params)
|
||||
|
||||
@pytest.mark.parametrize("load_params, load_type", [(LoadScenario.VERIFY, LoadType.S3)], indirect=True)
|
||||
def test_argument_parsing_for_s3_verify_scenario(self, load_params: LoadParams):
|
||||
expected_env_vars = {
|
||||
|
|
@ -592,6 +621,7 @@ class TestLoadConfig:
|
|||
"--acl ''",
|
||||
]
|
||||
expected_env_vars = {
|
||||
"CONFIG_DIR": "",
|
||||
"CONFIG_FILE": "",
|
||||
"DURATION": 0,
|
||||
"WRITE_OBJ_SIZE": 0,
|
||||
|
|
@ -599,6 +629,7 @@ class TestLoadConfig:
|
|||
"K6_OUT": "",
|
||||
"K6_MIN_ITERATION_DURATION": "",
|
||||
"K6_SETUP_TIMEOUT": "",
|
||||
"MAX_TOTAL_SIZE_GB": 0,
|
||||
"WRITERS": 0,
|
||||
"READERS": 0,
|
||||
"DELETERS": 0,
|
||||
|
|
@ -689,9 +720,7 @@ class TestLoadConfig:
|
|||
value = getattr(dataclass, field.name)
|
||||
assert value is not None, f"{field.name} is not None"
|
||||
|
||||
def _get_filled_load_params(
|
||||
self, load_type: LoadType, load_scenario: LoadScenario, set_emtpy: bool = False
|
||||
) -> LoadParams:
|
||||
def _get_filled_load_params(self, load_type: LoadType, load_scenario: LoadScenario, set_emtpy: bool = False) -> LoadParams:
|
||||
load_type_map = {
|
||||
LoadScenario.S3: LoadType.S3,
|
||||
LoadScenario.S3_CAR: LoadType.S3,
|
||||
|
|
@ -708,13 +737,12 @@ class TestLoadConfig:
|
|||
|
||||
meta_fields = self._get_meta_fields(load_params)
|
||||
for field in meta_fields:
|
||||
if (
|
||||
getattr(field.instance, field.field.name) is None
|
||||
and load_params.scenario in field.field.metadata["applicable_scenarios"]
|
||||
):
|
||||
if getattr(field.instance, field.field.name) is None and load_params.scenario in field.field.metadata["applicable_scenarios"]:
|
||||
value_to_set_map = {
|
||||
int: 0 if set_emtpy else len(field.field.name),
|
||||
float: 0 if set_emtpy else len(field.field.name),
|
||||
str: "" if set_emtpy else field.field.name,
|
||||
list[str]: "" if set_emtpy else [field.field.name, f"{field.field.name}_2"],
|
||||
bool: False if set_emtpy else True,
|
||||
}
|
||||
value_to_set = value_to_set_map[field.field_type]
|
||||
|
|
@ -727,11 +755,7 @@ class TestLoadConfig:
|
|||
|
||||
def _get_meta_fields(self, instance):
|
||||
data_fields = fields(instance)
|
||||
fields_with_data = [
|
||||
MetaTestField(field, self._get_actual_field_type(field), instance)
|
||||
for field in data_fields
|
||||
if field.metadata
|
||||
]
|
||||
fields_with_data = [MetaTestField(field, self._get_actual_field_type(field), instance) for field in data_fields if field.metadata]
|
||||
|
||||
for field in data_fields:
|
||||
actual_field_type = self._get_actual_field_type(field)
|
||||
|
|
|
|||
Loading…
Reference in a new issue