TrueCloudLab/frostfs-testlib
25
0
Fork 19
Code Issues Pull requests Releases 1 Activity Actions

[#256] Allow to set mix of policies for containers and buckets #256

Merged
abereziny merged 1 commit from abereziny/frostfs-testlib:feature-multi-load-policy into master 2024-07-03 12:19:58 +00:00
Conversation 0 Commits 1 Files changed 2 +95 -54
2 changed files with 95 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

65
src/frostfs_testlib/load/load_config.py
View file

@ -25,6 +25,16 @@ def convert_time_to_seconds(time: int | str | None) -> int:
return seconds
def force_list(input: str | list[str]):
if input is None:
return None
if isinstance(input, list):
return list(map(str.strip, input))
return [input.strip()]
class LoadType(Enum):
gRPC = "grpc"
S3 = "s3"
@ -142,8 +152,29 @@ class K6ProcessAllocationStrategy(Enum):
PER_ENDPOINT = "PER_ENDPOINT"
class MetaConfig:
def _get_field_formatter(self, field_name: str) -> Callable | None:
data_fields = fields(self)
formatters = [
field.metadata["formatter"]
for field in data_fields
if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None
]
if formatters:
return formatters[0]
return None
def __setattr__(self, field_name, value):
formatter = self._get_field_formatter(field_name)
if formatter:
value = formatter(value)
super().__setattr__(field_name, value)
@dataclass
class Preset:
class Preset(MetaConfig):
# ------ COMMON ------
# Amount of objects which should be created
objects_count: Optional[int] = metadata_field(all_load_scenarios, "preload_obj", None, False)
@ -158,13 +189,13 @@ class Preset:
# Amount of containers which should be created
containers_count: Optional[int] = metadata_field(grpc_preset_scenarios, "containers", None, False)
# Container placement policy for containers for gRPC
container_placement_policy: Optional[str] = metadata_field(grpc_preset_scenarios, "policy", None, False)
container_placement_policy: Optional[list[str]] = metadata_field(grpc_preset_scenarios, "policy", None, False, formatter=force_list)
# ------ S3 ------
# Amount of buckets which should be created
buckets_count: Optional[int] = metadata_field(s3_preset_scenarios, "buckets", None, False)
# S3 region (AKA placement policy for S3 buckets)
s3_location: Optional[str] = metadata_field(s3_preset_scenarios, "location", None, False)
s3_location: Optional[list[str]] = metadata_field(s3_preset_scenarios, "location", None, False, formatter=force_list)
# Delay between containers creation and object upload for preset
object_upload_delay: Optional[int] = metadata_field(all_load_scenarios, "sleep", None, False)
@ -177,7 +208,7 @@ class Preset:
@dataclass
class PrometheusParams:
class PrometheusParams(MetaConfig):
# Prometheus server URL
server_url: Optional[str] = metadata_field(all_load_scenarios, env_variable="K6_PROMETHEUS_RW_SERVER_URL", string_repr=False)
# Prometheus trend stats
@ -187,7 +218,7 @@ class PrometheusParams:
@dataclass
class LoadParams:
class LoadParams(MetaConfig):
# ------- CONTROL PARAMS -------
# Load type can be gRPC, HTTP, S3.
load_type: LoadType
@ -412,6 +443,11 @@ class LoadParams:
# For preset calls, bool values are passed with just --<argument_name> if the value is True
return f"--{meta_field.metadata['preset_argument']}" if meta_field.value else ""
if isinstance(meta_field.value, list):
return (
" ".join(f"--{meta_field.metadata['preset_argument']} '{value}'" for value in meta_field.value) if meta_field.value else ""
)
return f"--{meta_field.metadata['preset_argument']} '{meta_field.value}'"
@staticmethod
@ -431,25 +467,6 @@ class LoadParams:
return fields_with_data or []
def _get_field_formatter(self, field_name: str) -> Callable | None:
data_fields = fields(self)
formatters = [
field.metadata["formatter"]
for field in data_fields
if field.name == field_name and "formatter" in field.metadata and field.metadata["formatter"] != None
]
if formatters:
return formatters[0]
return None
def __setattr__(self, field_name, value):
formatter = self._get_field_formatter(field_name)
if formatter:
value = formatter(value)
super().__setattr__(field_name, value)
def __str__(self) -> str:
load_type_str = self.scenario.value if self.scenario else self.load_type.value
# TODO: migrate load_params defaults to testlib

84
tests/test_load_config.py
View file

@ -3,14 +3,7 @@ from typing import Any, get_args
import pytest
from frostfs_testlib.load.load_config import (
EndpointSelectionStrategy,
LoadParams,
LoadScenario,
LoadType,
Preset,
ReadFrom,
)
from frostfs_testlib.load.load_config import EndpointSelectionStrategy, LoadParams, LoadScenario, LoadType, Preset, ReadFrom
from frostfs_testlib.load.runners import DefaultRunner
from frostfs_testlib.resources.load_params import BACKGROUND_LOAD_DEFAULT_VU_INIT_TIME
from frostfs_testlib.storage.cluster import ClusterNode
@ -99,9 +92,7 @@ class TestLoadConfig:
def test_load_controller_string_representation(self, load_params: LoadParams):
load_params.endpoint_selection_strategy = EndpointSelectionStrategy.ALL
load_params.object_size = 512
background_load_controller = BackgroundLoadController(
"tmp", load_params, "wallet", None, None, DefaultRunner(None)
)
background_load_controller = BackgroundLoadController("tmp", load_params, None, None, DefaultRunner(None))
expected = "grpc 512 KiB, writers=7, readers=7, deleters=8"
assert f"{background_load_controller}" == expected
assert repr(background_load_controller) == expected
@ -141,7 +132,7 @@ class TestLoadConfig:
"--out 'pregen_json'",
"--workers '7'",
"--containers '16'",
"--policy 'container_placement_policy'",
"--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
"--ignore-errors",
"--sleep '19'",
"--local",
@ -173,7 +164,7 @@ class TestLoadConfig:
"--out 'pregen_json'",
"--workers '7'",
"--containers '16'",
"--policy 'container_placement_policy'",
"--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
"--ignore-errors",
"--sleep '19'",
"--local",
@ -214,7 +205,7 @@ class TestLoadConfig:
"--out 'pregen_json'",
"--workers '7'",
"--buckets '13'",
"--location 's3_location'",
"--location 's3_location' --location 's3_location_2'",
"--ignore-errors",
"--sleep '19'",
"--acl 'acl'",
@ -248,7 +239,7 @@ class TestLoadConfig:
"--out 'pregen_json'",
"--workers '7'",
"--buckets '13'",
"--location 's3_location'",
"--location 's3_location' --location 's3_location_2'",
"--ignore-errors",
"--sleep '19'",
"--acl 'acl'",
@ -288,7 +279,7 @@ class TestLoadConfig:
"--out 'pregen_json'",
"--workers '7'",
"--buckets '13'",
"--location 's3_location'",
"--location 's3_location' --location 's3_location_2'",
"--ignore-errors",
"--sleep '19'",
"--acl 'acl'",
@ -329,7 +320,7 @@ class TestLoadConfig:
"--out 'pregen_json'",
"--workers '7'",
"--containers '16'",
"--policy 'container_placement_policy'",
"--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
"--ignore-errors",
"--sleep '19'",
"--acl 'acl'",
@ -362,12 +353,13 @@ class TestLoadConfig:
"--out 'pregen_json'",
"--workers '7'",
"--containers '16'",
"--policy 'container_placement_policy'",
"--policy 'container_placement_policy' --policy 'container_placement_policy_2'",
"--ignore-errors",
"--sleep '19'",
"--acl 'acl'",
]
expected_env_vars = {
"CONFIG_DIR": "config_dir",
"CONFIG_FILE": "config_file",
"DURATION": 9,
"WRITE_OBJ_SIZE": 11,
@ -380,12 +372,49 @@ class TestLoadConfig:
"DELETERS": 8,
"READ_AGE": 8,
"STREAMING": 9,
"MAX_TOTAL_SIZE_GB": 17,
"PREGEN_JSON": "pregen_json",
}
self._check_preset_params(load_params, expected_preset_args)
self._check_env_vars(load_params, expected_env_vars)
@pytest.mark.parametrize(
"input, value, params",
[
(["A C ", " B"], ["A C", "B"], [f"--policy 'A C' --policy 'B'"]),
(" A ", ["A"], ["--policy 'A'"]),
(" A , B ", ["A , B"], ["--policy 'A , B'"]),
([" A", "B "], ["A", "B"], ["--policy 'A' --policy 'B'"]),
(None, None, []),
],
)
def test_grpc_list_parsing_formatter(self, input, value, params):
load_params = LoadParams(LoadType.gRPC)
load_params.preset = Preset()
load_params.preset.container_placement_policy = input
assert load_params.preset.container_placement_policy == value
self._check_preset_params(load_params, params)
@pytest.mark.parametrize(
"input, value, params",
[
(["A C ", " B"], ["A C", "B"], [f"--location 'A C' --location 'B'"]),
(" A ", ["A"], ["--location 'A'"]),
(" A , B ", ["A , B"], ["--location 'A , B'"]),
([" A", "B "], ["A", "B"], ["--location 'A' --location 'B'"]),
(None, None, []),
],
)
def test_s3_list_parsing_formatter(self, input, value, params):
load_params = LoadParams(LoadType.S3)
load_params.preset = Preset()
load_params.preset.s3_location = input
assert load_params.preset.s3_location == value
self._check_preset_params(load_params, params)
@pytest.mark.parametrize("load_params, load_type", [(LoadScenario.VERIFY, LoadType.S3)], indirect=True)
def test_argument_parsing_for_s3_verify_scenario(self, load_params: LoadParams):
expected_env_vars = {
@ -592,6 +621,7 @@ class TestLoadConfig:
"--acl ''",
]
expected_env_vars = {
"CONFIG_DIR": "",
"CONFIG_FILE": "",
"DURATION": 0,
"WRITE_OBJ_SIZE": 0,
@ -599,6 +629,7 @@ class TestLoadConfig:
"K6_OUT": "",
"K6_MIN_ITERATION_DURATION": "",
"K6_SETUP_TIMEOUT": "",
"MAX_TOTAL_SIZE_GB": 0,
"WRITERS": 0,
"READERS": 0,
"DELETERS": 0,
@ -689,9 +720,7 @@ class TestLoadConfig:
value = getattr(dataclass, field.name)
assert value is not None, f"{field.name} is not None"
def _get_filled_load_params(
self, load_type: LoadType, load_scenario: LoadScenario, set_emtpy: bool = False
) -> LoadParams:
def _get_filled_load_params(self, load_type: LoadType, load_scenario: LoadScenario, set_emtpy: bool = False) -> LoadParams:
load_type_map = {
LoadScenario.S3: LoadType.S3,
LoadScenario.S3_CAR: LoadType.S3,
@ -708,13 +737,12 @@ class TestLoadConfig:
meta_fields = self._get_meta_fields(load_params)
for field in meta_fields:
if (
getattr(field.instance, field.field.name) is None
and load_params.scenario in field.field.metadata["applicable_scenarios"]
):
if getattr(field.instance, field.field.name) is None and load_params.scenario in field.field.metadata["applicable_scenarios"]:
value_to_set_map = {
int: 0 if set_emtpy else len(field.field.name),
float: 0 if set_emtpy else len(field.field.name),
str: "" if set_emtpy else field.field.name,
list[str]: "" if set_emtpy else [field.field.name, f"{field.field.name}_2"],
bool: False if set_emtpy else True,
}
value_to_set = value_to_set_map[field.field_type]
@ -727,11 +755,7 @@ class TestLoadConfig:
def _get_meta_fields(self, instance):
data_fields = fields(instance)
fields_with_data = [
MetaTestField(field, self._get_actual_field_type(field), instance)
for field in data_fields
if field.metadata
]
fields_with_data = [MetaTestField(field, self._get_actual_field_type(field), instance) for field in data_fields if field.metadata]
for field in data_fields:
actual_field_type = self._get_actual_field_type(field)