[#88] Add read from switch

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
Andrey Berezin 2023-09-25 16:26:45 +03:00
parent 602de43bff
commit eb37573df8
3 changed files with 39 additions and 9 deletions

View file

@ -7,6 +7,7 @@ from frostfs_testlib.load.load_config import (
LoadType, LoadType,
NodesSelectionStrategy, NodesSelectionStrategy,
Preset, Preset,
ReadFrom,
) )
from frostfs_testlib.load.load_report import LoadReport from frostfs_testlib.load.load_report import LoadReport
from frostfs_testlib.load.loaders import NodeLoader, RemoteLoader from frostfs_testlib.load.loaders import NodeLoader, RemoteLoader

View file

@ -23,6 +23,12 @@ class LoadScenario(Enum):
LOCAL = "local" LOCAL = "local"
class ReadFrom(Enum):
REGISTRY = "registry"
PRESET = "preset"
MANUAL = "manual"
all_load_scenarios = [ all_load_scenarios = [
LoadScenario.gRPC, LoadScenario.gRPC,
LoadScenario.S3, LoadScenario.S3,
@ -170,6 +176,8 @@ class LoadParams:
load_time: Optional[int] = metadata_field(all_load_scenarios, None, "DURATION", False) load_time: Optional[int] = metadata_field(all_load_scenarios, None, "DURATION", False)
# Object size in KB for load and preset. # Object size in KB for load and preset.
object_size: Optional[int] = metadata_field(all_load_scenarios, "size", "WRITE_OBJ_SIZE", False) object_size: Optional[int] = metadata_field(all_load_scenarios, "size", "WRITE_OBJ_SIZE", False)
# For read operations, controls from which set get objects to read
read_from: Optional[ReadFrom] = None
# Output registry K6 file. Filled automatically. # Output registry K6 file. Filled automatically.
registry_file: Optional[str] = metadata_field(all_scenarios, None, "REGISTRY_FILE", False) registry_file: Optional[str] = metadata_field(all_scenarios, None, "REGISTRY_FILE", False)
# Specifies the minimum duration of every single execution (i.e. iteration). # Specifies the minimum duration of every single execution (i.e. iteration).
@ -256,7 +264,12 @@ class LoadParams:
def set_id(self, load_id): def set_id(self, load_id):
self.load_id = load_id self.load_id = load_id
if self.read_from == ReadFrom.REGISTRY:
self.registry_file = os.path.join(self.working_dir, f"{load_id}_registry.bolt") self.registry_file = os.path.join(self.working_dir, f"{load_id}_registry.bolt")
if self.read_from == ReadFrom.PRESET:
self.registry_file = None
if self.preset: if self.preset:
self.preset.pregen_json = os.path.join(self.working_dir, f"{load_id}_prepare.json") self.preset.pregen_json = os.path.join(self.working_dir, f"{load_id}_prepare.json")

View file

@ -9,7 +9,9 @@ from frostfs_testlib.load.load_config import (
LoadScenario, LoadScenario,
LoadType, LoadType,
Preset, Preset,
ReadFrom,
) )
from frostfs_testlib.load.runners import DefaultRunner
from frostfs_testlib.storage.cluster import ClusterNode from frostfs_testlib.storage.cluster import ClusterNode
from frostfs_testlib.storage.controllers.background_load_controller import BackgroundLoadController from frostfs_testlib.storage.controllers.background_load_controller import BackgroundLoadController
from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode
@ -47,13 +49,13 @@ class TestLoadConfig:
def test_load_params_only_load_type_required(self): def test_load_params_only_load_type_required(self):
load_params = LoadParams(load_type=LoadType.S3) load_params = LoadParams(load_type=LoadType.S3)
expected = "load: s3" expected = "s3"
assert repr(load_params) == expected assert repr(load_params) == expected
assert f"{load_params}" == expected assert f"{load_params}" == expected
def test_load_params_initially_have_all_values_none(self): def test_load_params_initially_have_all_values_none(self):
load_params = LoadParams(load_type=LoadType.S3) load_params = LoadParams(load_type=LoadType.S3)
self._check_all_values_none(load_params, ["load_type"]) self._check_all_values_none(load_params, ["load_type", "scenario"])
def test_preset_initially_have_all_values_none(self): def test_preset_initially_have_all_values_none(self):
preset = Preset() preset = Preset()
@ -62,14 +64,14 @@ class TestLoadConfig:
@pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True) @pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True)
def test_string_representation_s3_car(self, load_params: LoadParams): def test_string_representation_s3_car(self, load_params: LoadParams):
load_params.object_size = 524288 load_params.object_size = 524288
expected = "load: s3_car (512 MiB), write_rate=10, read_rate=9, delete_rate=11, preallocated_writers=20, preallocated_readers=20, preallocated_deleters=21" expected = "s3_car 512 MiB, write_rate=10, read_rate=9, delete_rate=11, preallocated_writers=20, preallocated_readers=20, preallocated_deleters=21"
assert f"{load_params}" == expected assert f"{load_params}" == expected
assert repr(load_params) == expected assert repr(load_params) == expected
@pytest.mark.parametrize("load_params", [LoadScenario.gRPC], indirect=True) @pytest.mark.parametrize("load_params", [LoadScenario.gRPC], indirect=True)
def test_string_representation_grpc(self, load_params: LoadParams): def test_string_representation_grpc(self, load_params: LoadParams):
load_params.object_size = 512 load_params.object_size = 512
expected = "load: grpc (512 KiB), writers=7, readers=7, deleters=8" expected = "grpc 512 KiB, writers=7, readers=7, deleters=8"
assert f"{load_params}" == expected assert f"{load_params}" == expected
assert repr(load_params) == expected assert repr(load_params) == expected
@ -78,15 +80,16 @@ class TestLoadConfig:
load_params.endpoint_selection_strategy = EndpointSelectionStrategy.ALL load_params.endpoint_selection_strategy = EndpointSelectionStrategy.ALL
load_params.object_size = 512 load_params.object_size = 512
background_load_controller = BackgroundLoadController( background_load_controller = BackgroundLoadController(
"tmp", load_params, "wallet", None, None "tmp", load_params, "wallet", None, None, DefaultRunner(None)
) )
expected = "load: grpc (512 KiB), writers=7, readers=7, deleters=8" expected = "grpc 512 KiB, writers=7, readers=7, deleters=8"
assert f"{background_load_controller}" == expected assert f"{background_load_controller}" == expected
assert repr(background_load_controller) == expected assert repr(background_load_controller) == expected
def test_load_set_id_changes_fields(self): def test_load_set_id_changes_fields(self):
load_params = LoadParams(load_type=LoadType.S3) load_params = LoadParams(load_type=LoadType.S3)
load_params.preset = Preset() load_params.preset = Preset()
load_params.read_from = ReadFrom["REGISTRY"]
load_params.working_dir = "/tmp" load_params.working_dir = "/tmp"
load_params.set_id("test_id") load_params.set_id("test_id")
@ -96,9 +99,18 @@ class TestLoadConfig:
# No other values should be changed # No other values should be changed
self._check_all_values_none( self._check_all_values_none(
load_params, ["load_type", "working_dir", "load_id", "registry_file", "preset"] load_params,
[
"load_type",
"working_dir",
"load_id",
"registry_file",
"preset",
"scenario",
"read_from",
],
) )
self._check_all_values_none(load_params.preset, ["pregen_json"]) self._check_all_values_none(load_params.preset, ["pregen_json", "scenario"])
@pytest.mark.parametrize("load_params", [LoadScenario.gRPC], indirect=True) @pytest.mark.parametrize("load_params", [LoadScenario.gRPC], indirect=True)
def test_argument_parsing_for_grpc_scenario(self, load_params: LoadParams): def test_argument_parsing_for_grpc_scenario(self, load_params: LoadParams):
@ -120,6 +132,7 @@ class TestLoadConfig:
"READERS": 7, "READERS": 7,
"DELETERS": 8, "DELETERS": 8,
"PREGEN_JSON": "pregen_json", "PREGEN_JSON": "pregen_json",
"PREPARE_LOCALLY": True,
} }
self._check_preset_params(load_params, expected_preset_args) self._check_preset_params(load_params, expected_preset_args)
@ -152,6 +165,7 @@ class TestLoadConfig:
"WRITE_RATE": 10, "WRITE_RATE": 10,
"READ_RATE": 9, "READ_RATE": 9,
"DELETE_RATE": 11, "DELETE_RATE": 11,
"PREPARE_LOCALLY": True,
} }
self._check_preset_params(load_params, expected_preset_args) self._check_preset_params(load_params, expected_preset_args)
@ -319,6 +333,7 @@ class TestLoadConfig:
"READERS": 0, "READERS": 0,
"DELETERS": 0, "DELETERS": 0,
"PREGEN_JSON": "", "PREGEN_JSON": "",
"PREPARE_LOCALLY": False,
} }
self._check_preset_params(load_params, expected_preset_args) self._check_preset_params(load_params, expected_preset_args)
@ -353,6 +368,7 @@ class TestLoadConfig:
"WRITE_RATE": 0, "WRITE_RATE": 0,
"READ_RATE": 0, "READ_RATE": 0,
"DELETE_RATE": 0, "DELETE_RATE": 0,
"PREPARE_LOCALLY": False,
} }
self._check_preset_params(load_params, expected_preset_args) self._check_preset_params(load_params, expected_preset_args)