From eb37573df898dacb539a4d9227902925ae91711b Mon Sep 17 00:00:00 2001 From: Andrey Berezin Date: Mon, 25 Sep 2023 16:26:45 +0300 Subject: [PATCH] [#88] Add read from switch Signed-off-by: Andrey Berezin --- src/frostfs_testlib/load/__init__.py | 1 + src/frostfs_testlib/load/load_config.py | 15 +++++++++++- tests/test_load_config.py | 32 ++++++++++++++++++------- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/frostfs_testlib/load/__init__.py b/src/frostfs_testlib/load/__init__.py index e8ed75e..74b710f 100644 --- a/src/frostfs_testlib/load/__init__.py +++ b/src/frostfs_testlib/load/__init__.py @@ -7,6 +7,7 @@ from frostfs_testlib.load.load_config import ( LoadType, NodesSelectionStrategy, Preset, + ReadFrom, ) from frostfs_testlib.load.load_report import LoadReport from frostfs_testlib.load.loaders import NodeLoader, RemoteLoader diff --git a/src/frostfs_testlib/load/load_config.py b/src/frostfs_testlib/load/load_config.py index 9023f87..97f5dd6 100644 --- a/src/frostfs_testlib/load/load_config.py +++ b/src/frostfs_testlib/load/load_config.py @@ -23,6 +23,12 @@ class LoadScenario(Enum): LOCAL = "local" +class ReadFrom(Enum): + REGISTRY = "registry" + PRESET = "preset" + MANUAL = "manual" + + all_load_scenarios = [ LoadScenario.gRPC, LoadScenario.S3, @@ -170,6 +176,8 @@ class LoadParams: load_time: Optional[int] = metadata_field(all_load_scenarios, None, "DURATION", False) # Object size in KB for load and preset. object_size: Optional[int] = metadata_field(all_load_scenarios, "size", "WRITE_OBJ_SIZE", False) + # For read operations, controls from which set get objects to read + read_from: Optional[ReadFrom] = None # Output registry K6 file. Filled automatically. registry_file: Optional[str] = metadata_field(all_scenarios, None, "REGISTRY_FILE", False) # Specifies the minimum duration of every single execution (i.e. iteration). @@ -256,7 +264,12 @@ class LoadParams: def set_id(self, load_id): self.load_id = load_id - self.registry_file = os.path.join(self.working_dir, f"{load_id}_registry.bolt") + + if self.read_from == ReadFrom.REGISTRY: + self.registry_file = os.path.join(self.working_dir, f"{load_id}_registry.bolt") + if self.read_from == ReadFrom.PRESET: + self.registry_file = None + if self.preset: self.preset.pregen_json = os.path.join(self.working_dir, f"{load_id}_prepare.json") diff --git a/tests/test_load_config.py b/tests/test_load_config.py index a84a188..256a04b 100644 --- a/tests/test_load_config.py +++ b/tests/test_load_config.py @@ -9,7 +9,9 @@ from frostfs_testlib.load.load_config import ( LoadScenario, LoadType, Preset, + ReadFrom, ) +from frostfs_testlib.load.runners import DefaultRunner from frostfs_testlib.storage.cluster import ClusterNode from frostfs_testlib.storage.controllers.background_load_controller import BackgroundLoadController from frostfs_testlib.storage.dataclasses.frostfs_services import StorageNode @@ -47,13 +49,13 @@ class TestLoadConfig: def test_load_params_only_load_type_required(self): load_params = LoadParams(load_type=LoadType.S3) - expected = "load: s3" + expected = "s3" assert repr(load_params) == expected assert f"{load_params}" == expected def test_load_params_initially_have_all_values_none(self): load_params = LoadParams(load_type=LoadType.S3) - self._check_all_values_none(load_params, ["load_type"]) + self._check_all_values_none(load_params, ["load_type", "scenario"]) def test_preset_initially_have_all_values_none(self): preset = Preset() @@ -62,14 +64,14 @@ class TestLoadConfig: @pytest.mark.parametrize("load_params", [LoadScenario.S3_CAR], indirect=True) def test_string_representation_s3_car(self, load_params: LoadParams): load_params.object_size = 524288 - expected = "load: s3_car (512 MiB), write_rate=10, read_rate=9, delete_rate=11, preallocated_writers=20, preallocated_readers=20, preallocated_deleters=21" + expected = "s3_car 512 MiB, write_rate=10, read_rate=9, delete_rate=11, preallocated_writers=20, preallocated_readers=20, preallocated_deleters=21" assert f"{load_params}" == expected assert repr(load_params) == expected @pytest.mark.parametrize("load_params", [LoadScenario.gRPC], indirect=True) def test_string_representation_grpc(self, load_params: LoadParams): load_params.object_size = 512 - expected = "load: grpc (512 KiB), writers=7, readers=7, deleters=8" + expected = "grpc 512 KiB, writers=7, readers=7, deleters=8" assert f"{load_params}" == expected assert repr(load_params) == expected @@ -78,15 +80,16 @@ class TestLoadConfig: load_params.endpoint_selection_strategy = EndpointSelectionStrategy.ALL load_params.object_size = 512 background_load_controller = BackgroundLoadController( - "tmp", load_params, "wallet", None, None + "tmp", load_params, "wallet", None, None, DefaultRunner(None) ) - expected = "load: grpc (512 KiB), writers=7, readers=7, deleters=8" + expected = "grpc 512 KiB, writers=7, readers=7, deleters=8" assert f"{background_load_controller}" == expected assert repr(background_load_controller) == expected def test_load_set_id_changes_fields(self): load_params = LoadParams(load_type=LoadType.S3) load_params.preset = Preset() + load_params.read_from = ReadFrom["REGISTRY"] load_params.working_dir = "/tmp" load_params.set_id("test_id") @@ -96,9 +99,18 @@ class TestLoadConfig: # No other values should be changed self._check_all_values_none( - load_params, ["load_type", "working_dir", "load_id", "registry_file", "preset"] + load_params, + [ + "load_type", + "working_dir", + "load_id", + "registry_file", + "preset", + "scenario", + "read_from", + ], ) - self._check_all_values_none(load_params.preset, ["pregen_json"]) + self._check_all_values_none(load_params.preset, ["pregen_json", "scenario"]) @pytest.mark.parametrize("load_params", [LoadScenario.gRPC], indirect=True) def test_argument_parsing_for_grpc_scenario(self, load_params: LoadParams): @@ -120,6 +132,7 @@ class TestLoadConfig: "READERS": 7, "DELETERS": 8, "PREGEN_JSON": "pregen_json", + "PREPARE_LOCALLY": True, } self._check_preset_params(load_params, expected_preset_args) @@ -152,6 +165,7 @@ class TestLoadConfig: "WRITE_RATE": 10, "READ_RATE": 9, "DELETE_RATE": 11, + "PREPARE_LOCALLY": True, } self._check_preset_params(load_params, expected_preset_args) @@ -319,6 +333,7 @@ class TestLoadConfig: "READERS": 0, "DELETERS": 0, "PREGEN_JSON": "", + "PREPARE_LOCALLY": False, } self._check_preset_params(load_params, expected_preset_args) @@ -353,6 +368,7 @@ class TestLoadConfig: "WRITE_RATE": 0, "READ_RATE": 0, "DELETE_RATE": 0, + "PREPARE_LOCALLY": False, } self._check_preset_params(load_params, expected_preset_args)