forked from TrueCloudLab/frostfs-testlib
Move shared code to testlib
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
parent
d97a02d1d3
commit
997e768e92
69 changed files with 9213 additions and 64 deletions
211
src/frostfs_testlib/load/load_config.py
Normal file
211
src/frostfs_testlib/load/load_config.py
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class LoadType(Enum):
|
||||
gRPC = "grpc"
|
||||
S3 = "s3"
|
||||
HTTP = "http"
|
||||
|
||||
|
||||
class LoadScenario(Enum):
|
||||
gRPC = "grpc"
|
||||
gRPC_CAR = "grpc_car"
|
||||
S3 = "s3"
|
||||
S3_CAR = "s3_car"
|
||||
HTTP = "http"
|
||||
VERIFY = "verify"
|
||||
|
||||
|
||||
all_load_scenarios = [
|
||||
LoadScenario.gRPC,
|
||||
LoadScenario.S3,
|
||||
LoadScenario.HTTP,
|
||||
LoadScenario.S3_CAR,
|
||||
LoadScenario.gRPC_CAR,
|
||||
]
|
||||
all_scenarios = all_load_scenarios.copy() + [LoadScenario.VERIFY]
|
||||
|
||||
constant_vus_scenarios = [LoadScenario.gRPC, LoadScenario.S3, LoadScenario.HTTP]
|
||||
constant_arrival_rate_scenarios = [LoadScenario.gRPC_CAR, LoadScenario.S3_CAR]
|
||||
|
||||
grpc_preset_scenarios = [LoadScenario.gRPC, LoadScenario.HTTP, LoadScenario.gRPC_CAR]
|
||||
s3_preset_scenarios = [LoadScenario.S3, LoadScenario.S3_CAR]
|
||||
|
||||
|
||||
def metadata_field(
|
||||
applicable_scenarios: list[LoadScenario],
|
||||
preset_param: Optional[str] = None,
|
||||
scenario_variable: Optional[str] = None,
|
||||
distributed: Optional[bool] = False,
|
||||
):
|
||||
return field(
|
||||
default=None,
|
||||
metadata={
|
||||
"applicable_scenarios": applicable_scenarios,
|
||||
"preset_argument": preset_param,
|
||||
"env_variable": scenario_variable,
|
||||
"distributed": distributed,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class NodesSelectionStrategy(Enum):
|
||||
# Select ONE random node from cluster nodes.
|
||||
RANDOM_SINGLE = "RANDOM_SINGLE"
|
||||
# Select All nodes.
|
||||
ALL = "ALL"
|
||||
# Select All nodes except node under test (useful for failover). This is DEFAULT one
|
||||
ALL_EXCEPT_UNDER_TEST = "ALL_EXCEPT_UNDER_TEST"
|
||||
# Select ONE random node except under test (useful for failover).
|
||||
RANDOM_SINGLE_EXCEPT_UNDER_TEST = "RANDOM_SINGLE_EXCEPT_UNDER_TEST"
|
||||
|
||||
|
||||
class EndpointSelectionStrategy(Enum):
|
||||
"""Enum which defines which endpoint to select from each storage node"""
|
||||
|
||||
# Select All endpoints.
|
||||
ALL = "ALL"
|
||||
# Select first endpoint from node
|
||||
FIRST = "FIRST"
|
||||
|
||||
|
||||
class K6ProcessAllocationStrategy(Enum):
|
||||
"""Enum which defines how K6 processes should be allocated"""
|
||||
|
||||
# Each load node will get one k6 process with all endpoints (Default)
|
||||
PER_LOAD_NODE = "PER_LOAD_NODE"
|
||||
# Each endpoint will get it's own k6 process regardless of number of load nodes.
|
||||
# If there is not enough load nodes, some nodes may have multiple k6 processes
|
||||
PER_ENDPOINT = "PER_ENDPOINT"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Preset:
|
||||
# ------ COMMON ------
|
||||
# Amount of objects which should be created
|
||||
objects_count: Optional[int] = metadata_field(all_load_scenarios, "preload_obj", None)
|
||||
# Preset json. Filled automatically.
|
||||
pregen_json: Optional[str] = metadata_field(all_load_scenarios, "out", "PREGEN_JSON")
|
||||
# Workers count for preset
|
||||
workers: Optional[int] = metadata_field(all_load_scenarios, "workers", None)
|
||||
|
||||
# ------ GRPC ------
|
||||
# Amount of containers which should be created
|
||||
containers_count: Optional[int] = metadata_field(grpc_preset_scenarios, "containers", None)
|
||||
# Container placement policy for containers for gRPC
|
||||
container_placement_policy: Optional[str] = metadata_field(
|
||||
grpc_preset_scenarios, "policy", None
|
||||
)
|
||||
|
||||
# ------ S3 ------
|
||||
# Amount of buckets which should be created
|
||||
buckets_count: Optional[int] = metadata_field(s3_preset_scenarios, "buckets", None)
|
||||
# S3 region (AKA placement policy for S3 buckets)
|
||||
s3_location: Optional[str] = metadata_field(s3_preset_scenarios, "location", None)
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadParams:
|
||||
# ------- CONTROL PARAMS -------
|
||||
# Load type can be gRPC, HTTP, S3.
|
||||
load_type: LoadType
|
||||
# Load scenario from k6 scenarios
|
||||
scenario: Optional[LoadScenario] = None
|
||||
# Strategy to select nodes under load. See NodesSelectionStrategy class for more details.
|
||||
# default is ALL_EXCEPT_UNDER_TEST
|
||||
nodes_selection_strategy: Optional[NodesSelectionStrategy] = None
|
||||
# Strategy which defines which endpoint to select from each storage node
|
||||
endpoint_selection_strategy: Optional[EndpointSelectionStrategy] = None
|
||||
# Strategy which defines how K6 processes should be allocated
|
||||
k6_process_allocation_strategy: Optional[K6ProcessAllocationStrategy] = None
|
||||
# Set to true in order to verify uploaded objects after K6 load finish. Default is True.
|
||||
verify: Optional[bool] = None
|
||||
# Just id for load so distinct it between runs. Filled automatically.
|
||||
load_id: Optional[str] = None
|
||||
# Working directory
|
||||
working_dir: Optional[str] = None
|
||||
# Preset for the k6 run
|
||||
preset: Optional[Preset] = None
|
||||
|
||||
# ------- COMMON SCENARIO PARAMS -------
|
||||
# Load time is the maximum duration for k6 to give load. Default is the BACKGROUND_LOAD_DEFAULT_TIME value.
|
||||
load_time: Optional[int] = metadata_field(all_load_scenarios, None, "DURATION")
|
||||
# Object size in KB for load and preset.
|
||||
object_size: Optional[int] = metadata_field(all_load_scenarios, "size", "WRITE_OBJ_SIZE")
|
||||
# Output registry K6 file. Filled automatically.
|
||||
registry_file: Optional[str] = metadata_field(all_scenarios, None, "REGISTRY_FILE")
|
||||
# Specifies the minimum duration of every single execution (i.e. iteration).
|
||||
# Any iterations that are shorter than this value will cause that VU to
|
||||
# sleep for the remainder of the time until the specified minimum duration is reached.
|
||||
min_iteration_duration: Optional[str] = metadata_field(
|
||||
all_load_scenarios, None, "K6_MIN_ITERATION_DURATION"
|
||||
)
|
||||
|
||||
# ------- CONSTANT VUS SCENARIO PARAMS -------
|
||||
# Amount of Writers VU.
|
||||
writers: Optional[int] = metadata_field(constant_vus_scenarios, None, "WRITERS", True)
|
||||
# Amount of Readers VU.
|
||||
readers: Optional[int] = metadata_field(constant_vus_scenarios, None, "READERS", True)
|
||||
# Amount of Deleters VU.
|
||||
deleters: Optional[int] = metadata_field(constant_vus_scenarios, None, "DELETERS", True)
|
||||
|
||||
# ------- CONSTANT ARRIVAL RATE SCENARIO PARAMS -------
|
||||
# Number of iterations to start during each timeUnit period for write.
|
||||
write_rate: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "WRITE_RATE", True
|
||||
)
|
||||
|
||||
# Number of iterations to start during each timeUnit period for read.
|
||||
read_rate: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "READ_RATE", True
|
||||
)
|
||||
|
||||
# Number of iterations to start during each timeUnit period for delete.
|
||||
delete_rate: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "DELETE_RATE", True
|
||||
)
|
||||
|
||||
# Amount of preAllocatedVUs for write operations.
|
||||
preallocated_writers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "PRE_ALLOC_WRITERS", True
|
||||
)
|
||||
# Amount of maxVUs for write operations.
|
||||
max_writers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "MAX_WRITERS", True
|
||||
)
|
||||
|
||||
# Amount of preAllocatedVUs for read operations.
|
||||
preallocated_readers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "PRE_ALLOC_READERS", True
|
||||
)
|
||||
# Amount of maxVUs for read operations.
|
||||
max_readers: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "MAX_READERS", True
|
||||
)
|
||||
|
||||
# Amount of preAllocatedVUs for read operations.
|
||||
preallocated_deleters: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "PRE_ALLOC_DELETERS", True
|
||||
)
|
||||
# Amount of maxVUs for delete operations.
|
||||
max_deleters: Optional[int] = metadata_field(
|
||||
constant_arrival_rate_scenarios, None, "MAX_DELETERS", True
|
||||
)
|
||||
|
||||
# Period of time to apply the rate value.
|
||||
time_unit: Optional[str] = metadata_field(constant_arrival_rate_scenarios, None, "TIME_UNIT")
|
||||
|
||||
# ------- VERIFY SCENARIO PARAMS -------
|
||||
# Maximum verification time for k6 to verify objects. Default is BACKGROUND_LOAD_MAX_VERIFY_TIME (3600).
|
||||
verify_time: Optional[int] = metadata_field([LoadScenario.VERIFY], None, "TIME_LIMIT")
|
||||
# Amount of Verification VU.
|
||||
clients: Optional[int] = metadata_field([LoadScenario.VERIFY], None, "CLIENTS")
|
||||
|
||||
def set_id(self, load_id):
|
||||
self.load_id = load_id
|
||||
self.registry_file = os.path.join(self.working_dir, f"{load_id}_registry.bolt")
|
||||
if self.preset:
|
||||
self.preset.pregen_json = os.path.join(self.working_dir, f"{load_id}_prepare.json")
|
||||
Loading…
Add table
Add a link
Reference in a new issue