forked from TrueCloudLab/frostfs-testlib
Make load things parallel
Signed-off-by: Andrey Berezin <a.berezin@yadro.com>
This commit is contained in:
parent
05ac392485
commit
a14b082a4d
8 changed files with 284 additions and 178 deletions
|
|
@ -28,15 +28,31 @@ from frostfs_testlib.storage.cluster import ClusterNode
|
|||
from frostfs_testlib.storage.controllers.cluster_state_controller import ClusterStateController
|
||||
from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate, StorageNode
|
||||
from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
|
||||
from frostfs_testlib.testing.test_control import run_optionally
|
||||
from frostfs_testlib.utils import datetime_utils
|
||||
from frostfs_testlib.utils.file_keeper import FileKeeper
|
||||
from frostfs_testlib.testing import parallel, run_optionally
|
||||
from frostfs_testlib.utils import FileKeeper, datetime_utils
|
||||
|
||||
reporter = get_reporter()
|
||||
|
||||
|
||||
class DefaultRunner(ScenarioRunner):
|
||||
class RunnerBase(ScenarioRunner):
|
||||
k6_instances: list[K6]
|
||||
|
||||
@reporter.step_deco("Run preset on loaders")
|
||||
def preset(self):
|
||||
parallel([k6.preset for k6 in self.k6_instances])
|
||||
|
||||
@reporter.step_deco("Wait until load finish")
|
||||
def wait_until_finish(self):
|
||||
parallel([k6.wait_until_finished for k6 in self.k6_instances])
|
||||
|
||||
@property
|
||||
def is_running(self):
|
||||
futures = parallel([k6.is_running for k6 in self.k6_instances])
|
||||
|
||||
return any([future.result() for future in futures])
|
||||
|
||||
|
||||
class DefaultRunner(RunnerBase):
|
||||
loaders: list[Loader]
|
||||
loaders_wallet: WalletInfo
|
||||
|
||||
|
|
@ -51,7 +67,7 @@ class DefaultRunner(ScenarioRunner):
|
|||
self.loaders_wallet = loaders_wallet
|
||||
|
||||
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
|
||||
@reporter.step_deco("Prepare load instances")
|
||||
@reporter.step_deco("Preparation steps")
|
||||
def prepare(
|
||||
self,
|
||||
load_params: LoadParams,
|
||||
|
|
@ -68,48 +84,52 @@ class DefaultRunner(ScenarioRunner):
|
|||
]
|
||||
grpc_peer = storage_node.get_rpc_endpoint()
|
||||
|
||||
for loader in self.loaders:
|
||||
with reporter.step(f"Init s3 client on {loader.ip}"):
|
||||
shell = loader.get_shell()
|
||||
frostfs_authmate_exec: FrostfsAuthmate = FrostfsAuthmate(
|
||||
shell, FROSTFS_AUTHMATE_EXEC
|
||||
)
|
||||
issue_secret_output = frostfs_authmate_exec.secret.issue(
|
||||
wallet=self.loaders_wallet.path,
|
||||
peer=grpc_peer,
|
||||
gate_public_key=s3_public_keys,
|
||||
container_placement_policy=load_params.preset.container_placement_policy,
|
||||
container_policy=f"{k6_dir}/scenarios/files/policy.json",
|
||||
wallet_password=self.loaders_wallet.password,
|
||||
).stdout
|
||||
aws_access_key_id = str(
|
||||
re.search(
|
||||
r"access_key_id.*:\s.(?P<aws_access_key_id>\w*)", issue_secret_output
|
||||
).group("aws_access_key_id")
|
||||
)
|
||||
aws_secret_access_key = str(
|
||||
re.search(
|
||||
r"secret_access_key.*:\s.(?P<aws_secret_access_key>\w*)",
|
||||
issue_secret_output,
|
||||
).group("aws_secret_access_key")
|
||||
)
|
||||
parallel(
|
||||
self._prepare_loader, self.loaders, load_params, grpc_peer, s3_public_keys, k6_dir
|
||||
)
|
||||
|
||||
configure_input = [
|
||||
InteractiveInput(
|
||||
prompt_pattern=r"AWS Access Key ID.*", input=aws_access_key_id
|
||||
),
|
||||
InteractiveInput(
|
||||
prompt_pattern=r"AWS Secret Access Key.*", input=aws_secret_access_key
|
||||
),
|
||||
InteractiveInput(prompt_pattern=r".*", input=""),
|
||||
InteractiveInput(prompt_pattern=r".*", input=""),
|
||||
]
|
||||
shell.exec("aws configure", CommandOptions(interactive_inputs=configure_input))
|
||||
def _prepare_loader(
|
||||
self,
|
||||
loader: Loader,
|
||||
load_params: LoadParams,
|
||||
grpc_peer: str,
|
||||
s3_public_keys: list[str],
|
||||
k6_dir: str,
|
||||
):
|
||||
with reporter.step(f"Init s3 client on {loader.ip}"):
|
||||
shell = loader.get_shell()
|
||||
frostfs_authmate_exec: FrostfsAuthmate = FrostfsAuthmate(shell, FROSTFS_AUTHMATE_EXEC)
|
||||
issue_secret_output = frostfs_authmate_exec.secret.issue(
|
||||
wallet=self.loaders_wallet.path,
|
||||
peer=grpc_peer,
|
||||
gate_public_key=s3_public_keys,
|
||||
container_placement_policy=load_params.preset.container_placement_policy,
|
||||
container_policy=f"{k6_dir}/scenarios/files/policy.json",
|
||||
wallet_password=self.loaders_wallet.password,
|
||||
).stdout
|
||||
aws_access_key_id = str(
|
||||
re.search(
|
||||
r"access_key_id.*:\s.(?P<aws_access_key_id>\w*)", issue_secret_output
|
||||
).group("aws_access_key_id")
|
||||
)
|
||||
aws_secret_access_key = str(
|
||||
re.search(
|
||||
r"secret_access_key.*:\s.(?P<aws_secret_access_key>\w*)",
|
||||
issue_secret_output,
|
||||
).group("aws_secret_access_key")
|
||||
)
|
||||
|
||||
def wait_until_finish(self):
|
||||
for k6_instance in self.k6_instances:
|
||||
k6_instance.wait_until_finished()
|
||||
configure_input = [
|
||||
InteractiveInput(prompt_pattern=r"AWS Access Key ID.*", input=aws_access_key_id),
|
||||
InteractiveInput(
|
||||
prompt_pattern=r"AWS Secret Access Key.*", input=aws_secret_access_key
|
||||
),
|
||||
InteractiveInput(prompt_pattern=r".*", input=""),
|
||||
InteractiveInput(prompt_pattern=r".*", input=""),
|
||||
]
|
||||
shell.exec("aws configure", CommandOptions(interactive_inputs=configure_input))
|
||||
|
||||
@reporter.step_deco("Init k6 instances")
|
||||
def init_k6_instances(self, load_params: LoadParams, endpoints: list[str], k6_dir: str):
|
||||
self.k6_instances = []
|
||||
cycled_loaders = itertools.cycle(self.loaders)
|
||||
|
|
@ -131,29 +151,32 @@ class DefaultRunner(ScenarioRunner):
|
|||
load_params, k6_processes_count
|
||||
)
|
||||
|
||||
for distributed_load_params in distributed_load_params_list:
|
||||
loader = next(cycled_loaders)
|
||||
shell = loader.get_shell()
|
||||
with reporter.step(
|
||||
f"Init K6 instances on {loader.ip} for load id {distributed_load_params.load_id}"
|
||||
):
|
||||
with reporter.step(f"Make working directory"):
|
||||
shell.exec(f"sudo mkdir -p {distributed_load_params.working_dir}")
|
||||
shell.exec(
|
||||
f"sudo chown {LOAD_NODE_SSH_USER} {distributed_load_params.working_dir}"
|
||||
)
|
||||
futures = parallel(
|
||||
self._init_k6_instance,
|
||||
distributed_load_params_list,
|
||||
loader=cycled_loaders,
|
||||
endpoints=endpoints_gen,
|
||||
k6_dir=k6_dir,
|
||||
)
|
||||
self.k6_instances = [future.result() for future in futures]
|
||||
|
||||
k6_instance = K6(
|
||||
distributed_load_params,
|
||||
next(endpoints_gen),
|
||||
k6_dir,
|
||||
shell,
|
||||
loader,
|
||||
self.loaders_wallet,
|
||||
)
|
||||
self.k6_instances.append(k6_instance)
|
||||
if load_params.preset:
|
||||
k6_instance.preset()
|
||||
def _init_k6_instance(
|
||||
self, load_params_for_loader: LoadParams, loader: Loader, endpoints: list[str], k6_dir: str
|
||||
):
|
||||
shell = loader.get_shell()
|
||||
with reporter.step(f"Init K6 instance on {loader.ip} for endpoints {endpoints}"):
|
||||
with reporter.step(f"Make working directory"):
|
||||
shell.exec(f"sudo mkdir -p {load_params_for_loader.working_dir}")
|
||||
shell.exec(f"sudo chown {LOAD_NODE_SSH_USER} {load_params_for_loader.working_dir}")
|
||||
|
||||
return K6(
|
||||
load_params_for_loader,
|
||||
endpoints,
|
||||
k6_dir,
|
||||
shell,
|
||||
loader,
|
||||
self.loaders_wallet,
|
||||
)
|
||||
|
||||
def _get_distributed_load_params_list(
|
||||
self, original_load_params: LoadParams, workers_count: int
|
||||
|
|
@ -215,15 +238,7 @@ class DefaultRunner(ScenarioRunner):
|
|||
def start(self):
|
||||
load_params = self.k6_instances[0].load_params
|
||||
|
||||
with ThreadPoolExecutor(max_workers=len(self.k6_instances)) as executor:
|
||||
futures = [executor.submit(k6.start) for k6 in self.k6_instances]
|
||||
|
||||
# Check for exceptions
|
||||
exceptions = [future.exception() for future in futures if future.exception()]
|
||||
if exceptions:
|
||||
raise RuntimeError(
|
||||
f"The following exceptions occured during start of k6: {exceptions}"
|
||||
)
|
||||
parallel([k6.start for k6 in self.k6_instances])
|
||||
|
||||
wait_after_start_time = datetime_utils.parse_time(load_params.setup_timeout) + 5
|
||||
with reporter.step(
|
||||
|
|
@ -251,17 +266,8 @@ class DefaultRunner(ScenarioRunner):
|
|||
|
||||
return results
|
||||
|
||||
@property
|
||||
def is_running(self):
|
||||
for k6_instance in self.k6_instances:
|
||||
if not k6_instance.is_running:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class LocalRunner(ScenarioRunner):
|
||||
k6_instances: list[K6]
|
||||
class LocalRunner(RunnerBase):
|
||||
loaders: list[Loader]
|
||||
cluster_state_controller: ClusterStateController
|
||||
file_keeper: FileKeeper
|
||||
|
|
@ -278,7 +284,7 @@ class LocalRunner(ScenarioRunner):
|
|||
self.loaders = [NodeLoader(node) for node in nodes_under_load]
|
||||
|
||||
@run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
|
||||
@reporter.step_deco("Prepare load instances")
|
||||
@reporter.step_deco("Preparation steps")
|
||||
def prepare(
|
||||
self,
|
||||
load_params: LoadParams,
|
||||
|
|
@ -319,37 +325,39 @@ class LocalRunner(ScenarioRunner):
|
|||
for _ in result:
|
||||
pass
|
||||
|
||||
def wait_until_finish(self):
|
||||
for k6_instance in self.k6_instances:
|
||||
k6_instance.wait_until_finished()
|
||||
|
||||
@reporter.step_deco("Init k6 instances")
|
||||
def init_k6_instances(self, load_params: LoadParams, endpoints: list[str], k6_dir: str):
|
||||
self.k6_instances = []
|
||||
for loader in self.loaders:
|
||||
shell = loader.get_shell()
|
||||
with reporter.step(f"Init K6 instances on {loader.ip}"):
|
||||
with reporter.step(f"Make working directory"):
|
||||
shell.exec(f"sudo mkdir -p {load_params.working_dir}")
|
||||
# If we chmod /home/<user_name> folder we can no longer ssh to the node
|
||||
# !! IMPORTANT !!
|
||||
if (
|
||||
load_params.working_dir
|
||||
and not load_params.working_dir == f"/home/{LOAD_NODE_SSH_USER}"
|
||||
and not load_params.working_dir == f"/home/{LOAD_NODE_SSH_USER}/"
|
||||
):
|
||||
shell.exec(f"sudo chmod -R 777 {load_params.working_dir}")
|
||||
futures = parallel(
|
||||
self._init_k6_instance,
|
||||
self.loaders,
|
||||
load_params,
|
||||
k6_dir,
|
||||
)
|
||||
self.k6_instances = [future.result() for future in futures]
|
||||
|
||||
k6_instance = K6(
|
||||
load_params,
|
||||
["localhost:8080"],
|
||||
k6_dir,
|
||||
shell,
|
||||
loader,
|
||||
self.wallet,
|
||||
)
|
||||
self.k6_instances.append(k6_instance)
|
||||
if load_params.preset:
|
||||
k6_instance.preset()
|
||||
def _init_k6_instance(self, loader: Loader, load_params: LoadParams, k6_dir: str):
|
||||
shell = loader.get_shell()
|
||||
with reporter.step(f"Init K6 instance on {loader.ip}"):
|
||||
with reporter.step(f"Make working directory"):
|
||||
shell.exec(f"sudo mkdir -p {load_params.working_dir}")
|
||||
# If we chmod /home/<user_name> folder we can no longer ssh to the node
|
||||
# !! IMPORTANT !!
|
||||
if (
|
||||
load_params.working_dir
|
||||
and not load_params.working_dir == f"/home/{LOAD_NODE_SSH_USER}"
|
||||
and not load_params.working_dir == f"/home/{LOAD_NODE_SSH_USER}/"
|
||||
):
|
||||
shell.exec(f"sudo chmod -R 777 {load_params.working_dir}")
|
||||
|
||||
return K6(
|
||||
load_params,
|
||||
["localhost:8080"],
|
||||
k6_dir,
|
||||
shell,
|
||||
loader,
|
||||
self.wallet,
|
||||
)
|
||||
|
||||
def start(self):
|
||||
load_params = self.k6_instances[0].load_params
|
||||
|
|
@ -357,15 +365,7 @@ class LocalRunner(ScenarioRunner):
|
|||
self.cluster_state_controller.stop_all_s3_gates()
|
||||
self.cluster_state_controller.stop_all_storage_services()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=len(self.k6_instances)) as executor:
|
||||
futures = [executor.submit(k6.start) for k6 in self.k6_instances]
|
||||
|
||||
# Check for exceptions
|
||||
exceptions = [future.exception() for future in futures if future.exception()]
|
||||
if exceptions:
|
||||
raise RuntimeError(
|
||||
f"The following exceptions occured during start of k6: {exceptions}"
|
||||
)
|
||||
parallel([k6.start for k6 in self.k6_instances])
|
||||
|
||||
wait_after_start_time = datetime_utils.parse_time(load_params.setup_timeout) + 5
|
||||
with reporter.step(
|
||||
|
|
@ -387,11 +387,3 @@ class LocalRunner(ScenarioRunner):
|
|||
results[k6_instance.loader.ip] = result
|
||||
|
||||
return results
|
||||
|
||||
@property
|
||||
def is_running(self):
|
||||
for k6_instance in self.k6_instances:
|
||||
if not k6_instance.is_running:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue