From d9e881001e2e2f82366851ca26a3c0a448b3b64c Mon Sep 17 00:00:00 2001 From: Vladimir Avdeev Date: Mon, 28 Nov 2022 22:27:37 +0300 Subject: [PATCH] Add background load fixture Signed-off-by: Vladimir Avdeev --- pytest_tests/helpers/k6.py | 52 ++++++++---- pytest_tests/steps/load.py | 9 +- pytest_tests/testsuites/conftest.py | 83 ++++++++++++++++++- .../network/test_node_management.py | 20 +++-- robot/variables/common.py | 6 +- 5 files changed, 142 insertions(+), 28 deletions(-) diff --git a/pytest_tests/helpers/k6.py b/pytest_tests/helpers/k6.py index 67a2668..e6e5a72 100644 --- a/pytest_tests/helpers/k6.py +++ b/pytest_tests/helpers/k6.py @@ -2,6 +2,7 @@ import re from contextlib import contextmanager from dataclasses import dataclass from time import sleep +from typing import Optional import allure from neofs_testlib.shell import Shell @@ -23,16 +24,18 @@ LOAD_RESULTS_PATTERNS = { @dataclass class LoadParams: - obj_size: int - containers_count: int - out_file: str - obj_count: int - writers: int - readers: int - deleters: int - load_time: int load_type: str endpoint: str + writers: Optional[int] = None + readers: Optional[int] = None + deleters: Optional[int] = None + clients: Optional[int] = None + containers_count: Optional[int] = None + out_file: Optional[str] = None + load_time: Optional[int] = None + obj_count: Optional[int] = None + obj_size: Optional[int] = None + registry_file: Optional[str] = None @dataclass @@ -97,19 +100,36 @@ class K6: else: raise AssertionError("Wrong K6 load type") + @allure.step("Generate K6 command") + def _generate_env_variables(self, load_params: LoadParams, k6_dir: str) -> str: + env_vars = { + "DURATION": load_params.load_time or None, + "WRITE_OBJ_SIZE": load_params.obj_size or None, + "WRITERS": load_params.writers or 0, + "READERS": load_params.readers or 0, + "DELETERS": load_params.deleters or 0, + "REGISTRY_FILE": load_params.registry_file or None, + "CLIENTS": load_params.clients or None, + f"{self.load_params.load_type.upper()}_ENDPOINTS": self.load_params.endpoint, + "PREGEN_JSON": f"{self.k6_dir}/{self.load_params.load_type}_{self.load_params.out_file}" + if load_params.out_file + else None, + } + allure.attach( + "\n".join(f"{param}: {value}" for param, value in env_vars.items()), + "K6 ENV variables", + allure.attachment_type.TEXT, + ) + return " ".join( + [f"-e {param}={value}" for param, value in env_vars.items() if value is not None] + ) + @allure.step("Start K6 on initiator") def start(self) -> None: self._k6_dir = self.k6_dir command = ( - f"{self.k6_dir}/k6 run " - f"-e DURATION={self.load_params.load_time} " - f"-e WRITE_OBJ_SIZE={self.load_params.obj_size} " - f"-e WRITERS={self.load_params.writers} -e READERS={self.load_params.readers} " - f"-e DELETERS={self.load_params.deleters} " - f"-e {self.load_params.load_type.upper()}_ENDPOINTS={self.load_params.endpoint} " - f"-e PREGEN_JSON={self.k6_dir}/" - f"{self.load_params.load_type}_{self.load_params.out_file} " + f"{self.k6_dir}/k6 run {self._generate_env_variables(self.load_params, self.k6_dir)} " f"{self.k6_dir}/scenarios/{self.load_params.load_type}.js" ) self._k6_process = RemoteProcess.create(command, self.shell) diff --git a/pytest_tests/steps/load.py b/pytest_tests/steps/load.py index 0877993..5e70c8c 100644 --- a/pytest_tests/steps/load.py +++ b/pytest_tests/steps/load.py @@ -86,15 +86,18 @@ def prepare_objects(k6_instance: K6): @allure.title("Prepare K6 instances and objects") -def prepare_k6_instances(load_nodes: list, login: str, pkey: str, load_params: LoadParams) -> list: +def prepare_k6_instances( + load_nodes: list, login: str, pkey: str, load_params: LoadParams, prepare: bool = True +) -> list: k6_load_objects = [] for load_node in load_nodes: ssh_client = SSHShell(host=load_node, login=login, private_key_path=pkey) k6_load_object = K6(load_params, ssh_client) k6_load_objects.append(k6_load_object) for k6_load_object in k6_load_objects: - with allure.step("Prepare objects"): - prepare_objects(k6_load_object) + if prepare: + with allure.step("Prepare objects"): + prepare_objects(k6_load_object) return k6_load_objects diff --git a/pytest_tests/testsuites/conftest.py b/pytest_tests/testsuites/conftest.py index cbd8e41..ebd8ca6 100644 --- a/pytest_tests/testsuites/conftest.py +++ b/pytest_tests/testsuites/conftest.py @@ -9,8 +9,24 @@ import allure import pytest import yaml from binary_version_helper import get_local_binaries_versions, get_remote_binaries_versions -from common import ASSETS_DIR, FREE_STORAGE, HOSTING_CONFIG_FILE, NEOFS_NETMAP_DICT, WALLET_PASS +from common import ( + ASSETS_DIR, + BACKGROUND_LOAD_MAX_TIME, + BACKGROUND_OBJ_SIZE, + BACKGROUND_READERS_COUNT, + BACKGROUND_WRITERS_COUNT, + FREE_STORAGE, + HOSTING_CONFIG_FILE, + LOAD_NODE_SSH_PRIVATE_KEY_PATH, + LOAD_NODE_SSH_USER, + LOAD_NODES, + NEOFS_NETMAP_DICT, + STORAGE_NODE_SERVICE_NAME_REGEX, + WALLET_PASS, +) from env_properties import save_env_properties +from k6 import LoadParams +from load import get_services_endpoints, prepare_k6_instances from neofs_testlib.hosting import Hosting from neofs_testlib.reporter import AllureHandler, get_reporter from neofs_testlib.shell import LocalShell, Shell @@ -138,6 +154,71 @@ def run_health_check(collect_logs, hosting: Hosting): raise AssertionError(f"Nodes {failed_nodes} are not healthy") +@pytest.fixture(scope="session") +def background_grpc_load(client_shell, prepare_wallet_and_deposit): + registry_file = os.path.join("/tmp/", f"{str(uuid.uuid4())}.bolt") + prepare_file = os.path.join("/tmp/", f"{str(uuid.uuid4())}.json") + allure.dynamic.title( + f"Start background load with parameters: " + f"writers = {BACKGROUND_WRITERS_COUNT}, " + f"obj_size = {BACKGROUND_OBJ_SIZE}, " + f"load_time = {BACKGROUND_LOAD_MAX_TIME}" + f"prepare_json = {prepare_file}" + ) + with allure.step("Get endpoints"): + endpoints_list = get_services_endpoints( + hosting=hosting, + service_name_regex=STORAGE_NODE_SERVICE_NAME_REGEX, + endpoint_attribute="rpc_endpoint", + ) + endpoints = ",".join(endpoints_list) + load_params = LoadParams( + endpoint=endpoints, + obj_size=BACKGROUND_OBJ_SIZE, + registry_file=registry_file, + containers_count=1, + obj_count=0, + out_file=prepare_file, + readers=0, + writers=BACKGROUND_WRITERS_COUNT, + deleters=0, + load_time=BACKGROUND_LOAD_MAX_TIME, + load_type="grpc", + ) + k6_load_instances = prepare_k6_instances( + load_nodes=LOAD_NODES, + login=LOAD_NODE_SSH_USER, + pkey=LOAD_NODE_SSH_PRIVATE_KEY_PATH, + load_params=load_params, + ) + with allure.step("Run background load"): + for k6_load_instance in k6_load_instances: + k6_load_instance.start() + yield + with allure.step("Stop background load"): + for k6_load_instance in k6_load_instances: + k6_load_instance.stop() + with allure.step("Verify background load data"): + verify_params = LoadParams( + endpoint=endpoints, + clients=BACKGROUND_READERS_COUNT, + registry_file=registry_file, + load_time=BACKGROUND_LOAD_MAX_TIME, + load_type="verify", + ) + k6_verify_instances = prepare_k6_instances( + load_nodes=LOAD_NODES, + login=LOAD_NODE_SSH_USER, + pkey=LOAD_NODE_SSH_PRIVATE_KEY_PATH, + load_params=verify_params, + prepare=False, + ) + with allure.step("Run verify background load data"): + for k6_verify_instance in k6_verify_instances: + k6_verify_instance.start() + k6_verify_instance.wait_until_finished(BACKGROUND_LOAD_MAX_TIME) + + @pytest.fixture(scope="session") @allure.title("Prepare wallet and deposit") def prepare_wallet_and_deposit(client_shell, prepare_tmp_dir): diff --git a/pytest_tests/testsuites/network/test_node_management.py b/pytest_tests/testsuites/network/test_node_management.py index 3065ef4..e615e79 100644 --- a/pytest_tests/testsuites/network/test_node_management.py +++ b/pytest_tests/testsuites/network/test_node_management.py @@ -89,6 +89,18 @@ def return_nodes_after_test_run(client_shell: Shell, hosting: Hosting): return_nodes(client_shell, hosting) +@allure.step("Tick epoch with retries") +def tick_epoch_with_retries(shell: Shell, attempts: int = 3, timeout: int = 3): + for __attempt in range(attempts): + try: + tick_epoch(shell=shell) + except RuntimeError: + sleep(timeout) + continue + return + raise + + @allure.step("Return node to cluster") def return_nodes(shell: Shell, hosting: Hosting, alive_node: Optional[str] = None) -> None: for node in list(check_nodes): @@ -107,13 +119,7 @@ def return_nodes(shell: Shell, hosting: Hosting, alive_node: Optional[str] = Non check_nodes.remove(node) sleep(parse_time(MORPH_BLOCK_TIME)) - for __attempt in range(3): - try: - tick_epoch(shell=shell) - break - except RuntimeError: - sleep(3) - + tick_epoch_with_retries(attempts=3) check_node_in_map(node, shell=shell, alive_node=alive_node) diff --git a/robot/variables/common.py b/robot/variables/common.py index 2f34309..8c9295b 100644 --- a/robot/variables/common.py +++ b/robot/variables/common.py @@ -36,8 +36,12 @@ WALLET_PASS = os.getenv("WALLET_PASS", "") # Load node parameters LOAD_NODES = os.getenv("LOAD_NODES", "").split(",") -LOAD_NODE_SSH_USER = os.getenv("LOAD_NODE_SSH_USER") +LOAD_NODE_SSH_USER = os.getenv("LOAD_NODE_SSH_USER", "root") LOAD_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PATH") +BACKGROUND_WRITERS_COUNT = os.getenv("BACKGROUND_WRITERS_COUNT", 10) +BACKGROUND_READERS_COUNT = os.getenv("BACKGROUND_READERS_COUNT", 10) +BACKGROUND_OBJ_SIZE = os.getenv("BACKGROUND_OBJ_SIZE", 1024) +BACKGROUND_LOAD_MAX_TIME = os.getenv("BACKGROUND_LOAD_MAX_TIME", 600) # Configuration of storage nodes # TODO: we should use hosting instead of all these variables