From 93e5cb5f46b9745c4f7a5126f512d4ad8148b780 Mon Sep 17 00:00:00 2001 From: "a.lipay" Date: Wed, 19 Oct 2022 01:01:42 +0300 Subject: [PATCH] Add Load library, new params for common.py, new load tests, Adapt K6, remote_process for Hosting Signed-off-by: a.lipay --- pytest_tests/helpers/k6.py | 14 ++-- pytest_tests/helpers/remote_process.py | 50 +++++++------ pytest_tests/steps/load.py | 86 +++++++++++++++++++++++ pytest_tests/testsuites/load/test_load.py | 66 +++++++++++++++++ robot/variables/common.py | 6 ++ 5 files changed, 189 insertions(+), 33 deletions(-) create mode 100644 pytest_tests/steps/load.py create mode 100644 pytest_tests/testsuites/load/test_load.py diff --git a/pytest_tests/helpers/k6.py b/pytest_tests/helpers/k6.py index ca55a2d..370fc7b 100644 --- a/pytest_tests/helpers/k6.py +++ b/pytest_tests/helpers/k6.py @@ -4,9 +4,9 @@ from dataclasses import dataclass from time import sleep import allure +from neofs_testlib.shell import Shell from pytest_tests.helpers.remote_process import RemoteProcess -from pytest_tests.helpers.ssh_helper import HostClient EXIT_RESULT_CODE = 0 @@ -40,10 +40,10 @@ class LoadResults: class K6: - def __init__(self, load_params: LoadParams, host_client: HostClient): + def __init__(self, load_params: LoadParams, shell: Shell): self.load_params = load_params - self.host_client = host_client + self.shell = shell self._k6_dir = None self._k6_result = None @@ -59,7 +59,7 @@ class K6: @property def k6_dir(self) -> str: if not self._k6_dir: - self._k6_dir = self.host_client.exec("locate -l 1 'k6'").stdout.strip("\n") + self._k6_dir = self.shell.exec("locate -l 1 'k6'").stdout.strip("\n") return self._k6_dir @allure.step("Prepare containers and objects") @@ -74,7 +74,7 @@ class K6: f"--endpoint {self.load_params.endpoint.split(',')[0]} " f"--preload_obj {self.load_params.obj_count} " ) - terminal = self.host_client.exec(command) + terminal = self.shell.exec(command) return terminal.stdout.strip("\n") elif self.load_params.load_type == "s3": command = ( @@ -85,7 +85,7 @@ class K6: f"--preload_obj {self.load_params.obj_count} " f"--location load-1-1" ) - terminal = self.host_client.exec(command) + terminal = self.shell.exec(command) return terminal.stdout.strip("\n") else: raise AssertionError("Wrong K6 load type") @@ -105,7 +105,7 @@ class K6: f"{self.load_params.load_type}_{self.load_params.out_file} " f"{self.k6_dir}/scenarios/{self.load_params.load_type}.js" ) - self._k6_process = RemoteProcess.create(command, self.host_client) + self._k6_process = RemoteProcess.create(command, self.shell) @allure.step("Wait until K6 is finished") def wait_until_finished(self, timeout: int = 0, k6_should_be_running: bool = False) -> None: diff --git a/pytest_tests/helpers/remote_process.py b/pytest_tests/helpers/remote_process.py index b0e898f..1f803c8 100644 --- a/pytest_tests/helpers/remote_process.py +++ b/pytest_tests/helpers/remote_process.py @@ -4,17 +4,13 @@ import uuid from typing import Optional import allure - -# This file is broken, because tenacity is not registered in requirements.txt -# So, the file won't be fixed in scope of this PR, alipay will fix himself by -# switching RemoteProcess and K6 classes from HostClient to shell from hosting +from neofs_testlib.shell.interfaces import CommandOptions +from neofs_testlib.shell import Shell from tenacity import retry, stop_after_attempt, wait_fixed -from pytest_tests.helpers.ssh_helper import HostClient - class RemoteProcess: - def __init__(self, cmd: str, process_dir: str, host_client: HostClient): + def __init__(self, cmd: str, process_dir: str, shell: Shell): self.process_dir = process_dir self.cmd = cmd self.stdout_last_line_number = 0 @@ -23,11 +19,11 @@ class RemoteProcess: self.proc_rc: Optional[int] = None self.saved_stdout: Optional[str] = None self.saved_stderr: Optional[str] = None - self.host_client = host_client + self.shell = shell @classmethod @allure.step("Create remote process") - def create(cls, command: str, host_client: HostClient) -> RemoteProcess: + def create(cls, command: str, shell: Shell) -> RemoteProcess: """ Create a process on a remote host. @@ -39,14 +35,14 @@ class RemoteProcess: stdout: contains script output Args: - host_client: Host client instance + shell: Shell instance command: command to be run on a remote host Returns: RemoteProcess instance for further examination """ remote_process = cls( - cmd=command, process_dir=f"/tmp/proc_{uuid.uuid4()}", host_client=host_client + cmd=command, process_dir=f"/tmp/proc_{uuid.uuid4()}", shell=shell ) remote_process._create_process_dir() remote_process._generate_command_script(command) @@ -69,7 +65,7 @@ class RemoteProcess: if self.saved_stdout is not None: cur_stdout = self.saved_stdout else: - terminal = self.host_client.exec(f"cat {self.process_dir}/stdout") + terminal = self.shell.exec(f"cat {self.process_dir}/stdout") if self.proc_rc is not None: self.saved_stdout = terminal.stdout cur_stdout = terminal.stdout @@ -98,7 +94,7 @@ class RemoteProcess: if self.saved_stderr is not None: cur_stderr = self.saved_stderr else: - terminal = self.host_client.exec(f"cat {self.process_dir}/stderr") + terminal = self.shell.exec(f"cat {self.process_dir}/stderr") if self.proc_rc is not None: self.saved_stderr = terminal.stdout cur_stderr = terminal.stdout @@ -116,10 +112,10 @@ class RemoteProcess: if self.proc_rc is not None: return self.proc_rc - terminal = self.host_client.exec(f"cat {self.process_dir}/rc", verify=False) + terminal = self.shell.exec(f"cat {self.process_dir}/rc", CommandOptions(check=False)) if "No such file or directory" in terminal.stderr: return None - elif terminal.stderr or terminal.rc != 0: + elif terminal.stderr or terminal.return_code != 0: raise AssertionError(f"cat process rc was not successfull: {terminal.stderr}") self.proc_rc = int(terminal.stdout) @@ -131,11 +127,13 @@ class RemoteProcess: @allure.step("Send signal to process") def send_signal(self, signal: int) -> None: - kill_res = self.host_client.exec(f"kill -{signal} {self.pid}", verify=False) + kill_res = self.shell.exec(f"kill -{signal} {self.pid}", CommandOptions(check=False)) if "No such process" in kill_res.stderr: return - if kill_res.rc: - raise AssertionError(f"Signal {signal} not sent. Return code of kill: {kill_res.rc}") + if kill_res.return_code: + raise AssertionError( + f"Signal {signal} not sent. Return code of kill: {kill_res.return_code}" + ) @allure.step("Stop process") def stop(self) -> None: @@ -149,11 +147,11 @@ class RemoteProcess: def clear(self) -> None: if self.process_dir == "/": raise AssertionError(f"Invalid path to delete: {self.process_dir}") - self.host_client.exec(f"rm -rf {self.process_dir}") + self.shell.exec(f"rm -rf {self.process_dir}") @allure.step("Start remote process") def _start_process(self) -> None: - self.host_client.exec( + self.shell.exec( f"nohup {self.process_dir}/command.sh {self.process_dir}/stdout " f"2>{self.process_dir}/stderr &" @@ -161,14 +159,14 @@ class RemoteProcess: @allure.step("Create process directory") def _create_process_dir(self) -> None: - self.host_client.exec(f"mkdir {self.process_dir}; chmod 777 {self.process_dir}") - terminal = self.host_client.exec(f"realpath {self.process_dir}") + self.shell.exec(f"mkdir {self.process_dir}; chmod 777 {self.process_dir}") + terminal = self.shell.exec(f"realpath {self.process_dir}") self.process_dir = terminal.stdout.strip() @allure.step("Get pid") @retry(wait=wait_fixed(10), stop=stop_after_attempt(5), reraise=True) def _get_pid(self) -> str: - terminal = self.host_client.exec(f"cat {self.process_dir}/pid") + terminal = self.shell.exec(f"cat {self.process_dir}/pid") assert terminal.stdout, f"invalid pid: {terminal.stdout}" return terminal.stdout.strip() @@ -186,6 +184,6 @@ class RemoteProcess: f"echo $? > {self.process_dir}/rc" ) - self.host_client.exec(f'echo "{script}" > {self.process_dir}/command.sh') - self.host_client.exec(f"cat {self.process_dir}/command.sh") - self.host_client.exec(f"chmod +x {self.process_dir}/command.sh") + self.shell.exec(f'echo "{script}" > {self.process_dir}/command.sh') + self.shell.exec(f"cat {self.process_dir}/command.sh") + self.shell.exec(f"chmod +x {self.process_dir}/command.sh") diff --git a/pytest_tests/steps/load.py b/pytest_tests/steps/load.py new file mode 100644 index 0000000..31478de --- /dev/null +++ b/pytest_tests/steps/load.py @@ -0,0 +1,86 @@ +import concurrent.futures +from dataclasses import asdict + +import allure +from common import STORAGE_NODE_SERVICE_NAME_REGEX +from neofs_testlib.hosting import Hosting +from neofs_testlib.shell import SSHShell + +from pytest_tests.helpers.k6 import K6, LoadParams, LoadResults + + +@allure.title("Get storage host endpoints") +def get_storage_host_endpoints(hosting: Hosting) -> list: + service_configs = hosting.find_service_configs(STORAGE_NODE_SERVICE_NAME_REGEX) + return [service_config.attributes["rpc_endpoint"] for service_config in service_configs] + + +@allure.title("Clear cache and data from storage nodes") +def clear_cache_and_data(hosting: Hosting): + service_configs = hosting.find_service_configs(STORAGE_NODE_SERVICE_NAME_REGEX) + for service_config in service_configs: + host = hosting.get_host_by_service(service_config.name) + host.stop_service(service_config.name) + host.delete_storage_node_data(service_config.name) + host.start_service(service_config.name) + + +@allure.title("Prepare objects") +def prepare_objects(k6_instance: K6): + k6_instance.prepare() + + +@allure.title("Prepare K6 instances and objects") +def prepare_k6_instances(load_nodes: list, login: str, pkey: str, load_params: LoadParams) -> list: + k6_load_objects = [] + for load_node in load_nodes: + ssh_client = SSHShell(host=load_node, login=login, private_key_path=pkey) + k6_load_object = K6(load_params, ssh_client) + k6_load_objects.append(k6_load_object) + for k6_load_object in k6_load_objects: + with allure.step("Prepare objects"): + prepare_objects(k6_load_object) + return k6_load_objects + + +@allure.title("Run K6") +def run_k6_load(k6_instance: K6) -> LoadResults: + with allure.step("Executing load"): + k6_instance.start() + k6_instance.wait_until_finished(k6_instance.load_params.load_time * 2) + with allure.step("Printing results"): + k6_instance.get_k6_results() + return k6_instance.parsing_results() + + +@allure.title("MultiNode K6 Run") +def multi_node_k6_run(k6_instances: list) -> dict: + results = [] + avg_results = {} + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [] + for k6_instance in k6_instances: + futures.append(executor.submit(run_k6_load, k6_instance)) + for future in concurrent.futures.as_completed(futures): + results.append(asdict(future.result())) + for k6_result in results: + for key in k6_result: + try: + avg_results[key] += k6_result[key] / len(results) + except KeyError: + avg_results[key] = k6_result[key] / len(results) + return avg_results + + +@allure.title("Compare results") +def compare_load_results(result: dict, result_new: dict): + for key in result: + if result[key] != 0 and result_new[key] != 0: + if (abs(result[key] - result_new[key]) / min(result[key], result_new[key])) < 0.25: + continue + else: + raise AssertionError(f"Difference in {key} values more than 25%") + elif result[key] == 0 and result_new[key] == 0: + continue + else: + raise AssertionError(f"Unexpected zero value in {key}") diff --git a/pytest_tests/testsuites/load/test_load.py b/pytest_tests/testsuites/load/test_load.py new file mode 100644 index 0000000..9fe7493 --- /dev/null +++ b/pytest_tests/testsuites/load/test_load.py @@ -0,0 +1,66 @@ +import allure +import pytest +from common import LOAD_NODE_SSH_PRIVATE_KEY_PATH, LOAD_NODE_SSH_USER, LOAD_NODES +from neofs_testlib.hosting import Hosting + +from pytest_tests.helpers.k6 import LoadParams +from pytest_tests.steps.load import ( + clear_cache_and_data, + get_storage_host_endpoints, + multi_node_k6_run, + prepare_k6_instances, +) + +CONTAINERS_COUNT = 1 +OBJ_COUNT = 3 + + +class TestLoad: + @pytest.fixture(autouse=True) + def clear_cache_and_data(self, hosting: Hosting): + clear_cache_and_data(hosting=hosting) + + @pytest.mark.parametrize("obj_size, out_file", [(1000, "1mb_200.json")]) + @pytest.mark.parametrize("writers, readers, deleters", [(140, 60, 0), (200, 0, 0)]) + @pytest.mark.parametrize("load_time", [200, 900]) + @pytest.mark.parametrize("node_count", [4]) + def test_grpc_benchmark( + self, + obj_size, + out_file, + writers, + readers, + deleters, + load_time, + node_count, + hosting: Hosting, + ): + allure.dynamic.title( + f"Benchmark test - node_count = {node_count}, " + f"writers = {writers} readers = {readers}, " + f"deleters = {deleters}, obj_size = {obj_size}, " + f"load_time = {load_time}" + ) + with allure.step("Get endpoints"): + endpoints_list = get_storage_host_endpoints(hosting=hosting) + endpoints = ",".join(endpoints_list[:node_count]) + load_params = LoadParams( + endpoint=endpoints, + obj_size=obj_size, + containers_count=CONTAINERS_COUNT, + out_file=out_file, + obj_count=OBJ_COUNT, + writers=writers, + readers=readers, + deleters=deleters, + load_time=load_time, + load_type="grpc", + ) + k6_load_instances = prepare_k6_instances( + load_nodes=LOAD_NODES.split(','), + login=LOAD_NODE_SSH_USER, + pkey=LOAD_NODE_SSH_PRIVATE_KEY_PATH, + load_params=load_params, + ) + with allure.step("Run load"): + multi_node_k6_run(k6_load_instances) diff --git a/robot/variables/common.py b/robot/variables/common.py index d753104..7a588f0 100644 --- a/robot/variables/common.py +++ b/robot/variables/common.py @@ -34,6 +34,11 @@ DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "neofs-dev-env")) # Password of wallet owned by user on behalf of whom we are running tests WALLET_PASS = os.getenv("WALLET_PASS", "") +# Load node parameters +LOAD_NODES = os.getenv("LOAD_NODES") +LOAD_NODE_SSH_USER = os.getenv("LOAD_NODE_SSH_USER") +LOAD_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PATH") + # Configuration of storage nodes # TODO: we should use hosting instead of all these variables STORAGE_RPC_ENDPOINT_1 = os.getenv("STORAGE_RPC_ENDPOINT_1", "s01.neofs.devenv:8080") @@ -116,6 +121,7 @@ FREE_STORAGE = os.getenv("FREE_STORAGE", "false").lower() == "true" BIN_VERSIONS_FILE = os.getenv("BIN_VERSIONS_FILE") HOSTING_CONFIG_FILE = os.getenv("HOSTING_CONFIG_FILE", ".devenv.hosting.yaml") +STORAGE_NODE_SERVICE_NAME_REGEX = r"s\d\d" # Generate wallet configs # TODO: we should move all info about wallet configs to fixtures