Add Load library, new params for common.py, new load tests, Adapt K6, remote_process for Hosting

Signed-off-by: a.lipay <a.lipay@yadro.com>

pytest_tests/helpers/k6.py

@@ -4,9 +4,9 @@ from dataclasses import dataclass
 from time import sleep
 
 import allure
+from neofs_testlib.shell import Shell
 
 from pytest_tests.helpers.remote_process import RemoteProcess
-from pytest_tests.helpers.ssh_helper import HostClient
 
 EXIT_RESULT_CODE = 0
 
@@ -40,10 +40,10 @@ class LoadResults:
 
 
 class K6:
-    def __init__(self, load_params: LoadParams, host_client: HostClient):
+    def __init__(self, load_params: LoadParams, shell: Shell):
 
         self.load_params = load_params
-        self.host_client = host_client
+        self.shell = shell
 
         self._k6_dir = None
         self._k6_result = None
@@ -59,7 +59,7 @@ class K6:
     @property
     def k6_dir(self) -> str:
         if not self._k6_dir:
-            self._k6_dir = self.host_client.exec("locate -l 1 'k6'").stdout.strip("\n")
+            self._k6_dir = self.shell.exec("locate -l 1 'k6'").stdout.strip("\n")
         return self._k6_dir
 
     @allure.step("Prepare containers and objects")
@@ -74,7 +74,7 @@ class K6:
                 f"--endpoint {self.load_params.endpoint.split(',')[0]} "
                 f"--preload_obj {self.load_params.obj_count} "
             )
-            terminal = self.host_client.exec(command)
+            terminal = self.shell.exec(command)
             return terminal.stdout.strip("\n")
         elif self.load_params.load_type == "s3":
             command = (
@@ -85,7 +85,7 @@ class K6:
                 f"--preload_obj {self.load_params.obj_count} "
                 f"--location load-1-1"
             )
-            terminal = self.host_client.exec(command)
+            terminal = self.shell.exec(command)
             return terminal.stdout.strip("\n")
         else:
             raise AssertionError("Wrong K6 load type")
@@ -105,7 +105,7 @@ class K6:
             f"{self.load_params.load_type}_{self.load_params.out_file} "
             f"{self.k6_dir}/scenarios/{self.load_params.load_type}.js"
         )
-        self._k6_process = RemoteProcess.create(command, self.host_client)
+        self._k6_process = RemoteProcess.create(command, self.shell)
 
     @allure.step("Wait until K6 is finished")
     def wait_until_finished(self, timeout: int = 0, k6_should_be_running: bool = False) -> None:

pytest_tests/helpers/remote_process.py

@@ -4,17 +4,13 @@ import uuid
 from typing import Optional
 
 import allure
-
+from neofs_testlib.shell.interfaces import CommandOptions
-# This file is broken, because tenacity is not registered in requirements.txt
+from neofs_testlib.shell import Shell
-# So, the file won't be fixed in scope of this PR, alipay will fix himself by
-# switching RemoteProcess and K6 classes from HostClient to shell from hosting
 from tenacity import retry, stop_after_attempt, wait_fixed
 
-from pytest_tests.helpers.ssh_helper import HostClient
-
 
 class RemoteProcess:
-    def __init__(self, cmd: str, process_dir: str, host_client: HostClient):
+    def __init__(self, cmd: str, process_dir: str, shell: Shell):
         self.process_dir = process_dir
         self.cmd = cmd
         self.stdout_last_line_number = 0
@@ -23,11 +19,11 @@ class RemoteProcess:
         self.proc_rc: Optional[int] = None
         self.saved_stdout: Optional[str] = None
         self.saved_stderr: Optional[str] = None
-        self.host_client = host_client
+        self.shell = shell
 
     @classmethod
     @allure.step("Create remote process")
-    def create(cls, command: str, host_client: HostClient) -> RemoteProcess:
+    def create(cls, command: str, shell: Shell) -> RemoteProcess:
         """
         Create a process on a remote host.
 
@@ -39,14 +35,14 @@ class RemoteProcess:
             stdout: contains script output
 
         Args:
-            host_client: Host client instance
+            shell: Shell instance
             command: command to be run on a remote host
 
         Returns:
             RemoteProcess instance for further examination
         """
         remote_process = cls(
-            cmd=command, process_dir=f"/tmp/proc_{uuid.uuid4()}", host_client=host_client
+            cmd=command, process_dir=f"/tmp/proc_{uuid.uuid4()}", shell=shell
         )
         remote_process._create_process_dir()
         remote_process._generate_command_script(command)
@@ -69,7 +65,7 @@ class RemoteProcess:
         if self.saved_stdout is not None:
             cur_stdout = self.saved_stdout
         else:
-            terminal = self.host_client.exec(f"cat {self.process_dir}/stdout")
+            terminal = self.shell.exec(f"cat {self.process_dir}/stdout")
             if self.proc_rc is not None:
                 self.saved_stdout = terminal.stdout
             cur_stdout = terminal.stdout
@@ -98,7 +94,7 @@ class RemoteProcess:
         if self.saved_stderr is not None:
             cur_stderr = self.saved_stderr
         else:
-            terminal = self.host_client.exec(f"cat {self.process_dir}/stderr")
+            terminal = self.shell.exec(f"cat {self.process_dir}/stderr")
             if self.proc_rc is not None:
                 self.saved_stderr = terminal.stdout
             cur_stderr = terminal.stdout
@@ -116,10 +112,10 @@ class RemoteProcess:
         if self.proc_rc is not None:
             return self.proc_rc
 
-        terminal = self.host_client.exec(f"cat {self.process_dir}/rc", verify=False)
+        terminal = self.shell.exec(f"cat {self.process_dir}/rc", CommandOptions(check=False))
         if "No such file or directory" in terminal.stderr:
             return None
-        elif terminal.stderr or terminal.rc != 0:
+        elif terminal.stderr or terminal.return_code != 0:
             raise AssertionError(f"cat process rc was not successfull: {terminal.stderr}")
 
         self.proc_rc = int(terminal.stdout)
@@ -131,11 +127,13 @@ class RemoteProcess:
 
     @allure.step("Send signal to process")
     def send_signal(self, signal: int) -> None:
-        kill_res = self.host_client.exec(f"kill -{signal} {self.pid}", verify=False)
+        kill_res = self.shell.exec(f"kill -{signal} {self.pid}", CommandOptions(check=False))
         if "No such process" in kill_res.stderr:
             return
-        if kill_res.rc:
+        if kill_res.return_code:
-            raise AssertionError(f"Signal {signal} not sent. Return code of kill: {kill_res.rc}")
+            raise AssertionError(
+                f"Signal {signal} not sent. Return code of kill: {kill_res.return_code}"
+            )
 
     @allure.step("Stop process")
     def stop(self) -> None:
@@ -149,11 +147,11 @@ class RemoteProcess:
     def clear(self) -> None:
         if self.process_dir == "/":
             raise AssertionError(f"Invalid path to delete: {self.process_dir}")
-        self.host_client.exec(f"rm -rf {self.process_dir}")
+        self.shell.exec(f"rm -rf {self.process_dir}")
 
     @allure.step("Start remote process")
     def _start_process(self) -> None:
-        self.host_client.exec(
+        self.shell.exec(
             f"nohup {self.process_dir}/command.sh </dev/null "
             f">{self.process_dir}/stdout "
             f"2>{self.process_dir}/stderr &"
@@ -161,14 +159,14 @@ class RemoteProcess:
 
     @allure.step("Create process directory")
     def _create_process_dir(self) -> None:
-        self.host_client.exec(f"mkdir {self.process_dir}; chmod 777 {self.process_dir}")
+        self.shell.exec(f"mkdir {self.process_dir}; chmod 777 {self.process_dir}")
-        terminal = self.host_client.exec(f"realpath {self.process_dir}")
+        terminal = self.shell.exec(f"realpath {self.process_dir}")
         self.process_dir = terminal.stdout.strip()
 
     @allure.step("Get pid")
     @retry(wait=wait_fixed(10), stop=stop_after_attempt(5), reraise=True)
     def _get_pid(self) -> str:
-        terminal = self.host_client.exec(f"cat {self.process_dir}/pid")
+        terminal = self.shell.exec(f"cat {self.process_dir}/pid")
         assert terminal.stdout, f"invalid pid: {terminal.stdout}"
         return terminal.stdout.strip()
 
@@ -186,6 +184,6 @@ class RemoteProcess:
             f"echo $? > {self.process_dir}/rc"
         )
 
-        self.host_client.exec(f'echo "{script}" > {self.process_dir}/command.sh')
+        self.shell.exec(f'echo "{script}" > {self.process_dir}/command.sh')
-        self.host_client.exec(f"cat {self.process_dir}/command.sh")
+        self.shell.exec(f"cat {self.process_dir}/command.sh")
-        self.host_client.exec(f"chmod +x {self.process_dir}/command.sh")
+        self.shell.exec(f"chmod +x {self.process_dir}/command.sh")

pytest_tests/steps/load.py

@@ -0,0 +1,86 @@
+import concurrent.futures
+from dataclasses import asdict
+
+import allure
+from common import STORAGE_NODE_SERVICE_NAME_REGEX
+from neofs_testlib.hosting import Hosting
+from neofs_testlib.shell import SSHShell
+
+from pytest_tests.helpers.k6 import K6, LoadParams, LoadResults
+
+
+@allure.title("Get storage host endpoints")
+def get_storage_host_endpoints(hosting: Hosting) -> list:
+    service_configs = hosting.find_service_configs(STORAGE_NODE_SERVICE_NAME_REGEX)
+    return [service_config.attributes["rpc_endpoint"] for service_config in service_configs]
+
+
+@allure.title("Clear cache and data from storage nodes")
+def clear_cache_and_data(hosting: Hosting):
+    service_configs = hosting.find_service_configs(STORAGE_NODE_SERVICE_NAME_REGEX)
+    for service_config in service_configs:
+        host = hosting.get_host_by_service(service_config.name)
+        host.stop_service(service_config.name)
+        host.delete_storage_node_data(service_config.name)
+        host.start_service(service_config.name)
+
+
+@allure.title("Prepare objects")
+def prepare_objects(k6_instance: K6):
+    k6_instance.prepare()
+
+
+@allure.title("Prepare K6 instances and objects")
+def prepare_k6_instances(load_nodes: list, login: str, pkey: str, load_params: LoadParams) -> list:
+    k6_load_objects = []
+    for load_node in load_nodes:
+        ssh_client = SSHShell(host=load_node, login=login, private_key_path=pkey)
+        k6_load_object = K6(load_params, ssh_client)
+        k6_load_objects.append(k6_load_object)
+    for k6_load_object in k6_load_objects:
+        with allure.step("Prepare objects"):
+            prepare_objects(k6_load_object)
+    return k6_load_objects
+
+
+@allure.title("Run K6")
+def run_k6_load(k6_instance: K6) -> LoadResults:
+    with allure.step("Executing load"):
+        k6_instance.start()
+        k6_instance.wait_until_finished(k6_instance.load_params.load_time * 2)
+    with allure.step("Printing results"):
+        k6_instance.get_k6_results()
+        return k6_instance.parsing_results()
+
+
+@allure.title("MultiNode K6 Run")
+def multi_node_k6_run(k6_instances: list) -> dict:
+    results = []
+    avg_results = {}
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        futures = []
+        for k6_instance in k6_instances:
+            futures.append(executor.submit(run_k6_load, k6_instance))
+        for future in concurrent.futures.as_completed(futures):
+            results.append(asdict(future.result()))
+    for k6_result in results:
+        for key in k6_result:
+            try:
+                avg_results[key] += k6_result[key] / len(results)
+            except KeyError:
+                avg_results[key] = k6_result[key] / len(results)
+    return avg_results
+
+
+@allure.title("Compare results")
+def compare_load_results(result: dict, result_new: dict):
+    for key in result:
+        if result[key] != 0 and result_new[key] != 0:
+            if (abs(result[key] - result_new[key]) / min(result[key], result_new[key])) < 0.25:
+                continue
+            else:
+                raise AssertionError(f"Difference in {key} values more than 25%")
+        elif result[key] == 0 and result_new[key] == 0:
+            continue
+        else:
+            raise AssertionError(f"Unexpected zero value in {key}")

pytest_tests/testsuites/load/test_load.py

@@ -0,0 +1,66 @@
+import allure
+import pytest
+from common import LOAD_NODE_SSH_PRIVATE_KEY_PATH, LOAD_NODE_SSH_USER, LOAD_NODES
+from neofs_testlib.hosting import Hosting
+
+from pytest_tests.helpers.k6 import LoadParams
+from pytest_tests.steps.load import (
+    clear_cache_and_data,
+    get_storage_host_endpoints,
+    multi_node_k6_run,
+    prepare_k6_instances,
+)
+
+CONTAINERS_COUNT = 1
+OBJ_COUNT = 3
+
+
+class TestLoad:
+    @pytest.fixture(autouse=True)
+    def clear_cache_and_data(self, hosting: Hosting):
+        clear_cache_and_data(hosting=hosting)
+
+    @pytest.mark.parametrize("obj_size, out_file", [(1000, "1mb_200.json")])
+    @pytest.mark.parametrize("writers, readers, deleters", [(140, 60, 0), (200, 0, 0)])
+    @pytest.mark.parametrize("load_time", [200, 900])
+    @pytest.mark.parametrize("node_count", [4])
+    def test_grpc_benchmark(
+        self,
+        obj_size,
+        out_file,
+        writers,
+        readers,
+        deleters,
+        load_time,
+        node_count,
+        hosting: Hosting,
+    ):
+        allure.dynamic.title(
+            f"Benchmark test - node_count = {node_count}, "
+            f"writers = {writers} readers = {readers}, "
+            f"deleters = {deleters}, obj_size = {obj_size}, "
+            f"load_time = {load_time}"
+        )
+        with allure.step("Get endpoints"):
+            endpoints_list = get_storage_host_endpoints(hosting=hosting)
+            endpoints = ",".join(endpoints_list[:node_count])
+        load_params = LoadParams(
+            endpoint=endpoints,
+            obj_size=obj_size,
+            containers_count=CONTAINERS_COUNT,
+            out_file=out_file,
+            obj_count=OBJ_COUNT,
+            writers=writers,
+            readers=readers,
+            deleters=deleters,
+            load_time=load_time,
+            load_type="grpc",
+        )
+        k6_load_instances = prepare_k6_instances(
+            load_nodes=LOAD_NODES.split(','),
+            login=LOAD_NODE_SSH_USER,
+            pkey=LOAD_NODE_SSH_PRIVATE_KEY_PATH,
+            load_params=load_params,
+        )
+        with allure.step("Run load"):
+            multi_node_k6_run(k6_load_instances)

robot/variables/common.py

@@ -34,6 +34,11 @@ DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "neofs-dev-env"))
 # Password of wallet owned by user on behalf of whom we are running tests
 WALLET_PASS = os.getenv("WALLET_PASS", "")
 
+# Load node parameters
+LOAD_NODES = os.getenv("LOAD_NODES")
+LOAD_NODE_SSH_USER = os.getenv("LOAD_NODE_SSH_USER")
+LOAD_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PATH")
+
 # Configuration of storage nodes
 # TODO: we should use hosting instead of all these variables
 STORAGE_RPC_ENDPOINT_1 = os.getenv("STORAGE_RPC_ENDPOINT_1", "s01.neofs.devenv:8080")
@@ -116,6 +121,7 @@ FREE_STORAGE = os.getenv("FREE_STORAGE", "false").lower() == "true"
 BIN_VERSIONS_FILE = os.getenv("BIN_VERSIONS_FILE")
 
 HOSTING_CONFIG_FILE = os.getenv("HOSTING_CONFIG_FILE", ".devenv.hosting.yaml")
+STORAGE_NODE_SERVICE_NAME_REGEX = r"s\d\d"
 
 # Generate wallet configs
 # TODO: we should move all info about wallet configs to fixtures