mmalygina/frostfs-testcases
1
0
Fork 0
forked from TrueCloudLab/frostfs-testcases
Code Pull requests Activity

Add Load library, new params for common.py, new load tests, Adapt K6, remote_process for Hosting

Browse source 
Signed-off-by: a.lipay <a.lipay@yadro.com>
This commit is contained in:
a.lipay 2022-10-19 01:01:42 +03:00 committed by Alipayy
parent 805e014c2f
commit 93e5cb5f46
5 changed files with 189 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
pytest_tests/helpers/k6.py
View file

@ -4,9 +4,9 @@ from dataclasses import dataclass
from time import sleep
import allure
from neofs_testlib.shell import Shell
from pytest_tests.helpers.remote_process import RemoteProcess
from pytest_tests.helpers.ssh_helper import HostClient
EXIT_RESULT_CODE = 0
@ -40,10 +40,10 @@ class LoadResults:
class K6:
def __init__(self, load_params: LoadParams, host_client: HostClient):
def __init__(self, load_params: LoadParams, shell: Shell):
self.load_params = load_params
self.host_client = host_client
self.shell = shell
self._k6_dir = None
self._k6_result = None
@ -59,7 +59,7 @@ class K6:
@property
def k6_dir(self) -> str:
if not self._k6_dir:
self._k6_dir = self.host_client.exec("locate -l 1 'k6'").stdout.strip("\n")
self._k6_dir = self.shell.exec("locate -l 1 'k6'").stdout.strip("\n")
return self._k6_dir
@allure.step("Prepare containers and objects")
@ -74,7 +74,7 @@ class K6:
f"--endpoint {self.load_params.endpoint.split(',')[0]} "
f"--preload_obj {self.load_params.obj_count} "
)
terminal = self.host_client.exec(command)
terminal = self.shell.exec(command)
return terminal.stdout.strip("\n")
elif self.load_params.load_type == "s3":
command = (
@ -85,7 +85,7 @@ class K6:
f"--preload_obj {self.load_params.obj_count} "
f"--location load-1-1"
)
terminal = self.host_client.exec(command)
terminal = self.shell.exec(command)
return terminal.stdout.strip("\n")
else:
raise AssertionError("Wrong K6 load type")
@ -105,7 +105,7 @@ class K6:
f"{self.load_params.load_type}_{self.load_params.out_file} "
f"{self.k6_dir}/scenarios/{self.load_params.load_type}.js"
)
self._k6_process = RemoteProcess.create(command, self.host_client)
self._k6_process = RemoteProcess.create(command, self.shell)
@allure.step("Wait until K6 is finished")
def wait_until_finished(self, timeout: int = 0, k6_should_be_running: bool = False) -> None:

50
pytest_tests/helpers/remote_process.py
View file

@ -4,17 +4,13 @@ import uuid
from typing import Optional
import allure
# This file is broken, because tenacity is not registered in requirements.txt
# So, the file won't be fixed in scope of this PR, alipay will fix himself by
# switching RemoteProcess and K6 classes from HostClient to shell from hosting
from neofs_testlib.shell.interfaces import CommandOptions
from neofs_testlib.shell import Shell
from tenacity import retry, stop_after_attempt, wait_fixed
from pytest_tests.helpers.ssh_helper import HostClient
class RemoteProcess:
def __init__(self, cmd: str, process_dir: str, host_client: HostClient):
def __init__(self, cmd: str, process_dir: str, shell: Shell):
self.process_dir = process_dir
self.cmd = cmd
self.stdout_last_line_number = 0
@ -23,11 +19,11 @@ class RemoteProcess:
self.proc_rc: Optional[int] = None
self.saved_stdout: Optional[str] = None
self.saved_stderr: Optional[str] = None
self.host_client = host_client
self.shell = shell
@classmethod
@allure.step("Create remote process")
def create(cls, command: str, host_client: HostClient) -> RemoteProcess:
def create(cls, command: str, shell: Shell) -> RemoteProcess:
"""
Create a process on a remote host.
@ -39,14 +35,14 @@ class RemoteProcess:
stdout: contains script output
Args:
host_client: Host client instance
shell: Shell instance
command: command to be run on a remote host
Returns:
RemoteProcess instance for further examination
"""
remote_process = cls(
cmd=command, process_dir=f"/tmp/proc_{uuid.uuid4()}", host_client=host_client
cmd=command, process_dir=f"/tmp/proc_{uuid.uuid4()}", shell=shell
)
remote_process._create_process_dir()
remote_process._generate_command_script(command)
@ -69,7 +65,7 @@ class RemoteProcess:
if self.saved_stdout is not None:
cur_stdout = self.saved_stdout
else:
terminal = self.host_client.exec(f"cat {self.process_dir}/stdout")
terminal = self.shell.exec(f"cat {self.process_dir}/stdout")
if self.proc_rc is not None:
self.saved_stdout = terminal.stdout
cur_stdout = terminal.stdout
@ -98,7 +94,7 @@ class RemoteProcess:
if self.saved_stderr is not None:
cur_stderr = self.saved_stderr
else:
terminal = self.host_client.exec(f"cat {self.process_dir}/stderr")
terminal = self.shell.exec(f"cat {self.process_dir}/stderr")
if self.proc_rc is not None:
self.saved_stderr = terminal.stdout
cur_stderr = terminal.stdout
@ -116,10 +112,10 @@ class RemoteProcess:
if self.proc_rc is not None:
return self.proc_rc
terminal = self.host_client.exec(f"cat {self.process_dir}/rc", verify=False)
terminal = self.shell.exec(f"cat {self.process_dir}/rc", CommandOptions(check=False))
if "No such file or directory" in terminal.stderr:
return None
elif terminal.stderr or terminal.rc != 0:
elif terminal.stderr or terminal.return_code != 0:
raise AssertionError(f"cat process rc was not successfull: {terminal.stderr}")
self.proc_rc = int(terminal.stdout)
@ -131,11 +127,13 @@ class RemoteProcess:
@allure.step("Send signal to process")
def send_signal(self, signal: int) -> None:
kill_res = self.host_client.exec(f"kill -{signal} {self.pid}", verify=False)
kill_res = self.shell.exec(f"kill -{signal} {self.pid}", CommandOptions(check=False))
if "No such process" in kill_res.stderr:
return
if kill_res.rc:
raise AssertionError(f"Signal {signal} not sent. Return code of kill: {kill_res.rc}")
if kill_res.return_code:
raise AssertionError(
f"Signal {signal} not sent. Return code of kill: {kill_res.return_code}"
)
@allure.step("Stop process")
def stop(self) -> None:
@ -149,11 +147,11 @@ class RemoteProcess:
def clear(self) -> None:
if self.process_dir == "/":
raise AssertionError(f"Invalid path to delete: {self.process_dir}")
self.host_client.exec(f"rm -rf {self.process_dir}")
self.shell.exec(f"rm -rf {self.process_dir}")
@allure.step("Start remote process")
def _start_process(self) -> None:
self.host_client.exec(
self.shell.exec(
f"nohup {self.process_dir}/command.sh </dev/null "
f">{self.process_dir}/stdout "
f"2>{self.process_dir}/stderr &"
@ -161,14 +159,14 @@ class RemoteProcess:
@allure.step("Create process directory")
def _create_process_dir(self) -> None:
self.host_client.exec(f"mkdir {self.process_dir}; chmod 777 {self.process_dir}")
terminal = self.host_client.exec(f"realpath {self.process_dir}")
self.shell.exec(f"mkdir {self.process_dir}; chmod 777 {self.process_dir}")
terminal = self.shell.exec(f"realpath {self.process_dir}")
self.process_dir = terminal.stdout.strip()
@allure.step("Get pid")
@retry(wait=wait_fixed(10), stop=stop_after_attempt(5), reraise=True)
def _get_pid(self) -> str:
terminal = self.host_client.exec(f"cat {self.process_dir}/pid")
terminal = self.shell.exec(f"cat {self.process_dir}/pid")
assert terminal.stdout, f"invalid pid: {terminal.stdout}"
return terminal.stdout.strip()
@ -186,6 +184,6 @@ class RemoteProcess:
f"echo $? > {self.process_dir}/rc"
)
self.host_client.exec(f'echo "{script}" > {self.process_dir}/command.sh')
self.host_client.exec(f"cat {self.process_dir}/command.sh")
self.host_client.exec(f"chmod +x {self.process_dir}/command.sh")
self.shell.exec(f'echo "{script}" > {self.process_dir}/command.sh')
self.shell.exec(f"cat {self.process_dir}/command.sh")
self.shell.exec(f"chmod +x {self.process_dir}/command.sh")

86
pytest_tests/steps/load.py Normal file
View file

@ -0,0 +1,86 @@
import concurrent.futures
from dataclasses import asdict
import allure
from common import STORAGE_NODE_SERVICE_NAME_REGEX
from neofs_testlib.hosting import Hosting
from neofs_testlib.shell import SSHShell
from pytest_tests.helpers.k6 import K6, LoadParams, LoadResults
@allure.title("Get storage host endpoints")
def get_storage_host_endpoints(hosting: Hosting) -> list:
service_configs = hosting.find_service_configs(STORAGE_NODE_SERVICE_NAME_REGEX)
return [service_config.attributes["rpc_endpoint"] for service_config in service_configs]
@allure.title("Clear cache and data from storage nodes")
def clear_cache_and_data(hosting: Hosting):
service_configs = hosting.find_service_configs(STORAGE_NODE_SERVICE_NAME_REGEX)
for service_config in service_configs:
host = hosting.get_host_by_service(service_config.name)
host.stop_service(service_config.name)
host.delete_storage_node_data(service_config.name)
host.start_service(service_config.name)
@allure.title("Prepare objects")
def prepare_objects(k6_instance: K6):
k6_instance.prepare()
@allure.title("Prepare K6 instances and objects")
def prepare_k6_instances(load_nodes: list, login: str, pkey: str, load_params: LoadParams) -> list:
k6_load_objects = []
for load_node in load_nodes:
ssh_client = SSHShell(host=load_node, login=login, private_key_path=pkey)
k6_load_object = K6(load_params, ssh_client)
k6_load_objects.append(k6_load_object)
for k6_load_object in k6_load_objects:
with allure.step("Prepare objects"):
prepare_objects(k6_load_object)
return k6_load_objects
@allure.title("Run K6")
def run_k6_load(k6_instance: K6) -> LoadResults:
with allure.step("Executing load"):
k6_instance.start()
k6_instance.wait_until_finished(k6_instance.load_params.load_time * 2)
with allure.step("Printing results"):
k6_instance.get_k6_results()
return k6_instance.parsing_results()
@allure.title("MultiNode K6 Run")
def multi_node_k6_run(k6_instances: list) -> dict:
results = []
avg_results = {}
with concurrent.futures.ThreadPoolExecutor() as executor:
futures = []
for k6_instance in k6_instances:
futures.append(executor.submit(run_k6_load, k6_instance))
for future in concurrent.futures.as_completed(futures):
results.append(asdict(future.result()))
for k6_result in results:
for key in k6_result:
try:
avg_results[key] += k6_result[key] / len(results)
except KeyError:
avg_results[key] = k6_result[key] / len(results)
return avg_results
@allure.title("Compare results")
def compare_load_results(result: dict, result_new: dict):
for key in result:
if result[key] != 0 and result_new[key] != 0:
if (abs(result[key] - result_new[key]) / min(result[key], result_new[key])) < 0.25:
continue
else:
raise AssertionError(f"Difference in {key} values more than 25%")
elif result[key] == 0 and result_new[key] == 0:
continue
else:
raise AssertionError(f"Unexpected zero value in {key}")

66
pytest_tests/testsuites/load/test_load.py Normal file
View file

@ -0,0 +1,66 @@
import allure
import pytest
from common import LOAD_NODE_SSH_PRIVATE_KEY_PATH, LOAD_NODE_SSH_USER, LOAD_NODES
from neofs_testlib.hosting import Hosting
from pytest_tests.helpers.k6 import LoadParams
from pytest_tests.steps.load import (
clear_cache_and_data,
get_storage_host_endpoints,
multi_node_k6_run,
prepare_k6_instances,
)
CONTAINERS_COUNT = 1
OBJ_COUNT = 3
class TestLoad:
@pytest.fixture(autouse=True)
def clear_cache_and_data(self, hosting: Hosting):
clear_cache_and_data(hosting=hosting)
@pytest.mark.parametrize("obj_size, out_file", [(1000, "1mb_200.json")])
@pytest.mark.parametrize("writers, readers, deleters", [(140, 60, 0), (200, 0, 0)])
@pytest.mark.parametrize("load_time", [200, 900])
@pytest.mark.parametrize("node_count", [4])
def test_grpc_benchmark(
self,
obj_size,
out_file,
writers,
readers,
deleters,
load_time,
node_count,
hosting: Hosting,
):
allure.dynamic.title(
f"Benchmark test - node_count = {node_count}, "
f"writers = {writers} readers = {readers}, "
f"deleters = {deleters}, obj_size = {obj_size}, "
f"load_time = {load_time}"
)
with allure.step("Get endpoints"):
endpoints_list = get_storage_host_endpoints(hosting=hosting)
endpoints = ",".join(endpoints_list[:node_count])
load_params = LoadParams(
endpoint=endpoints,
obj_size=obj_size,
containers_count=CONTAINERS_COUNT,
out_file=out_file,
obj_count=OBJ_COUNT,
writers=writers,
readers=readers,
deleters=deleters,
load_time=load_time,
load_type="grpc",
)
k6_load_instances = prepare_k6_instances(
load_nodes=LOAD_NODES.split(','),
login=LOAD_NODE_SSH_USER,
pkey=LOAD_NODE_SSH_PRIVATE_KEY_PATH,
load_params=load_params,
)
with allure.step("Run load"):
multi_node_k6_run(k6_load_instances)

6
robot/variables/common.py
View file

@ -34,6 +34,11 @@ DEVENV_PATH = os.getenv("DEVENV_PATH", os.path.join("..", "neofs-dev-env"))
# Password of wallet owned by user on behalf of whom we are running tests
WALLET_PASS = os.getenv("WALLET_PASS", "")
# Load node parameters
LOAD_NODES = os.getenv("LOAD_NODES")
LOAD_NODE_SSH_USER = os.getenv("LOAD_NODE_SSH_USER")
LOAD_NODE_SSH_PRIVATE_KEY_PATH = os.getenv("LOAD_NODE_SSH_PRIVATE_KEY_PATH")
# Configuration of storage nodes
# TODO: we should use hosting instead of all these variables
STORAGE_RPC_ENDPOINT_1 = os.getenv("STORAGE_RPC_ENDPOINT_1", "s01.neofs.devenv:8080")
@ -116,6 +121,7 @@ FREE_STORAGE = os.getenv("FREE_STORAGE", "false").lower() == "true"
BIN_VERSIONS_FILE = os.getenv("BIN_VERSIONS_FILE")
HOSTING_CONFIG_FILE = os.getenv("HOSTING_CONFIG_FILE", ".devenv.hosting.yaml")
STORAGE_NODE_SERVICE_NAME_REGEX = r"s\d\d"
# Generate wallet configs
# TODO: we should move all info about wallet configs to fixtures