TrueCloudLab/frostfs-testcases
26
0
Fork 21
Code Issues 2 Pull requests 3 Releases Activity Actions

[#314] Format all files with black and isort

Browse source 
Signed-off-by: Vladimir Domnich <v.domnich@yadro.com>
This commit is contained in:
Vladimir Domnich 2022-09-28 16:07:16 +04:00 • committed by Vladimir
parent 26032a67ec
commit 147cac0ebc
46 changed files with 1506 additions and 1100 deletions
Download patch file Download diff file Expand all files Collapse all files

115
pytest_tests/helpers/service_helper.py
View file

@ -6,18 +6,23 @@ import time
from contextlib import contextmanager
from datetime import datetime
from typing import Optional
from requests import HTTPError
import docker
from cli_helpers import _cmd_run
from common import (INFRASTRUCTURE_TYPE, NEOFS_CLI_EXEC, NEOFS_NETMAP_DICT, STORAGE_NODE_BIN_PATH,
STORAGE_NODE_SSH_PASSWORD, STORAGE_NODE_SSH_PRIVATE_KEY_PATH,
STORAGE_NODE_SSH_USER, WALLET_CONFIG)
from common import (
INFRASTRUCTURE_TYPE,
NEOFS_CLI_EXEC,
NEOFS_NETMAP_DICT,
STORAGE_NODE_BIN_PATH,
STORAGE_NODE_SSH_PASSWORD,
STORAGE_NODE_SSH_PRIVATE_KEY_PATH,
STORAGE_NODE_SSH_USER,
WALLET_CONFIG,
)
from requests import HTTPError
from ssh_helper import HostClient
logger = logging.getLogger('NeoLogger')
logger = logging.getLogger("NeoLogger")
class LocalDevEnvStorageServiceHelper:
@ -59,8 +64,8 @@ class LocalDevEnvStorageServiceHelper:
wallet_path = NEOFS_NETMAP_DICT[node_name]["wallet_path"]
cmd = (
f'{NEOFS_CLI_EXEC} {command} --endpoint {control_endpoint} '
f'--wallet {wallet_path} --config {WALLET_CONFIG}'
f"{NEOFS_CLI_EXEC} {command} --endpoint {control_endpoint} "
f"--wallet {wallet_path} --config {WALLET_CONFIG}"
)
output = _cmd_run(cmd)
return output
@ -77,8 +82,9 @@ class LocalDevEnvStorageServiceHelper:
def get_binaries_version(self) -> dict:
return {}
def dump_logs(self, directory_path: str, since: Optional[datetime],
until: Optional[datetime]) -> None:
def dump_logs(
self, directory_path: str, since: Optional[datetime], until: Optional[datetime]
) -> None:
# All containers are running on the same host, so we can use 1st node to collect all logs
first_node_name = next(iter(NEOFS_NETMAP_DICT))
client = self._get_docker_client(first_node_name)
@ -107,8 +113,9 @@ class LocalDevEnvStorageServiceHelper:
return container
return None
def _wait_for_container_to_be_in_state(self, node_name: str, container_name: str,
expected_state: str) -> None:
def _wait_for_container_to_be_in_state(
self, node_name: str, container_name: str, expected_state: str
) -> None:
for __attempt in range(10):
container = self._get_container_by_name(node_name, container_name)
logger.info(f"Container info:\n{json.dumps(container, indent=2)}")
@ -116,7 +123,7 @@ class LocalDevEnvStorageServiceHelper:
return
time.sleep(5)
raise AssertionError(f'Container {container_name} is not in {expected_state} state.')
raise AssertionError(f"Container {container_name} is not in {expected_state} state.")
def _get_docker_client(self, node_name: str) -> docker.APIClient:
# For local docker we use default docker client that talks to unix socket
@ -143,7 +150,9 @@ class CloudVmStorageServiceHelper:
logger.info(f"Start command output: {output.stdout}")
if wait:
self._wait_for_service_to_be_in_state(node_name, self.STORAGE_SERVICE, "active (running)")
self._wait_for_service_to_be_in_state(
node_name, self.STORAGE_SERVICE, "active (running)"
)
def run_control_command(self, node_name: str, command: str) -> str:
control_endpoint = NEOFS_NETMAP_DICT[node_name]["control"]
@ -161,28 +170,31 @@ class CloudVmStorageServiceHelper:
# Put config on storage node host
remote_config_path = f"/tmp/{node_name}-config.yaml"
remote_config = 'password: ""'
ssh_client.exec_with_confirmation(f"echo '{remote_config}' > {remote_config_path}", [""])
ssh_client.exec_with_confirmation(
f"echo '{remote_config}' > {remote_config_path}", [""]
)
# Execute command
cmd = (
f'sudo {STORAGE_NODE_BIN_PATH}/neofs-cli {command} --endpoint {control_endpoint} '
f'--wallet {remote_wallet_path} --config {remote_config_path}'
f"sudo {STORAGE_NODE_BIN_PATH}/neofs-cli {command} --endpoint {control_endpoint} "
f"--wallet {remote_wallet_path} --config {remote_config_path}"
)
output = ssh_client.exec_with_confirmation(cmd, [""])
return output.stdout
def _wait_for_service_to_be_in_state(self, node_name: str, service_name: str,
expected_state: str) -> None:
def _wait_for_service_to_be_in_state(
self, node_name: str, service_name: str, expected_state: str
) -> None:
with _create_ssh_client(node_name) as ssh_client:
for __attempt in range(10):
# Run command to get service status (set --lines=0 to suppress logs output)
# Also we don't verify return code, because for an inactive service return code will be 3
command = f'sudo systemctl status {service_name} --lines=0'
command = f"sudo systemctl status {service_name} --lines=0"
output = ssh_client.exec(command, verify=False)
if expected_state in output.stdout:
return
time.sleep(3)
raise AssertionError(f'Service {service_name} is not in {expected_state} state')
raise AssertionError(f"Service {service_name} is not in {expected_state} state")
def delete_node_data(self, node_name: str) -> None:
with _create_ssh_client(node_name) as ssh_client:
@ -190,16 +202,16 @@ class CloudVmStorageServiceHelper:
def get_binaries_version(self, binaries: list = None) -> dict:
default_binaries = [
'neo-go',
'neofs-adm',
'neofs-cli',
'neofs-http-gw',
'neofs-ir',
'neofs-lens',
'neofs-node',
'neofs-s3-authmate',
'neofs-s3-gw',
'neogo-morph-cn',
"neo-go",
"neofs-adm",
"neofs-cli",
"neofs-http-gw",
"neofs-ir",
"neofs-lens",
"neofs-node",
"neofs-s3-authmate",
"neofs-s3-gw",
"neogo-morph-cn",
]
binaries = binaries or default_binaries
@ -208,31 +220,35 @@ class CloudVmStorageServiceHelper:
with _create_ssh_client(node_name) as ssh_client:
for binary in binaries:
try:
out = ssh_client.exec(f'sudo {binary} --version').stdout
out = ssh_client.exec(f"sudo {binary} --version").stdout
except AssertionError as err:
logger.error(f'Can not get version for {binary} because of\n{err}')
version_map[binary] = 'Can not get version'
logger.error(f"Can not get version for {binary} because of\n{err}")
version_map[binary] = "Can not get version"
continue
version = re.search(r'version[:\s]*v?(.+)', out, re.IGNORECASE)
version = version.group(1).strip() if version else 'Unknown'
version = re.search(r"version[:\s]*v?(.+)", out, re.IGNORECASE)
version = version.group(1).strip() if version else "Unknown"
if not version_map.get(binary):
version_map[binary] = version
else:
assert version_map[binary] == version, \
f'Expected binary {binary} to have identical version on all nodes ' \
f'(mismatch on node {node_name})'
assert version_map[binary] == version, (
f"Expected binary {binary} to have identical version on all nodes "
f"(mismatch on node {node_name})"
)
return version_map
def dump_logs(self, directory_path: str, since: Optional[datetime],
until: Optional[datetime]) -> None:
def dump_logs(
self, directory_path: str, since: Optional[datetime], until: Optional[datetime]
) -> None:
for node_name, node_info in NEOFS_NETMAP_DICT.items():
with _create_ssh_client(node_name) as ssh_client:
# We do not filter out logs of neofs services, because system logs might contain
# information that is useful for troubleshooting
filters = " ".join([
f"--since '{since:%Y-%m-%d %H:%M:%S}'" if since else "",
f"--until '{until:%Y-%m-%d %H:%M:%S}'" if until else "",
])
filters = " ".join(
[
f"--since '{since:%Y-%m-%d %H:%M:%S}'" if since else "",
f"--until '{until:%Y-%m-%d %H:%M:%S}'" if until else "",
]
)
result = ssh_client.exec(f"journalctl --no-pager {filters}")
logs = result.stdout
@ -250,6 +266,7 @@ class RemoteDevEnvStorageServiceHelper(LocalDevEnvStorageServiceHelper):
Most of operations are identical to local devenv, however, any interactions
with host resources (files, etc.) require ssh into the remote host machine.
"""
def _get_docker_client(self, node_name: str) -> docker.APIClient:
# For remote devenv we use docker client that talks to tcp socket 2375:
# https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-socket-option
@ -299,12 +316,12 @@ def _create_ssh_client(node_name: str) -> HostClient:
def _get_node_host(node_name: str) -> str:
if node_name not in NEOFS_NETMAP_DICT:
raise AssertionError(f'Node {node_name} is not found!')
raise AssertionError(f"Node {node_name} is not found!")
# We use rpc endpoint to determine host address, because control endpoint
# (if it is private) will be a local address on the host machine
node_config = NEOFS_NETMAP_DICT.get(node_name)
host = node_config.get('rpc').split(':')[0]
host = node_config.get("rpc").split(":")[0]
return host
@ -313,7 +330,7 @@ def _get_storage_container_name(node_name: str) -> str:
Converts name of storage node (as it is listed in netmap) into the name of docker container
that runs instance of this storage node.
"""
return node_name.split('.')[0]
return node_name.split(".")[0]
def _get_storage_volume_name(node_name: str) -> str: