Rename neofs to frostfs

Signed-off-by: Yulia Kovshova <y.kovshova@yadro.com>

src/frostfs_testlib/hosting/__init__.py

@@ -0,0 +1,3 @@
+from frostfs_testlib.hosting.config import CLIConfig, HostConfig, ServiceConfig
+from frostfs_testlib.hosting.hosting import Hosting
+from frostfs_testlib.hosting.interfaces import Host

src/frostfs_testlib/hosting/config.py

@@ -0,0 +1,70 @@
+from dataclasses import dataclass, field, fields
+from typing import Any
+
+
+@dataclass
+class ParsedAttributes:
+    """Base class for data structures representing parsed attributes from configs."""
+
+    @classmethod
+    def parse(cls, attributes: dict[str, Any]):
+        # Pick attributes supported by the class
+        field_names = set(field.name for field in fields(cls))
+        supported_attributes = {
+            key: value for key, value in attributes.items() if key in field_names
+        }
+        return cls(**supported_attributes)
+
+
+@dataclass
+class CLIConfig:
+    """Describes CLI tool on some host.
+
+    Attributes:
+        name: Name of the tool.
+        exec_path: Path to executable file of the tool.
+        attributes: Dict with extra information about the tool.
+    """
+
+    name: str
+    exec_path: str
+    attributes: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class ServiceConfig:
+    """Describes neoFS service on some host.
+
+    Attributes:
+        name: Name of the service that uniquely identifies it across all hosts.
+        attributes: Dict with extra information about the service. For example, we can store
+            name of docker container (or name of systemd service), endpoints, path to wallet,
+            path to configuration file, etc.
+    """
+
+    name: str
+    attributes: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class HostConfig:
+    """Describes machine that hosts neoFS services.
+
+    Attributes:
+        plugin_name: Name of plugin that should be used to manage the host.
+        address: Address of the machine (IP or DNS name).
+        services: List of services hosted on the machine.
+        clis: List of CLI tools available on the machine.
+        attributes: Dict with extra information about the host. For example, we can store
+            connection parameters in this dict.
+    """
+
+    plugin_name: str
+    address: str
+    services: list[ServiceConfig] = field(default_factory=list)
+    clis: list[CLIConfig] = field(default_factory=list)
+    attributes: dict[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        self.services = [ServiceConfig(**service) for service in self.services or []]
+        self.clis = [CLIConfig(**cli) for cli in self.clis or []]

src/frostfs_testlib/hosting/docker_host.py

@@ -0,0 +1,238 @@
+import json
+import logging
+import os
+import re
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Optional
+
+import docker
+from requests import HTTPError
+
+from frostfs_testlib.hosting.config import ParsedAttributes
+from frostfs_testlib.hosting.interfaces import Host
+from frostfs_testlib.shell import LocalShell, Shell, SSHShell
+from frostfs_testlib.shell.command_inspectors import SudoInspector
+
+logger = logging.getLogger("frostfs.testlib.hosting")
+
+
+@dataclass
+class HostAttributes(ParsedAttributes):
+    """Represents attributes of host where Docker with neoFS runs.
+
+    Attributes:
+        sudo_shell: Specifies whether shell commands should be auto-prefixed with sudo.
+        docker_endpoint: Protocol, address and port of docker where neoFS runs. Recommended format
+            is tcp socket (https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-socket-option),
+            for example: tcp://{address}:2375 (where 2375 is default docker port).
+        ssh_login: Login for SSH connection to the machine where docker runs.
+        ssh_password: Password for SSH connection.
+        ssh_private_key_path: Path to private key for SSH connection.
+        ssh_private_key_passphrase: Passphrase for the private key.
+    """
+
+    sudo_shell: bool = False
+    docker_endpoint: Optional[str] = None
+    ssh_login: Optional[str] = None
+    ssh_password: Optional[str] = None
+    ssh_private_key_path: Optional[str] = None
+    ssh_private_key_passphrase: Optional[str] = None
+
+
+@dataclass
+class ServiceAttributes(ParsedAttributes):
+    """Represents attributes of service running as Docker container.
+
+    Attributes:
+        container_name: Name of Docker container where the service runs.
+        volume_name: Name of volume where storage node service stores the data.
+        start_timeout: Timeout (in seconds) for service to start.
+        stop_timeout: Timeout (in seconds) for service to stop.
+    """
+
+    container_name: str
+    volume_name: Optional[str] = None
+    start_timeout: int = 90
+    stop_timeout: int = 90
+
+
+class DockerHost(Host):
+    """Manages services hosted in Docker containers running on a local or remote machine."""
+
+    def get_shell(self) -> Shell:
+        host_attributes = HostAttributes.parse(self._config.attributes)
+        command_inspectors = []
+        if host_attributes.sudo_shell:
+            command_inspectors.append(SudoInspector())
+
+        if not host_attributes.ssh_login:
+            # If there is no SSH connection to the host, use local shell
+            return LocalShell(command_inspectors)
+
+        # If there is SSH connection to the host, use SSH shell
+        return SSHShell(
+            host=self._config.address,
+            login=host_attributes.ssh_login,
+            password=host_attributes.ssh_password,
+            private_key_path=host_attributes.ssh_private_key_path,
+            private_key_passphrase=host_attributes.ssh_private_key_passphrase,
+            command_inspectors=command_inspectors,
+        )
+
+    def start_host(self) -> None:
+        # We emulate starting machine by starting all services
+        # As an alternative we can probably try to stop docker service...
+        for service_config in self._config.services:
+            self.start_service(service_config.name)
+
+    def stop_host(self) -> None:
+        # We emulate stopping machine by stopping all services
+        # As an alternative we can probably try to stop docker service...
+        for service_config in self._config.services:
+            self.stop_service(service_config.name)
+
+    def start_service(self, service_name: str) -> None:
+        service_attributes = self._get_service_attributes(service_name)
+
+        client = self._get_docker_client()
+        client.start(service_attributes.container_name)
+
+        self._wait_for_container_to_be_in_state(
+            container_name=service_attributes.container_name,
+            expected_state="running",
+            timeout=service_attributes.start_timeout,
+        )
+
+    def stop_service(self, service_name: str) -> None:
+        service_attributes = self._get_service_attributes(service_name)
+
+        client = self._get_docker_client()
+        client.stop(service_attributes.container_name)
+
+        self._wait_for_container_to_be_in_state(
+            container_name=service_attributes.container_name,
+            expected_state="exited",
+            timeout=service_attributes.stop_timeout,
+        )
+
+    def restart_service(self, service_name: str) -> None:
+        service_attributes = self._get_service_attributes(service_name)
+
+        client = self._get_docker_client()
+        client.restart(service_attributes.container_name)
+
+        self._wait_for_container_to_be_in_state(
+            container_name=service_attributes.container_name,
+            expected_state="running",
+            timeout=service_attributes.start_timeout,
+        )
+
+    def delete_storage_node_data(self, service_name: str, cache_only: bool = False) -> None:
+        service_attributes = self._get_service_attributes(service_name)
+
+        client = self._get_docker_client()
+        volume_info = client.inspect_volume(service_attributes.volume_name)
+        volume_path = volume_info["Mountpoint"]
+
+        shell = self.get_shell()
+        meta_clean_cmd = f"rm -rf {volume_path}/meta*/*"
+        data_clean_cmd = f"; rm -rf {volume_path}/data*/*" if not cache_only else ""
+        cmd = f"{meta_clean_cmd}{data_clean_cmd}"
+        shell.exec(cmd)
+
+    def dump_logs(
+        self,
+        directory_path: str,
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+        filter_regex: Optional[str] = None,
+    ) -> None:
+        client = self._get_docker_client()
+        for service_config in self._config.services:
+            container_name = self._get_service_attributes(service_config.name).container_name
+            try:
+                logs = client.logs(container_name, since=since, until=until)
+            except HTTPError as exc:
+                logger.info(f"Got exception while dumping logs of '{container_name}': {exc}")
+                continue
+
+            if filter_regex:
+                logs = (
+                    "\n".join(match[0] for match in re.findall(filter_regex, logs, re.IGNORECASE))
+                    or f"No matches found in logs based on given filter '{filter_regex}'"
+                )
+
+            # Save logs to the directory
+            file_path = os.path.join(
+                directory_path,
+                f"{self._config.address}-{container_name}-log.txt",
+            )
+            with open(file_path, "wb") as file:
+                file.write(logs)
+
+    def is_message_in_logs(
+        self,
+        message_regex: str,
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+    ) -> bool:
+        client = self._get_docker_client()
+        for service_config in self._config.services:
+            container_name = self._get_service_attributes(service_config.name).container_name
+            try:
+                logs = client.logs(container_name, since=since, until=until)
+            except HTTPError as exc:
+                logger.info(f"Got exception while dumping logs of '{container_name}': {exc}")
+                continue
+
+            if message_regex:
+                matches = re.findall(message_regex, logs, re.IGNORECASE)
+                if matches:
+                    return True
+
+        return False
+
+    def _get_service_attributes(self, service_name) -> ServiceAttributes:
+        service_config = self.get_service_config(service_name)
+        return ServiceAttributes.parse(service_config.attributes)
+
+    def _get_docker_client(self) -> docker.APIClient:
+        docker_endpoint = HostAttributes.parse(self._config.attributes).docker_endpoint
+
+        if not docker_endpoint:
+            # Use default docker client that talks to unix socket
+            return docker.APIClient()
+
+        # Otherwise use docker client that talks to specified endpoint
+        return docker.APIClient(base_url=docker_endpoint)
+
+    def _get_container_by_name(self, container_name: str) -> dict[str, Any]:
+        client = self._get_docker_client()
+        containers = client.containers(all=True)
+
+        for container in containers:
+            # Names in local docker environment are prefixed with /
+            clean_names = set(name.strip("/") for name in container["Names"])
+            if container_name in clean_names:
+                return container
+        return None
+
+    def _wait_for_container_to_be_in_state(
+        self, container_name: str, expected_state: str, timeout: int
+    ) -> None:
+        iterations = 10
+        iteration_wait_time = timeout / iterations
+
+        # To speed things up, we break timeout in smaller iterations and check container state
+        # several times. This way waiting stops as soon as container reaches the expected state
+        for _ in range(iterations):
+            container = self._get_container_by_name(container_name)
+            logger.debug(f"Current container state\n:{json.dumps(container, indent=2)}")
+
+            if container and container["State"] == expected_state:
+                return
+            time.sleep(iteration_wait_time)
+
+        raise RuntimeError(f"Container {container_name} is not in {expected_state} state.")

src/frostfs_testlib/hosting/hosting.py

@@ -0,0 +1,107 @@
+import re
+from typing import Any
+
+from frostfs_testlib.hosting.config import HostConfig, ServiceConfig
+from frostfs_testlib.hosting.interfaces import Host
+from frostfs_testlib.plugins import load_plugin
+
+
+class Hosting:
+    """Hosting manages infrastructure  where neoFS runs (machines and neoFS services)."""
+
+    _hosts: list[Host]
+    _host_by_address: dict[str, Host]
+    _host_by_service_name: dict[str, Host]
+
+    @property
+    def hosts(self) -> list[Host]:
+        """Returns all hosts registered in the hosting.
+
+        Returns:
+            List of hosts.
+        """
+        return self._hosts
+
+    def configure(self, config: dict[str, Any]) -> None:
+        """Configures hosts from specified config.
+
+        All existing hosts will be removed from the hosting.
+
+        Args:
+            config: Dictionary with hosting configuration.
+        """
+        hosts = []
+        host_by_address = {}
+        host_by_service_name = {}
+
+        host_configs = [HostConfig(**host_config) for host_config in config["hosts"]]
+        for host_config in host_configs:
+            host_class = load_plugin("frostfs.testlib.hosting", host_config.plugin_name)
+            host = host_class(host_config)
+
+            hosts.append(host)
+            host_by_address[host_config.address] = host
+
+            for service_config in host_config.services:
+                host_by_service_name[service_config.name] = host
+
+        self._hosts = hosts
+        self._host_by_address = host_by_address
+        self._host_by_service_name = host_by_service_name
+
+    def get_host_by_address(self, host_address: str) -> Host:
+        """Returns host with specified address.
+
+        Args:
+            host_address: Address of the host.
+
+        Returns:
+            Host that manages machine with specified address.
+        """
+        host = self._host_by_address.get(host_address)
+        if host is None:
+            raise ValueError(f"Unknown host address: '{host_address}'")
+        return host
+
+    def get_host_by_service(self, service_name: str) -> Host:
+        """Returns host where service with specified name is located.
+
+        Args:
+            service_name: Name of the service.
+
+        Returns:
+            Host that manages machine where service is located.
+        """
+        host = self._host_by_service_name.get(service_name)
+        if host is None:
+            raise ValueError(f"Unknown service name: '{service_name}'")
+        return host
+
+    def get_service_config(self, service_name: str) -> ServiceConfig:
+        """Returns config of service with specified name.
+
+        Args:
+            service_name: Name of the service.
+
+        Returns:
+            Config of the service.
+        """
+        host = self.get_host_by_service(service_name)
+        return host.get_service_config(service_name)
+
+    def find_service_configs(self, service_name_pattern: str) -> list[ServiceConfig]:
+        """Finds configs of services where service name matches specified regular expression.
+
+        Args:
+            service_name_pattern - regular expression for service names.
+
+        Returns:
+            List of service configs matched with the regular expression.
+        """
+        service_configs = [
+            service_config
+            for host in self.hosts
+            for service_config in host.config.services
+            if re.match(service_name_pattern, service_config.name)
+        ]
+        return service_configs

src/frostfs_testlib/hosting/interfaces.py

@@ -0,0 +1,192 @@
+from abc import ABC, abstractmethod
+from datetime import datetime
+from typing import Any, Optional
+
+from frostfs_testlib.hosting.config import CLIConfig, HostConfig, ServiceConfig
+from frostfs_testlib.shell.interfaces import Shell
+
+
+class DiskInfo(dict):
+    """Dict wrapper for disk_info for disk management commands."""
+
+
+class Host(ABC):
+    """Interface of a host machine where neoFS services are running.
+
+    Allows to manage the machine and neoFS services that are hosted on it.
+    """
+
+    def __init__(self, config: HostConfig) -> None:
+        self._config = config
+        self._service_config_by_name = {
+            service_config.name: service_config for service_config in config.services
+        }
+        self._cli_config_by_name = {cli_config.name: cli_config for cli_config in config.clis}
+
+    @property
+    def config(self) -> HostConfig:
+        """Returns config of the host.
+
+        Returns:
+            Config of this host.
+        """
+        return self._config
+
+    def get_service_config(self, service_name: str) -> ServiceConfig:
+        """Returns config of service with specified name.
+
+        The service must be hosted on this host.
+
+        Args:
+            service_name: Name of the service.
+
+        Returns:
+            Config of the service.
+        """
+        service_config = self._service_config_by_name.get(service_name)
+        if service_config is None:
+            raise ValueError(f"Unknown service name: '{service_name}'")
+        return service_config
+
+    def get_cli_config(self, cli_name: str) -> CLIConfig:
+        """Returns config of CLI tool with specified name.
+
+        The CLI must be located on this host.
+
+        Args:
+            cli_name: Name of the CLI tool.
+
+        Returns:
+            Config of the CLI tool.
+        """
+        cli_config = self._cli_config_by_name.get(cli_name)
+        if cli_config is None:
+            raise ValueError(f"Unknown CLI name: '{cli_name}'")
+        return cli_config
+
+    @abstractmethod
+    def get_shell(self) -> Shell:
+        """Returns shell to this host.
+
+        Returns:
+            Shell that executes commands on this host.
+        """
+
+    @abstractmethod
+    def start_host(self) -> None:
+        """Starts the host machine."""
+
+    @abstractmethod
+    def stop_host(self, mode: str) -> None:
+        """Stops the host machine.
+
+        Args:
+            mode: Specifies mode how host should be stopped. Mode might be host-specific.
+        """
+
+    @abstractmethod
+    def start_service(self, service_name: str) -> None:
+        """Starts the service with specified name and waits until it starts.
+
+        The service must be hosted on this host.
+
+        Args:
+            service_name: Name of the service to start.
+        """
+
+    @abstractmethod
+    def stop_service(self, service_name: str) -> None:
+        """Stops the service with specified name and waits until it stops.
+
+        The service must be hosted on this host.
+
+        Args:
+            service_name: Name of the service to stop.
+        """
+
+    @abstractmethod
+    def restart_service(self, service_name: str) -> None:
+        """Restarts the service with specified name and waits until it starts.
+        The service must be hosted on this host.
+        Args:
+            service_name: Name of the service to restart.
+        """
+
+    @abstractmethod
+    def delete_storage_node_data(self, service_name: str, cache_only: bool = False) -> None:
+        """Erases all data of the storage node with specified name.
+
+        Args:
+            service_name: Name of storage node service.
+            cache_only: To delete cache only.
+        """
+
+    @abstractmethod
+    def detach_disk(self, device: str) -> DiskInfo:
+        """Detaches disk device to simulate disk offline/failover scenario.
+
+        Args:
+            device: Device name to detach.
+
+        Returns:
+            internal service disk info related to host plugin (i.e. volume id for cloud devices),
+            which may be used to identify or re-attach existing volume back.
+        """
+
+    @abstractmethod
+    def attach_disk(self, device: str, disk_info: DiskInfo) -> None:
+        """Attaches disk device back.
+
+        Args:
+            device: Device name to attach.
+            service_info: any info required for host plugin to identify/attach disk.
+        """
+
+    @abstractmethod
+    def is_disk_attached(self, device: str, disk_info: DiskInfo) -> bool:
+        """Checks if disk device is attached.
+
+        Args:
+            device: Device name to check.
+            service_info: any info required for host plugin to identify disk.
+
+        Returns:
+            True if attached.
+            False if detached.
+        """
+
+    @abstractmethod
+    def dump_logs(
+        self,
+        directory_path: str,
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+        filter_regex: Optional[str] = None,
+    ) -> None:
+        """Dumps logs of all services on the host to specified directory.
+
+        Args:
+            directory_path: Path to the directory where logs should be stored.
+            since: If set, limits the time from which logs should be collected. Must be in UTC.
+            until: If set, limits the time until which logs should be collected. Must be in UTC.
+            filter_regex: regex to filter output
+        """
+
+    @abstractmethod
+    def is_message_in_logs(
+        self,
+        message_regex: str,
+        since: Optional[datetime] = None,
+        until: Optional[datetime] = None,
+    ) -> bool:
+        """Checks logs on host for specified message regex.
+
+        Args:
+            message_regex: message to find.
+            since: If set, limits the time from which logs should be collected. Must be in UTC.
+            until: If set, limits the time until which logs should be collected. Must be in UTC.
+
+        Returns:
+            True if message found in logs in the given time frame.
+            False otherwise.
+        """