Move shared code to testlib

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>

src/frostfs_testlib/storage/__init__.py

@@ -0,0 +1,33 @@
+from frostfs_testlib.storage.constants import _FrostfsServicesNames
+from frostfs_testlib.storage.dataclasses.frostfs_services import (
+    HTTPGate,
+    InnerRing,
+    MainChain,
+    MorphChain,
+    S3Gate,
+    StorageNode,
+)
+from frostfs_testlib.storage.service_registry import ServiceRegistry
+
+__class_registry = ServiceRegistry()
+
+# Register default public services
+__class_registry.register_service(_FrostfsServicesNames.STORAGE, StorageNode)
+__class_registry.register_service(_FrostfsServicesNames.INNER_RING, InnerRing)
+__class_registry.register_service(_FrostfsServicesNames.MORPH_CHAIN, MorphChain)
+__class_registry.register_service(_FrostfsServicesNames.S3_GATE, S3Gate)
+__class_registry.register_service(_FrostfsServicesNames.HTTP_GATE, HTTPGate)
+# # TODO: Remove this since we are no longer have main chain
+__class_registry.register_service(_FrostfsServicesNames.MAIN_CHAIN, MainChain)
+
+
+def get_service_registry() -> ServiceRegistry:
+    """Returns registry with registered classes related to cluster and cluster nodes.
+
+    ServiceClassRegistry is a singleton instance that can be configured with multiple classes that
+    represents service on the cluster physical node.
+
+    Returns:
+        Singleton ServiceClassRegistry instance.
+    """
+    return __class_registry

src/frostfs_testlib/storage/cluster.py

@@ -0,0 +1,237 @@
+import random
+import re
+
+import yaml
+
+from frostfs_testlib.hosting import Host, Hosting
+from frostfs_testlib.hosting.config import ServiceConfig
+from frostfs_testlib.storage import get_service_registry
+from frostfs_testlib.storage.constants import ConfigAttributes
+from frostfs_testlib.storage.dataclasses.frostfs_services import (
+    HTTPGate,
+    InnerRing,
+    MorphChain,
+    S3Gate,
+    StorageNode,
+)
+from frostfs_testlib.storage.dataclasses.node_base import NodeBase, ServiceClass
+from frostfs_testlib.storage.service_registry import ServiceRegistry
+
+
+class ClusterNode:
+    """
+    Represents physical node where multiple different services may be located
+    """
+
+    class_registry: ServiceRegistry
+    id: int
+    host: Host
+
+    def __init__(self, host: Host, id: int) -> None:
+        self.host = host
+        self.id = id
+        self.class_registry = get_service_registry()
+
+    @property
+    def host_ip(self):
+        return self.host.config.address
+
+    def __eq__(self, other):
+        return self.host.config.address == other.host.config.address
+
+    def __hash__(self):
+        return id(self.host.config.address)
+
+    def __str__(self):
+        return self.host.config.address
+
+    def __repr__(self) -> str:
+        return self.host.config.address
+
+    # for backward compatibility and to not touch other codebase too much
+    @property
+    def storage_node(self) -> StorageNode:
+        return self.service(StorageNode)
+
+    # for backward compatibility and to not touch other codebase too much
+    @property
+    def ir_node(self) -> InnerRing:
+        return self.service(InnerRing)
+
+    # for backward compatibility and to not touch other codebase too much
+    @property
+    def morph_chain(self) -> MorphChain:
+        return self.service(MorphChain)
+
+    # for backward compatibility and to not touch other codebase too much
+    @property
+    def http_gate(self) -> HTTPGate:
+        return self.service(HTTPGate)
+
+    # for backward compatibility and to not touch other codebase too much
+    @property
+    def s3_gate(self) -> S3Gate:
+        return self.service(S3Gate)
+
+    def service(self, service_type: type[ServiceClass]) -> ServiceClass:
+        """
+        Get a service cluster node of specified type.
+
+        Args:
+            service_type: type of the service which should be returned,
+                          for frostfs it can be StorageNode, S3Gate, HttpGate, MorphChain and InnerRing.
+
+        Returns:
+            service of service_type class.
+        """
+
+        service_entry = self.class_registry.get_entry(service_type)
+        service_name = service_entry["hosting_service_name"]
+
+        pattern = f"{service_name}{self.id}"
+        config = self.host.get_service_config(pattern)
+
+        return service_type(
+            self.id,
+            config.name,
+            self.host,
+        )
+
+    def get_list_of_services(self) -> list[str]:
+        return [
+            config.attributes[ConfigAttributes.SERVICE_NAME] for config in self.host.config.services
+        ]
+
+
+class Cluster:
+    """
+    This class represents a Cluster object for the whole storage based on provided hosting
+    """
+
+    default_rpc_endpoint: str
+    default_s3_gate_endpoint: str
+    default_http_gate_endpoint: str
+
+    def __init__(self, hosting: Hosting) -> None:
+        self._hosting = hosting
+
+        self.class_registry = get_service_registry()
+        self.default_rpc_endpoint = self.services(StorageNode)[0].get_rpc_endpoint()
+        self.default_s3_gate_endpoint = self.services(S3Gate)[0].get_endpoint()
+        self.default_http_gate_endpoint = self.services(HTTPGate)[0].get_endpoint()
+
+    @property
+    def hosts(self) -> list[Host]:
+        """
+        Returns list of Hosts
+        """
+        return self._hosting.hosts
+
+    # for backward compatibility and to not touch other codebase too much
+    @property
+    def storage_nodes(self) -> list[StorageNode]:
+        return self.services(StorageNode)
+
+    # for backward compatibility and to not touch other codebase too much
+    @property
+    def ir_nodes(self) -> list[InnerRing]:
+        return self.services(InnerRing)
+
+    # for backward compatibility and to not touch other codebase too much
+    @property
+    def s3_gates(self) -> list[S3Gate]:
+        return self.services(S3Gate)
+
+    @property
+    def http_gates(self) -> list[HTTPGate]:
+        return self.services(HTTPGate)
+
+    @property
+    def morph_chain(self) -> list[MorphChain]:
+        return self.services(MorphChain)
+
+    def services(self, service_type: type[ServiceClass]) -> list[ServiceClass]:
+        """
+        Get all services in a cluster of specified type.
+
+        Args:
+            service_type: type of the services which should be returned,
+                          for frostfs it can be StorageNode, S3Gate, HttpGate, MorphChain and InnerRing.
+
+        Returns:
+            list of services of service_type class.
+        """
+
+        service = self.class_registry.get_entry(service_type)
+        service_name = service["hosting_service_name"]
+        cls: type[NodeBase] = service["cls"]
+
+        pattern = f"{service_name}\d*$"
+        configs = self.hosting.find_service_configs(pattern)
+
+        found_nodes = []
+        for config in configs:
+            # config.name is something like s3-gate01. Cut last digits to know service type
+            service_type = re.findall(".*\D", config.name)[0]
+            # exclude unsupported services
+            if service_type != service_name:
+                continue
+
+            found_nodes.append(
+                cls(
+                    self._get_id(config.name),
+                    config.name,
+                    self.hosting.get_host_by_service(config.name),
+                )
+            )
+        return found_nodes
+
+    @property
+    def cluster_nodes(self) -> list[ClusterNode]:
+        """
+        Returns list of Cluster Nodes
+        """
+
+        return [ClusterNode(host, id) for id, host in enumerate(self.hosts, start=1)]
+
+    @property
+    def hosting(self) -> Hosting:
+        return self._hosting
+
+    def _create_wallet_config(self, service: ServiceConfig) -> None:
+        wallet_path = service.attributes[ConfigAttributes.LOCAL_WALLET_CONFIG]
+        wallet_password = service.attributes[ConfigAttributes.WALLET_PASSWORD]
+        with open(wallet_path, "w") as file:
+            yaml.dump({"password": wallet_password}, file)
+
+    def create_wallet_configs(self, hosting: Hosting) -> None:
+        configs = hosting.find_service_configs(".*")
+        for config in configs:
+            if ConfigAttributes.LOCAL_WALLET_CONFIG in config.attributes:
+                self._create_wallet_config(config)
+
+    def is_local_devenv(self) -> bool:
+        if len(self.hosting.hosts) == 1:
+            host = self.hosting.hosts[0]
+            if host.config.address == "localhost" and host.config.plugin_name == "docker":
+                return True
+        return False
+
+    def _get_id(self, node_name) -> int:
+        pattern = "\d*$"
+
+        matches = re.search(pattern, node_name)
+        if not matches:
+            raise RuntimeError(f"Can't parse Id of the node {node_name}")
+        return int(matches.group())
+
+    def get_random_storage_rpc_endpoint(self) -> str:
+        return random.choice(self.get_storage_rpc_endpoints())
+
+    def get_storage_rpc_endpoints(self) -> list[str]:
+        nodes: list[StorageNode] = self.services(StorageNode)
+        return [node.get_rpc_endpoint() for node in nodes]
+
+    def get_morph_endpoints(self) -> list[str]:
+        nodes: list[MorphChain] = self.services(MorphChain)
+        return [node.get_endpoint() for node in nodes]

src/frostfs_testlib/storage/constants.py

@@ -0,0 +1,22 @@
+class ConfigAttributes:
+    SERVICE_NAME = "systemd_service_name"
+    WALLET_PASSWORD = "wallet_password"
+    WALLET_PATH = "wallet_path"
+    WALLET_CONFIG = "wallet_config"
+    CONFIG_PATH = "config_path"
+    LOCAL_WALLET_PATH = "local_wallet_path"
+    LOCAL_WALLET_CONFIG = "local_config_path"
+    ENDPOINT_DATA_0 = "endpoint_data0"
+    ENDPOINT_DATA_1 = "endpoint_data1"
+    ENDPOINT_INTERNAL = "endpoint_internal0"
+    CONTROL_ENDPOINT = "control_endpoint"
+    UN_LOCODE = "un_locode"
+
+
+class _FrostfsServicesNames:
+    STORAGE = "s"
+    S3_GATE = "s3-gate"
+    HTTP_GATE = "http-gate"
+    MORPH_CHAIN = "morph-chain"
+    INNER_RING = "ir"
+    MAIN_CHAIN = "main-chain"

src/frostfs_testlib/storage/controllers/background_load_controller.py

@@ -0,0 +1,207 @@
+import frostfs_testlib.resources.optionals as optionals
+from frostfs_testlib.load.k6 import K6
+from frostfs_testlib.load.load_config import (
+    EndpointSelectionStrategy,
+    K6ProcessAllocationStrategy,
+    LoadParams,
+    LoadScenario,
+    LoadType,
+)
+from frostfs_testlib.load.load_steps import init_s3_client, prepare_k6_instances
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.resources.load_params import (
+    K6_TEARDOWN_PERIOD,
+    LOAD_NODE_SSH_PASSWORD,
+    LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE,
+    LOAD_NODE_SSH_PRIVATE_KEY_PATH,
+    LOAD_NODE_SSH_USER,
+    LOAD_NODES,
+)
+from frostfs_testlib.shell.interfaces import SshCredentials
+from frostfs_testlib.storage.cluster import ClusterNode
+from frostfs_testlib.storage.dataclasses.frostfs_services import S3Gate, StorageNode
+from frostfs_testlib.storage.dataclasses.wallet import WalletInfo
+from frostfs_testlib.testing.test_control import run_optionally
+
+reporter = get_reporter()
+
+
+class BackgroundLoadController:
+    k6_instances: list[K6]
+    k6_dir: str
+    load_params: LoadParams
+    load_nodes: list[str]
+    verification_params: LoadParams
+    nodes_under_load: list[ClusterNode]
+    ssh_credentials: SshCredentials
+    loaders_wallet: WalletInfo
+    endpoints: list[str]
+
+    def __init__(
+        self,
+        k6_dir: str,
+        load_params: LoadParams,
+        loaders_wallet: WalletInfo,
+        nodes_under_load: list[ClusterNode],
+    ) -> None:
+        self.k6_dir = k6_dir
+        self.load_params = load_params
+        self.nodes_under_load = nodes_under_load
+        self.load_nodes = LOAD_NODES
+        self.loaders_wallet = loaders_wallet
+
+        if load_params.endpoint_selection_strategy is None:
+            raise RuntimeError("endpoint_selection_strategy should not be None")
+
+        self.endpoints = self._get_endpoints(
+            load_params.load_type, load_params.endpoint_selection_strategy
+        )
+        self.verification_params = LoadParams(
+            clients=load_params.readers,
+            scenario=LoadScenario.VERIFY,
+            registry_file=load_params.registry_file,
+            verify_time=load_params.verify_time,
+            load_type=load_params.load_type,
+            load_id=load_params.load_id,
+            working_dir=load_params.working_dir,
+            endpoint_selection_strategy=load_params.endpoint_selection_strategy,
+            k6_process_allocation_strategy=load_params.k6_process_allocation_strategy,
+        )
+        self.ssh_credentials = SshCredentials(
+            LOAD_NODE_SSH_USER,
+            LOAD_NODE_SSH_PASSWORD,
+            LOAD_NODE_SSH_PRIVATE_KEY_PATH,
+            LOAD_NODE_SSH_PRIVATE_KEY_PASSPHRASE,
+        )
+
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED, [])
+    def _get_endpoints(
+        self, load_type: LoadType, endpoint_selection_strategy: EndpointSelectionStrategy
+    ):
+        all_endpoints = {
+            LoadType.gRPC: {
+                EndpointSelectionStrategy.ALL: list(
+                    set(
+                        endpoint
+                        for node_under_load in self.nodes_under_load
+                        for endpoint in node_under_load.service(StorageNode).get_all_rpc_endpoint()
+                    )
+                ),
+                EndpointSelectionStrategy.FIRST: list(
+                    set(
+                        node_under_load.service(StorageNode).get_rpc_endpoint()
+                        for node_under_load in self.nodes_under_load
+                    )
+                ),
+            },
+            # for some reason xk6 appends http protocol on its own
+            LoadType.S3: {
+                EndpointSelectionStrategy.ALL: list(
+                    set(
+                        endpoint.replace("http://", "")
+                        for node_under_load in self.nodes_under_load
+                        for endpoint in node_under_load.service(S3Gate).get_all_endpoints()
+                    )
+                ),
+                EndpointSelectionStrategy.FIRST: list(
+                    set(
+                        node_under_load.service(S3Gate).get_endpoint().replace("http://", "")
+                        for node_under_load in self.nodes_under_load
+                    )
+                ),
+            },
+        }
+
+        return all_endpoints[load_type][endpoint_selection_strategy]
+
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
+    @reporter.step_deco("Prepare background load instances")
+    def prepare(self):
+        if self.load_params.load_type == LoadType.S3:
+            init_s3_client(
+                self.load_nodes,
+                self.load_params,
+                self.k6_dir,
+                self.ssh_credentials,
+                self.nodes_under_load,
+                self.loaders_wallet,
+            )
+
+        self._prepare(self.load_params)
+
+    def _prepare(self, load_params: LoadParams):
+        self.k6_instances = prepare_k6_instances(
+            load_nodes=LOAD_NODES,
+            ssh_credentials=self.ssh_credentials,
+            k6_dir=self.k6_dir,
+            load_params=load_params,
+            endpoints=self.endpoints,
+            loaders_wallet=self.loaders_wallet,
+        )
+
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
+    @reporter.step_deco("Start background load")
+    def start(self):
+        if self.load_params.preset is None:
+            raise RuntimeError("Preset should not be none at the moment of start")
+
+        with reporter.step(
+            f"Start background load on nodes {self.nodes_under_load}: "
+            f"writers = {self.load_params.writers}, "
+            f"obj_size = {self.load_params.object_size}, "
+            f"load_time = {self.load_params.load_time}, "
+            f"prepare_json = {self.load_params.preset.pregen_json}, "
+            f"endpoints = {self.endpoints}"
+        ):
+            for k6_load_instance in self.k6_instances:
+                k6_load_instance.start()
+
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
+    @reporter.step_deco("Stop background load")
+    def stop(self):
+        for k6_load_instance in self.k6_instances:
+            k6_load_instance.stop()
+
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED, True)
+    def is_running(self):
+        for k6_load_instance in self.k6_instances:
+            if not k6_load_instance.is_running:
+                return False
+
+        return True
+
+    def wait_until_finish(self):
+        if self.load_params.load_time is None:
+            raise RuntimeError("LoadTime should not be none")
+
+        for k6_instance in self.k6_instances:
+            k6_instance.wait_until_finished(self.load_params.load_time + int(K6_TEARDOWN_PERIOD))
+
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
+    def verify(self):
+        if self.verification_params.verify_time is None:
+            raise RuntimeError("verify_time should not be none")
+
+        self._prepare(self.verification_params)
+        with reporter.step("Run verify background load data"):
+            for k6_verify_instance in self.k6_instances:
+                k6_verify_instance.start()
+                k6_verify_instance.wait_until_finished(self.verification_params.verify_time)
+
+    @run_optionally(optionals.OPTIONAL_BACKGROUND_LOAD_ENABLED)
+    @reporter.step_deco("K6 run results")
+    def get_results(self) -> dict:
+        results = {}
+        for k6_instance in self.k6_instances:
+            if k6_instance.load_params.k6_process_allocation_strategy is None:
+                raise RuntimeError("k6_process_allocation_strategy should not be none")
+
+            result = k6_instance.get_results()
+            keys_map = {
+                K6ProcessAllocationStrategy.PER_LOAD_NODE: k6_instance.load_node,
+                K6ProcessAllocationStrategy.PER_ENDPOINT: k6_instance.endpoints[0],
+            }
+            key = keys_map[k6_instance.load_params.k6_process_allocation_strategy]
+            results[key] = result
+
+        return results

src/frostfs_testlib/storage/controllers/cluster_state_controller.py

@@ -0,0 +1,130 @@
+import time
+
+import allure
+
+import frostfs_testlib.resources.optionals as optionals
+from frostfs_testlib.reporter import get_reporter
+from frostfs_testlib.shell import CommandOptions, Shell
+from frostfs_testlib.steps import epoch
+from frostfs_testlib.storage.cluster import Cluster, ClusterNode, StorageNode
+from frostfs_testlib.storage.controllers.disk_controller import DiskController
+from frostfs_testlib.testing.test_control import run_optionally, wait_for_success
+from frostfs_testlib.utils.failover_utils import (
+    wait_all_storage_nodes_returned,
+    wait_for_host_offline,
+    wait_for_host_online,
+    wait_for_node_online,
+)
+
+reporter = get_reporter()
+
+
+class ClusterStateController:
+    def __init__(self, shell: Shell, cluster: Cluster) -> None:
+        self.stopped_nodes: list[ClusterNode] = []
+        self.detached_disks: dict[str, DiskController] = {}
+        self.stopped_storage_nodes: list[StorageNode] = []
+        self.cluster = cluster
+        self.shell = shell
+
+    @run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
+    @reporter.step_deco("Stop host of node {node}")
+    def stop_node_host(self, node: ClusterNode, mode: str):
+        with allure.step(f"Stop host {node.host.config.address}"):
+            node.host.stop_host(mode=mode)
+            wait_for_host_offline(self.shell, node.storage_node)
+        self.stopped_nodes.append(node)
+
+    @run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
+    @reporter.step_deco("Start host of node {node}")
+    def start_node_host(self, node: ClusterNode):
+        with allure.step(f"Start host {node.host.config.address}"):
+            node.host.start_host()
+            wait_for_host_online(self.shell, node.storage_node)
+            wait_for_node_online(node.storage_node)
+        self.stopped_nodes.remove(node)
+
+    @run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
+    @reporter.step_deco("Start stopped hosts")
+    def start_stopped_hosts(self):
+        for node in self.stopped_nodes:
+            node.host.start_host()
+        self.stopped_nodes = []
+        wait_all_storage_nodes_returned(self.shell, self.cluster)
+
+    @run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
+    @reporter.step_deco("Detach disk {device} at {mountpoint} on node {node}")
+    def detach_disk(self, node: StorageNode, device: str, mountpoint: str):
+        disk_controller = self._get_disk_controller(node, device, mountpoint)
+        self.detached_disks[disk_controller.id] = disk_controller
+        disk_controller.detach()
+
+    @run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
+    @reporter.step_deco("Attach disk {device} at {mountpoint} on node {node}")
+    def attach_disk(self, node: StorageNode, device: str, mountpoint: str):
+        disk_controller = self._get_disk_controller(node, device, mountpoint)
+        disk_controller.attach()
+        self.detached_disks.pop(disk_controller.id, None)
+
+    @run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
+    @reporter.step_deco("Restore detached disks")
+    def restore_disks(self):
+        for disk_controller in self.detached_disks.values():
+            disk_controller.attach()
+        self.detached_disks = {}
+
+    @run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
+    @reporter.step_deco("Stop storage service on {node}")
+    def stop_storage_service(self, node: ClusterNode):
+        node.storage_node.stop_service()
+        self.stopped_storage_nodes.append(node.storage_node)
+
+    @run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
+    @reporter.step_deco("Start storage service on {node}")
+    def start_storage_service(self, node: ClusterNode):
+        node.storage_node.start_service()
+        self.stopped_storage_nodes.remove(node.storage_node)
+
+    @run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
+    @reporter.step_deco("Start stopped storage services")
+    def start_stopped_storage_services(self):
+        for node in self.stopped_storage_nodes:
+            node.start_service()
+        self.stopped_storage_nodes = []
+
+    @run_optionally(optionals.OPTIONAL_FAILOVER_ENABLED)
+    @reporter.step_deco("Hard reboot host {node} via magic SysRq option")
+    def panic_reboot_host(self, node: ClusterNode):
+        shell = node.host.get_shell()
+        shell.exec('sudo sh -c "echo 1 > /proc/sys/kernel/sysrq"')
+
+        options = CommandOptions(close_stdin=True, timeout=1, check=False)
+        shell.exec('sudo sh -c "echo b > /proc/sysrq-trigger"', options)
+
+        # Let the things to be settled
+        # A little wait here to prevent ssh stuck during panic
+        time.sleep(10)
+        wait_for_host_online(self.shell, node.storage_node)
+        wait_for_node_online(node.storage_node)
+
+    @reporter.step_deco("Wait up to {timeout} seconds for nodes on cluster to align epochs")
+    def wait_for_epochs_align(self, timeout=60):
+        @wait_for_success(timeout, 5, None, True)
+        def check_epochs():
+            epochs_by_node = epoch.get_epochs_from_nodes(self.shell, self.cluster)
+            assert (
+                len(set(epochs_by_node.values())) == 1
+            ), f"unaligned epochs found: {epochs_by_node}"
+
+        check_epochs()
+
+    def _get_disk_controller(
+        self, node: StorageNode, device: str, mountpoint: str
+    ) -> DiskController:
+        disk_controller_id = DiskController.get_id(node, device)
+        if disk_controller_id in self.detached_disks.keys():
+            disk_controller = self.detached_disks[disk_controller_id]
+        else:
+            disk_controller = DiskController(node, device, mountpoint)
+
+        return disk_controller

src/frostfs_testlib/storage/controllers/disk_controller.py

@@ -0,0 +1,41 @@
+from frostfs_testlib.hosting.interfaces import DiskInfo
+from frostfs_testlib.shell import CommandOptions
+from frostfs_testlib.storage.cluster import StorageNode
+from frostfs_testlib.testing.test_control import wait_for_success
+
+
+class DiskController:
+    def __init__(self, node: StorageNode, device: str, mountpoint: str) -> None:
+        self.node: StorageNode = node
+        self.device: str = device
+        self.device_by_label: str
+        self.mountpoint: str = mountpoint.strip()
+        self.disk_info: DiskInfo = DiskInfo()
+        self.id = self.get_id(node, device)
+
+        shell = node.host.get_shell()
+        cmd = f"sudo udevadm info -n {device} | egrep \"S:.*label\" | awk '{{print $2}}'"
+        self.device_by_label = f"/dev/{shell.exec(cmd).stdout.strip()}"
+
+    @wait_for_success(60, 3, False)
+    def _wait_until_detached(self):
+        return self.node.host.is_disk_attached(self.device, self.disk_info)
+
+    @wait_for_success(60, 3, True)
+    def _wait_until_attached(self):
+        return self.node.host.is_disk_attached(self.device, self.disk_info)
+
+    def detach(self):
+        self.disk_info = self.node.host.detach_disk(self.device)
+        self._wait_until_detached()
+
+    def attach(self):
+        self.node.host.attach_disk(self.device, self.disk_info)
+        self._wait_until_attached()
+        remote_shell = self.node.host.get_shell()
+        remote_shell.exec(f"sudo umount -l {self.device}", options=CommandOptions(check=False))
+        remote_shell.exec(f"sudo mount {self.device_by_label} {self.mountpoint}")
+
+    @staticmethod
+    def get_id(node: StorageNode, device: str):
+        return f"{node.host.config.address} - {device}"

src/frostfs_testlib/storage/controllers/shards_watcher.py

@@ -0,0 +1,118 @@
+import json
+from typing import Any
+
+from frostfs_testlib.cli.frostfs_cli.shards import FrostfsCliShards
+from frostfs_testlib.storage.cluster import ClusterNode
+from frostfs_testlib.testing.test_control import wait_for_success
+
+
+class ShardsWatcher:
+    shards_snapshots: list[dict[str, Any]] = []
+
+    def __init__(self, node_under_test: ClusterNode) -> None:
+        self.storage_node = node_under_test.storage_node
+        self.take_shards_snapshot()
+
+    def take_shards_snapshot(self):
+        snapshot = self.get_shards_snapshot()
+        self.shards_snapshots.append(snapshot)
+
+    def get_shards_snapshot(self):
+        shards_snapshot: dict[str, Any] = {}
+
+        shards = self.get_shards()
+        for shard in shards:
+            shards_snapshot[shard["shard_id"]] = shard
+
+        return shards_snapshot
+
+    def _get_current_snapshot(self):
+        return self.shards_snapshots[-1]
+
+    def _get_previous_snapshot(self):
+        return self.shards_snapshots[-2]
+
+    def _is_shard_present(self, shard_id):
+        snapshot = self._get_current_snapshot()
+        return shard_id in snapshot
+
+    def get_shards_with_new_errors(self):
+        current_snapshot = self._get_current_snapshot()
+        previous_snapshot = self._get_previous_snapshot()
+        shards_with_new_errors: dict[str, Any] = {}
+        for shard_id, shard in previous_snapshot.items():
+            if current_snapshot[shard_id]["error_count"] > shard["error_count"]:
+                shards_with_new_errors[shard_id] = current_snapshot[shard_id]
+
+        return shards_with_new_errors
+
+    def get_shards_with_errors(self):
+        snapshot = self.get_shards_snapshot()
+        shards_with_errors: dict[str, Any] = {}
+        for shard_id, shard in snapshot.items():
+            if shard["error_count"] > 0:
+                shards_with_errors[shard_id] = shard
+
+        return shards_with_errors
+
+    def get_shard_status(self, shard_id: str):
+        snapshot = self.get_shards_snapshot()
+
+        assert shard_id in snapshot, f"Shard {shard_id} is missing: {snapshot}"
+
+        return snapshot[shard_id]["mode"]
+
+    @wait_for_success(60, 2)
+    def await_for_all_shards_status(self, status: str):
+        snapshot = self.get_shards_snapshot()
+
+        for shard_id in snapshot:
+            assert snapshot[shard_id]["mode"] == status, f"Shard {shard_id} have wrong shard status"
+
+    @wait_for_success(60, 2)
+    def await_for_shard_status(self, shard_id: str, status: str):
+        assert self.get_shard_status(shard_id) == status
+
+    @wait_for_success(60, 2)
+    def await_for_shard_have_new_errors(self, shard_id: str):
+        self.take_shards_snapshot()
+        assert self._is_shard_present(shard_id)
+        shards_with_new_errors = self.get_shards_with_new_errors()
+
+        assert (
+            shard_id in shards_with_new_errors
+        ), f"Expected shard {shard_id} to have new errors, but haven't {self.shards_snapshots[-1]}"
+
+    @wait_for_success(300, 5)
+    def await_for_shards_have_no_new_errors(self):
+        self.take_shards_snapshot()
+        shards_with_new_errors = self.get_shards_with_new_errors()
+        assert len(shards_with_new_errors) == 0
+
+    def get_shards(self) -> dict[str, Any]:
+        shards_cli = FrostfsCliShards(
+            self.storage_node.host.get_shell(),
+            self.storage_node.host.get_cli_config("frostfs-cli").exec_path,
+        )
+
+        response = shards_cli.list(
+            endpoint=self.storage_node.get_control_endpoint(),
+            wallet=self.storage_node.get_remote_wallet_path(),
+            wallet_password=self.storage_node.get_wallet_password(),
+        )
+
+        return json.loads(response.stdout.split(">", 1)[1])
+
+    def set_shard_mode(self, shard_id: str, mode: str, clear_errors: bool = True):
+        shards_cli = FrostfsCliShards(
+            self.storage_node.host.get_shell(),
+            self.storage_node.host.get_cli_config("frostfs-cli").exec_path,
+        )
+        return shards_cli.set_mode(
+            self.storage_node.get_control_endpoint(),
+            self.storage_node.get_remote_wallet_path(),
+            self.storage_node.get_wallet_password(),
+            mode=mode,
+            id=[shard_id],
+            clear_errors=clear_errors,
+        )

src/frostfs_testlib/storage/dataclasses/acl.py

@@ -0,0 +1,103 @@
+import logging
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from frostfs_testlib.utils import wallet_utils
+
+logger = logging.getLogger("NeoLogger")
+EACL_LIFETIME = 100500
+FROSTFS_CONTRACT_CACHE_TIMEOUT = 30
+
+
+class EACLOperation(Enum):
+    PUT = "put"
+    GET = "get"
+    HEAD = "head"
+    GET_RANGE = "getrange"
+    GET_RANGE_HASH = "getrangehash"
+    SEARCH = "search"
+    DELETE = "delete"
+
+
+class EACLAccess(Enum):
+    ALLOW = "allow"
+    DENY = "deny"
+
+
+class EACLRole(Enum):
+    OTHERS = "others"
+    USER = "user"
+    SYSTEM = "system"
+
+
+class EACLHeaderType(Enum):
+    REQUEST = "req"  # Filter request headers
+    OBJECT = "obj"  # Filter object headers
+    SERVICE = "SERVICE"  # Filter service headers. These are not processed by FrostFS nodes and exist for service use only
+
+
+class EACLMatchType(Enum):
+    STRING_EQUAL = "="  # Return true if strings are equal
+    STRING_NOT_EQUAL = "!="  # Return true if strings are different
+
+
+@dataclass
+class EACLFilter:
+    header_type: EACLHeaderType = EACLHeaderType.REQUEST
+    match_type: EACLMatchType = EACLMatchType.STRING_EQUAL
+    key: Optional[str] = None
+    value: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "headerType": self.header_type,
+            "matchType": self.match_type,
+            "key": self.key,
+            "value": self.value,
+        }
+
+
+@dataclass
+class EACLFilters:
+    filters: Optional[List[EACLFilter]] = None
+
+    def __str__(self):
+        return ",".join(
+            [
+                f"{filter.header_type.value}:"
+                f"{filter.key}{filter.match_type.value}{filter.value}"
+                for filter in self.filters
+            ]
+            if self.filters
+            else []
+        )
+
+
+@dataclass
+class EACLPubKey:
+    keys: Optional[List[str]] = None
+
+
+@dataclass
+class EACLRule:
+    operation: Optional[EACLOperation] = None
+    access: Optional[EACLAccess] = None
+    role: Optional[Union[EACLRole, str]] = None
+    filters: Optional[EACLFilters] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "Operation": self.operation,
+            "Access": self.access,
+            "Role": self.role,
+            "Filters": self.filters or [],
+        }
+
+    def __str__(self):
+        role = (
+            self.role.value
+            if isinstance(self.role, EACLRole)
+            else f'pubkey:{wallet_utils.get_wallet_public_key(self.role, "")}'
+        )
+        return f'{self.access.value} {self.operation.value} {self.filters or ""} {role}'

src/frostfs_testlib/storage/dataclasses/frostfs_services.py

@@ -0,0 +1,173 @@
+import yaml
+
+from frostfs_testlib.blockchain import RPCClient
+from frostfs_testlib.storage.constants import ConfigAttributes
+from frostfs_testlib.storage.dataclasses.node_base import NodeBase
+
+
+class InnerRing(NodeBase):
+    """
+    Class represents inner ring node in a cluster
+
+    Inner ring node is not always the same as physical host (or physical node, if you will):
+        It can be service running in a container or on physical host
+    For testing perspective, it's not relevant how it is actually running,
+        since frostfs network will still treat it as "node"
+    """
+
+    def service_healthcheck(self) -> bool:
+        health_metric = "frostfs_node_ir_health"
+        output = (
+            self.host.get_shell()
+            .exec(f"curl -s localhost:6662 | grep {health_metric} | sed 1,2d")
+            .stdout
+        )
+        return health_metric in output
+
+    def get_netmap_cleaner_threshold(self) -> str:
+        config_file = self.get_remote_config_path()
+        contents = self.host.get_shell().exec(f"cat {config_file}").stdout
+
+        config = yaml.safe_load(contents)
+        value = config["netmap_cleaner"]["threshold"]
+
+        return value
+
+
+class S3Gate(NodeBase):
+    """
+    Class represents S3 gateway in a cluster
+    """
+
+    def get_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.ENDPOINT_DATA_0)
+
+    def get_all_endpoints(self) -> list[str]:
+        return [
+            self._get_attribute(ConfigAttributes.ENDPOINT_DATA_0),
+            self._get_attribute(ConfigAttributes.ENDPOINT_DATA_1),
+        ]
+
+    def service_healthcheck(self) -> bool:
+        health_metric = "frostfs_s3_gw_state_health"
+        output = (
+            self.host.get_shell()
+            .exec(f"curl -s localhost:8086 | grep {health_metric} | sed 1,2d")
+            .stdout
+        )
+        return health_metric in output
+
+    @property
+    def label(self) -> str:
+        return f"{self.name}: {self.get_endpoint()}"
+
+
+class HTTPGate(NodeBase):
+    """
+    Class represents HTTP gateway in a cluster
+    """
+
+    def get_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.ENDPOINT_DATA_0)
+
+    def service_healthcheck(self) -> bool:
+        health_metric = "frostfs_http_gw_state_health"
+        output = (
+            self.host.get_shell()
+            .exec(f"curl -s localhost:5662 | grep {health_metric}  | sed 1,2d")
+            .stdout
+        )
+        return health_metric in output
+
+    @property
+    def label(self) -> str:
+        return f"{self.name}: {self.get_endpoint()}"
+
+
+class MorphChain(NodeBase):
+    """
+    Class represents side-chain aka morph-chain consensus node in a cluster
+
+    Consensus node is not always the same as physical host (or physical node, if you will):
+        It can be service running in a container or on physical host
+    For testing perspective, it's not relevant how it is actually running,
+        since frostfs network will still treat it as "node"
+    """
+
+    rpc_client: RPCClient
+
+    def construct(self):
+        self.rpc_client = RPCClient(self.get_endpoint())
+
+    def get_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.ENDPOINT_INTERNAL)
+
+    def service_healthcheck(self) -> bool:
+        # TODO Rework in 1.3 Release when metrics for each service will be available
+        return True
+
+    @property
+    def label(self) -> str:
+        return f"{self.name}: {self.get_endpoint()}"
+
+
+class MainChain(NodeBase):
+    """
+    Class represents main-chain consensus node in a cluster
+
+    Consensus node is not always the same as physical host:
+        It can be service running in a container or on physical host (or physical node, if you will):
+    For testing perspective, it's not relevant how it is actually running,
+        since frostfs network will still treat it as "node"
+    """
+
+    rpc_client: RPCClient
+
+    def construct(self):
+        self.rpc_client = RPCClient(self.get_endpoint())
+
+    def get_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.ENDPOINT_INTERNAL)
+
+    @property
+    def label(self) -> str:
+        return f"{self.name}: {self.get_endpoint()}"
+
+
+class StorageNode(NodeBase):
+    """
+    Class represents storage node in a storage cluster
+
+    Storage node is not always the same as physical host:
+        It can be service running in a container or on physical host (or physical node, if you will):
+    For testing perspective, it's not relevant how it is actually running,
+        since frostfs network will still treat it as "node"
+    """
+
+    def get_rpc_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.ENDPOINT_DATA_0)
+
+    def get_all_rpc_endpoint(self) -> list[str]:
+        return [
+            self._get_attribute(ConfigAttributes.ENDPOINT_DATA_0),
+            self._get_attribute(ConfigAttributes.ENDPOINT_DATA_1),
+        ]
+
+    def service_healthcheck(self) -> bool:
+        health_metric = "frostfs_node_state_health"
+        output = (
+            self.host.get_shell()
+            .exec(f"curl -s localhost:6672 | grep {health_metric} | sed 1,2d")
+            .stdout
+        )
+        return health_metric in output
+
+    def get_control_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.CONTROL_ENDPOINT)
+
+    def get_un_locode(self):
+        return self._get_attribute(ConfigAttributes.UN_LOCODE)
+
+    @property
+    def label(self) -> str:
+        return f"{self.name}: {self.get_rpc_endpoint()}"

src/frostfs_testlib/storage/dataclasses/node_base.py

@@ -0,0 +1,122 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Optional, TypedDict, TypeVar
+
+from frostfs_testlib.hosting.config import ServiceConfig
+from frostfs_testlib.hosting.interfaces import Host
+from frostfs_testlib.storage.constants import ConfigAttributes
+from frostfs_testlib.utils import wallet_utils
+
+
+@dataclass
+class NodeBase(ABC):
+    """
+    Represents a node of some underlying service
+    """
+
+    id: str
+    name: str
+    host: Host
+
+    def __init__(self, id, name, host) -> None:
+        self.id = id
+        self.name = name
+        self.host = host
+        self.construct()
+
+    def construct(self):
+        pass
+
+    def __eq__(self, other):
+        return self.name == other.name
+
+    def __hash__(self):
+        return id(self.name)
+
+    def __str__(self):
+        return self.label
+
+    def __repr__(self) -> str:
+        return self.label
+
+    @property
+    def label(self) -> str:
+        return self.name
+
+    def get_service_systemctl_name(self) -> str:
+        return self._get_attribute(ConfigAttributes.SERVICE_NAME)
+
+    def start_service(self):
+        self.host.start_service(self.name)
+
+    @abstractmethod
+    def service_healthcheck(self) -> bool:
+        """Service healthcheck."""
+
+    def stop_service(self):
+        self.host.stop_service(self.name)
+
+    def restart_service(self):
+        self.host.restart_service(self.name)
+
+    def get_wallet_password(self) -> str:
+        return self._get_attribute(ConfigAttributes.WALLET_PASSWORD)
+
+    def get_wallet_path(self) -> str:
+        return self._get_attribute(
+            ConfigAttributes.LOCAL_WALLET_PATH,
+            ConfigAttributes.WALLET_PATH,
+        )
+
+    def get_remote_wallet_path(self) -> str:
+        """
+        Returns node wallet file path located on remote host
+        """
+        return self._get_attribute(
+            ConfigAttributes.WALLET_PATH,
+        )
+
+    def get_remote_config_path(self) -> str:
+        """
+        Returns node config file path located on remote host
+        """
+        return self._get_attribute(
+            ConfigAttributes.CONFIG_PATH,
+        )
+
+    def get_wallet_config_path(self):
+        return self._get_attribute(
+            ConfigAttributes.LOCAL_WALLET_CONFIG,
+            ConfigAttributes.WALLET_CONFIG,
+        )
+
+    def get_wallet_public_key(self):
+        storage_wallet_path = self.get_wallet_path()
+        storage_wallet_pass = self.get_wallet_password()
+        return wallet_utils.get_wallet_public_key(storage_wallet_path, storage_wallet_pass)
+
+    def _get_attribute(
+        self, attribute_name: str, default_attribute_name: Optional[str] = None
+    ) -> str:
+        config = self.host.get_service_config(self.name)
+
+        if attribute_name not in config.attributes:
+            if default_attribute_name is None:
+                raise RuntimeError(
+                    f"Service {self.name} has no {attribute_name} in config and fallback attribute isn't set either"
+                )
+
+            return config.attributes[default_attribute_name]
+
+        return config.attributes[attribute_name]
+
+    def _get_service_config(self) -> ServiceConfig:
+        return self.host.get_service_config(self.name)
+
+
+ServiceClass = TypeVar("ServiceClass", bound=NodeBase)
+
+
+class NodeClassDict(TypedDict):
+    hosting_service_name: str
+    cls: type[NodeBase]

src/frostfs_testlib/storage/dataclasses/storage_object_info.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class ObjectRef:
+    cid: str
+    oid: str
+
+
+@dataclass
+class LockObjectInfo(ObjectRef):
+    lifetime: Optional[int] = None
+    expire_at: Optional[int] = None
+
+
+@dataclass
+class StorageObjectInfo(ObjectRef):
+    size: Optional[int] = None
+    wallet_file_path: Optional[str] = None
+    file_path: Optional[str] = None
+    file_hash: Optional[str] = None
+    attributes: Optional[list[dict[str, str]]] = None
+    tombstone: Optional[str] = None
+    locks: Optional[list[LockObjectInfo]] = None

src/frostfs_testlib/storage/dataclasses/wallet.py

@@ -0,0 +1,90 @@
+import json
+import logging
+import os
+import uuid
+from dataclasses import dataclass
+from typing import Optional
+
+from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG, DEFAULT_WALLET_PASS
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.storage.cluster import Cluster, NodeBase
+from frostfs_testlib.utils.wallet_utils import get_last_address_from_wallet, init_wallet
+
+logger = logging.getLogger("frostfs.testlib.utils")
+
+
+@dataclass
+class WalletInfo:
+    path: str
+    password: str = DEFAULT_WALLET_PASS
+    config_path: str = DEFAULT_WALLET_CONFIG
+
+    @staticmethod
+    def from_node(node: NodeBase):
+        return WalletInfo(
+            node.get_wallet_path(), node.get_wallet_password(), node.get_wallet_config_path()
+        )
+
+    def get_address(self) -> str:
+        """
+        Extracts the last address from wallet via neo3 lib.
+
+        Returns:
+            The address of the wallet.
+        """
+        return get_last_address_from_wallet(self.path, self.password)
+
+    def get_address_from_json(self, account_id: int = 0) -> str:
+        """
+        Extracts address of the given account id from wallet using json lookup.
+        (Useful if neo3 fails for some reason and can't be used).
+
+        Args:
+            account_id: id of the account to get address.
+
+        Returns:
+            address string.
+        """
+        with open(self.path, "r") as wallet:
+            wallet_json = json.load(wallet)
+        assert abs(account_id) + 1 <= len(
+            wallet_json["accounts"]
+        ), f"There is no index '{account_id}' in wallet: {wallet_json}"
+
+        return wallet_json["accounts"][account_id]["address"]
+
+
+class WalletFactory:
+    def __init__(self, wallets_dir: str, shell: Shell, cluster: Cluster) -> None:
+        self.shell = shell
+        self.wallets_dir = wallets_dir
+        self.cluster = cluster
+
+    def create_wallet(
+        self, file_name: Optional[str] = None, password: Optional[str] = None
+    ) -> WalletInfo:
+        """
+        Creates new default wallet.
+
+        Args:
+            file_name: output wallet file name.
+            password: wallet password.
+
+        Returns:
+            WalletInfo object of new wallet.
+        """
+
+        if file_name is None:
+            file_name = str(uuid.uuid4())
+        if password is None:
+            password = ""
+
+        base_path = os.path.join(self.wallets_dir, file_name)
+        wallet_path = f"{base_path}.json"
+        wallet_config_path = f"{base_path}.yaml"
+        init_wallet(wallet_path, password)
+
+        with open(wallet_config_path, "w") as config_file:
+            config_file.write(f'password: "{password}"')
+
+        return WalletInfo(wallet_path, password, wallet_config_path)

src/frostfs_testlib/storage/service_registry.py

@@ -0,0 +1,21 @@
+from frostfs_testlib.storage.dataclasses.node_base import NodeBase, NodeClassDict, ServiceClass
+
+
+class ServiceRegistry:
+    _class_mapping: dict[str, NodeClassDict] = {}
+
+    def get_entry(self, service_type: type[ServiceClass]) -> NodeClassDict:
+        key = service_type.__name__
+
+        if key not in self._class_mapping:
+            raise RuntimeError(
+                f"Unregistered service type requested: {key}. At this moment registered services are: {self._class_mapping.keys()}"
+            )
+
+        return self._class_mapping[key]
+
+    def register_service(self, service_name: str, service_class: type[NodeBase]):
+        self._class_mapping[service_class.__name__] = {
+            "cls": service_class,
+            "hosting_service_name": service_name,
+        }