Move shared code to testlib

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>

src/frostfs_testlib/storage/dataclasses/acl.py

@@ -0,0 +1,103 @@
+import logging
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from frostfs_testlib.utils import wallet_utils
+
+logger = logging.getLogger("NeoLogger")
+EACL_LIFETIME = 100500
+FROSTFS_CONTRACT_CACHE_TIMEOUT = 30
+
+
+class EACLOperation(Enum):
+    PUT = "put"
+    GET = "get"
+    HEAD = "head"
+    GET_RANGE = "getrange"
+    GET_RANGE_HASH = "getrangehash"
+    SEARCH = "search"
+    DELETE = "delete"
+
+
+class EACLAccess(Enum):
+    ALLOW = "allow"
+    DENY = "deny"
+
+
+class EACLRole(Enum):
+    OTHERS = "others"
+    USER = "user"
+    SYSTEM = "system"
+
+
+class EACLHeaderType(Enum):
+    REQUEST = "req"  # Filter request headers
+    OBJECT = "obj"  # Filter object headers
+    SERVICE = "SERVICE"  # Filter service headers. These are not processed by FrostFS nodes and exist for service use only
+
+
+class EACLMatchType(Enum):
+    STRING_EQUAL = "="  # Return true if strings are equal
+    STRING_NOT_EQUAL = "!="  # Return true if strings are different
+
+
+@dataclass
+class EACLFilter:
+    header_type: EACLHeaderType = EACLHeaderType.REQUEST
+    match_type: EACLMatchType = EACLMatchType.STRING_EQUAL
+    key: Optional[str] = None
+    value: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "headerType": self.header_type,
+            "matchType": self.match_type,
+            "key": self.key,
+            "value": self.value,
+        }
+
+
+@dataclass
+class EACLFilters:
+    filters: Optional[List[EACLFilter]] = None
+
+    def __str__(self):
+        return ",".join(
+            [
+                f"{filter.header_type.value}:"
+                f"{filter.key}{filter.match_type.value}{filter.value}"
+                for filter in self.filters
+            ]
+            if self.filters
+            else []
+        )
+
+
+@dataclass
+class EACLPubKey:
+    keys: Optional[List[str]] = None
+
+
+@dataclass
+class EACLRule:
+    operation: Optional[EACLOperation] = None
+    access: Optional[EACLAccess] = None
+    role: Optional[Union[EACLRole, str]] = None
+    filters: Optional[EACLFilters] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "Operation": self.operation,
+            "Access": self.access,
+            "Role": self.role,
+            "Filters": self.filters or [],
+        }
+
+    def __str__(self):
+        role = (
+            self.role.value
+            if isinstance(self.role, EACLRole)
+            else f'pubkey:{wallet_utils.get_wallet_public_key(self.role, "")}'
+        )
+        return f'{self.access.value} {self.operation.value} {self.filters or ""} {role}'

src/frostfs_testlib/storage/dataclasses/frostfs_services.py

@@ -0,0 +1,173 @@
+import yaml
+
+from frostfs_testlib.blockchain import RPCClient
+from frostfs_testlib.storage.constants import ConfigAttributes
+from frostfs_testlib.storage.dataclasses.node_base import NodeBase
+
+
+class InnerRing(NodeBase):
+    """
+    Class represents inner ring node in a cluster
+
+    Inner ring node is not always the same as physical host (or physical node, if you will):
+        It can be service running in a container or on physical host
+    For testing perspective, it's not relevant how it is actually running,
+        since frostfs network will still treat it as "node"
+    """
+
+    def service_healthcheck(self) -> bool:
+        health_metric = "frostfs_node_ir_health"
+        output = (
+            self.host.get_shell()
+            .exec(f"curl -s localhost:6662 | grep {health_metric} | sed 1,2d")
+            .stdout
+        )
+        return health_metric in output
+
+    def get_netmap_cleaner_threshold(self) -> str:
+        config_file = self.get_remote_config_path()
+        contents = self.host.get_shell().exec(f"cat {config_file}").stdout
+
+        config = yaml.safe_load(contents)
+        value = config["netmap_cleaner"]["threshold"]
+
+        return value
+
+
+class S3Gate(NodeBase):
+    """
+    Class represents S3 gateway in a cluster
+    """
+
+    def get_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.ENDPOINT_DATA_0)
+
+    def get_all_endpoints(self) -> list[str]:
+        return [
+            self._get_attribute(ConfigAttributes.ENDPOINT_DATA_0),
+            self._get_attribute(ConfigAttributes.ENDPOINT_DATA_1),
+        ]
+
+    def service_healthcheck(self) -> bool:
+        health_metric = "frostfs_s3_gw_state_health"
+        output = (
+            self.host.get_shell()
+            .exec(f"curl -s localhost:8086 | grep {health_metric} | sed 1,2d")
+            .stdout
+        )
+        return health_metric in output
+
+    @property
+    def label(self) -> str:
+        return f"{self.name}: {self.get_endpoint()}"
+
+
+class HTTPGate(NodeBase):
+    """
+    Class represents HTTP gateway in a cluster
+    """
+
+    def get_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.ENDPOINT_DATA_0)
+
+    def service_healthcheck(self) -> bool:
+        health_metric = "frostfs_http_gw_state_health"
+        output = (
+            self.host.get_shell()
+            .exec(f"curl -s localhost:5662 | grep {health_metric}  | sed 1,2d")
+            .stdout
+        )
+        return health_metric in output
+
+    @property
+    def label(self) -> str:
+        return f"{self.name}: {self.get_endpoint()}"
+
+
+class MorphChain(NodeBase):
+    """
+    Class represents side-chain aka morph-chain consensus node in a cluster
+
+    Consensus node is not always the same as physical host (or physical node, if you will):
+        It can be service running in a container or on physical host
+    For testing perspective, it's not relevant how it is actually running,
+        since frostfs network will still treat it as "node"
+    """
+
+    rpc_client: RPCClient
+
+    def construct(self):
+        self.rpc_client = RPCClient(self.get_endpoint())
+
+    def get_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.ENDPOINT_INTERNAL)
+
+    def service_healthcheck(self) -> bool:
+        # TODO Rework in 1.3 Release when metrics for each service will be available
+        return True
+
+    @property
+    def label(self) -> str:
+        return f"{self.name}: {self.get_endpoint()}"
+
+
+class MainChain(NodeBase):
+    """
+    Class represents main-chain consensus node in a cluster
+
+    Consensus node is not always the same as physical host:
+        It can be service running in a container or on physical host (or physical node, if you will):
+    For testing perspective, it's not relevant how it is actually running,
+        since frostfs network will still treat it as "node"
+    """
+
+    rpc_client: RPCClient
+
+    def construct(self):
+        self.rpc_client = RPCClient(self.get_endpoint())
+
+    def get_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.ENDPOINT_INTERNAL)
+
+    @property
+    def label(self) -> str:
+        return f"{self.name}: {self.get_endpoint()}"
+
+
+class StorageNode(NodeBase):
+    """
+    Class represents storage node in a storage cluster
+
+    Storage node is not always the same as physical host:
+        It can be service running in a container or on physical host (or physical node, if you will):
+    For testing perspective, it's not relevant how it is actually running,
+        since frostfs network will still treat it as "node"
+    """
+
+    def get_rpc_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.ENDPOINT_DATA_0)
+
+    def get_all_rpc_endpoint(self) -> list[str]:
+        return [
+            self._get_attribute(ConfigAttributes.ENDPOINT_DATA_0),
+            self._get_attribute(ConfigAttributes.ENDPOINT_DATA_1),
+        ]
+
+    def service_healthcheck(self) -> bool:
+        health_metric = "frostfs_node_state_health"
+        output = (
+            self.host.get_shell()
+            .exec(f"curl -s localhost:6672 | grep {health_metric} | sed 1,2d")
+            .stdout
+        )
+        return health_metric in output
+
+    def get_control_endpoint(self) -> str:
+        return self._get_attribute(ConfigAttributes.CONTROL_ENDPOINT)
+
+    def get_un_locode(self):
+        return self._get_attribute(ConfigAttributes.UN_LOCODE)
+
+    @property
+    def label(self) -> str:
+        return f"{self.name}: {self.get_rpc_endpoint()}"

src/frostfs_testlib/storage/dataclasses/node_base.py

@@ -0,0 +1,122 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Optional, TypedDict, TypeVar
+
+from frostfs_testlib.hosting.config import ServiceConfig
+from frostfs_testlib.hosting.interfaces import Host
+from frostfs_testlib.storage.constants import ConfigAttributes
+from frostfs_testlib.utils import wallet_utils
+
+
+@dataclass
+class NodeBase(ABC):
+    """
+    Represents a node of some underlying service
+    """
+
+    id: str
+    name: str
+    host: Host
+
+    def __init__(self, id, name, host) -> None:
+        self.id = id
+        self.name = name
+        self.host = host
+        self.construct()
+
+    def construct(self):
+        pass
+
+    def __eq__(self, other):
+        return self.name == other.name
+
+    def __hash__(self):
+        return id(self.name)
+
+    def __str__(self):
+        return self.label
+
+    def __repr__(self) -> str:
+        return self.label
+
+    @property
+    def label(self) -> str:
+        return self.name
+
+    def get_service_systemctl_name(self) -> str:
+        return self._get_attribute(ConfigAttributes.SERVICE_NAME)
+
+    def start_service(self):
+        self.host.start_service(self.name)
+
+    @abstractmethod
+    def service_healthcheck(self) -> bool:
+        """Service healthcheck."""
+
+    def stop_service(self):
+        self.host.stop_service(self.name)
+
+    def restart_service(self):
+        self.host.restart_service(self.name)
+
+    def get_wallet_password(self) -> str:
+        return self._get_attribute(ConfigAttributes.WALLET_PASSWORD)
+
+    def get_wallet_path(self) -> str:
+        return self._get_attribute(
+            ConfigAttributes.LOCAL_WALLET_PATH,
+            ConfigAttributes.WALLET_PATH,
+        )
+
+    def get_remote_wallet_path(self) -> str:
+        """
+        Returns node wallet file path located on remote host
+        """
+        return self._get_attribute(
+            ConfigAttributes.WALLET_PATH,
+        )
+
+    def get_remote_config_path(self) -> str:
+        """
+        Returns node config file path located on remote host
+        """
+        return self._get_attribute(
+            ConfigAttributes.CONFIG_PATH,
+        )
+
+    def get_wallet_config_path(self):
+        return self._get_attribute(
+            ConfigAttributes.LOCAL_WALLET_CONFIG,
+            ConfigAttributes.WALLET_CONFIG,
+        )
+
+    def get_wallet_public_key(self):
+        storage_wallet_path = self.get_wallet_path()
+        storage_wallet_pass = self.get_wallet_password()
+        return wallet_utils.get_wallet_public_key(storage_wallet_path, storage_wallet_pass)
+
+    def _get_attribute(
+        self, attribute_name: str, default_attribute_name: Optional[str] = None
+    ) -> str:
+        config = self.host.get_service_config(self.name)
+
+        if attribute_name not in config.attributes:
+            if default_attribute_name is None:
+                raise RuntimeError(
+                    f"Service {self.name} has no {attribute_name} in config and fallback attribute isn't set either"
+                )
+
+            return config.attributes[default_attribute_name]
+
+        return config.attributes[attribute_name]
+
+    def _get_service_config(self) -> ServiceConfig:
+        return self.host.get_service_config(self.name)
+
+
+ServiceClass = TypeVar("ServiceClass", bound=NodeBase)
+
+
+class NodeClassDict(TypedDict):
+    hosting_service_name: str
+    cls: type[NodeBase]

src/frostfs_testlib/storage/dataclasses/storage_object_info.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class ObjectRef:
+    cid: str
+    oid: str
+
+
+@dataclass
+class LockObjectInfo(ObjectRef):
+    lifetime: Optional[int] = None
+    expire_at: Optional[int] = None
+
+
+@dataclass
+class StorageObjectInfo(ObjectRef):
+    size: Optional[int] = None
+    wallet_file_path: Optional[str] = None
+    file_path: Optional[str] = None
+    file_hash: Optional[str] = None
+    attributes: Optional[list[dict[str, str]]] = None
+    tombstone: Optional[str] = None
+    locks: Optional[list[LockObjectInfo]] = None

src/frostfs_testlib/storage/dataclasses/wallet.py

@@ -0,0 +1,90 @@
+import json
+import logging
+import os
+import uuid
+from dataclasses import dataclass
+from typing import Optional
+
+from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG, DEFAULT_WALLET_PASS
+from frostfs_testlib.shell import Shell
+from frostfs_testlib.storage.cluster import Cluster, NodeBase
+from frostfs_testlib.utils.wallet_utils import get_last_address_from_wallet, init_wallet
+
+logger = logging.getLogger("frostfs.testlib.utils")
+
+
+@dataclass
+class WalletInfo:
+    path: str
+    password: str = DEFAULT_WALLET_PASS
+    config_path: str = DEFAULT_WALLET_CONFIG
+
+    @staticmethod
+    def from_node(node: NodeBase):
+        return WalletInfo(
+            node.get_wallet_path(), node.get_wallet_password(), node.get_wallet_config_path()
+        )
+
+    def get_address(self) -> str:
+        """
+        Extracts the last address from wallet via neo3 lib.
+
+        Returns:
+            The address of the wallet.
+        """
+        return get_last_address_from_wallet(self.path, self.password)
+
+    def get_address_from_json(self, account_id: int = 0) -> str:
+        """
+        Extracts address of the given account id from wallet using json lookup.
+        (Useful if neo3 fails for some reason and can't be used).
+
+        Args:
+            account_id: id of the account to get address.
+
+        Returns:
+            address string.
+        """
+        with open(self.path, "r") as wallet:
+            wallet_json = json.load(wallet)
+        assert abs(account_id) + 1 <= len(
+            wallet_json["accounts"]
+        ), f"There is no index '{account_id}' in wallet: {wallet_json}"
+
+        return wallet_json["accounts"][account_id]["address"]
+
+
+class WalletFactory:
+    def __init__(self, wallets_dir: str, shell: Shell, cluster: Cluster) -> None:
+        self.shell = shell
+        self.wallets_dir = wallets_dir
+        self.cluster = cluster
+
+    def create_wallet(
+        self, file_name: Optional[str] = None, password: Optional[str] = None
+    ) -> WalletInfo:
+        """
+        Creates new default wallet.
+
+        Args:
+            file_name: output wallet file name.
+            password: wallet password.
+
+        Returns:
+            WalletInfo object of new wallet.
+        """
+
+        if file_name is None:
+            file_name = str(uuid.uuid4())
+        if password is None:
+            password = ""
+
+        base_path = os.path.join(self.wallets_dir, file_name)
+        wallet_path = f"{base_path}.json"
+        wallet_config_path = f"{base_path}.yaml"
+        init_wallet(wallet_path, password)
+
+        with open(wallet_config_path, "w") as config_file:
+            config_file.write(f'password: "{password}"')
+
+        return WalletInfo(wallet_path, password, wallet_config_path)