From 58b14717aaefee356f6a9bac4cc939e5fca732ce Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Wed, 7 May 2025 13:30:39 +0300 Subject: [PATCH 1/5] [#379] Add missing dependencies: configobj, httpx Some packages were missing from the dependency list even though they were being imported from our code. Signed-off-by: Vitaliy Potyarkin --- pyproject.toml | 5 ++++- requirements.txt | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d62f04b..2d107c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,9 @@ dependencies = [ "tenacity==8.0.1", "boto3==1.35.30", "boto3-stubs[s3,iam,sts]==1.35.30", + "pydantic==2.10.6", + "configobj==5.0.6", + "httpx==0.28.1", ] requires-python = ">=3.10" @@ -92,4 +95,4 @@ filterwarnings = [ testpaths = ["tests"] [project.entry-points.pytest11] -testlib = "frostfs_testlib" \ No newline at end of file +testlib = "frostfs_testlib" diff --git a/requirements.txt b/requirements.txt index 56d9b83..b97919c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,8 @@ pytest==7.1.2 boto3==1.35.30 boto3-stubs[s3,iam,sts]==1.35.30 pydantic==2.10.6 +configobj==5.0.6 +httpx==0.28.1 # Dev dependencies black==22.8.0 @@ -22,4 +24,4 @@ pylint==2.17.4 # Packaging dependencies build==0.8.0 setuptools==65.3.0 -twine==4.0.1 \ No newline at end of file +twine==4.0.1 From 9261d46ed574d9391ad57638d875330ae01cb6bb Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 6 May 2025 16:57:40 +0300 Subject: [PATCH 2/5] Upgrade docker API bindings Version in pyproject.toml was not strictly bound anyways. Now requirements.txt uses the same version spec. Signed-off-by: Vitaliy Potyarkin --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b97919c..10eafa3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ allure-python-commons==2.13.2 -docker==4.4.0 +docker>=4.4.0 neo-mamba==1.0.0 paramiko==2.10.3 pexpect==4.8.0 From 1fbd7b7de1274a5240a6dd5cbe9014b38ef21884 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Tue, 6 May 2025 16:55:24 +0300 Subject: [PATCH 3/5] Allow ClusterNode.__init__ with missing services Not all component test environments will provide full set of services. It's ok for gateways and even storage nodes to be completely missing. Signed-off-by: Vitaliy Potyarkin --- src/frostfs_testlib/storage/cluster.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/frostfs_testlib/storage/cluster.py b/src/frostfs_testlib/storage/cluster.py index b67e34d..7110038 100644 --- a/src/frostfs_testlib/storage/cluster.py +++ b/src/frostfs_testlib/storage/cluster.py @@ -161,17 +161,24 @@ class Cluster: This class represents a Cluster object for the whole storage based on provided hosting """ - default_rpc_endpoint: str - default_s3_gate_endpoint: str - default_http_gate_endpoint: str + default_rpc_endpoint: str = "not deployed" + default_s3_gate_endpoint: str = "not deployed" + default_http_gate_endpoint: str = "not deployed" def __init__(self, hosting: Hosting) -> None: self._hosting = hosting self.class_registry = get_service_registry() - self.default_rpc_endpoint = self.services(StorageNode)[0].get_rpc_endpoint() - self.default_s3_gate_endpoint = self.services(S3Gate)[0].get_endpoint() - self.default_http_gate_endpoint = self.services(HTTPGate)[0].get_endpoint() + + storage = self.services(StorageNode) + if storage: + self.default_rpc_endpoint = storage[0].get_rpc_endpoint() + s3gate = self.services(S3Gate) + if s3gate: + self.default_s3_gate_endpoint = s3gate[0].get_endpoint() + http_gate = self.services(HTTPGate) + if http_gate: + self.default_http_gate_endpoint = http_gate[0].get_endpoint() @property def hosts(self) -> list[Host]: From 211f9a0abd60c15e08d18f9487b58b44ec151fb8 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Wed, 7 May 2025 14:35:37 +0300 Subject: [PATCH 4/5] Implement fixtures for deploying FrostFS components Exported from a private playground repo @ commit ba8c88d7e11e8e8c17e54ca1317bc2dbf8b52204 Signed-off-by: Vitaliy Potyarkin --- pyproject.toml | 2 + requirements.txt | 1 + .../component_tests/__init__.py | 1 + .../component_tests/container.py | 267 +++++++ .../component_tests/fixtures.py | 707 ++++++++++++++++++ .../component_tests/templates/__init__.py | 0 .../component_tests/templates/httpgw.yml | 119 +++ .../component_tests/templates/innerring.yml | 88 +++ .../component_tests/templates/neogo.yml | 70 ++ .../component_tests/templates/storage.yml | 103 +++ tests/test_component_container_wrapper.py | 44 ++ tests/test_component_fixture_demo.py | 24 + tests/test_component_fixtures.py | 74 ++ 13 files changed, 1500 insertions(+) create mode 100644 src/frostfs_testlib/component_tests/__init__.py create mode 100644 src/frostfs_testlib/component_tests/container.py create mode 100644 src/frostfs_testlib/component_tests/fixtures.py create mode 100644 src/frostfs_testlib/component_tests/templates/__init__.py create mode 100644 src/frostfs_testlib/component_tests/templates/httpgw.yml create mode 100644 src/frostfs_testlib/component_tests/templates/innerring.yml create mode 100644 src/frostfs_testlib/component_tests/templates/neogo.yml create mode 100644 src/frostfs_testlib/component_tests/templates/storage.yml create mode 100644 tests/test_component_container_wrapper.py create mode 100644 tests/test_component_fixture_demo.py create mode 100644 tests/test_component_fixtures.py diff --git a/pyproject.toml b/pyproject.toml index 2d107c7..7c8dfd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "pydantic==2.10.6", "configobj==5.0.6", "httpx==0.28.1", + "testcontainers==4.10.0", ] requires-python = ">=3.10" @@ -95,4 +96,5 @@ filterwarnings = [ testpaths = ["tests"] [project.entry-points.pytest11] +component_tests = "frostfs_testlib.component_tests.fixtures" testlib = "frostfs_testlib" diff --git a/requirements.txt b/requirements.txt index 10eafa3..8a14b91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ boto3-stubs[s3,iam,sts]==1.35.30 pydantic==2.10.6 configobj==5.0.6 httpx==0.28.1 +testcontainers==4.10.0 # Dev dependencies black==22.8.0 diff --git a/src/frostfs_testlib/component_tests/__init__.py b/src/frostfs_testlib/component_tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/frostfs_testlib/component_tests/__init__.py @@ -0,0 +1 @@ + diff --git a/src/frostfs_testlib/component_tests/container.py b/src/frostfs_testlib/component_tests/container.py new file mode 100644 index 0000000..f6561da --- /dev/null +++ b/src/frostfs_testlib/component_tests/container.py @@ -0,0 +1,267 @@ +import codecs +import io +import os +import re +import tarfile +import threading +from collections.abc import Mapping +from pathlib import Path +from textwrap import dedent + +from docker.models.containers import Container, ExecResult +from testcontainers.core.container import DockerContainer +from testcontainers.core.network import Network + + +class ContainerizedService: + """ + Testcontainers wrapper specialized for our use case. + + Extra features that testcontainers do not provide: + + - Copy files into container prior to executing the entrypoint. + - Stop and restart the service running in container. + """ + + _testcontainer: DockerContainer = None + _container: Container = None + _network: Network = None + name: str + image: str + command: str + _default_cmd_template: str = "" + _default_cmd_rewrite = None + + def __init__(self, image: str, command: str, name: str = None, network: Network = None, default_command: str = "{command}") -> None: + self.name = name + self.image = image + self.command = command + self._network = network + self._default_cmd_template = default_command + self._ip = {} + + def start(self): + if self._testcontainer is None: + self.create() + self.stop() + self._pid1(f"{{ {self.command} ; }} &") + + def stop(self): + self._pid1( + """ + kill -9 $(jobs -p) + wait + """ + ) + + @property + def ip(self) -> str: + if self._container.id in self._ip: + return self._ip[self._container.id] + inspect = self._container.client.api.inspect_container(self._container.id) + for network, options in inspect["NetworkSettings"]["Networks"].items(): + if network == self._network.name: + self._ip[self._container.id] = options["IPAddress"] + return self._ip[self._container.id] + else: + raise RuntimeError(f"container not attached to {self._network.name}: {self._container.name}") + + def signal(self, signal): + self._pid1(f"kill -{signal} $(jobs -p)") + + def create(self) -> None: + c = DockerContainer( + self.image, + entrypoint="", + tty=True, + stdin_open=True, + user=0, + working_dir="/", + ) + c.with_network(self._network) + c.with_name(self.name) + c.with_command(["sh", "-ims"]) + c.start() + self._testcontainer = c + self._container = c._container + self.name = self._container.name + self._pid1("uname -a; whoami; date") + + def destroy(self) -> None: + self._testcontainer.stop() + self._testcontainer = None + self._container = None + + def _pid1(self, command: str) -> None: + """ + Execute a shell command in PID1. No feedback is provided, use with extreme caution! + """ + command = f"{dedent(command).strip()}\n" + socket = self._container.attach_socket(params=dict(stdin=True, stream=True)) + socket._sock.send(command.encode()) + socket._sock.close() + socket.close() + + def add_file(self, src: Path, dest: Path) -> None: + """ + Add file from local filesystem into a running container. + Keeps a copy of the whole file in memory (TODO: stream directly from disk). + """ + file = tarfile.TarInfo(str(dest)) + file.size = Path(src).stat().st_size + if file.size > (64 << 20): + raise ValueError(f"file too large for current add_file implementation: {src} ({file.size >> 20}MB)") + if self._container is None: + self.create() + archive = io.BytesIO() + with tarfile.open(fileobj=archive, mode="w|") as tar: + with open(src, "rb") as f: + tar.addfile(file, f) + archive.seek(0) + self._container.put_archive("/", archive) + + def add_directory(self, src: Path, dest: Path) -> None: + """Add all files from directory (one by one).""" + dest = Path(dest) + for root, _, files in os.walk(src): + root = Path(root) + subdir = root.relative_to(src) + for file in files: + file = Path(file) + self.add_file(root / file, dest / subdir / file) + + def fetch(self, src: Path, dest: Path) -> None: + """ + Fetch file or directory from a running container. + """ + src = Path(src) + dest = Path(dest) + + stream, stat = self._container.get_archive(str(src)) + with tarfile.open(fileobj=generator_to_stream(stream), mode="r|*") as tar: + tar.extractall(dest.parent, filter=_tar_rename(src, dest)) + + def logs(self, stdout=True, stderr=True, tail="all", since=None, timeout=None): + """ + Log stream from ContainerizedService. + You should probably call close() on received object after you're done. + """ + return LogStream( + self._container.logs( + stream=True, + stdout=stdout, + stderr=stderr, + tail=tail, + since=since, + ), + timeout=timeout, + ) + + def wait(self, regex, **kwargs): + """ + Wait until a line appears in container logs that matches provided regex. + """ + if isinstance(regex, str): + regex = re.compile(regex) + for line in self.logs(**kwargs): + if regex.search(line): + return + raise TimeoutError("log stream was closed before a matching line appeared") + + def exec(self, command: str, env: Mapping = None) -> ExecResult: + """ + Execute a command in container shell. + """ + return self._container.exec_run(["sh", "-c", command], detach=False, environment=env) + + def __call__(self, command: str, env: Mapping = None) -> str: + """ + Execute default templated command in container shell. + """ + if not self._default_cmd_template: + raise ValueError(f"default command was not specified during container initialization") + if self._default_cmd_rewrite: + command = self._default_cmd_rewrite(command) + result = self.exec( + command=self._default_cmd_template.format(command=command), + env=env, + ) + if result.exit_code != 0: + raise ValueError(f"exit code {result.exit_code}") + return result.output.decode() + + +class LogStream: + def __init__(self, stream, timeout=None): + self._stream = stream + self._buffer = bytearray() + self._cancel = threading.Lock() + if timeout: + + def _cancel(): + "Close the stream either when timeout is reached or when the lock is released." + self._cancel.acquire(timeout=timeout) + self.close() + + self._cancel.acquire() + threading.Thread(target=_cancel).run() + + def __iter__(self): + return self + + def __next__(self) -> str: + self._buffer.clear() + for chunk in self._stream: + self._buffer += chunk + if chunk == b"\n" or len(chunk) > 1: + break + if len(self._buffer) == 0: + raise StopIteration + return self._buffer.decode() + + def close(self): + try: + self._cancel.release() + except RuntimeError: # release unlocked lock + pass + return self._stream.close() + + +def _tar_rename(src: Path, dest: Path) -> tarfile.TarInfo | None: + def _filter(member: tarfile.TarInfo, path: str) -> tarfile.TarInfo | None: + archive_path = Path(member.name) + if archive_path.is_absolute(): + subpath = archive_path.relative_to(src).name + member = member.replace(name=subpath) + if member.name == src.name: + member = member.replace(name=dest.name) + elif member.name.startswith(f"{src.name}/"): + member = member.replace(name=member.name.replace(src.name, dest.name, 1)) + return tarfile.data_filter(member, path) + + return _filter + + +def generator_to_stream(generator, buffer_size=io.DEFAULT_BUFFER_SIZE): + """ + https://stackoverflow.com/a/51546783 + """ + + class GeneratorStream(io.RawIOBase): + def __init__(self): + self.leftover = None + + def readable(self): + return True + + def readinto(self, b): + try: + l = len(b) # : We're supposed to return at most this much + chunk = self.leftover or next(generator) + output, self.leftover = chunk[:l], chunk[l:] + b[: len(output)] = output + return len(output) + except StopIteration: + return 0 # : Indicate EOF + + return io.BufferedReader(GeneratorStream()) diff --git a/src/frostfs_testlib/component_tests/fixtures.py b/src/frostfs_testlib/component_tests/fixtures.py new file mode 100644 index 0000000..919d4a2 --- /dev/null +++ b/src/frostfs_testlib/component_tests/fixtures.py @@ -0,0 +1,707 @@ +""" +Reusable fixtures for deploying FrostFS components with all the dependencies. +""" + +# TODO: This file is larger that desirable. +# TODO: If anyone knows how to break it into fixtures/base.py, fixtures/alphabet.py, fixtures/... - be my guest + +import gzip +import importlib.resources +import json +import random +import re +import shlex +import shutil +import string +import subprocess +import tarfile +import tempfile +from base64 import b64decode +from collections.abc import Mapping +from enum import Enum +from itertools import chain +from pathlib import Path +from types import SimpleNamespace +from typing import List +from urllib.request import urlopen + +import pytest +import yaml +from neo3.wallet.account import Account +from neo3.wallet.wallet import Wallet +from testcontainers.core.network import Network + +from .container import ContainerizedService, ExecResult + +_SCOPE = "session" +_PREFIX = "frostfs-" + + +glagolic = [ + "az", + "buky", + "vedi", + "glagoli", + "dobro", + "yest", + "zhivete", + "dzelo", + "zemlja", + "izhe", + "izhei", + "gerv", + "kako", + "ljudi", + "mislete", + "nash", + "on", + "pokoj", + "rtsi", + "slovo", + "tverdo", + "uk", +] + + +class Component(Enum): + + ADM = "adm" + ALPHABET = "alphabet" + CONTRACT = "contract" + HTTPGW = "httpgw" + INNERRING = "innerring" + LOCODE = "locode" + NEOGO = "neogo" + S3GW = "s3gw" + STORAGE = "storage" + + def __str__(self): + return self.value + + def __len__(self): + return len(self.value) + + +@pytest.fixture(scope=_SCOPE) +def _deployment(_deployment_dir) -> dict: + """ + Read deployment options from environment. + + DO NOT REFERENCE THIS FIXTURE DIRECTLY FROM TESTS! + This fixture is to be referenced only from other *_deployment fixtures. + + Runtime overrides will not be applied to _deployment() - + only to specific service fixtures, e.g. alphabet_deployment(). + """ + with importlib.resources.path("frostfs_testlib.component_tests.templates", ".") as template_dir: + default = { + "dir": _deployment_dir, + "template": template_dir, + } + for service in Component: + default[f"{service}_dir"] = _deployment_dir / str(service) + default[f"{service}_template"] = template_dir / f"{service}.yml" + config = { # TODO: replace hardcoded values with reading from a config file + "adm_image": "git.frostfs.info/truecloudlab/frostfs-adm", + "adm_version": "0.44.9", + "alphabet_foo": "bar", # FIXME + "alphabet_node_count": 4, + "contract_archive_url": "https://git.frostfs.info/TrueCloudLab/frostfs-contract/releases/download/v{version}/frostfs-contract-v{version}.tar.gz", + "contract_version": "0.21.1", + "httpgw_image": "git.frostfs.info/truecloudlab/frostfs-http-gw", + "httpgw_node_count": 3, + "httpgw_version": "0.32.1-debian", # TODO: none of the published images work: either POSIX shell is missing or CORS container is required + "innerring_image": "git.frostfs.info/truecloudlab/frostfs-ir", + "innerring_version": "0.44.9", + "locode_archive_url": "https://git.frostfs.info/TrueCloudLab/frostfs-locode-db/releases/download/v{version}/locode_db.gz", + "locode_version": "0.5.2", + "neogo_image": "nspccdev/neo-go", + "neogo_min_peers": 3, + "neogo_version": "0.106.3", + "storage_image": "git.frostfs.info/truecloudlab/frostfs-storage", + "storage_node_count": 2, + "storage_version": "0.44.9", + } + default.update(config) + yield default + + +def _customizable_deployment(service: Component, _deployment, request): + """Test fixture builder that allows overriding some deployment parameters later.""" + config = {} + for key, value in _deployment.items(): + if not key.startswith(f"{service}_"): + continue + config[key[len(service) + 1 :]] = value + override = getattr(request, "param", {}) + config.update(override) + directory = config.get("dir") + if directory: + directory = Path(directory) + basename = directory.name + index = 0 + while directory.exists(): + index += 1 + directory = directory.with_name(basename + f"-{index}") + config["dir"] = directory + config["prefix"] = f"{directory.parent.name}-{directory.name}-" + for key in ["dir", "template"]: + if isinstance(config[key], str): + config[key] = Path(config[key]) + return SimpleNamespace(**config) + + +def _customize_decorator(service: Component, options): + """ + Test decorator that overrides deployment options for the specific service. + + Docs: https://docs.pytest.org/en/latest/example/parametrize.html#indirect-parametrization + """ + return pytest.mark.parametrize(f"{service}_deployment", [options], indirect=[f"{service}_deployment"], ids=[f"custom_{service}"]) + + +@pytest.fixture(scope=_SCOPE) +def alphabet_deployment(_deployment, request): + """Alphabet node parameters.""" + return _customizable_deployment(Component.ALPHABET, _deployment, request) + + +def alphabet_customize(**options): + """Test decorator that overrides deployment options for alphabet nodes.""" + return _customize_decorator(Component.ALPHABET, options) + + +@pytest.fixture(scope=_SCOPE) +def contract_deployment(_deployment, request): + """Contract deployment parameters.""" + return _customizable_deployment(Component.CONTRACT, _deployment, request) + + +def contract_customize(**options): + """Test decorator that overrides deployment options for frostfs-contracts.""" + return _customize_decorator(Component.CONTRACT, options) + + +@pytest.fixture(scope=_SCOPE) +def neogo_deployment(_deployment, request, alphabet_deployment): + """neo-go deployment parameters.""" + deployment = _customizable_deployment(Component.NEOGO, _deployment, request) + deployment.node_count = alphabet_deployment.node_count + return deployment + + +def neogo_customize(**options): + """Test decorator that overrides deployment options for neo-go nodes.""" + return _customize_decorator(Component.NEOGO, options) + + +@pytest.fixture(scope=_SCOPE) +def adm_deployment(_deployment, request): + """Frostfs-adm container parameters.""" + return _customizable_deployment(Component.ADM, _deployment, request) + + +def adm_customize(**options): + """Test decorator that overrides deployment options for frostfs-adm container.""" + return _customize_decorator(Component.ADM, options) + + +@pytest.fixture(scope=_SCOPE) +def locode_deployment(_deployment, request): + """Frostfs locode database parameters.""" + return _customizable_deployment(Component.LOCODE, _deployment, request) + + +def locode_customize(**options): + """Test decorator that overrides deployment options for frostfs-locode-db archive.""" + return _customize_decorator(Component.LOCODE, options) + + +@pytest.fixture(scope=_SCOPE) +def innerring_deployment(_deployment, request): + """Innerring node parameters.""" + return _customizable_deployment(Component.INNERRING, _deployment, request) + + +def innerring_customize(**options): + """Test decorator that overrides deployment options for innerring nodes.""" + return _customize_decorator(Component.INNERRING, options) + + +@pytest.fixture(scope=_SCOPE) +def storage_deployment(_deployment, request): + """Storage node parameters.""" + return _customizable_deployment(Component.STORAGE, _deployment, request) + + +def storage_customize(**options): + """Test decorator that overrides deployment options for storage nodes.""" + return _customize_decorator(Component.STORAGE, options) + + +@pytest.fixture(scope=_SCOPE) +def httpgw_deployment(_deployment, request): + """HTTP gateway deployment parameters.""" + return _customizable_deployment(Component.HTTPGW, _deployment, request) + + +def httpgw_customize(**options): + """Test decorator that overrides deployment options for HTTP gateways.""" + return _customize_decorator(Component.HTTPGW, options) + + +@pytest.fixture(scope=_SCOPE) +def _network(): + """Docker container network fixture. Should not be referenced directly from tests.""" + network = Network() + network.name = f"{_PREFIX}{network.name}" + network.create() + yield network + network.remove() + + +@pytest.fixture(scope=_SCOPE) +def _deployment_dir() -> Path: + """Temporary directory for a dynamic deployment. Should not be referenced directly from tests.""" + tmp = Path(tempfile.mkdtemp(prefix=f"{_PREFIX}test-")).absolute() + yield tmp + shutil.rmtree(tmp) + + +@pytest.fixture(scope=_SCOPE) +def adm_config(alphabet_deployment): + alphabet_deployment.dir.mkdir(mode=0o700, exist_ok=False) + file = alphabet_deployment.dir / "_frostfs_adm.json" + tree = { + "alphabet-wallets": str(alphabet_deployment.dir), + "credentials": {}, + } + rnd = random.SystemRandom() + for key in chain(["contract"], glagolic[: alphabet_deployment.node_count]): + tree["credentials"][key] = "".join(rnd.choice(string.ascii_letters) for _ in range(12)) + with open(file, "w") as f: + json.dump(tree, f, indent=True, sort_keys=True, ensure_ascii=False) + yield tree, file + shutil.rmtree(alphabet_deployment.dir) + + +@pytest.fixture(scope=_SCOPE) +def alphabet_wallets(alphabet_deployment, frostfs_adm): + dest = alphabet_deployment.dir + count = alphabet_deployment.node_count + frostfs_adm(f"morph generate-alphabet --size {count}") + frostfs_adm.fetch(dest, dest) + pubkeys = _read_alphabet_public_keys(dest, count) + return pubkeys, dest + + +def _read_alphabet_public_keys(directory, count) -> List[str]: + public = [] + for index in range(count): + letter = glagolic[index] + file = directory / f"{letter}.json" + public.append(_wallet_public_key(file)) + return public + + +def _wallet_address(path: Path, account=0) -> str: + """Read account address from Neo NEP-6 wallet.""" + with open(path) as f: + wallet = json.load(f) + account = wallet["accounts"][account] + return account["address"] + + +def _wallet_public_key(path: Path, account=0) -> str: + """Read public key from Neo NEP-6 wallet.""" + with open(path) as f: + wallet = json.load(f) + account = wallet["accounts"][account] + script = b64decode(account["contract"]["script"]) + if not _is_signature_contract(script): + raise ValueError(f"not a signature contract: {account['contract']['script']}") + return script[2:35].hex() + + +def _is_signature_contract(script: bytes) -> bool: + """ + Test if the provided script is a (single) signature contract. + + Args: + script: contract script. + + Copied from neo-mamba (neo3.contracts.utils.is_signature_contract). + """ + + PUSHDATA1 = 0x0C + SYSCALL = 0x41 + SYSTEM_CRYPTO_CHECK_STANDARD_ACCOUNT = bytes((0x56, 0xE7, 0xB3, 0x27)) + + if len(script) != 40: + return False + + if script[0] != PUSHDATA1 or script[1] != 33 or script[35] != SYSCALL or script[36:40] != SYSTEM_CRYPTO_CHECK_STANDARD_ACCOUNT: + return False + return True + + +@pytest.fixture(scope=_SCOPE) +def neogo_config(neogo_deployment, adm_config, alphabet_wallets): + neogo_deployment.dir.mkdir(mode=0o700, exist_ok=False) + with open(neogo_deployment.template) as f: + template = f.read() + alphabet, alphabet_dir = alphabet_wallets + adm, _ = adm_config + credentials = adm["credentials"] + + seedlist = [f"{neogo_deployment.prefix}{index}:20333" for index in range(len(alphabet))] + override = getattr(neogo_deployment, "override", {}) + fields = vars(neogo_deployment) + configs = {} + for index in range(len(alphabet)): + letter = glagolic[index] + fields.update( + dict( + letter=letter, + index=index, + password=credentials[letter], + ) + ) + config = yaml.load(template.format(**fields), yaml.SafeLoader) + config["ProtocolConfiguration"]["Hardforks"] = {} # kludge: templating collision for {} + config["ProtocolConfiguration"]["StandbyCommittee"] = alphabet + config["ProtocolConfiguration"]["SeedList"] = seedlist[:index] + seedlist[index + 1 :] + _update(config, override) + with open(neogo_deployment.dir / f"{letter}.json", "w") as c: + json.dump(config, c, ensure_ascii=False, indent=True, sort_keys=True) + configs[letter] = config + yield configs, neogo_deployment.dir + shutil.rmtree(neogo_deployment.dir) + + +def _update(old: Mapping, new: Mapping) -> None: + """Recursive version of dict.update.""" + for key in new: + if key in old and isinstance(old[key], Mapping) and isinstance(new[key], Mapping): + _update(old[key], new[key]) + continue + old[key] = new[key] + + +@pytest.fixture(scope=_SCOPE) +def neogo(neogo_deployment, neogo_config, alphabet_deployment, frostfs_adm, _network): + wallet_dir = alphabet_deployment.dir + _, config_dir = neogo_config + nodes = [] + for index in range(neogo_deployment.node_count): + letter = glagolic[index] + node = ContainerizedService( + command=f"neo-go node --config-file /neogo/{letter}.json --privnet --debug", + image=f"{neogo_deployment.image}:{neogo_deployment.version}", + name=f"{neogo_deployment.prefix}{index+1}", + network=_network, + ) + node.add_file(wallet_dir / f"{letter}.json", f"/wallet/{letter}.json") + node.add_file(config_dir / f"{letter}.json", f"/neogo/{letter}.json") + node.start() + nodes.append(node) + + def add_rpc_endpoint(command): + for arg in shlex.split(command): + # Check that there is at least one non-flag argument + # (--version does not work with --rpc-endpoint) + if not arg.startswith("-"): + break + else: + return command + return f"--rpc-endpoint 'http://{random.choice(nodes).name}:30333' {command}" + + frostfs_adm._default_cmd_rewrite = add_rpc_endpoint + yield nodes + for node in nodes: + node.destroy() + + +@pytest.fixture(scope=_SCOPE) +def frostfs_adm(adm_deployment, adm_config, alphabet_deployment, _network): + _, config_file = adm_config + adm = ContainerizedService( + command="sleep infinity", + default_command=f'frostfs-adm --config "{config_file}" ' "{command}", + image=f"{adm_deployment.image}:{adm_deployment.version}", + name=f"{adm_deployment.prefix.strip('-')}", + network=_network, + ) + wallet_dir = alphabet_deployment.dir + adm.add_directory(wallet_dir, wallet_dir) + adm.add_file(config_file, config_file) + yield adm + adm.destroy() + + +@pytest.fixture(scope=_SCOPE) +def frostfs_bootstrap(frostfs_contract, frostfs_adm, neogo) -> Mapping[str, str]: + output = {} + + def morph(command: str) -> str: + output[command] = frostfs_adm(f"morph {command}") + + frostfs_adm.add_directory(frostfs_contract, frostfs_contract) + morph(f"init --contracts '{frostfs_contract}'") + morph( + "ape add-rule-chain " + "--target-type namespace " + "--target-name '' " + "--rule 'allow Container.* *' " + "--chain-id 'allow_container_ops'" + ) + morph("set-config ContainerFee=0") + morph("set-config ContainerAliasFee=0") + morph("set-config InnerRingCandidateFee=13") + morph("set-config WithdrawFee=17") + return output + + +@pytest.fixture(scope=_SCOPE) +def frostfs_contract(contract_deployment): + if contract_deployment.dir.exists(): + return contract_deployment.dir + contract_deployment.dir.mkdir(mode=0o700, exist_ok=False) + with urlopen( + contract_deployment.archive_url.format( + version=contract_deployment.version, + ) + ) as request: + with tarfile.open(fileobj=request, mode="r|*") as tar: + tar.extractall(path=contract_deployment.dir, filter=_tar_strip_components(1)) + return contract_deployment.dir + + +@pytest.fixture(scope=_SCOPE) +def frostfs_locode(locode_deployment): + locode = locode_deployment.dir / "locode_db" + if locode.exists(): + return locode + locode_deployment.dir.mkdir(mode=0o700, exist_ok=False) + with urlopen( + locode_deployment.archive_url.format( + version=locode_deployment.version, + ) + ) as request: + with gzip.GzipFile(fileobj=request, mode="rb") as archive: + with open(locode, "wb") as destination: + shutil.copyfileobj(archive, destination) + return locode + + +@pytest.fixture(scope=_SCOPE) +def innerring_config(innerring_deployment, neogo, adm_config, alphabet_wallets): + innerring_deployment.dir.mkdir(mode=0o700, exist_ok=False) + with open(innerring_deployment.template) as f: + template = f.read() + alphabet, alphabet_dir = alphabet_wallets + adm, _ = adm_config + credentials = adm["credentials"] + + override = getattr(innerring_deployment, "override", {}) + fields = vars(innerring_deployment) + configs = {} + for index in range(len(alphabet)): + letter = glagolic[index] + fields.update( + dict( + letter=letter, + index=index, + password=credentials[letter], + neogo=neogo[index].name, + ) + ) + config = yaml.load(template.format(**fields), yaml.SafeLoader) + config["morph"]["validators"] = alphabet + _update(config, override) + with open(innerring_deployment.dir / f"{letter}.json", "w") as c: + json.dump(config, c, ensure_ascii=False, indent=True, sort_keys=True) + configs[letter] = config + yield configs, innerring_deployment.dir + shutil.rmtree(innerring_deployment.dir) + + +@pytest.fixture(scope=_SCOPE) +def innerring(innerring_deployment, innerring_config, frostfs_locode, frostfs_bootstrap, alphabet_deployment, _network): + wallet_dir = alphabet_deployment.dir + _, config_dir = innerring_config + nodes = [] + for index in range(alphabet_deployment.node_count): + letter = glagolic[index] + node = ContainerizedService( + command=f"frostfs-ir --config /innerring/{letter}.json", + image=f"{innerring_deployment.image}:{innerring_deployment.version}", + name=f"{innerring_deployment.prefix}{index+1}", + network=_network, + ) + node.add_file(wallet_dir / f"{letter}.json", f"/wallet/{letter}.json") + node.add_file(config_dir / f"{letter}.json", f"/innerring/{letter}.json") + node.add_file(frostfs_locode, f"/innerring/locode.db") + node.start() + nodes.append(node) + yield nodes + for node in nodes: + node.destroy() + + +@pytest.fixture(scope=_SCOPE) +def storage_config(storage_deployment, neogo, frostfs_adm, innerring): + storage_deployment.dir.mkdir(mode=0o700, exist_ok=False) + with open(storage_deployment.template) as f: + template = f.read() + sidechain = [] + for node in neogo: + sidechain.append( + dict( + address=f"ws://{node.name}:30333/ws", + priority=0, + ) + ) + override = getattr(storage_deployment, "override", {}) + fields = vars(storage_deployment) + configs = [] + for index in range(storage_deployment.node_count): + wallet = storage_deployment.dir / f"wallet-{index}.json" + _, password = _new_wallet(wallet) + fields.update( + dict( + index=index, + prefix=storage_deployment.prefix, + wallet=str(wallet), + password=password, + price=42, + ) + ) + config = yaml.load(template.format(**fields), yaml.SafeLoader) + config["morph"]["rpc_endpoint"] = sidechain + _update(config, override) + with open(storage_deployment.dir / f"config-{index}.json", "w") as c: + json.dump(config, c, ensure_ascii=False, indent=True, sort_keys=True) + frostfs_adm.add_file(wallet, wallet) + frostfs_adm(f"morph refill-gas --storage-wallet '{wallet}' --gas 50.0") + configs.append(config) + yield configs, storage_deployment.dir + shutil.rmtree(storage_deployment.dir) + + +@pytest.fixture(scope=_SCOPE) +def storage(storage_deployment, storage_config, frostfs_adm, _network): + nodes = [] + configs, _ = storage_config + for index, config in enumerate(configs): + node = ContainerizedService( + command=f"frostfs-node --config /storage/config.json", + image=f"{storage_deployment.image}:{storage_deployment.version}", + name=f"{storage_deployment.prefix}{index+1}", + network=_network, + ) + node.add_file(config["node"]["wallet"]["path"], config["node"]["wallet"]["path"]) + node.add_file(storage_deployment.dir / f"config-{index}.json", f"/storage/config.json") + node.start() + nodes.append(node) + for index, node in enumerate(nodes): + # Adding storage node account to proxy contract is required to be able to use apemanager: + # https://chat.yadro.com/yadro/pl/eet5jxiuabn1i8omg6jz4yeeso + address = _wallet_address(configs[index]["node"]["wallet"]["path"]) + frostfs_adm(f"morph proxy-add-account --account {address}") + frostfs_adm("morph force-new-epoch") + yield nodes + for node in nodes: + node.destroy() + + +@pytest.fixture(scope=_SCOPE) +def httpgw_config(httpgw_deployment, storage, neogo): + httpgw_deployment.dir.mkdir(mode=0o700, exist_ok=False) + with open(httpgw_deployment.template) as f: + template = f.read() + peers = {} + for index, node in enumerate(storage): + peers[index] = dict( + address=f"grpc://{node.name}:8802", + priority=1, + weight=1, + ) + override = getattr(httpgw_deployment, "override", {}) + fields = vars(httpgw_deployment) + configs = [] + for index in range(httpgw_deployment.node_count): + wallet = httpgw_deployment.dir / f"wallet-{index}.json" + _, password = _new_wallet(wallet) + fields.update( + dict( + wallet=str(wallet), + password=password, + morph=neogo[index % len(neogo)].name, + ) + ) + config = yaml.load(template.format(**fields), yaml.SafeLoader) + config["peers"] = peers + _update(config, override) + with open(httpgw_deployment.dir / f"config-{index}.json", "w") as c: + json.dump(config, c, ensure_ascii=False, indent=True, sort_keys=True) + configs.append(config) + yield configs, httpgw_deployment.dir + shutil.rmtree(httpgw_deployment.dir) + + +@pytest.fixture(scope=_SCOPE) +def httpgw(httpgw_deployment, httpgw_config, _network): + nodes = [] + configs, _ = httpgw_config + for index, config in enumerate(configs): + node = ContainerizedService( + command=f"frostfs-http-gw --config /httpgw/config.json", + image=f"{httpgw_deployment.image}:{httpgw_deployment.version}", + name=f"{httpgw_deployment.prefix}{index+1}", + network=_network, + ) + node.add_file(config["wallet"]["path"], config["wallet"]["path"]) + node.add_file(httpgw_deployment.dir / f"config-{index}.json", f"/httpgw/config.json") + node.start() + nodes.append(node) + ready = re.compile(r"starting server.*\:80") + for node in nodes: + node.wait(ready, timeout=10) + yield nodes + for node in nodes: + node.destroy() + + +def _new_wallet(path: Path, password: str = None) -> (Wallet, str): + """ + Create new wallet and new account. + """ + wallet = Wallet() + if password is None: + password = "".join(random.choice(string.ascii_letters) for _ in range(12)) + account = Account.create_new(password) + wallet.account_add(account) + with open(path, "w") as out: + json.dump(wallet.to_json(), out) + return wallet, password + + +def _tar_strip_components(number=1): + """ + See --strip-components in `man tar`. + """ + sep = "/" + + def _filter(member: tarfile.TarInfo, path: str) -> tarfile.TarInfo | None: + components = member.name.split(sep) + for _ in range(number): + if not components: + break + components.pop(0) + if not components: + return None + member = member.replace(name=sep.join(components)) + return tarfile.data_filter(member, path) + + return _filter diff --git a/src/frostfs_testlib/component_tests/templates/__init__.py b/src/frostfs_testlib/component_tests/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/frostfs_testlib/component_tests/templates/httpgw.yml b/src/frostfs_testlib/component_tests/templates/httpgw.yml new file mode 100644 index 0000000..91af0ca --- /dev/null +++ b/src/frostfs_testlib/component_tests/templates/httpgw.yml @@ -0,0 +1,119 @@ +wallet: + path: "{wallet}" + passphrase: "{password}" + +pprof: + enabled: false + address: :8083 +prometheus: + enabled: false + address: :8084 +tracing: + enabled: false + exporter: "otlp_grpc" + endpoint: :4317 + trusted_ca: "" + +logger: + level: debug + destination: stdout + +server: + - address: :80 + tls: + enabled: false + +peers: # This config branch is replaced completely during template parsing + 0: + address: grpc://storage1:8802 + priority: 1 + weight: 1 + 1: + address: grpc://storage2:8802 + priority: 1 + weight: 1 + 2: + address: grpc://storage3:8802 + priority: 1 + weight: 1 + +reconnect_interval: 1m + +web: + # Per-connection buffer size for requests' reading. + # This also limits the maximum header size. + read_buffer_size: 4096 + + # Per-connection buffer size for responses' writing. + write_buffer_size: 4096 + + # ReadTimeout is the amount of time allowed to read + # the full request including body. The connection's read + # deadline is reset when the connection opens, or for + # keep-alive connections after the first byte has been read. + read_timeout: 10m + + # WriteTimeout is the maximum duration before timing out + # writes of the response. It is reset after the request handler + # has returned. + write_timeout: 5m + + # StreamRequestBody enables request body streaming, + # and calls the handler sooner when given body is + # larger then the current limit. + stream_request_body: true + + # Maximum request body size. + # The server rejects requests with bodies exceeding this limit. + max_request_body_size: 4194304 + +# RPC endpoint to be able to use nns container resolving. +rpc_endpoint: http://{morph}:30333 +# The order in which resolvers are used to find an container id by name. +resolve_order: + - nns + - dns + +upload_header: + use_default_timestamp: false # Create timestamp for object if it isn't provided by header. + +connect_timeout: 5s # Timeout to dial node. +stream_timeout: 10s # Timeout for individual operations in streaming RPC. +request_timeout: 5s # Timeout to check node health during rebalance. +rebalance_timer: 30s # Interval to check nodes health. +pool_error_threshold: 100 # The number of errors on connection after which node is considered as unhealthy. + +zip: + compression: false # Enable zip compression to download files by common prefix. + +runtime: + soft_memory_limit: 1gb + +# Parameters of requests to FrostFS +frostfs: + # This flag enables client side object preparing. + client_cut: false + # Sets max buffer size for read payload in put operations. + buffer_max_size_for_put: 1048576 + # Max attempt to make successful tree request. + # default value is 0 that means the number of attempts equals to number of nodes in pool. + tree_pool_max_attempts: 0 + +# Caching +cache: + # Cache which contains mapping of bucket name to bucket info + buckets: + lifetime: 1m + size: 1000 + +resolve_bucket: + namespace_header: X-Frostfs-Namespace + default_namespaces: [ "", "root" ] + +cors: + allow_origin: "" + allow_methods: [] + allow_headers: [] + expose_headers: [] + allow_credentials: false + max_age: 600 diff --git a/src/frostfs_testlib/component_tests/templates/innerring.yml b/src/frostfs_testlib/component_tests/templates/innerring.yml new file mode 100644 index 0000000..f153c1c --- /dev/null +++ b/src/frostfs_testlib/component_tests/templates/innerring.yml @@ -0,0 +1,88 @@ +# Logger section +logger: + level: debug # Minimum enabled logging level + +control: + authorized_keys: # Q: Node keys are always assumed to be trusted? + grpc: + endpoint: :8099 + +# Wallet settings +wallet: + path: /wallet/{letter}.json + password: {password} + +# Profiler section +pprof: + enabled: true + address: :6060 # Endpoint for application pprof profiling; disabled by default + shutdown_timeout: 30s # Timeout for profiling HTTP server graceful shutdown + +# Application metrics section +prometheus: + enabled: true + address: :9090 # Endpoint for application prometheus metrics; disabled by default + shutdown_timeout: 30s # Timeout for metrics HTTP server graceful shutdown + +# Toggling the sidechain-only mode +without_mainnet: true + +# Neo main chain RPC settings +mainnet: + endpoint: + +# Neo side chain RPC settings +morph: + endpoint: + client: # List of websocket RPC endpoints in sidechain + - address: ws://{neogo}:30333/ws + validators: # This config branch is replaced completely during template parsing + - 03aa8d8a0b8f9f0c5b36ce37975cfbcab3df75e1f12c501e3ead6d5f49d6e0b6f2 # az + - 02a51fd92f9e518f46ad009cc4b1e6f6d2879c33c252e12368ca9a431f4aabcd4f # buky + - 02856ed40a58b1e4bec8e1288ef8d5b2b4d8652557c89f03da72cc3988f7b4cf61 # vedi + - 0398803ed9999ddcbe8eba0d88805fd6b2ee92553af78ab0e4364446e7cb7229b3 # glagoli + - 036f8f0e9c2cd033c5c7bf38f5fc45926950988cecbc6f6d47575781217786166a # dobro + - 023790dcc88bef1f500f549342d1ec7a4b7e48912555d93b71ed6561a309142893 # yest + - 0244cb3df7b5a81810b8bb9ff2f0442434b5e8ab7c278c09723c07b127ff30e347 # zhivete + +# Network time settings +timers: + emit: 50 # Number of sidechain blocks between GAS emission cycles; disabled by default + stop_estimation: + mul: 1 # Multiplier in x/y relation of when to stop basic income estimation within the epoch + div: 4 # Divider in x/y relation of when to stop basic income estimation within the epoch + collect_basic_income: + mul: 1 # Multiplier in x/y relation of when to start basic income asset collection within the epoch + div: 2 # Divider in x/y relation of when to start basic income asset collecting within the epoch + distribute_basic_income: + mul: 3 # Multiplier in x/y relation of when to start basic income asset distribution within the epoch + div: 4 # Divider in x/y relation of when to start basic income asset distribution within the epoch + +# Storage node GAS emission settings +emit: + storage: + amount: 1000000000 # Fixed8 value of sidechain GAS emitted to all storage nodes once per GAS emission cycle; disabled by default + +# Storage node removal settings +netmap_cleaner: + enabled: true # Enable voting for removing stale storage nodes from network map + threshold: 3 # Number of FrostFS epoch without bootstrap request from storage node before it considered stale + +# Audit settings +audit: + pdp: + max_sleep_interval: 100ms # Maximum timeout between object.RangeHash requests to the storage node + +# Settlement settings +settlement: + basic_income_rate: 100000000 # Optional: override basic income rate value from network config; applied only in debug mode + audit_fee: 100000 # Optional: override audit fee value from network config; applied only in debug mode + +# LOCODE database +locode: + db: + path: /innerring/locode.db # Path to UN/LOCODE database file + +node: + persistent_state: + path: /innerring/{letter}.state diff --git a/src/frostfs_testlib/component_tests/templates/neogo.yml b/src/frostfs_testlib/component_tests/templates/neogo.yml new file mode 100644 index 0000000..745fe23 --- /dev/null +++ b/src/frostfs_testlib/component_tests/templates/neogo.yml @@ -0,0 +1,70 @@ +ProtocolConfiguration: + Magic: 2025042112 + MaxTraceableBlocks: 200000 + TimePerBlock: 1s + MemPoolSize: 50000 + StandbyCommittee: # This config branch is replaced completely during parsing + - 03aa8d8a0b8f9f0c5b36ce37975cfbcab3df75e1f12c501e3ead6d5f49d6e0b6f2 # az + - 02a51fd92f9e518f46ad009cc4b1e6f6d2879c33c252e12368ca9a431f4aabcd4f # buky + - 02856ed40a58b1e4bec8e1288ef8d5b2b4d8652557c89f03da72cc3988f7b4cf61 # vedi + - 0398803ed9999ddcbe8eba0d88805fd6b2ee92553af78ab0e4364446e7cb7229b3 # glagoli + - 036f8f0e9c2cd033c5c7bf38f5fc45926950988cecbc6f6d47575781217786166a # dobro + - 023790dcc88bef1f500f549342d1ec7a4b7e48912555d93b71ed6561a309142893 # yest + - 0244cb3df7b5a81810b8bb9ff2f0442434b5e8ab7c278c09723c07b127ff30e347 # zhivete + + ValidatorsCount: {node_count} + VerifyTransactions: true + StateRootInHeader: true + P2PSigExtensions: true + Hardforks: # This config branch is replaced completely during parsing + SeedList: # This config branch is replaced completely during parsing + - "{prefix}1:20333" + - "{prefix}2:20333" + - "{prefix}3:20333" + - "{prefix}4:20333" + - "{prefix}5:20333" + - "{prefix}6:20333" + - "{prefix}7:20333" + +ApplicationConfiguration: + SkipBlockVerification: false + DBConfiguration: + Type: "boltdb" + BoltDBOptions: + FilePath: "/chain/{letter}.bolt" + P2P: + Addresses: + - ":20333" + DialTimeout: 3s + ProtoTickInterval: 2s + PingInterval: 30s + PingTimeout: 90s + MaxPeers: 10 + AttemptConnPeers: 5 + MinPeers: {min_peers} + Relay: true + Consensus: + Enabled: true + UnlockWallet: + Path: "/wallet/{letter}.json" + Password: "{password}" + RPC: + Addresses: + - ":30333" + Enabled: true + SessionEnabled: true + EnableCORSWorkaround: false + MaxGasInvoke: 100 + P2PNotary: + Enabled: true + UnlockWallet: + Path: "/wallet/{letter}.json" + Password: "{password}" + Prometheus: + Addresses: + - ":20001" + Enabled: true + Pprof: + Addresses: + - ":20011" + Enabled: true diff --git a/src/frostfs_testlib/component_tests/templates/storage.yml b/src/frostfs_testlib/component_tests/templates/storage.yml new file mode 100644 index 0000000..6e58c1c --- /dev/null +++ b/src/frostfs_testlib/component_tests/templates/storage.yml @@ -0,0 +1,103 @@ +# Logger section +logger: + level: debug # Minimum enabled logging level + +# Profiler section +pprof: + enabled: true + address: :6060 # Server address + shutdown_timeout: 15s # Timeout for profiling HTTP server graceful shutdown + +# Application metrics section +prometheus: + enabled: true + address: :9090 # Server address + shutdown_timeout: 15s # Timeout for metrics HTTP server graceful shutdown + +# Morph section +morph: + dial_timeout: 30s # Timeout for side chain NEO RPC client connection + rpc_endpoint: # This config branch is replaced completely during template parsing + - address: ws://morph1:30333/ws + priority: 0 + - address: ws://morph2:30333/ws + priority: 0 + - address: ws://morph3:30333/ws + priority: 0 + - address: ws://morph4:30333/ws + priority: 0 + - address: ws://morph5:30333/ws + priority: 1 + - address: ws://morph6:30333/ws + priority: 1 + - address: ws://morph7:30333/ws + priority: 1 + +# Common storage node settings +node: + wallet: + path: "{wallet}" + password: "{password}" + addresses: + - grpc://{prefix}{index}:8802 + attribute_0: "User-Agent:FrostFS component tests" + attribute_1: "Price:{price}" + persistent_state: + path: /storage/state + +grpc: + - endpoint: :8802 + tls: + enabled: false + +control: + grpc: + endpoint: :8801 + +# Tree section +tree: + enabled: true + +# Storage engine configuration +storage: + shard: + 0: + writecache: + enabled: true + path: /storage/data/wc0 # Write-cache root directory + + metabase: + path: /storage/data/meta0 # Path to the metabase + + blobstor: + - type: blobovnicza + path: /storage/data/blobovnicza0 # Blobovnicza root directory + depth: 2 + width: 4 + - type: fstree + path: /storage/data/fstree0 # FSTree root directory + depth: 2 + + pilorama: + path: /storage/data/pilorama0 # Path to the pilorama database + + 1: + writecache: + enabled: true + path: /storage/data/wc1 # Write-cache root directory + + metabase: + path: /storage/data/meta1 # Path to the metabase + + blobstor: + - type: blobovnicza + path: /storage/data/blobovnicza1 # Blobovnicza root directory + depth: 2 + width: 4 + - type: fstree + path: /storage/data/fstree1 # FSTree root directory + depth: 2 + + pilorama: + path: /storage/data/pilorama1 # Path to the pilorama database + diff --git a/tests/test_component_container_wrapper.py b/tests/test_component_container_wrapper.py new file mode 100644 index 0000000..2824b25 --- /dev/null +++ b/tests/test_component_container_wrapper.py @@ -0,0 +1,44 @@ +import pytest + +from frostfs_testlib.component_tests.container import ContainerizedService + + +@pytest.mark.parametrize( + "image", + [ + "busybox:musl", + "golang:1.24", + "python:3.12", + ], +) +def test_containerized_wrapper(image, neogo_deployment): + demo = ContainerizedService( + image=image, + command='tick=0; while true; do tick=$((tick+1)); echo "tick=$tick"; sleep 0.1; done', + ) + demo.add_file(neogo_deployment.template, "/neogo/config.yml") + demo.start() + result = demo.exec("cat /neogo/config.yml") + assert result.exit_code == 0 + assert "StandbyCommittee" in str(result.output) + print(f"Checking logs for {demo.name}") + seen = [] + for line in demo.logs(timeout=20): + if "tick=3" in line: + break + seen.append(line) + if len(seen) > 15: + assert False, "expected output did not appear in container logs" + demo.destroy() + + +@pytest.mark.timeout(15) +def test_wait_for_logs(): + demo = ContainerizedService( + image="busybox:musl", + command='tick=0; while true; do tick=$((tick+1)); echo "tick=$tick"; sleep 0.1; done', + ) + demo.start() + demo.wait("tick=1", timeout=5) + with pytest.raises(TimeoutError): + demo.wait("impossible line", timeout=5) diff --git a/tests/test_component_fixture_demo.py b/tests/test_component_fixture_demo.py new file mode 100644 index 0000000..d1ad046 --- /dev/null +++ b/tests/test_component_fixture_demo.py @@ -0,0 +1,24 @@ +""" +Demonstration of overrides for *_deployment fixtures +""" +import pytest + +from frostfs_testlib.component_tests.fixtures import alphabet_customize + + +@pytest.fixture(scope="session") +def deployment(alphabet_deployment): + print(f"{alphabet_deployment.prefix=}") + return alphabet_deployment + + +def test_deployment(deployment): + assert deployment.node_count == 4 + assert deployment.foo == "bar" + + +@alphabet_customize(node_count=5) +def test_deployment_custom(deployment): + assert deployment.node_count == 5 + assert deployment.foo == "bar" + print(deployment.dir) diff --git a/tests/test_component_fixtures.py b/tests/test_component_fixtures.py new file mode 100644 index 0000000..38f54e4 --- /dev/null +++ b/tests/test_component_fixtures.py @@ -0,0 +1,74 @@ +import random +import re + +import pytest + +from frostfs_testlib.component_tests.fixtures import adm_customize + + +@pytest.mark.timeout(30) +def test_sidechain(neogo): + """ + Launch sidechain and wait for blocks to tick up to eleven. + """ + node = neogo[3] + needle = re.compile(r'"height": 11') + node.wait(needle) + + +@adm_customize(version="0.45.0-rc.10") +def test_frostfs_adm(frostfs_adm): + """ + This is just a demonstration. + + Overriding frostfs-adm version from component tests is not recommended + because it will effectively redeploy everything just for that test case: + alphabet wallets depend upon frostfs-adm, and everything else depends on + alphabet wallets. + """ + output = frostfs_adm("--version") + assert "0.45.0-rc.10" in output + + +def test_contract_fetch(frostfs_contract): + assert (frostfs_contract / "alphabet" / "alphabet_contract.nef").exists() + + +def test_frostfs_bootstrap(frostfs_bootstrap): + assert len(frostfs_bootstrap) == 6 + + +@pytest.mark.timeout(60) +def test_innerring_startup(innerring): + ir = random.choice(innerring) + block = re.compile(r'new block.*"index": \d+') + blocks_seen = 0 + for line in ir.logs(): + if block.search(line): + blocks_seen += 1 + if blocks_seen > 3: + break + + +@pytest.mark.timeout(90) +def test_storage_startup(storage): + node = random.choice(storage) + block = re.compile(r'new block.*"index": \d+') + blocks_seen = 0 + for line in node.logs(): + if block.search(line): + blocks_seen += 1 + if blocks_seen > 3: + break + + +@pytest.mark.timeout(90) +def test_httpgw_startup(httpgw, frostfs_adm): + dummy_cid = "Dxhf4PNprrJHWWTG5RGLdfLkJiSQ3AQqit1MSnEPRkDZ" + dummy_oid = "2m8PtaoricLouCn5zE8hAFr3gZEBDCZFe9BEgVJTSocY" + gateway = random.choice(httpgw) + rc, output = frostfs_adm.exec(f"wget -O - http://{gateway.name}/get/{dummy_cid}/{dummy_oid}") + assert rc == 1 + assert "404 Not Found" in str(output) + expected = re.compile(f"{dummy_cid}.*code = 3072.*container not found") + gateway.wait(expected) From cf950f6313f75624694382b14764023005f9a256 Mon Sep 17 00:00:00 2001 From: Vitaliy Potyarkin Date: Wed, 7 May 2025 13:11:55 +0300 Subject: [PATCH 5/5] WIP: Integrate component test fixtures into testlib Signed-off-by: Vitaliy Potyarkin --- pyproject.toml | 2 + .../component_tests/hosting.py | 171 ++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 src/frostfs_testlib/component_tests/hosting.py diff --git a/pyproject.toml b/pyproject.toml index 7c8dfd3..1b7fb32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ allure = "frostfs_testlib.reporter.allure_handler:AllureHandler" [project.entry-points."frostfs.testlib.hosting"] docker = "frostfs_testlib.hosting.docker_host:DockerHost" +component_tests = "frostfs_testlib.component_tests.hosting:ContainerHost" [project.entry-points."frostfs.testlib.healthcheck"] basic = "frostfs_testlib.healthcheck.basic_healthcheck:BasicHealthcheck" @@ -64,6 +65,7 @@ frostfs-ir = "frostfs_testlib.storage.dataclasses.frostfs_services:InnerRing" [project.entry-points."frostfs.testlib.credentials_providers"] authmate = "frostfs_testlib.credentials.authmate_s3_provider:AuthmateS3CredentialsProvider" wallet_factory = "frostfs_testlib.credentials.wallet_factory_provider:WalletFactoryProvider" +component_tests = "frostfs_testlib.component_tests.hosting:ClientWalletFactory" [project.entry-points."frostfs.testlib.bucket_cid_resolver"] frostfs = "frostfs_testlib.clients.s3.curl_bucket_resolver:CurlBucketContainerResolver" diff --git a/src/frostfs_testlib/component_tests/hosting.py b/src/frostfs_testlib/component_tests/hosting.py new file mode 100644 index 0000000..715bc71 --- /dev/null +++ b/src/frostfs_testlib/component_tests/hosting.py @@ -0,0 +1,171 @@ +""" +Hosting object for the fixture based dynamic component test environment. +Based on testcontainers and Docker. +""" + +from pathlib import Path +from typing import Any + +from frostfs_testlib.credentials.interfaces import GrpcCredentialsProvider, User +from frostfs_testlib.hosting.interfaces import Host +from frostfs_testlib.resources.common import ASSETS_DIR, DEFAULT_WALLET_PASS +from frostfs_testlib.storage.cluster import Cluster, ClusterNode +from frostfs_testlib.storage.dataclasses.wallet import WalletInfo + +from .container import ContainerizedService # TODO: move fixtures into testlib +from .fixtures import _new_wallet, _wallet_public_key + + +def dynamic_hosting_config(**fixtures) -> dict[str, Any]: + """ + Translate ContainerizedService fixtures into a hosting configuration tree. + """ + config = { + "hosts": [ + { + "plugin_name": "component_tests", + "grpc_creds_plugin_name": "component_tests", + "healthcheck_plugin_name": "basic", + "hostname": "component_tests", + "address": "component_tests", + "attributes": { + "force_transactions": True, + "skip_readiness_check": True, + "sudo_shell": False, + "frostfs_adm": fixtures.get("frostfs_adm"), + }, + "services": [], + "clis": [], + } + ], + } + services = config["hosts"][0]["services"] + for name, nodes in fixtures.items(): + services.extend(_services(name, nodes)) + return config + + +def _services(name: str, nodes: list) -> None: + if name == "storage": + for index, node in enumerate(nodes, 1): + yield { + "name": f"frostfs-storage_{index:02}", + "attributes": { + "control_endpoint": f"{node.ip}:8801", + "endpoint_data0": f"{node.ip}:8802", + "endpoint_prometheus": f"{node.ip}:9090", + }, + } + + +class ContainerHost(Host): + """ + Exposes services running in testcontainers. + """ + + def get_shell(self, sudo=True): + raise NotImplementedError + + def start_host(self): + raise NotImplementedError + + def get_host_status(self): + raise NotImplementedError + + def stop_host(self, mode): + raise NotImplementedError + + def start_service(self, service_name): + raise NotImplementedError + + def stop_service(self, service_name): + raise NotImplementedError + + def send_signal_to_service(self, service_name, signal): + raise NotImplementedError + + def mask_service(self, service_name): + raise NotImplementedError + + def unmask_service(self, service_name): + raise NotImplementedError + + def restart_service(self, service_name): + raise NotImplementedError + + def get_data_directory(self, service_name): + raise NotImplementedError + + def wait_success_suspend_process(self, process_name): + raise NotImplementedError + + def wait_success_resume_process(self, process_name): + raise NotImplementedError + + def delete_storage_node_data(self, service_name, cache_only=False): + raise NotImplementedError + + def wipefs_storage_node_data(self, service_name): + raise NotImplementedError + + def delete_fstree(self, service_name): + raise NotImplementedError + + def delete_metabase(self, service_name): + raise NotImplementedError + + def delete_write_cache(self, service_name): + raise NotImplementedError + + def delete_blobovnicza(self, service_name): + raise NotImplementedError + + def delete_file(self, file_path): + raise NotImplementedError + + def is_file_exist(self, file_path): + raise NotImplementedError + + def detach_disk(self, device): + raise NotImplementedError + + def attach_disk(self, device, disk_info): + raise NotImplementedError + + def is_disk_attached(self, device, disk_info): + raise NotImplementedError + + def dump_logs(self, directory_path, since=None, until=None, filter_regex=None): + raise NotImplementedError + + def get_filtered_logs(self, filter_regex, since=None, until=None, unit=None, exclude_filter=None, priority=None, word_count=None): + raise NotImplementedError + + def is_message_in_logs(self, message_regex, since=None, until=None, unit=None): + raise NotImplementedError + + def wait_for_service_to_be_in_state(self, systemd_service_name, expected_state, timeout): + raise NotImplementedError + + +class ClientWalletFactory(GrpcCredentialsProvider): + def provide(self, user: User, cluster_node: ClusterNode, **kwargs) -> WalletInfo: + if user.wallet is not None: + return user.wallet + frostfs_adm = cluster_node.host.config.attributes.get("frostfs_adm") + if not frostfs_adm: + raise RuntimeError("hosting fixture must depend directly on frostfs_adm") + directory = Path(ASSETS_DIR).absolute() + wallet = WalletInfo( + path=str(directory / f"{user.name}-wallet.json"), + password=DEFAULT_WALLET_PASS, + config_path=str(directory / f"{user.name}-config.yml"), + ) + _new_wallet(wallet.path, wallet.password) + pubkey = _wallet_public_key(wallet.path) + frostfs_adm.add_file(wallet.path, wallet.path) + frostfs_adm(f"morph frostfsid create-subject --namespace '' --subject-key '{pubkey}' --subject-name '{user.name}'") + with open(wallet.config_path, "w") as config: + config.write(f"wallet: {wallet.path!r}\npassword: {wallet.password!r}\n") + user.wallet = wallet + return user.wallet