From 55cebc042c49a59f699136d9258be6682ef0fbff Mon Sep 17 00:00:00 2001 From: Andrey Berezin Date: Mon, 19 Feb 2024 17:48:09 +0300 Subject: [PATCH] [#183] Read all configuration files for service config Signed-off-by: Andrey Berezin --- src/frostfs_testlib/storage/cluster.py | 5 +- .../configuration/service_configuration.py | 75 ++++++++++++------- .../storage/dataclasses/frostfs_services.py | 18 ++--- .../storage/dataclasses/node_base.py | 9 ++- .../storage/dataclasses/shard.py | 13 +--- 5 files changed, 68 insertions(+), 52 deletions(-) diff --git a/src/frostfs_testlib/storage/cluster.py b/src/frostfs_testlib/storage/cluster.py index c867515..23130cb 100644 --- a/src/frostfs_testlib/storage/cluster.py +++ b/src/frostfs_testlib/storage/cluster.py @@ -9,7 +9,6 @@ from frostfs_testlib.hosting import Host, Hosting from frostfs_testlib.hosting.config import ServiceConfig from frostfs_testlib.storage import get_service_registry from frostfs_testlib.storage.configuration.interfaces import ServiceConfigurationYml -from frostfs_testlib.storage.configuration.service_configuration import ServiceConfiguration from frostfs_testlib.storage.constants import ConfigAttributes from frostfs_testlib.storage.dataclasses.frostfs_services import HTTPGate, InnerRing, MorphChain, S3Gate, StorageNode from frostfs_testlib.storage.dataclasses.node_base import NodeBase, ServiceClass @@ -72,6 +71,7 @@ class ClusterNode: def s3_gate(self) -> S3Gate: return self.service(S3Gate) + # TODO: Deprecated. Use config with ServiceConfigurationYml interface def get_config(self, config_file_path: str) -> dict: shell = self.host.get_shell() @@ -81,6 +81,7 @@ class ClusterNode: config = yaml.safe_load(config_text) return config + # TODO: Deprecated. Use config with ServiceConfigurationYml interface def save_config(self, new_config: dict, config_file_path: str) -> None: shell = self.host.get_shell() @@ -88,7 +89,7 @@ class ClusterNode: shell.exec(f"echo '{config_str}' | sudo tee {config_file_path}") def config(self, service_type: type[ServiceClass]) -> ServiceConfigurationYml: - return ServiceConfiguration(self.service(service_type)) + return self.service(service_type).config def service(self, service_type: type[ServiceClass]) -> ServiceClass: """ diff --git a/src/frostfs_testlib/storage/configuration/service_configuration.py b/src/frostfs_testlib/storage/configuration/service_configuration.py index f7b3be7..fddd64a 100644 --- a/src/frostfs_testlib/storage/configuration/service_configuration.py +++ b/src/frostfs_testlib/storage/configuration/service_configuration.py @@ -5,51 +5,74 @@ from typing import Any import yaml from frostfs_testlib import reporter -from frostfs_testlib.shell.interfaces import CommandOptions +from frostfs_testlib.shell.interfaces import CommandOptions, Shell from frostfs_testlib.storage.configuration.interfaces import ServiceConfigurationYml -from frostfs_testlib.storage.dataclasses.node_base import ServiceClass + + +def extend_dict(extend_me: dict, extend_by: dict): + if isinstance(extend_by, dict): + for k, v in extend_by.items(): + if k in extend_me: + extend_dict(extend_me.get(k), v) + else: + extend_me[k] = v + else: + extend_me += extend_by class ServiceConfiguration(ServiceConfigurationYml): - def __init__(self, service: "ServiceClass") -> None: - self.service = service - self.shell = self.service.host.get_shell() - self.confd_path = os.path.join(self.service.config_dir, "conf.d") + def __init__(self, service_name: str, shell: Shell, config_dir: str, main_config_path: str) -> None: + self.service_name = service_name + self.shell = shell + self.main_config_path = main_config_path + self.confd_path = os.path.join(config_dir, "conf.d") self.custom_file = os.path.join(self.confd_path, "99_changes.yml") def _path_exists(self, path: str) -> bool: return not self.shell.exec(f"test -e {path}", options=CommandOptions(check=False)).return_code - def _get_data_from_file(self, path: str) -> dict: - content = self.shell.exec(f"cat {path}").stdout - data = yaml.safe_load(content) - return data + def _get_config_files(self): + config_files = [self.main_config_path] - def get(self, key: str) -> str: - with reporter.step(f"Get {key} configuration value for {self.service}"): - config_files = [self.service.main_config_path] + if self._path_exists(self.confd_path): + files = self.shell.exec(f"find {self.confd_path} -type f").stdout.strip().split() + # Sorting files in backwards order from latest to first one + config_files.extend(sorted(files, key=lambda x: -int(re.findall("^\d+", os.path.basename(x))[0]))) - if self._path_exists(self.confd_path): - files = self.shell.exec(f"find {self.confd_path} -type f").stdout.strip().split() - # Sorting files in backwards order from latest to first one - config_files.extend(sorted(files, key=lambda x: -int(re.findall("^\d+", os.path.basename(x))[0]))) + return config_files - result = None - for file in files: - data = self._get_data_from_file(file) - result = self._find_option(key, data) - if result is not None: - break + def _get_configuration(self, config_files: list[str]) -> dict: + if not config_files: + return [{}] + splitter = "+++++" + files_str = " ".join(config_files) + all_content = self.shell.exec( + f"echo Getting config files; for file in {files_str}; do (echo {splitter}; sudo cat ${{file}}); done" + ).stdout + files_content = all_content.split("+++++")[1:] + files_data = [yaml.safe_load(file_content) for file_content in files_content] + + mergedData = {} + for data in files_data: + extend_dict(mergedData, data) + + return mergedData + + def get(self, key: str) -> str | Any: + with reporter.step(f"Get {key} configuration value for {self.service_name}"): + config_files = self._get_config_files() + configuration = self._get_configuration(config_files) + result = self._find_option(key, configuration) return result def set(self, values: dict[str, Any]): - with reporter.step(f"Change configuration for {self.service}"): + with reporter.step(f"Change configuration for {self.service_name}"): if not self._path_exists(self.confd_path): self.shell.exec(f"mkdir {self.confd_path}") if self._path_exists(self.custom_file): - data = self._get_data_from_file(self.custom_file) + data = self._get_configuration([self.custom_file]) else: data = {} @@ -61,5 +84,5 @@ class ServiceConfiguration(ServiceConfigurationYml): self.shell.exec(f"chmod 777 {self.custom_file}") def revert(self): - with reporter.step(f"Revert changed options for {self.service}"): + with reporter.step(f"Revert changed options for {self.service_name}"): self.shell.exec(f"rm -rf {self.custom_file}") diff --git a/src/frostfs_testlib/storage/dataclasses/frostfs_services.py b/src/frostfs_testlib/storage/dataclasses/frostfs_services.py index ddc650a..9e671d5 100644 --- a/src/frostfs_testlib/storage/dataclasses/frostfs_services.py +++ b/src/frostfs_testlib/storage/dataclasses/frostfs_services.py @@ -127,25 +127,23 @@ class StorageNode(NodeBase): output = self.host.get_shell().exec(f"curl -s localhost:6672 | grep {health_metric} | sed 1,2d").stdout return health_metric in output + # TODO: Deprecated. Use new approach with config def get_shard_config_path(self) -> str: return self._get_attribute(ConfigAttributes.SHARD_CONFIG_PATH) + # TODO: Deprecated. Use new approach with config def get_shards_config(self) -> tuple[str, dict]: return self.get_config(self.get_shard_config_path()) def get_shards(self) -> list[Shard]: - config = self.get_shards_config()[1] - config["storage"]["shard"].pop("default") - return [Shard.from_object(shard) for shard in config["storage"]["shard"].values()] + shards = self.config.get("storage:shard") - def get_shards_from_env(self) -> list[Shard]: - config = self.get_shards_config()[1] - configObj = ConfigObj(StringIO(config)) + if not shards: + raise RuntimeError(f"Cannot get shards information for {self.name} on {self.host.config.address}") - pattern = f"{SHARD_PREFIX}\d*" - num_shards = len(set(re.findall(pattern, self.get_shards_config()))) - - return [Shard.from_config_object(configObj, shard_id) for shard_id in range(num_shards)] + if "default" in shards: + shards.pop("default") + return [Shard.from_object(shard) for shard in shards.values()] def get_control_endpoint(self) -> str: return self._get_attribute(ConfigAttributes.CONTROL_ENDPOINT) diff --git a/src/frostfs_testlib/storage/dataclasses/node_base.py b/src/frostfs_testlib/storage/dataclasses/node_base.py index 72b12a9..8291345 100644 --- a/src/frostfs_testlib/storage/dataclasses/node_base.py +++ b/src/frostfs_testlib/storage/dataclasses/node_base.py @@ -10,6 +10,7 @@ from frostfs_testlib import reporter from frostfs_testlib.hosting.config import ServiceConfig from frostfs_testlib.hosting.interfaces import Host from frostfs_testlib.shell.interfaces import CommandResult +from frostfs_testlib.storage.configuration.service_configuration import ServiceConfiguration, ServiceConfigurationYml from frostfs_testlib.storage.constants import ConfigAttributes from frostfs_testlib.testing.readable import HumanReadableABC from frostfs_testlib.utils import wallet_utils @@ -147,7 +148,11 @@ class NodeBase(HumanReadableABC): def main_config_path(self) -> str: return self._get_attribute(ConfigAttributes.CONFIG_PATH) - # TODO: Deprecated + @property + def config(self) -> ServiceConfigurationYml: + return ServiceConfiguration(self.name, self.host.get_shell(), self.config_dir, self.main_config_path) + + # TODO: Deprecated. Use config with ServiceConfigurationYml interface def get_config(self, config_file_path: Optional[str] = None) -> tuple[str, dict]: if config_file_path is None: config_file_path = self._get_attribute(ConfigAttributes.CONFIG_PATH) @@ -160,7 +165,7 @@ class NodeBase(HumanReadableABC): config = yaml.safe_load(config_text) return config_file_path, config - # TODO: Deprecated + # TODO: Deprecated. Use config with ServiceConfigurationYml interface def save_config(self, new_config: dict, config_file_path: Optional[str] = None) -> None: if config_file_path is None: config_file_path = self._get_attribute(ConfigAttributes.CONFIG_PATH) diff --git a/src/frostfs_testlib/storage/dataclasses/shard.py b/src/frostfs_testlib/storage/dataclasses/shard.py index 584138d..170a477 100644 --- a/src/frostfs_testlib/storage/dataclasses/shard.py +++ b/src/frostfs_testlib/storage/dataclasses/shard.py @@ -1,16 +1,6 @@ -import json -import pathlib -import re from dataclasses import dataclass -from io import StringIO -import allure -import pytest -import yaml from configobj import ConfigObj -from frostfs_testlib.cli import FrostfsCli -from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT -from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_" BLOBSTOR_PREFIX = "_BLOBSTOR_" @@ -94,6 +84,5 @@ class Shard: blobstor=[Blobstor(path=blobstor["path"], path_type=blobstor["type"]) for blobstor in shard["blobstor"]], metabase=metabase, writecache=writecache, - pilorama=pilorama + pilorama=pilorama, ) -