[#183] Read all configuration files for service config

Signed-off-by: Andrey Berezin <a.berezin@yadro.com>

src/frostfs_testlib/storage/cluster.py

@@ -9,7 +9,6 @@ from frostfs_testlib.hosting import Host, Hosting
 from frostfs_testlib.hosting.config import ServiceConfig
 from frostfs_testlib.storage import get_service_registry
 from frostfs_testlib.storage.configuration.interfaces import ServiceConfigurationYml
-from frostfs_testlib.storage.configuration.service_configuration import ServiceConfiguration
 from frostfs_testlib.storage.constants import ConfigAttributes
 from frostfs_testlib.storage.dataclasses.frostfs_services import HTTPGate, InnerRing, MorphChain, S3Gate, StorageNode
 from frostfs_testlib.storage.dataclasses.node_base import NodeBase, ServiceClass
@@ -72,6 +71,7 @@ class ClusterNode:
     def s3_gate(self) -> S3Gate:
         return self.service(S3Gate)
 
+    # TODO: Deprecated. Use config with ServiceConfigurationYml interface
     def get_config(self, config_file_path: str) -> dict:
         shell = self.host.get_shell()
 
@@ -81,6 +81,7 @@ class ClusterNode:
         config = yaml.safe_load(config_text)
         return config
 
+    # TODO: Deprecated. Use config with ServiceConfigurationYml interface
     def save_config(self, new_config: dict, config_file_path: str) -> None:
         shell = self.host.get_shell()
 
@@ -88,7 +89,7 @@ class ClusterNode:
         shell.exec(f"echo '{config_str}' | sudo tee {config_file_path}")
 
     def config(self, service_type: type[ServiceClass]) -> ServiceConfigurationYml:
-        return ServiceConfiguration(self.service(service_type))
+        return self.service(service_type).config
 
     def service(self, service_type: type[ServiceClass]) -> ServiceClass:
         """

src/frostfs_testlib/storage/configuration/service_configuration.py

@@ -5,51 +5,74 @@ from typing import Any
 import yaml
 
 from frostfs_testlib import reporter
-from frostfs_testlib.shell.interfaces import CommandOptions
+from frostfs_testlib.shell.interfaces import CommandOptions, Shell
 from frostfs_testlib.storage.configuration.interfaces import ServiceConfigurationYml
-from frostfs_testlib.storage.dataclasses.node_base import ServiceClass
+
+
+def extend_dict(extend_me: dict, extend_by: dict):
+    if isinstance(extend_by, dict):
+        for k, v in extend_by.items():
+            if k in extend_me:
+                extend_dict(extend_me.get(k), v)
+            else:
+                extend_me[k] = v
+    else:
+        extend_me += extend_by
 
 
 class ServiceConfiguration(ServiceConfigurationYml):
-    def __init__(self, service: "ServiceClass") -> None:
+    def __init__(self, service_name: str, shell: Shell, config_dir: str, main_config_path: str) -> None:
-        self.service = service
+        self.service_name = service_name
-        self.shell = self.service.host.get_shell()
+        self.shell = shell
-        self.confd_path = os.path.join(self.service.config_dir, "conf.d")
+        self.main_config_path = main_config_path
+        self.confd_path = os.path.join(config_dir, "conf.d")
         self.custom_file = os.path.join(self.confd_path, "99_changes.yml")
 
     def _path_exists(self, path: str) -> bool:
         return not self.shell.exec(f"test -e {path}", options=CommandOptions(check=False)).return_code
 
-    def _get_data_from_file(self, path: str) -> dict:
+    def _get_config_files(self):
-        content = self.shell.exec(f"cat {path}").stdout
+        config_files = [self.main_config_path]
-        data = yaml.safe_load(content)
-        return data
 
-    def get(self, key: str) -> str:
+        if self._path_exists(self.confd_path):
-        with reporter.step(f"Get {key} configuration value for {self.service}"):
+            files = self.shell.exec(f"find {self.confd_path} -type f").stdout.strip().split()
-            config_files = [self.service.main_config_path]
+            # Sorting files in backwards order from latest to first one
+            config_files.extend(sorted(files, key=lambda x: -int(re.findall("^\d+", os.path.basename(x))[0])))
 
-            if self._path_exists(self.confd_path):
+        return config_files
-                files = self.shell.exec(f"find {self.confd_path} -type f").stdout.strip().split()
-                # Sorting files in backwards order from latest to first one
-                config_files.extend(sorted(files, key=lambda x: -int(re.findall("^\d+", os.path.basename(x))[0])))
 
-            result = None
+    def _get_configuration(self, config_files: list[str]) -> dict:
-            for file in files:
+        if not config_files:
-                data = self._get_data_from_file(file)
+            return [{}]
-                result = self._find_option(key, data)
-                if result is not None:
-                    break
 
+        splitter = "+++++"
+        files_str = " ".join(config_files)
+        all_content = self.shell.exec(
+            f"echo Getting config files; for file in {files_str}; do (echo {splitter}; sudo cat ${{file}}); done"
+        ).stdout
+        files_content = all_content.split("+++++")[1:]
+        files_data = [yaml.safe_load(file_content) for file_content in files_content]
+
+        mergedData = {}
+        for data in files_data:
+            extend_dict(mergedData, data)
+
+        return mergedData
+
+    def get(self, key: str) -> str | Any:
+        with reporter.step(f"Get {key} configuration value for {self.service_name}"):
+            config_files = self._get_config_files()
+            configuration = self._get_configuration(config_files)
+            result = self._find_option(key, configuration)
         return result
 
     def set(self, values: dict[str, Any]):
-        with reporter.step(f"Change configuration for {self.service}"):
+        with reporter.step(f"Change configuration for {self.service_name}"):
             if not self._path_exists(self.confd_path):
                 self.shell.exec(f"mkdir {self.confd_path}")
 
             if self._path_exists(self.custom_file):
-                data = self._get_data_from_file(self.custom_file)
+                data = self._get_configuration([self.custom_file])
             else:
                 data = {}
 
@@ -61,5 +84,5 @@ class ServiceConfiguration(ServiceConfigurationYml):
             self.shell.exec(f"chmod 777 {self.custom_file}")
 
     def revert(self):
-        with reporter.step(f"Revert changed options for {self.service}"):
+        with reporter.step(f"Revert changed options for {self.service_name}"):
             self.shell.exec(f"rm -rf {self.custom_file}")

src/frostfs_testlib/storage/dataclasses/frostfs_services.py

@@ -127,25 +127,23 @@ class StorageNode(NodeBase):
         output = self.host.get_shell().exec(f"curl -s localhost:6672 | grep {health_metric} | sed 1,2d").stdout
         return health_metric in output
 
+    # TODO: Deprecated. Use new approach with config
     def get_shard_config_path(self) -> str:
         return self._get_attribute(ConfigAttributes.SHARD_CONFIG_PATH)
 
+    # TODO: Deprecated. Use new approach with config
     def get_shards_config(self) -> tuple[str, dict]:
         return self.get_config(self.get_shard_config_path())
 
     def get_shards(self) -> list[Shard]:
-        config = self.get_shards_config()[1]
+        shards = self.config.get("storage:shard")
-        config["storage"]["shard"].pop("default")
-        return [Shard.from_object(shard) for shard in config["storage"]["shard"].values()]
 
-    def get_shards_from_env(self) -> list[Shard]:
+        if not shards:
-        config = self.get_shards_config()[1]
+            raise RuntimeError(f"Cannot get shards information for {self.name} on {self.host.config.address}")
-        configObj = ConfigObj(StringIO(config))
 
-        pattern = f"{SHARD_PREFIX}\d*"
+        if "default" in shards:
-        num_shards = len(set(re.findall(pattern, self.get_shards_config())))
+            shards.pop("default")
-
+        return [Shard.from_object(shard) for shard in shards.values()]
-        return [Shard.from_config_object(configObj, shard_id) for shard_id in range(num_shards)]
 
     def get_control_endpoint(self) -> str:
         return self._get_attribute(ConfigAttributes.CONTROL_ENDPOINT)

src/frostfs_testlib/storage/dataclasses/node_base.py

@@ -10,6 +10,7 @@ from frostfs_testlib import reporter
 from frostfs_testlib.hosting.config import ServiceConfig
 from frostfs_testlib.hosting.interfaces import Host
 from frostfs_testlib.shell.interfaces import CommandResult
+from frostfs_testlib.storage.configuration.service_configuration import ServiceConfiguration, ServiceConfigurationYml
 from frostfs_testlib.storage.constants import ConfigAttributes
 from frostfs_testlib.testing.readable import HumanReadableABC
 from frostfs_testlib.utils import wallet_utils
@@ -147,7 +148,11 @@ class NodeBase(HumanReadableABC):
     def main_config_path(self) -> str:
         return self._get_attribute(ConfigAttributes.CONFIG_PATH)
 
-    # TODO: Deprecated
+    @property
+    def config(self) -> ServiceConfigurationYml:
+        return ServiceConfiguration(self.name, self.host.get_shell(), self.config_dir, self.main_config_path)
+
+    # TODO: Deprecated. Use config with ServiceConfigurationYml interface
     def get_config(self, config_file_path: Optional[str] = None) -> tuple[str, dict]:
         if config_file_path is None:
             config_file_path = self._get_attribute(ConfigAttributes.CONFIG_PATH)
@@ -160,7 +165,7 @@ class NodeBase(HumanReadableABC):
         config = yaml.safe_load(config_text)
         return config_file_path, config
 
-    # TODO: Deprecated
+    # TODO: Deprecated. Use config with ServiceConfigurationYml interface
     def save_config(self, new_config: dict, config_file_path: Optional[str] = None) -> None:
         if config_file_path is None:
             config_file_path = self._get_attribute(ConfigAttributes.CONFIG_PATH)

src/frostfs_testlib/storage/dataclasses/shard.py

@@ -1,16 +1,6 @@
-import json
-import pathlib
-import re
 from dataclasses import dataclass
-from io import StringIO
 
-import allure
-import pytest
-import yaml
 from configobj import ConfigObj
-from frostfs_testlib.cli import FrostfsCli
-from frostfs_testlib.resources.cli import CLI_DEFAULT_TIMEOUT
-from frostfs_testlib.resources.common import DEFAULT_WALLET_CONFIG
 
 SHARD_PREFIX = "FROSTFS_STORAGE_SHARD_"
 BLOBSTOR_PREFIX = "_BLOBSTOR_"
@@ -94,6 +84,5 @@ class Shard:
             blobstor=[Blobstor(path=blobstor["path"], path_type=blobstor["type"]) for blobstor in shard["blobstor"]],
             metabase=metabase,
             writecache=writecache,
-            pilorama=pilorama
+            pilorama=pilorama,
         )
-