[TrueCloudLab#21] Support multiple configs

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>

cmd/s3-gw/app.go

@@ -472,11 +472,11 @@ func shutdownContext() (context.Context, context.CancelFunc) {
 func (a *App) configReload() {
 	a.log.Info("SIGHUP config reload started")
 
-	if !a.cfg.IsSet(cmdConfig) {
+	if !a.cfg.IsSet(cmdConfig) && !a.cfg.IsSet(cmdConfigDir) {
 		a.log.Warn("failed to reload config because it's missed")
 		return
 	}
-	if err := readConfig(a.cfg); err != nil {
+	if err := readInConfig(a.cfg); err != nil {
 		a.log.Warn("failed to reload config", zap.Error(err))
 		return
 	}

cmd/s3-gw/app_settings.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"path"
 	"runtime"
 	"sort"
 	"strconv"
@@ -113,11 +114,12 @@ const ( // Settings.
 	cfgApplicationBuildTime = "app.build_time"
 
 	// Command line args.
-	cmdHelp    = "help"
-	cmdVersion = "version"
-	cmdConfig  = "config"
-	cmdPProf   = "pprof"
-	cmdMetrics = "metrics"
+	cmdHelp      = "help"
+	cmdVersion   = "version"
+	cmdConfig    = "config"
+	cmdConfigDir = "config-dir"
+	cmdPProf     = "pprof"
+	cmdMetrics   = "metrics"
 
 	cmdListenAddress = "listen_address"
 
@@ -214,7 +216,8 @@ func newSettings() *viper.Viper {
 
 	flags.StringP(cmdWallet, "w", "", `path to the wallet`)
 	flags.String(cmdAddress, "", `address of wallet account`)
-	flags.String(cmdConfig, "", "config path")
+	flags.StringArray(cmdConfig, nil, "config paths")
+	flags.String(cmdConfigDir, "", "config dir path")
 
 	flags.Duration(cfgHealthcheckTimeout, defaultHealthcheckTimeout, "set timeout to check node health during rebalance")
 	flags.Duration(cfgConnectTimeout, defaultConnectTimeout, "set timeout to connect to FrostFS nodes")
@@ -313,10 +316,8 @@ func newSettings() *viper.Viper {
 		os.Exit(0)
 	}
 
-	if v.IsSet(cmdConfig) {
-		if err := readConfig(v); err != nil {
-			panic(err)
-		}
+	if err := readInConfig(v); err != nil {
+		panic(err)
 	}
 
 	return v
@@ -332,6 +333,9 @@ func bindFlags(v *viper.Viper, flags *pflag.FlagSet) error {
 	if err := v.BindPFlag(cmdConfig, flags.Lookup(cmdConfig)); err != nil {
 		return err
 	}
+	if err := v.BindPFlag(cmdConfigDir, flags.Lookup(cmdConfigDir)); err != nil {
+		return err
+	}
 	if err := v.BindPFlag(cfgWalletPath, flags.Lookup(cmdWallet)); err != nil {
 		return err
 	}
@@ -370,17 +374,72 @@ func bindFlags(v *viper.Viper, flags *pflag.FlagSet) error {
 	return nil
 }
 
-func readConfig(v *viper.Viper) error {
-	cfgFileName := v.GetString(cmdConfig)
-	cfgFile, err := os.Open(cfgFileName)
+func readInConfig(v *viper.Viper) error {
+	if v.IsSet(cmdConfig) {
+		if err := readConfig(v); err != nil {
+			return err
+		}
+	}
+
+	if v.IsSet(cmdConfigDir) {
+		if err := readConfigDir(v); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func readConfigDir(v *viper.Viper) error {
+	cfgSubConfigDir := v.GetString(cmdConfigDir)
+	entries, err := os.ReadDir(cfgSubConfigDir)
 	if err != nil {
 		return err
 	}
-	if err = v.ReadConfig(cfgFile); err != nil {
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+		ext := path.Ext(entry.Name())
+		if ext != ".yaml" && ext != ".yml" {
+			continue
+		}
+
+		if err = mergeConfig(v, path.Join(cfgSubConfigDir, entry.Name())); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func readConfig(v *viper.Viper) error {
+	for _, fileName := range v.GetStringSlice(cmdConfig) {
+		if err := mergeConfig(v, fileName); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func mergeConfig(v *viper.Viper, fileName string) error {
+	cfgFile, err := os.Open(fileName)
+	if err != nil {
 		return err
 	}
 
-	return cfgFile.Close()
+	defer func() {
+		if errClose := cfgFile.Close(); errClose != nil {
+			panic(errClose)
+		}
+	}()
+
+	if err = v.MergeConfig(cfgFile); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // newLogger constructs a Logger instance for the current application.

config/config.yaml

@@ -55,11 +55,11 @@ resolve_order:
 
 # Metrics
 pprof:
-  enabled: true
+  enabled: false
   address: localhost:8085
 
 prometheus:
-  enabled: true
+  enabled: false
   address: localhost:8086
 
 # Timeout to connect to a node

config/dir/pprof.yaml

@@ -0,0 +1,3 @@
+pprof:
+  enabled: true
+  address: localhost:8085

config/dir/prometheus.yaml

@@ -0,0 +1,3 @@
+prometheus:
+  enabled: true
+  address: localhost:8086

docs/configuration.md

@@ -110,6 +110,23 @@ A path to a configuration file can be specified with `--config` parameter:
 $ frostfs-s3-gw --config your-config.yaml
 ```
 
+### Multiple configs
+
+You can use several config files when running application. It allows you to split configuration into parts.
+For example, you can use separate yaml file for pprof and prometheus section in config (see [config examples](../config)).
+You can either provide several files with repeating `--config` flag or provide path to the dir that contains all configs using `--config-dir` flag.
+Also, you can combine these flags:
+
+```shell
+$ frostfs-s3-gw --config ./config/config.yaml --config /your/partial/config.yaml --config-dir ./config/dir
+```
+
+**Note:** next file in `--config` flag overwrites values from the previous one.
+Files from `--config-dir` directory overwrite values from `--config` files.
+So the command above run `frostfs-s3-gw` to listen on `0.0.0.0:8080` address (value from `./config/config.yaml`),
+applies parameters from `/your/partial/config.yaml`,
+enable pprof (value from `./config/dir/pprof.yaml`) and prometheus (value from `./config/dir/prometheus.yaml`).
+
 ### Reload on SIGHUP
 
 Some config values can be reloaded on SIGHUP signal.