From 87e69b93498c2a2aadb96ac99eece81a3a272a34 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 6 Feb 2023 16:53:16 +0300 Subject: [PATCH] [#44] node: Support multiple configs Signed-off-by: Denis Kirillov --- cmd/frostfs-node/config/config.go | 13 +++++++ cmd/frostfs-node/config/opts.go | 11 +++++- cmd/frostfs-node/main.go | 3 +- config/example/README.md | 61 +++++++++++++++++++++++++++++++ pkg/util/config/dir.go | 55 ++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 pkg/util/config/dir.go diff --git a/cmd/frostfs-node/config/config.go b/cmd/frostfs-node/config/config.go index 6f06a374e..8b6069761 100644 --- a/cmd/frostfs-node/config/config.go +++ b/cmd/frostfs-node/config/config.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/internal" + "github.com/TrueCloudLab/frostfs-node/pkg/util/config" "github.com/spf13/viper" ) @@ -54,6 +55,12 @@ func New(_ Prm, opts ...Option) *Config { } } + if o.configDir != "" { + if err := config.ReadConfigDir(v, o.configDir); err != nil { + panic(fmt.Errorf("failed to read config dir: %w", err)) + } + } + return &Config{ v: v, opts: *o, @@ -69,5 +76,11 @@ func (x *Config) Reload() error { } } + if x.opts.configDir != "" { + if err := config.ReadConfigDir(x.v, x.opts.configDir); err != nil { + return fmt.Errorf("rereading configuration dir: %w", err) + } + } + return nil } diff --git a/cmd/frostfs-node/config/opts.go b/cmd/frostfs-node/config/opts.go index f7f7685f7..009c6efef 100644 --- a/cmd/frostfs-node/config/opts.go +++ b/cmd/frostfs-node/config/opts.go @@ -1,7 +1,8 @@ package config type opts struct { - path string + path string + configDir string } func defaultOpts() *opts { @@ -18,3 +19,11 @@ func WithConfigFile(path string) Option { o.path = path } } + +// WithConfigDir returns an option to set the system path +// to the directory with configuration files. +func WithConfigDir(path string) Option { + return func(o *opts) { + o.configDir = path + } +} diff --git a/cmd/frostfs-node/main.go b/cmd/frostfs-node/main.go index d6345d3b8..11ee7a97b 100644 --- a/cmd/frostfs-node/main.go +++ b/cmd/frostfs-node/main.go @@ -34,6 +34,7 @@ func fatalOnErrDetails(details string, err error) { func main() { configFile := flag.String("config", "", "path to config") + configDir := flag.String("config-dir", "", "path to config directory") versionFlag := flag.Bool("version", false, "frostfs node version") dryRunFlag := flag.Bool("check", false, "validate configuration and exit") flag.Parse() @@ -44,7 +45,7 @@ func main() { os.Exit(SuccessReturnCode) } - appCfg := config.New(config.Prm{}, config.WithConfigFile(*configFile)) + appCfg := config.New(config.Prm{}, config.WithConfigFile(*configFile), config.WithConfigDir(*configDir)) err := validateConfig(appCfg) fatalOnErr(err) diff --git a/config/example/README.md b/config/example/README.md index 48b614bae..cbca83731 100644 --- a/config/example/README.md +++ b/config/example/README.md @@ -16,6 +16,67 @@ It is not recommended transferring these configs for real application launches. - CLI - YAML: `cli.yaml` +### Multiple configs + +You can split your configuration to several files. +For example, you can use separate yaml file for each shard or each service (pprof, prometheus). +You must use `--config-dir` flag to process several configs: + +```shell +$ ./bin/frotsfs-node --config ./config/example/node.yaml --config-dir ./dir/with/additional/configs +``` + +When the `--config-dir` flag set, the application: +* reads all `*.y[a]ml` files from provided directory, +* use Viper's [MergeConfig](https://pkg.go.dev/github.com/spf13/viper#MergeConfig) functionality to produce the final configuration, +* files are being processing in alphanumerical order so that `01.yaml` may be extended with contents of `02.yaml`. + +So if we have the following files: +```yaml +# 00.yaml +logger: + level: debug +pprof: + enabled: true + address: localhost:6060 +prometheus: + enabled: true + address: localhost:9090 +``` + +```yaml +# dir/01.yaml +logger: + level: info +pprof: + enabled: false +``` + +```yaml +# dir/02.yaml +logger: + level: warn +prometheus: + address: localhost:9091 +``` + +and provide the following flags: +```shell +$ ./bin/frotsfs-node --config 00.yaml --config-dir dir +``` + +result config will be: +```yaml +logger: + level: warn +pprof: + enabled: false + address: localhost:6060 +prometheus: + enabled: true + address: localhost:9091 +``` + ## Environment variables - Storage node: `node.env` diff --git a/pkg/util/config/dir.go b/pkg/util/config/dir.go new file mode 100644 index 000000000..a74992d19 --- /dev/null +++ b/pkg/util/config/dir.go @@ -0,0 +1,55 @@ +package config + +import ( + "os" + "path" + + "github.com/spf13/viper" +) + +// ReadConfigDir reads all config files from provided directory in alphabetical order +// and merge its content with current viper configuration. +func ReadConfigDir(v *viper.Viper, configDir string) error { + entries, err := os.ReadDir(configDir) + if err != nil { + return err + } + + for _, entry := range entries { + if entry.IsDir() { + continue + } + ext := path.Ext(entry.Name()) + if ext != ".yaml" && ext != ".yml" { + continue + } + + if err = mergeConfig(v, path.Join(configDir, entry.Name())); err != nil { + return err + } + } + + return nil +} + +// mergeConfig reads config file and merge its content with current viper. +func mergeConfig(v *viper.Viper, fileName string) (err error) { + var cfgFile *os.File + cfgFile, err = os.Open(fileName) + if err != nil { + return err + } + + defer func() { + errClose := cfgFile.Close() + if err == nil { + err = errClose + } + }() + + if err = v.MergeConfig(cfgFile); err != nil { + return err + } + + return nil +}