fix/355-increase-tree-service-client-cache-size #359

Closed
ale64bit wants to merge 156 commits from ale64bit/frostfs-node:fix/355-increase-tree-service-client-cache-size into support/v0.36
7 changed files with 80 additions and 53 deletions
Showing only changes of commit ef222e2487 - Show all commits

View file

@ -14,7 +14,9 @@ import (
type Config struct { type Config struct {
v *viper.Viper v *viper.Viper
opts configViper.Opts configFile string
configDir string
envPrefix string
defaultPath []string defaultPath []string
path []string path []string
@ -28,11 +30,14 @@ const (
// New creates a new Config instance. // New creates a new Config instance.
// //
// If file option is provided (WithConfigFile), // If file option is provided,
// configuration values are read from it. // configuration values are read from it.
// Otherwise, Config is a degenerate tree. // Otherwise, Config is a degenerate tree.
func New(opts ...configViper.Option) *Config { func New(configFile, configDir, envPrefix string) *Config {
v, o, err := configViper.CreateViper(opts...) v, err := configViper.CreateViper(
configViper.WithConfigFile(configFile),
configViper.WithConfigDir(configDir),
configViper.WithEnvPrefix(envPrefix))
if err != nil { if err != nil {
panic(err) panic(err)
@ -40,11 +45,16 @@ func New(opts ...configViper.Option) *Config {
return &Config{ return &Config{
v: v, v: v,
opts: *o, configFile: configFile,
configDir: configDir,
envPrefix: envPrefix,
} }
} }
// Reload reads configuration path if it was provided to New. // Reload reads configuration path if it was provided to New.
func (x *Config) Reload() error { func (x *Config) Reload() error {
return configViper.ReloadViper(x.v, x.opts) return configViper.ReloadViper(
configViper.WithViper(x.v), configViper.WithConfigFile(x.configFile),
configViper.WithConfigDir(x.configDir),
configViper.WithEnvPrefix(x.envPrefix))
} }

View file

@ -5,7 +5,6 @@ import (
"path" "path"
"testing" "testing"
configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config"
"github.com/spf13/cast" "github.com/spf13/cast"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -19,7 +18,7 @@ func TestConfigDir(t *testing.T) {
require.NoError(t, os.WriteFile(cfgFileName0, []byte(`{"storage":{"shard_pool_size":15}}`), 0777)) require.NoError(t, os.WriteFile(cfgFileName0, []byte(`{"storage":{"shard_pool_size":15}}`), 0777))
require.NoError(t, os.WriteFile(cfgFileName1, []byte("logger:\n level: debug"), 0777)) require.NoError(t, os.WriteFile(cfgFileName1, []byte("logger:\n level: debug"), 0777))
c := New(configViper.WithConfigDir(dir)) c := New("", dir, "")
require.Equal(t, "debug", cast.ToString(c.Sub("logger").Value("level"))) require.Equal(t, "debug", cast.ToString(c.Sub("logger").Value("level")))
require.EqualValues(t, 15, cast.ToUint32(c.Sub("storage").Value("shard_pool_size"))) require.EqualValues(t, 15, cast.ToUint32(c.Sub("storage").Value("shard_pool_size")))
} }

View file

@ -7,22 +7,19 @@ import (
"testing" "testing"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func fromFile(path string) *config.Config { func fromFile(path string) *config.Config {
os.Clearenv() // ENVs have priority over config files, so we do this in tests os.Clearenv() // ENVs have priority over config files, so we do this in tests
return config.New( return config.New(path, "", "")
configViper.WithConfigFile(path),
)
} }
func fromEnvFile(t testing.TB, path string) *config.Config { func fromEnvFile(t testing.TB, path string) *config.Config {
loadEnv(t, path) // github.com/joho/godotenv can do that as well loadEnv(t, path) // github.com/joho/godotenv can do that as well
return config.New(configViper.WithEnvPrefix(config.EnvPrefix)) return config.New("", "", config.EnvPrefix)
} }
func forEachFile(paths []string, f func(*config.Config)) { func forEachFile(paths []string, f func(*config.Config)) {
@ -48,7 +45,7 @@ func ForEnvFileType(t testing.TB, pref string, f func(*config.Config)) {
// EmptyConfig returns config without any values and sections. // EmptyConfig returns config without any values and sections.
func EmptyConfig() *config.Config { func EmptyConfig() *config.Config {
return config.New(configViper.WithEnvPrefix(config.EnvPrefix)) return config.New("", "", config.EnvPrefix)
} }
// loadEnv reads .env file, parses `X=Y` records and sets OS ENVs. // loadEnv reads .env file, parses `X=Y` records and sets OS ENVs.

View file

@ -8,7 +8,6 @@ import (
"os" "os"
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/misc"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/control"
@ -47,9 +46,7 @@ func main() {
os.Exit(SuccessReturnCode) os.Exit(SuccessReturnCode)
} }
appCfg := config.New( appCfg := config.New(*configFile, *configDir, config.EnvPrefix)
configViper.WithConfigFile(*configFile), configViper.WithConfigDir(*configDir),
configViper.WithEnvPrefix(config.EnvPrefix))
err := validateConfig(appCfg) err := validateConfig(appCfg)
fatalOnErr(err) fatalOnErr(err)

View file

@ -7,7 +7,6 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config" "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config"
configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test" configtest "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/test"
configViper "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/internal/common/config"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -27,13 +26,13 @@ func TestValidate(t *testing.T) {
t.Run("mainnet", func(t *testing.T) { t.Run("mainnet", func(t *testing.T) {
os.Clearenv() // ENVs have priority over config files, so we do this in tests os.Clearenv() // ENVs have priority over config files, so we do this in tests
p := filepath.Join(exampleConfigPrefix, "mainnet/config.yml") p := filepath.Join(exampleConfigPrefix, "mainnet/config.yml")
c := config.New(configViper.WithConfigFile(p)) c := config.New(p, "", config.EnvPrefix)
require.NoError(t, validateConfig(c)) require.NoError(t, validateConfig(c))
}) })
t.Run("testnet", func(t *testing.T) { t.Run("testnet", func(t *testing.T) {
os.Clearenv() // ENVs have priority over config files, so we do this in tests os.Clearenv() // ENVs have priority over config files, so we do this in tests
p := filepath.Join(exampleConfigPrefix, "testnet/config.yml") p := filepath.Join(exampleConfigPrefix, "testnet/config.yml")
c := config.New(configViper.WithConfigFile(p)) c := config.New(p, "", config.EnvPrefix)
require.NoError(t, validateConfig(c)) require.NoError(t, validateConfig(c))
}) })
} }

View file

@ -1,38 +1,49 @@
package config package config
type Opts struct { import "github.com/spf13/viper"
Path string
ConfigDir string type opts struct {
EnvPrefix string path string
configDir string
envPrefix string
v *viper.Viper
} }
func DefaultOpts() *Opts { func defaultOpts() *opts {
return new(Opts) return new(opts)
} }
// Option allows to set an optional parameter of the Config. // Option allows to set an optional parameter of the Config.
type Option func(*Opts) type Option func(*opts)
// WithConfigFile returns an option to set the system path // WithConfigFile returns an option to set the system path
// to the configuration file. // to the configuration file.
func WithConfigFile(path string) Option { func WithConfigFile(path string) Option {
return func(o *Opts) { return func(o *opts) {
o.Path = path o.path = path
} }
} }
// WithConfigDir returns an option to set the system path // WithConfigDir returns an option to set the system path
// to the directory with configuration files. // to the directory with configuration files.
func WithConfigDir(path string) Option { func WithConfigDir(path string) Option {
return func(o *Opts) { return func(o *opts) {
o.ConfigDir = path o.configDir = path
} }
} }
// WithEnvPrefix returns an option to defines // WithEnvPrefix returns an option to defines
// a prefix that ENVIRONMENT variables will use. // a prefix that ENVIRONMENT variables will use.
func WithEnvPrefix(envPrefix string) Option { func WithEnvPrefix(envPrefix string) Option {
return func(o *Opts) { return func(o *opts) {
o.EnvPrefix = envPrefix o.envPrefix = envPrefix
}
}
// WithViper returns an option to defines
// a predefined viper.Viper.
func WithViper(v *viper.Viper) Option {
return func(o *opts) {
o.v = v
} }
} }

View file

@ -15,48 +15,62 @@ const (
EnvSeparator = "_" EnvSeparator = "_"
) )
func CreateViper(opts ...Option) (*viper.Viper, *Opts, error) { func CreateViper(opts ...Option) (*viper.Viper, error) {
v := viper.New() o := defaultOpts()
o := DefaultOpts()
for i := range opts { for i := range opts {
opts[i](o) opts[i](o)
} }
if o.EnvPrefix != "" { var v *viper.Viper
v.SetEnvPrefix(o.EnvPrefix) if o.v != nil {
v = o.v
} else {
v = viper.New()
}
if o.envPrefix != "" {
v.SetEnvPrefix(o.envPrefix)
v.AutomaticEnv() v.AutomaticEnv()
v.SetEnvKeyReplacer(strings.NewReplacer(Separator, EnvSeparator)) v.SetEnvKeyReplacer(strings.NewReplacer(Separator, EnvSeparator))
} }
if o.Path != "" { if o.path != "" {
v.SetConfigFile(o.Path) v.SetConfigFile(o.path)
err := v.ReadInConfig() err := v.ReadInConfig()
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("failed to read config: %w", err) return nil, fmt.Errorf("failed to read config: %w", err)
} }
} }
if o.ConfigDir != "" { if o.configDir != "" {
if err := config.ReadConfigDir(v, o.ConfigDir); err != nil { if err := config.ReadConfigDir(v, o.configDir); err != nil {
return nil, nil, fmt.Errorf("failed to read config dir: %w", err) return nil, fmt.Errorf("failed to read config dir: %w", err)
} }
} }
return v, o, nil return v, nil
} }
func ReloadViper(v *viper.Viper, o Opts) error { func ReloadViper(opts ...Option) error {
if o.Path != "" { o := defaultOpts()
err := v.ReadInConfig() for i := range opts {
opts[i](o)
}
if o.v == nil {
return fmt.Errorf("provide viper in opts")
}
if o.path != "" {
err := o.v.ReadInConfig()
if err != nil { if err != nil {
return fmt.Errorf("rereading configuration file: %w", err) return fmt.Errorf("rereading configuration file: %w", err)
} }
} }
if o.ConfigDir != "" { if o.configDir != "" {
if err := config.ReadConfigDir(v, o.ConfigDir); err != nil { if err := config.ReadConfigDir(o.v, o.configDir); err != nil {
return fmt.Errorf("rereading configuration dir: %w", err) return fmt.Errorf("rereading configuration dir: %w", err)
} }
} }