From 5ffed0b96fd9b601d14be70e3aabafa79edbf5f2 Mon Sep 17 00:00:00 2001 From: Denis Kirillov Date: Mon, 23 Sep 2024 16:27:27 +0300 Subject: [PATCH] [#20] Add log sampling configuration Signed-off-by: Denis Kirillov --- cmd/s3-lifecycler/logger.go | 56 +++++++++++++++++++++-------------- cmd/s3-lifecycler/settings.go | 9 ++++-- config/config.env | 3 ++ config/config.yaml | 4 +++ docs/configuration.md | 17 +++++++---- 5 files changed, 59 insertions(+), 30 deletions(-) diff --git a/cmd/s3-lifecycler/logger.go b/cmd/s3-lifecycler/logger.go index 44bfad9..9ca5be3 100644 --- a/cmd/s3-lifecycler/logger.go +++ b/cmd/s3-lifecycler/logger.go @@ -21,29 +21,19 @@ type Logger struct { } func pickLogger(v *viper.Viper) *Logger { - lvl, err := getLogLevel(v.GetString(cfgLoggerLevel)) - if err != nil { - panic(err) + switch dest := v.GetString(cfgLoggerDestination); dest { + case destinationStdout: + return newStdoutLogger(v) + case destinationJournald: + return newJournaldLogger(v) + default: + panic(fmt.Sprintf("wrong destination for logger: %s", dest)) } - - dest := v.GetString(cfgLoggerDestination) - - if dest == destinationStdout { - return newStdoutLogger(lvl) - } - - if dest == destinationJournald { - return newJournaldLogger(lvl) - } - - panic(fmt.Sprintf("wrong destination for logger: %s", dest)) } -func newStdoutLogger(lvl zapcore.Level) *Logger { - c := zap.NewProductionConfig() - c.Level = zap.NewAtomicLevelAt(lvl) +func newStdoutLogger(v *viper.Viper) *Logger { + c := newZapLogConfig(v) c.Encoding = "console" - c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder l, err := c.Build( zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)), @@ -55,10 +45,8 @@ func newStdoutLogger(lvl zapcore.Level) *Logger { return &Logger{logger: l, lvl: c.Level} } -func newJournaldLogger(lvl zapcore.Level) *Logger { - c := zap.NewProductionConfig() - c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - c.Level = zap.NewAtomicLevelAt(lvl) +func newJournaldLogger(v *viper.Viper) *Logger { + c := newZapLogConfig(v) // We can use NewJSONEncoder instead if, say, frontend // would like to access journald logs and parse them easily. @@ -74,6 +62,28 @@ func newJournaldLogger(lvl zapcore.Level) *Logger { return &Logger{logger: l, lvl: c.Level} } +func newZapLogConfig(v *viper.Viper) zap.Config { + lvl, err := getLogLevel(v.GetString(cfgLoggerLevel)) + if err != nil { + panic(err) + } + + c := zap.NewProductionConfig() + c.Level = zap.NewAtomicLevelAt(lvl) + c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + + if v.GetBool(cfgLoggerSamplingEnabled) { + c.Sampling = &zap.SamplingConfig{ + Initial: v.GetInt(cfgLoggerSamplingInitial), + Thereafter: v.GetInt(cfgLoggerSamplingThereafter), + } + } else { + c.Sampling = nil + } + + return c +} + func getLogLevel(lvlStr string) (zapcore.Level, error) { var lvl zapcore.Level err := lvl.UnmarshalText([]byte(lvlStr)) diff --git a/cmd/s3-lifecycler/settings.go b/cmd/s3-lifecycler/settings.go index 6b1d94c..cb650c1 100644 --- a/cmd/s3-lifecycler/settings.go +++ b/cmd/s3-lifecycler/settings.go @@ -36,8 +36,11 @@ const ( cfgPprofAddress = "pprof.address" // Logger. - cfgLoggerLevel = "logger.level" - cfgLoggerDestination = "logger.destination" + cfgLoggerLevel = "logger.level" + cfgLoggerDestination = "logger.destination" + cfgLoggerSamplingEnabled = "logger.sampling.enabled" + cfgLoggerSamplingInitial = "logger.sampling.initial" + cfgLoggerSamplingThereafter = "logger.sampling.thereafter" // Morph. cfgMorphRPCEndpointPrefixTmpl = "morph.rpc_endpoint.%d." @@ -125,6 +128,8 @@ func settings() *viper.Viper { // logger: v.SetDefault(cfgLoggerLevel, "info") v.SetDefault(cfgLoggerDestination, "stdout") + v.SetDefault(cfgLoggerSamplingThereafter, 100) + v.SetDefault(cfgLoggerSamplingInitial, 100) // services: v.SetDefault(cfgPrometheusEnabled, false) diff --git a/config/config.env b/config/config.env index b7799e7..26d7da9 100644 --- a/config/config.env +++ b/config/config.env @@ -9,6 +9,9 @@ S3_LIFECYCLER_WALLET_PASSPHRASE=pwd # Logger S3_LIFECYCLER_LOGGER_LEVEL=debug S3_LIFECYCLER_LOGGER_DESTINATION=stdout +S3_LIFECYCLER_LOGGER_SAMPLING_ENABLED=false +S3_LIFECYCLER_LOGGER_SAMPLING_INITIAL=100 +S3_LIFECYCLER_LOGGER_SAMPLING_THEREAFTER=100 # Metrics S3_LIFECYCLER_PPROF_ENABLED=false diff --git a/config/config.yaml b/config/config.yaml index 8b92c42..2f8c48d 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,6 +7,10 @@ wallet: logger: level: info # Log level. destination: stdout # Logging destination. + sampling: + enabled: false + initial: 100 + thereafter: 100 pprof: enabled: false diff --git a/docs/configuration.md b/docs/configuration.md index 684d1c0..00c3f96 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -47,14 +47,21 @@ wallet: ```yaml logger: - level: debug + level: info destination: stdout + sampling: + enabled: false + initial: 100 + thereafter: 100 ``` -| Parameter | Type | SIGHUP reload | Default value | Description | -|---------------|----------|---------------|---------------|--------------------------------------------------------------------------------------| -| `level` | `string` | yes | `info` | Logging level. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. | -| `destination` | `string` | no | `stdout` | Destination for logger: `stdout` or `journald` | +| Parameter | Type | SIGHUP reload | Default value | Description | +|-----------------------|----------|---------------|---------------|--------------------------------------------------------------------------------------------------| +| `level` | `string` | yes | `info` | Logging level. Possible values: `debug`, `info`, `warn`, `dpanic`, `panic`, `fatal`. | +| `destination` | `string` | no | `stdout` | Destination for logger: `stdout` or `journald` | +| `sampling.enabled` | `bool` | no | `false` | Enable sampling. | +| `sampling.initial` | `int` | no | `100` | Logs firth N of the same (level and message) log entries each second. | +| `sampling.thereafter` | `int` | no | `100` | Logs every Mth of the same (level and message) log entries after first N entries in that second. | # `pprof` section