[#461] Configure logger sampling policy

Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>

cmd/s3-gw/app_settings.go

@@ -78,8 +78,10 @@ var (
 
 const ( // Settings.
 	// Logger.
-	cfgLoggerLevel       = "logger.level"
-	cfgLoggerDestination = "logger.destination"
+	cfgLoggerLevel              = "logger.level"
+	cfgLoggerDestination        = "logger.destination"
+	cfgLoggerSamplingInitial    = "logger.sampling.initial"
+	cfgLoggerSamplingThereafter = "logger.sampling.thereafter"
 
 	// HttpLogging.
 	cfgHTTPLoggingEnabled     = "http_logging.enabled"
@@ -787,6 +789,8 @@ func newSettings() *viper.Viper {
 	// logger:
 	v.SetDefault(cfgLoggerLevel, "debug")
 	v.SetDefault(cfgLoggerDestination, "stdout")
+	v.SetDefault(cfgLoggerSamplingThereafter, 100)
+	v.SetDefault(cfgLoggerSamplingInitial, 100)
 
 	// http logger
 	v.SetDefault(cfgHTTPLoggingEnabled, false)
@@ -1030,7 +1034,7 @@ func pickLogger(v *viper.Viper) *Logger {
 
 	switch dest {
 	case destinationStdout:
-		return newStdoutLogger(lvl)
+		return newStdoutLogger(v, lvl)
 	case destinationJournald:
 		return newJournaldLogger(lvl)
 	default:
@@ -1045,13 +1049,14 @@ func pickLogger(v *viper.Viper) *Logger {
 //   - parameterized level (debug by default)
 //   - console encoding
 //   - ISO8601 time encoding
+//   - sampling intervals
 //
 // and atomic log level to dynamically change it.
 //
 // Logger records a stack trace for all messages at or above fatal level.
 //
 // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace.
-func newStdoutLogger(lvl zapcore.Level) *Logger {
+func newStdoutLogger(v *viper.Viper, lvl zapcore.Level) *Logger {
 	c := zap.NewProductionConfig()
 	c.Level = zap.NewAtomicLevelAt(lvl)
 	c.Encoding = "console"
@@ -1064,8 +1069,21 @@ func newStdoutLogger(lvl zapcore.Level) *Logger {
 		panic(fmt.Sprintf("build zap logger instance: %v", err))
 	}
 
+	// Zap samples by logging the first cgfLoggerSamplingInitial entries with a given level
+	// and message within the specified time interval.
+	// In the above config, only the first cgfLoggerSamplingInitial log entries with the same level and message
+	// are recorded in a one-second interval. Every other log entry will be dropped within the interval since
+	// cfgLoggerSamplingThereafter is specified here.
+
+	samplingCore := zapcore.NewSamplerWithOptions(
+		l.Core(),
+		time.Second,
+		v.GetInt(cfgLoggerSamplingInitial),
+		v.GetInt(cfgLoggerSamplingThereafter),
+	)
+
 	return &Logger{
-		logger: l,
+		logger: zap.New(samplingCore),
 		lvl:    c.Level,
 	}
 }

config/config.env

@@ -51,6 +51,8 @@ S3_GW_CONFIG=/path/to/config/yaml
 
 # Logger
 S3_GW_LOGGER_LEVEL=debug
+S3_GW_LOGGER_SAMPLING_INITIAL=100
+S3_GW_LOGGER_SAMPLING_THEREAFTER=100
 
 # HTTP logger
 S3_GW_HTTP_LOGGING_ENABLED=false

config/config.yaml

@@ -55,6 +55,9 @@ vhs:
 logger:
   level: debug
   destination: stdout
+  sampling:
+    initial: 100
+    thereafter: 100
 
 # log http request data (URI, headers, query, etc)
 http_logging: