[#150] Add dropped logs metric

Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>

cmd/http-gw/app.go

@@ -177,6 +177,7 @@ func newApp(ctx context.Context, opt ...Option) App {
 
 	a.initResolver()
 	a.initMetrics()
+	a.initLogs()
 	a.initTracing(ctx)
 
 	return a
@@ -340,6 +341,11 @@ func (a *app) initMetrics() {
 	a.metrics.SetHealth(metrics.HealthStatusStarting)
 }
 
+func (a *app) initLogs() {
+	coreWithContext := applyZapCoreMiddlewares(a.log.Core(), a.cfg, a.metrics.provider)
+	a.log = zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
+}
+
 func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics {
 	if !enabled {
 		logger.Warn(logs.MetricsAreDisabled)

cmd/http-gw/settings.go

@@ -17,6 +17,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
 	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
 	internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net"
+	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics"
 	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
 	grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
@@ -428,9 +429,9 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) {
 
 	switch dest {
 	case destinationStdout:
-		return newStdoutLogger(v, lvl)
+		return newStdoutLogger(lvl)
 	case destinationJournald:
-		return newJournaldLogger(v, lvl)
+		return newJournaldLogger(lvl)
 	default:
 		panic(fmt.Sprintf("wrong destination for logger: %s", dest))
 	}
@@ -447,18 +448,17 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) {
 // Logger records a stack trace for all messages at or above fatal level.
 //
 // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace.
-func newStdoutLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) {
+func newStdoutLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) {
 	stdout := zapcore.AddSync(os.Stderr)
 	level := zap.NewAtomicLevelAt(lvl)
 
 	consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level)
-	consoleOutCore = samplingEnabling(v, consoleOutCore)
 
 	l := zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
 	return l, level
 }
 
-func newJournaldLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) {
+func newJournaldLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) {
 	level := zap.NewAtomicLevelAt(lvl)
 
 	encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields)
@@ -470,8 +470,6 @@ func newJournaldLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.Atom
 		zapjournald.SyslogPid(),
 	})
 
-	coreWithContext = samplingEnabling(v, coreWithContext)
-
 	l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
 
 	return l, level
@@ -484,19 +482,17 @@ func newLogEncoder() zapcore.Encoder {
 	return zapcore.NewConsoleEncoder(c)
 }
 
-func samplingEnabling(v *viper.Viper, core zapcore.Core) zapcore.Core {
-	// Zap samples by logging the first cgfLoggerSamplingInitial entries with a given level
-	// and message within the specified time interval.
-	// In the above config, only the first cgfLoggerSamplingInitial log entries with the same level and message
-	// are recorded in cfgLoggerSamplingInterval interval. Every other log entry will be dropped within the interval since
-	// cfgLoggerSamplingThereafter is specified here.
+func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, appMetrics *metrics.GateMetrics) zapcore.Core {
 	if v.GetBool(cfgLoggerSamplingEnabled) {
-		core = zapcore.NewSamplerWithOptions(
-			core,
+		core = zapcore.NewSamplerWithOptions(core,
 			v.GetDuration(cfgLoggerSamplingInterval),
 			v.GetInt(cfgLoggerSamplingInitial),
 			v.GetInt(cfgLoggerSamplingThereafter),
-		)
+			zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
+				if dec&zapcore.LogDropped > 0 {
+					appMetrics.IncDroppedLogs()
+				}
+			}))
 	}
 
 	return core

metrics/desc.go

@@ -76,6 +76,15 @@ var appMetricsDesc = map[string]map[string]Description{
 			VariableLabels: []string{"endpoint"},
 		},
 	},
+	statisticSubsystem: {
+		droppedLogs: Description{
+			Type:      dto.MetricType_COUNTER,
+			Namespace: namespace,
+			Subsystem: statisticSubsystem,
+			Name:      droppedLogs,
+			Help:      "Dropped logs (by sampling) count",
+		},
+	},
 }
 
 type Description struct {
@@ -148,3 +157,12 @@ func mustNewGaugeVec(description Description) *prometheus.GaugeVec {
 		description.VariableLabels,
 	)
 }
+
+func mustNewCounter(description Description) prometheus.Counter {
+	if description.Type != dto.MetricType_COUNTER {
+		panic("invalid metric type")
+	}
+	return prometheus.NewCounter(
+		prometheus.CounterOpts(newOpts(description)),
+	)
+}

metrics/metrics.go

@@ -10,15 +10,17 @@ import (
 )
 
 const (
-	namespace       = "frostfs_http_gw"
-	stateSubsystem  = "state"
-	poolSubsystem   = "pool"
-	serverSubsystem = "server"
+	namespace          = "frostfs_http_gw"
+	stateSubsystem     = "state"
+	poolSubsystem      = "pool"
+	serverSubsystem    = "server"
+	statisticSubsystem = "statistic"
 )
 
 const (
 	healthMetric      = "health"
 	versionInfoMetric = "version_info"
+	droppedLogs       = "dropped_logs"
 )
 
 const (
@@ -30,21 +32,19 @@ const (
 )
 
 const (
-	methodGetBalance       = "get_balance"
-	methodPutContainer     = "put_container"
-	methodGetContainer     = "get_container"
-	methodListContainer    = "list_container"
-	methodDeleteContainer  = "delete_container"
-	methodGetContainerEacl = "get_container_eacl"
-	methodSetContainerEacl = "set_container_eacl"
-	methodEndpointInfo     = "endpoint_info"
-	methodNetworkInfo      = "network_info"
-	methodPutObject        = "put_object"
-	methodDeleteObject     = "delete_object"
-	methodGetObject        = "get_object"
-	methodHeadObject       = "head_object"
-	methodRangeObject      = "range_object"
-	methodCreateSession    = "create_session"
+	methodGetBalance      = "get_balance"
+	methodPutContainer    = "put_container"
+	methodGetContainer    = "get_container"
+	methodListContainer   = "list_container"
+	methodDeleteContainer = "delete_container"
+	methodEndpointInfo    = "endpoint_info"
+	methodNetworkInfo     = "network_info"
+	methodPutObject       = "put_object"
+	methodDeleteObject    = "delete_object"
+	methodGetObject       = "get_object"
+	methodHeadObject      = "head_object"
+	methodRangeObject     = "range_object"
+	methodCreateSession   = "create_session"
 )
 
 // HealthStatus of the gate application.
@@ -69,6 +69,7 @@ type GateMetrics struct {
 	stateMetrics
 	poolMetricsCollector
 	serverMetrics
+	statisticMetrics
 }
 
 type stateMetrics struct {
@@ -76,6 +77,10 @@ type stateMetrics struct {
 	versionInfo *prometheus.GaugeVec
 }
 
+type statisticMetrics struct {
+	droppedLogs prometheus.Counter
+}
+
 type poolMetricsCollector struct {
 	scraper             StatisticScraper
 	overallErrors       prometheus.Gauge
@@ -96,10 +101,14 @@ func NewGateMetrics(p StatisticScraper) *GateMetrics {
 	serverMetric := newServerMetrics()
 	serverMetric.register()
 
+	statsMetric := newStatisticMetrics()
+	statsMetric.register()
+
 	return &GateMetrics{
 		stateMetrics:         *stateMetric,
 		poolMetricsCollector: *poolMetric,
 		serverMetrics:        *serverMetric,
+		statisticMetrics:     *statsMetric,
 	}
 }
 
@@ -107,6 +116,7 @@ func (g *GateMetrics) Unregister() {
 	g.stateMetrics.unregister()
 	prometheus.Unregister(&g.poolMetricsCollector)
 	g.serverMetrics.unregister()
+	g.statisticMetrics.unregister()
 }
 
 func newStateMetrics() *stateMetrics {
@@ -116,6 +126,20 @@ func newStateMetrics() *stateMetrics {
 	}
 }
 
+func newStatisticMetrics() *statisticMetrics {
+	return &statisticMetrics{
+		droppedLogs: mustNewCounter(appMetricsDesc[statisticSubsystem][droppedLogs]),
+	}
+}
+
+func (s statisticMetrics) register() {
+	prometheus.MustRegister(s.droppedLogs)
+}
+
+func (s statisticMetrics) unregister() {
+	prometheus.Unregister(s.droppedLogs)
+}
+
 func (m stateMetrics) register() {
 	prometheus.MustRegister(m.healthCheck)
 	prometheus.MustRegister(m.versionInfo)
@@ -134,6 +158,10 @@ func (m stateMetrics) SetVersion(ver string) {
 	m.versionInfo.WithLabelValues(ver).Set(1)
 }
 
+func (s statisticMetrics) IncDroppedLogs() {
+	s.droppedLogs.Inc()
+}
+
 func newPoolMetricsCollector(p StatisticScraper) *poolMetricsCollector {
 	return &poolMetricsCollector{
 		scraper:             p,