[#150] Add dropped logs metric

Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>

cmd/http-gw/app.go

@@ -177,6 +177,7 @@ func newApp(ctx context.Context, opt ...Option) App {
 
 	a.initResolver()
 	a.initMetrics()
+	a.initLogs()
 	a.initTracing(ctx)
 
 	return a
@@ -340,6 +341,11 @@ func (a *app) initMetrics() {
 	a.metrics.SetHealth(metrics.HealthStatusStarting)
 }
 
+func (a *app) initLogs() {
+	coreWithContext := applyZapCoreMiddlewares(a.log.Core(), a.cfg, a.metrics.provider)
+	a.log = zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
+}
+
 func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics {
 	if !enabled {
 		logger.Warn(logs.MetricsAreDisabled)

cmd/http-gw/settings.go

@@ -17,6 +17,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache"
 	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs"
 	internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net"
+	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics"
 	"git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver"
 	grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
@@ -502,6 +503,22 @@ func samplingEnabling(v *viper.Viper, core zapcore.Core) zapcore.Core {
 	return core
 }
 
+func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, appMetrics *metrics.GateMetrics) zapcore.Core {
+	if v.GetBool(cfgLoggerSamplingEnabled) {
+		core = zapcore.NewSamplerWithOptions(core,
+			v.GetDuration(cfgLoggerSamplingInterval),
+			v.GetInt(cfgLoggerSamplingInitial),
+			v.GetInt(cfgLoggerSamplingThereafter),
+			zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
+				if dec&zapcore.LogDropped > 0 {
+					appMetrics.IncDroppedLogs()
+				}
+			}))
+	}
+
+	return core
+}
+
 func getLogLevel(v *viper.Viper) (zapcore.Level, error) {
 	var lvl zapcore.Level
 	lvlStr := v.GetString(cfgLoggerLevel)

metrics/desc.go

@@ -76,6 +76,15 @@ var appMetricsDesc = map[string]map[string]Description{
 			VariableLabels: []string{"endpoint"},
 		},
 	},
+	statisticSubsystem: {
+		droppedLogs: Description{
+			Type:      dto.MetricType_COUNTER,
+			Namespace: namespace,
+			Subsystem: statisticSubsystem,
+			Name:      droppedLogs,
+			Help:      "Dropped logs (by sampling) count",
+		},
+	},
 }
 
 type Description struct {
@@ -148,3 +157,12 @@ func mustNewGaugeVec(description Description) *prometheus.GaugeVec {
 		description.VariableLabels,
 	)
 }
+
+func mustNewCounter(description Description) prometheus.Counter {
+	if description.Type != dto.MetricType_COUNTER {
+		panic("invalid metric type")
+	}
+	return prometheus.NewCounter(
+		prometheus.CounterOpts(newOpts(description)),
+	)
+}

metrics/metrics.go

@@ -10,15 +10,17 @@ import (
 )
 
 const (
-	namespace       = "frostfs_http_gw"
-	stateSubsystem  = "state"
-	poolSubsystem   = "pool"
-	serverSubsystem = "server"
+	namespace          = "frostfs_http_gw"
+	stateSubsystem     = "state"
+	poolSubsystem      = "pool"
+	serverSubsystem    = "server"
+	statisticSubsystem = "statistic"
 )
 
 const (
 	healthMetric      = "health"
 	versionInfoMetric = "version_info"
+	droppedLogs       = "dropped_logs"
 )
 
 const (
@@ -30,21 +32,19 @@ const (
 )
 
 const (
-	methodGetBalance       = "get_balance"
-	methodPutContainer     = "put_container"
-	methodGetContainer     = "get_container"
-	methodListContainer    = "list_container"
-	methodDeleteContainer  = "delete_container"
-	methodGetContainerEacl = "get_container_eacl"
-	methodSetContainerEacl = "set_container_eacl"
-	methodEndpointInfo     = "endpoint_info"
-	methodNetworkInfo      = "network_info"
-	methodPutObject        = "put_object"
-	methodDeleteObject     = "delete_object"
-	methodGetObject        = "get_object"
-	methodHeadObject       = "head_object"
-	methodRangeObject      = "range_object"
-	methodCreateSession    = "create_session"
+	methodGetBalance      = "get_balance"
+	methodPutContainer    = "put_container"
+	methodGetContainer    = "get_container"
+	methodListContainer   = "list_container"
+	methodDeleteContainer = "delete_container"
+	methodEndpointInfo    = "endpoint_info"
+	methodNetworkInfo     = "network_info"
+	methodPutObject       = "put_object"
+	methodDeleteObject    = "delete_object"
+	methodGetObject       = "get_object"
+	methodHeadObject      = "head_object"
+	methodRangeObject     = "range_object"
+	methodCreateSession   = "create_session"
 )
 
 // HealthStatus of the gate application.
@@ -69,6 +69,7 @@ type GateMetrics struct {
 	stateMetrics
 	poolMetricsCollector
 	serverMetrics
+	statisticMetrics
 }
 
 type stateMetrics struct {
@@ -76,6 +77,10 @@ type stateMetrics struct {
 	versionInfo *prometheus.GaugeVec
 }
 
+type statisticMetrics struct {
+	droppedLogs prometheus.Counter
+}
+
 type poolMetricsCollector struct {
 	scraper             StatisticScraper
 	overallErrors       prometheus.Gauge
@@ -96,10 +101,14 @@ func NewGateMetrics(p StatisticScraper) *GateMetrics {
 	serverMetric := newServerMetrics()
 	serverMetric.register()
 
+	statsMetric := newStatisticMetrics()
+	statsMetric.register()
+
 	return &GateMetrics{
 		stateMetrics:         *stateMetric,
 		poolMetricsCollector: *poolMetric,
 		serverMetrics:        *serverMetric,
+		statisticMetrics:     *statsMetric,
 	}
 }
 
@@ -107,6 +116,7 @@ func (g *GateMetrics) Unregister() {
 	g.stateMetrics.unregister()
 	prometheus.Unregister(&g.poolMetricsCollector)
 	g.serverMetrics.unregister()
+	g.statisticMetrics.unregister()
 }
 
 func newStateMetrics() *stateMetrics {
@@ -116,6 +126,20 @@ func newStateMetrics() *stateMetrics {
 	}
 }
 
+func newStatisticMetrics() *statisticMetrics {
+	return &statisticMetrics{
+		droppedLogs: mustNewCounter(appMetricsDesc[statisticSubsystem][droppedLogs]),
+	}
+}
+
+func (s statisticMetrics) register() {
+	prometheus.MustRegister(s.droppedLogs)
+}
+
+func (s statisticMetrics) unregister() {
+	prometheus.Unregister(s.droppedLogs)
+}
+
 func (m stateMetrics) register() {
 	prometheus.MustRegister(m.healthCheck)
 	prometheus.MustRegister(m.versionInfo)
@@ -134,6 +158,10 @@ func (m stateMetrics) SetVersion(ver string) {
 	m.versionInfo.WithLabelValues(ver).Set(1)
 }
 
+func (s statisticMetrics) IncDroppedLogs() {
+	s.droppedLogs.Inc()
+}
+
 func newPoolMetricsCollector(p StatisticScraper) *poolMetricsCollector {
 	return &poolMetricsCollector{
 		scraper:             p,