[#502] Add Dropped logs (by sampling) metric

Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>

cmd/s3-gw/app.go

@@ -163,6 +163,7 @@ func (a *App) init(ctx context.Context) {
 	a.initPolicyStorage(ctx)
 	a.initAPI(ctx)
 	a.initMetrics()
+	a.initLogger()
 	a.initServers(ctx)
 	a.initTracing(ctx)
 }
@@ -523,6 +524,11 @@ func (a *App) initMetrics() {
 	a.metrics.State().SetHealth(metrics.HealthStatusStarting)
 }
 
+func (a *App) initLogger() {
+	coreWithContext := applyZapCoreMiddlewares(a.log.Core(), a.cfg, a.metrics)
+	a.log = zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
+}
+
 func (a *App) initFrostfsID(ctx context.Context) {
 	cli, err := ffidcontract.New(ctx, ffidcontract.Config{
 		RPCAddress:    a.cfg.GetString(cfgRPCEndpoint),

cmd/s3-gw/app_settings.go

@@ -18,6 +18,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
 	internalnet "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/net"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
 	"git.frostfs.info/TrueCloudLab/zapjournald"
@@ -1221,3 +1222,19 @@ LOOP:
 	}
 	return validDomains
 }
+
+func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, appMetrics *metrics.AppMetrics) zapcore.Core {
+	if v.GetBool(cfgLoggerSamplingEnabled) {
+		core = zapcore.NewSamplerWithOptions(core,
+			v.GetDuration(cfgLoggerSamplingInterval),
+			v.GetInt(cfgLoggerSamplingInitial),
+			v.GetInt(cfgLoggerSamplingThereafter),
+			zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
+				if dec&zapcore.LogDropped > 0 {
+					appMetrics.DroppedLogsInc()
+				}
+			}))
+	}
+
+	return core
+}

metrics/app.go

@@ -42,6 +42,10 @@ func NewAppMetrics(cfg AppMetricsConfig) *AppMetrics {
 	}
 }
 
+func (m *AppMetrics) DroppedLogsInc() {
+	m.gate.Logs.DroppedLogsInc()
+}
+
 func (m *AppMetrics) SetEnabled(enabled bool) {
 	if !enabled {
 		m.logger.Warn(logs.MetricsAreDisabled)

metrics/desc.go

@@ -93,6 +93,13 @@ var appMetricsDesc = map[string]map[string]Description{
 		},
 	},
 	statisticSubsystem: {
+		droppedLogs: Description{
+			Type:      dto.MetricType_COUNTER,
+			Namespace: namespace,
+			Subsystem: statisticSubsystem,
+			Name:      droppedLogs,
+			Help:      "Dropped logs (by sampling) count",
+		},
 		requestsSecondsMetric: Description{
 			Type:           dto.MetricType_HISTOGRAM,
 			Namespace:      namespace,
@@ -252,3 +259,12 @@ func mustNewHistogramVec(description Description, buckets []float64) *prometheus
 		description.VariableLabels,
 	)
 }
+
+func mustNewCounter(description Description) prometheus.Counter {
+	if description.Type != dto.MetricType_COUNTER {
+		panic("invalid metric type")
+	}
+	return prometheus.NewCounter(
+		prometheus.CounterOpts(newOpts(description)),
+	)
+}

metrics/gate.go

@@ -28,6 +28,7 @@ type GateMetrics struct {
 	Stats      *APIStatMetrics
 	HTTPServer *httpServerMetrics
 	TreePool   *treePoolMetricsCollector
+	Logs       *logsMetric
 }
 
 func NewGateMetrics(scraper StatisticScraper, treeScraper TreePoolStatistic, registry prometheus.Registerer) *GateMetrics {
@@ -49,6 +50,9 @@ func NewGateMetrics(scraper StatisticScraper, treeScraper TreePoolStatistic, reg
 	treePoolMetric := newTreePoolMetricsCollector(treeScraper)
 	registry.MustRegister(treePoolMetric)
 
+	logsMetrics := newLogsMetrics()
+	registry.MustRegister(logsMetrics)
+
 	return &GateMetrics{
 		registry:   registry,
 		State:      stateMetric,
@@ -57,6 +61,7 @@ func NewGateMetrics(scraper StatisticScraper, treeScraper TreePoolStatistic, reg
 		Stats:      statsMetric,
 		HTTPServer: serverMetric,
 		TreePool:   treePoolMetric,
+		Logs:       logsMetrics,
 	}
 }
 
@@ -66,6 +71,7 @@ func (g *GateMetrics) Unregister() {
 	g.Billing.Unregister()
 	g.registry.Unregister(g.Stats)
 	g.registry.Unregister(g.HTTPServer)
+	g.registry.Unregister(g.Logs)
 }
 
 func (g *GateMetrics) Handler() http.Handler {

metrics/logs.go

@@ -0,0 +1,38 @@
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const (
+	droppedLogs = "dropped_logs"
+)
+
+type logsMetric struct {
+	droppedLogs prometheus.Counter
+}
+
+func newLogsMetrics() *logsMetric {
+	return &logsMetric{
+		droppedLogs: mustNewCounter(appMetricsDesc[statisticSubsystem][droppedLogs]),
+	}
+}
+
+func (m *logsMetric) DroppedLogsInc() {
+	if m == nil {
+		return
+	}
+	m.droppedLogs.Inc()
+}
+
+func (m *logsMetric) Describe(descs chan<- *prometheus.Desc) {
+	if m == nil {
+		return
+	}
+	m.droppedLogs.Describe(descs)
+}
+
+func (m *logsMetric) Collect(metrics chan<- prometheus.Metric) {
+	if m == nil {
+		return
+	}
+	m.droppedLogs.Collect(metrics)
+}