From 99ff86a0d1fa7697d94b21dd8e89d26e1fb48e9e Mon Sep 17 00:00:00 2001
From: Pavel Pogodaev <p.pogodaev@yadro.com>
Date: Wed, 20 Nov 2024 15:05:04 +0300
Subject: [PATCH] [#502] Add Dropped logs (by sampling) metric

Signed-off-by: Pavel Pogodaev <p.pogodaev@yadro.com>
---
 cmd/s3-gw/app.go          |  6 ++++++
 cmd/s3-gw/app_settings.go | 17 +++++++++++++++++
 metrics/app.go            |  4 ++++
 metrics/desc.go           | 16 ++++++++++++++++
 metrics/gate.go           |  6 ++++++
 metrics/logs.go           | 38 ++++++++++++++++++++++++++++++++++++++
 6 files changed, 87 insertions(+)
 create mode 100644 metrics/logs.go

diff --git a/cmd/s3-gw/app.go b/cmd/s3-gw/app.go
index 1ac2b30..cfddda0 100644
--- a/cmd/s3-gw/app.go
+++ b/cmd/s3-gw/app.go
@@ -163,6 +163,7 @@ func (a *App) init(ctx context.Context) {
 	a.initPolicyStorage(ctx)
 	a.initAPI(ctx)
 	a.initMetrics()
+	a.initLogger()
 	a.initServers(ctx)
 	a.initTracing(ctx)
 }
@@ -523,6 +524,11 @@ func (a *App) initMetrics() {
 	a.metrics.State().SetHealth(metrics.HealthStatusStarting)
 }
 
+func (a *App) initLogger() {
+	coreWithContext := applyZapCoreMiddlewares(a.log.Core(), a.cfg, a.metrics)
+	a.log = zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel)))
+}
+
 func (a *App) initFrostfsID(ctx context.Context) {
 	cli, err := ffidcontract.New(ctx, ffidcontract.Config{
 		RPCAddress:    a.cfg.GetString(cfgRPCEndpoint),
diff --git a/cmd/s3-gw/app_settings.go b/cmd/s3-gw/app_settings.go
index 691e231..be7f95d 100644
--- a/cmd/s3-gw/app_settings.go
+++ b/cmd/s3-gw/app_settings.go
@@ -18,6 +18,7 @@ import (
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/logs"
 	internalnet "git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/net"
 	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/internal/version"
+	"git.frostfs.info/TrueCloudLab/frostfs-s3-gw/metrics"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/netmap"
 	"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
 	"git.frostfs.info/TrueCloudLab/zapjournald"
@@ -1221,3 +1222,19 @@ LOOP:
 	}
 	return validDomains
 }
+
+func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, appMetrics *metrics.AppMetrics) zapcore.Core {
+	if v.GetBool(cfgLoggerSamplingEnabled) {
+		core = zapcore.NewSamplerWithOptions(core,
+			v.GetDuration(cfgLoggerSamplingInterval),
+			v.GetInt(cfgLoggerSamplingInitial),
+			v.GetInt(cfgLoggerSamplingThereafter),
+			zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) {
+				if dec&zapcore.LogDropped > 0 {
+					appMetrics.DroppedLogsInc()
+				}
+			}))
+	}
+
+	return core
+}
diff --git a/metrics/app.go b/metrics/app.go
index 1a2ef07..363ac8a 100644
--- a/metrics/app.go
+++ b/metrics/app.go
@@ -42,6 +42,10 @@ func NewAppMetrics(cfg AppMetricsConfig) *AppMetrics {
 	}
 }
 
+func (m *AppMetrics) DroppedLogsInc() {
+	m.gate.Logs.DroppedLogsInc()
+}
+
 func (m *AppMetrics) SetEnabled(enabled bool) {
 	if !enabled {
 		m.logger.Warn(logs.MetricsAreDisabled)
diff --git a/metrics/desc.go b/metrics/desc.go
index 6affeef..e3827ec 100644
--- a/metrics/desc.go
+++ b/metrics/desc.go
@@ -93,6 +93,13 @@ var appMetricsDesc = map[string]map[string]Description{
 		},
 	},
 	statisticSubsystem: {
+		droppedLogs: Description{
+			Type:      dto.MetricType_COUNTER,
+			Namespace: namespace,
+			Subsystem: statisticSubsystem,
+			Name:      droppedLogs,
+			Help:      "Dropped logs (by sampling) count",
+		},
 		requestsSecondsMetric: Description{
 			Type:           dto.MetricType_HISTOGRAM,
 			Namespace:      namespace,
@@ -252,3 +259,12 @@ func mustNewHistogramVec(description Description, buckets []float64) *prometheus
 		description.VariableLabels,
 	)
 }
+
+func mustNewCounter(description Description) prometheus.Counter {
+	if description.Type != dto.MetricType_COUNTER {
+		panic("invalid metric type")
+	}
+	return prometheus.NewCounter(
+		prometheus.CounterOpts(newOpts(description)),
+	)
+}
diff --git a/metrics/gate.go b/metrics/gate.go
index 02d608b..4adea51 100644
--- a/metrics/gate.go
+++ b/metrics/gate.go
@@ -28,6 +28,7 @@ type GateMetrics struct {
 	Stats      *APIStatMetrics
 	HTTPServer *httpServerMetrics
 	TreePool   *treePoolMetricsCollector
+	Logs       *logsMetric
 }
 
 func NewGateMetrics(scraper StatisticScraper, treeScraper TreePoolStatistic, registry prometheus.Registerer) *GateMetrics {
@@ -49,6 +50,9 @@ func NewGateMetrics(scraper StatisticScraper, treeScraper TreePoolStatistic, reg
 	treePoolMetric := newTreePoolMetricsCollector(treeScraper)
 	registry.MustRegister(treePoolMetric)
 
+	logsMetrics := newLogsMetrics()
+	registry.MustRegister(logsMetrics)
+
 	return &GateMetrics{
 		registry:   registry,
 		State:      stateMetric,
@@ -57,6 +61,7 @@ func NewGateMetrics(scraper StatisticScraper, treeScraper TreePoolStatistic, reg
 		Stats:      statsMetric,
 		HTTPServer: serverMetric,
 		TreePool:   treePoolMetric,
+		Logs:       logsMetrics,
 	}
 }
 
@@ -66,6 +71,7 @@ func (g *GateMetrics) Unregister() {
 	g.Billing.Unregister()
 	g.registry.Unregister(g.Stats)
 	g.registry.Unregister(g.HTTPServer)
+	g.registry.Unregister(g.Logs)
 }
 
 func (g *GateMetrics) Handler() http.Handler {
diff --git a/metrics/logs.go b/metrics/logs.go
new file mode 100644
index 0000000..44f5357
--- /dev/null
+++ b/metrics/logs.go
@@ -0,0 +1,38 @@
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const (
+	droppedLogs = "dropped_logs"
+)
+
+type logsMetric struct {
+	droppedLogs prometheus.Counter
+}
+
+func newLogsMetrics() *logsMetric {
+	return &logsMetric{
+		droppedLogs: mustNewCounter(appMetricsDesc[statisticSubsystem][droppedLogs]),
+	}
+}
+
+func (m *logsMetric) DroppedLogsInc() {
+	if m == nil {
+		return
+	}
+	m.droppedLogs.Inc()
+}
+
+func (m *logsMetric) Describe(descs chan<- *prometheus.Desc) {
+	if m == nil {
+		return
+	}
+	m.droppedLogs.Describe(descs)
+}
+
+func (m *logsMetric) Collect(metrics chan<- prometheus.Metric) {
+	if m == nil {
+		return
+	}
+	m.droppedLogs.Collect(metrics)
+}