From 1858668e53d40fdcd23a0e6476227329f8710c9d Mon Sep 17 00:00:00 2001 From: Pavel Pogodaev Date: Sun, 24 Nov 2024 13:32:40 +0300 Subject: [PATCH] [#150] Add dropped logs metric Signed-off-by: Pavel Pogodaev --- cmd/http-gw/app.go | 6 ++++ cmd/http-gw/settings.go | 17 +++++++++++ metrics/desc.go | 18 +++++++++++ metrics/metrics.go | 66 +++++++++++++++++++++++++++++------------ 4 files changed, 88 insertions(+), 19 deletions(-) diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 0dd53a6..ea3716c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -177,6 +177,7 @@ func newApp(ctx context.Context, opt ...Option) App { a.initResolver() a.initMetrics() + a.initLogs() a.initTracing(ctx) return a @@ -340,6 +341,11 @@ func (a *app) initMetrics() { a.metrics.SetHealth(metrics.HealthStatusStarting) } +func (a *app) initLogs() { + coreWithContext := applyZapCoreMiddlewares(a.log.Core(), a.cfg, a.metrics.provider) + a.log = zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) +} + func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics { if !enabled { logger.Warn(logs.MetricsAreDisabled) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 316c500..ff14733 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -17,6 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" @@ -502,6 +503,22 @@ func samplingEnabling(v *viper.Viper, core zapcore.Core) zapcore.Core { return core } +func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, appMetrics *metrics.GateMetrics) zapcore.Core { + if v.GetBool(cfgLoggerSamplingEnabled) { + core = zapcore.NewSamplerWithOptions(core, + v.GetDuration(cfgLoggerSamplingInterval), + v.GetInt(cfgLoggerSamplingInitial), + v.GetInt(cfgLoggerSamplingThereafter), + zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) { + if dec&zapcore.LogDropped > 0 { + appMetrics.IncDroppedLogs() + } + })) + } + + return core +} + func getLogLevel(v *viper.Viper) (zapcore.Level, error) { var lvl zapcore.Level lvlStr := v.GetString(cfgLoggerLevel) diff --git a/metrics/desc.go b/metrics/desc.go index e10050c..a00ab3e 100644 --- a/metrics/desc.go +++ b/metrics/desc.go @@ -76,6 +76,15 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"endpoint"}, }, }, + statisticSubsystem: { + droppedLogs: Description{ + Type: dto.MetricType_COUNTER, + Namespace: namespace, + Subsystem: statisticSubsystem, + Name: droppedLogs, + Help: "Dropped logs (by sampling) count", + }, + }, } type Description struct { @@ -148,3 +157,12 @@ func mustNewGaugeVec(description Description) *prometheus.GaugeVec { description.VariableLabels, ) } + +func mustNewCounter(description Description) prometheus.Counter { + if description.Type != dto.MetricType_COUNTER { + panic("invalid metric type") + } + return prometheus.NewCounter( + prometheus.CounterOpts(newOpts(description)), + ) +} diff --git a/metrics/metrics.go b/metrics/metrics.go index b516477..001ba89 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -10,15 +10,17 @@ import ( ) const ( - namespace = "frostfs_http_gw" - stateSubsystem = "state" - poolSubsystem = "pool" - serverSubsystem = "server" + namespace = "frostfs_http_gw" + stateSubsystem = "state" + poolSubsystem = "pool" + serverSubsystem = "server" + statisticSubsystem = "statistic" ) const ( healthMetric = "health" versionInfoMetric = "version_info" + droppedLogs = "dropped_logs" ) const ( @@ -30,21 +32,19 @@ const ( ) const ( - methodGetBalance = "get_balance" - methodPutContainer = "put_container" - methodGetContainer = "get_container" - methodListContainer = "list_container" - methodDeleteContainer = "delete_container" - methodGetContainerEacl = "get_container_eacl" - methodSetContainerEacl = "set_container_eacl" - methodEndpointInfo = "endpoint_info" - methodNetworkInfo = "network_info" - methodPutObject = "put_object" - methodDeleteObject = "delete_object" - methodGetObject = "get_object" - methodHeadObject = "head_object" - methodRangeObject = "range_object" - methodCreateSession = "create_session" + methodGetBalance = "get_balance" + methodPutContainer = "put_container" + methodGetContainer = "get_container" + methodListContainer = "list_container" + methodDeleteContainer = "delete_container" + methodEndpointInfo = "endpoint_info" + methodNetworkInfo = "network_info" + methodPutObject = "put_object" + methodDeleteObject = "delete_object" + methodGetObject = "get_object" + methodHeadObject = "head_object" + methodRangeObject = "range_object" + methodCreateSession = "create_session" ) // HealthStatus of the gate application. @@ -69,6 +69,7 @@ type GateMetrics struct { stateMetrics poolMetricsCollector serverMetrics + statisticMetrics } type stateMetrics struct { @@ -76,6 +77,10 @@ type stateMetrics struct { versionInfo *prometheus.GaugeVec } +type statisticMetrics struct { + droppedLogs prometheus.Counter +} + type poolMetricsCollector struct { scraper StatisticScraper overallErrors prometheus.Gauge @@ -96,10 +101,14 @@ func NewGateMetrics(p StatisticScraper) *GateMetrics { serverMetric := newServerMetrics() serverMetric.register() + statsMetric := newStatisticMetrics() + statsMetric.register() + return &GateMetrics{ stateMetrics: *stateMetric, poolMetricsCollector: *poolMetric, serverMetrics: *serverMetric, + statisticMetrics: *statsMetric, } } @@ -107,6 +116,7 @@ func (g *GateMetrics) Unregister() { g.stateMetrics.unregister() prometheus.Unregister(&g.poolMetricsCollector) g.serverMetrics.unregister() + g.statisticMetrics.unregister() } func newStateMetrics() *stateMetrics { @@ -116,6 +126,20 @@ func newStateMetrics() *stateMetrics { } } +func newStatisticMetrics() *statisticMetrics { + return &statisticMetrics{ + droppedLogs: mustNewCounter(appMetricsDesc[statisticSubsystem][droppedLogs]), + } +} + +func (s statisticMetrics) register() { + prometheus.MustRegister(s.droppedLogs) +} + +func (s statisticMetrics) unregister() { + prometheus.Unregister(s.droppedLogs) +} + func (m stateMetrics) register() { prometheus.MustRegister(m.healthCheck) prometheus.MustRegister(m.versionInfo) @@ -134,6 +158,10 @@ func (m stateMetrics) SetVersion(ver string) { m.versionInfo.WithLabelValues(ver).Set(1) } +func (s statisticMetrics) IncDroppedLogs() { + s.droppedLogs.Inc() +} + func newPoolMetricsCollector(p StatisticScraper) *poolMetricsCollector { return &poolMetricsCollector{ scraper: p,