diff --git a/cmd/http-gw/app.go b/cmd/http-gw/app.go index 0dd53a6..ea3716c 100644 --- a/cmd/http-gw/app.go +++ b/cmd/http-gw/app.go @@ -177,6 +177,7 @@ func newApp(ctx context.Context, opt ...Option) App { a.initResolver() a.initMetrics() + a.initLogs() a.initTracing(ctx) return a @@ -340,6 +341,11 @@ func (a *app) initMetrics() { a.metrics.SetHealth(metrics.HealthStatusStarting) } +func (a *app) initLogs() { + coreWithContext := applyZapCoreMiddlewares(a.log.Core(), a.cfg, a.metrics.provider) + a.log = zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) +} + func newGateMetrics(logger *zap.Logger, provider *metrics.GateMetrics, enabled bool) *gateMetrics { if !enabled { logger.Warn(logs.MetricsAreDisabled) diff --git a/cmd/http-gw/settings.go b/cmd/http-gw/settings.go index 316c500..c9aaf3d 100644 --- a/cmd/http-gw/settings.go +++ b/cmd/http-gw/settings.go @@ -17,6 +17,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/cache" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/logs" internalnet "git.frostfs.info/TrueCloudLab/frostfs-http-gw/internal/net" + "git.frostfs.info/TrueCloudLab/frostfs-http-gw/metrics" "git.frostfs.info/TrueCloudLab/frostfs-http-gw/resolver" grpctracing "git.frostfs.info/TrueCloudLab/frostfs-observability/tracing/grpc" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" @@ -428,9 +429,9 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { switch dest { case destinationStdout: - return newStdoutLogger(v, lvl) + return newStdoutLogger(lvl) case destinationJournald: - return newJournaldLogger(v, lvl) + return newJournaldLogger(lvl) default: panic(fmt.Sprintf("wrong destination for logger: %s", dest)) } @@ -447,18 +448,17 @@ func pickLogger(v *viper.Viper) (*zap.Logger, zap.AtomicLevel) { // Logger records a stack trace for all messages at or above fatal level. // // See also zapcore.Level, zap.NewProductionConfig, zap.AddStacktrace. -func newStdoutLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { +func newStdoutLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { stdout := zapcore.AddSync(os.Stderr) level := zap.NewAtomicLevelAt(lvl) consoleOutCore := zapcore.NewCore(newLogEncoder(), stdout, level) - consoleOutCore = samplingEnabling(v, consoleOutCore) l := zap.New(consoleOutCore, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) return l, level } -func newJournaldLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { +func newJournaldLogger(lvl zapcore.Level) (*zap.Logger, zap.AtomicLevel) { level := zap.NewAtomicLevelAt(lvl) encoder := zapjournald.NewPartialEncoder(newLogEncoder(), zapjournald.SyslogFields) @@ -470,8 +470,6 @@ func newJournaldLogger(v *viper.Viper, lvl zapcore.Level) (*zap.Logger, zap.Atom zapjournald.SyslogPid(), }) - coreWithContext = samplingEnabling(v, coreWithContext) - l := zap.New(coreWithContext, zap.AddStacktrace(zap.NewAtomicLevelAt(zap.FatalLevel))) return l, level @@ -484,19 +482,17 @@ func newLogEncoder() zapcore.Encoder { return zapcore.NewConsoleEncoder(c) } -func samplingEnabling(v *viper.Viper, core zapcore.Core) zapcore.Core { - // Zap samples by logging the first cgfLoggerSamplingInitial entries with a given level - // and message within the specified time interval. - // In the above config, only the first cgfLoggerSamplingInitial log entries with the same level and message - // are recorded in cfgLoggerSamplingInterval interval. Every other log entry will be dropped within the interval since - // cfgLoggerSamplingThereafter is specified here. +func applyZapCoreMiddlewares(core zapcore.Core, v *viper.Viper, appMetrics *metrics.GateMetrics) zapcore.Core { if v.GetBool(cfgLoggerSamplingEnabled) { - core = zapcore.NewSamplerWithOptions( - core, + core = zapcore.NewSamplerWithOptions(core, v.GetDuration(cfgLoggerSamplingInterval), v.GetInt(cfgLoggerSamplingInitial), v.GetInt(cfgLoggerSamplingThereafter), - ) + zapcore.SamplerHook(func(_ zapcore.Entry, dec zapcore.SamplingDecision) { + if dec&zapcore.LogDropped > 0 { + appMetrics.IncDroppedLogs() + } + })) } return core diff --git a/metrics/desc.go b/metrics/desc.go index e10050c..a00ab3e 100644 --- a/metrics/desc.go +++ b/metrics/desc.go @@ -76,6 +76,15 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"endpoint"}, }, }, + statisticSubsystem: { + droppedLogs: Description{ + Type: dto.MetricType_COUNTER, + Namespace: namespace, + Subsystem: statisticSubsystem, + Name: droppedLogs, + Help: "Dropped logs (by sampling) count", + }, + }, } type Description struct { @@ -148,3 +157,12 @@ func mustNewGaugeVec(description Description) *prometheus.GaugeVec { description.VariableLabels, ) } + +func mustNewCounter(description Description) prometheus.Counter { + if description.Type != dto.MetricType_COUNTER { + panic("invalid metric type") + } + return prometheus.NewCounter( + prometheus.CounterOpts(newOpts(description)), + ) +} diff --git a/metrics/metrics.go b/metrics/metrics.go index b516477..001ba89 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -10,15 +10,17 @@ import ( ) const ( - namespace = "frostfs_http_gw" - stateSubsystem = "state" - poolSubsystem = "pool" - serverSubsystem = "server" + namespace = "frostfs_http_gw" + stateSubsystem = "state" + poolSubsystem = "pool" + serverSubsystem = "server" + statisticSubsystem = "statistic" ) const ( healthMetric = "health" versionInfoMetric = "version_info" + droppedLogs = "dropped_logs" ) const ( @@ -30,21 +32,19 @@ const ( ) const ( - methodGetBalance = "get_balance" - methodPutContainer = "put_container" - methodGetContainer = "get_container" - methodListContainer = "list_container" - methodDeleteContainer = "delete_container" - methodGetContainerEacl = "get_container_eacl" - methodSetContainerEacl = "set_container_eacl" - methodEndpointInfo = "endpoint_info" - methodNetworkInfo = "network_info" - methodPutObject = "put_object" - methodDeleteObject = "delete_object" - methodGetObject = "get_object" - methodHeadObject = "head_object" - methodRangeObject = "range_object" - methodCreateSession = "create_session" + methodGetBalance = "get_balance" + methodPutContainer = "put_container" + methodGetContainer = "get_container" + methodListContainer = "list_container" + methodDeleteContainer = "delete_container" + methodEndpointInfo = "endpoint_info" + methodNetworkInfo = "network_info" + methodPutObject = "put_object" + methodDeleteObject = "delete_object" + methodGetObject = "get_object" + methodHeadObject = "head_object" + methodRangeObject = "range_object" + methodCreateSession = "create_session" ) // HealthStatus of the gate application. @@ -69,6 +69,7 @@ type GateMetrics struct { stateMetrics poolMetricsCollector serverMetrics + statisticMetrics } type stateMetrics struct { @@ -76,6 +77,10 @@ type stateMetrics struct { versionInfo *prometheus.GaugeVec } +type statisticMetrics struct { + droppedLogs prometheus.Counter +} + type poolMetricsCollector struct { scraper StatisticScraper overallErrors prometheus.Gauge @@ -96,10 +101,14 @@ func NewGateMetrics(p StatisticScraper) *GateMetrics { serverMetric := newServerMetrics() serverMetric.register() + statsMetric := newStatisticMetrics() + statsMetric.register() + return &GateMetrics{ stateMetrics: *stateMetric, poolMetricsCollector: *poolMetric, serverMetrics: *serverMetric, + statisticMetrics: *statsMetric, } } @@ -107,6 +116,7 @@ func (g *GateMetrics) Unregister() { g.stateMetrics.unregister() prometheus.Unregister(&g.poolMetricsCollector) g.serverMetrics.unregister() + g.statisticMetrics.unregister() } func newStateMetrics() *stateMetrics { @@ -116,6 +126,20 @@ func newStateMetrics() *stateMetrics { } } +func newStatisticMetrics() *statisticMetrics { + return &statisticMetrics{ + droppedLogs: mustNewCounter(appMetricsDesc[statisticSubsystem][droppedLogs]), + } +} + +func (s statisticMetrics) register() { + prometheus.MustRegister(s.droppedLogs) +} + +func (s statisticMetrics) unregister() { + prometheus.Unregister(s.droppedLogs) +} + func (m stateMetrics) register() { prometheus.MustRegister(m.healthCheck) prometheus.MustRegister(m.versionInfo) @@ -134,6 +158,10 @@ func (m stateMetrics) SetVersion(ver string) { m.versionInfo.WithLabelValues(ver).Set(1) } +func (s statisticMetrics) IncDroppedLogs() { + s.droppedLogs.Inc() +} + func newPoolMetricsCollector(p StatisticScraper) *poolMetricsCollector { return &poolMetricsCollector{ scraper: p,