From 3a997d120767eef9636e11b878b805d0d19ffc68 Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Sun, 8 Oct 2023 12:32:00 +0300 Subject: [PATCH] [#680] metrics: Initialize log metrics together with services Signed-off-by: Anton Nikiforov --- cmd/frostfs-ir/main.go | 8 +++++--- cmd/frostfs-node/config.go | 37 ++++++++++++++++------------------ pkg/innerring/innerring.go | 5 +++-- pkg/metrics/innerring.go | 7 +++++++ pkg/metrics/node.go | 7 +++++++ pkg/util/logger/logger.go | 10 ++++----- pkg/util/logger/metrics.go | 19 +++++++++++++---- scripts/export-metrics/main.go | 3 --- 8 files changed, 58 insertions(+), 38 deletions(-) diff --git a/cmd/frostfs-ir/main.go b/cmd/frostfs-ir/main.go index 70199b094..2b39f78e3 100644 --- a/cmd/frostfs-ir/main.go +++ b/cmd/frostfs-ir/main.go @@ -10,6 +10,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/misc" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring" + irMetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "github.com/spf13/viper" "go.uber.org/zap" @@ -61,12 +62,13 @@ func main() { cfg, err = newConfig() exitErr(err) - logPrm.MetricsNamespace = "frostfs_ir" + metrics := irMetrics.NewInnerRingMetrics() + err = logPrm.SetLevelString( cfg.GetString("logger.level"), ) exitErr(err) - + logPrm.SamplingHook = metrics.LogMetrics().GetSamplingHook() log, err = logger.NewLogger(logPrm) exitErr(err) @@ -78,7 +80,7 @@ func main() { metricsCmp = newMetricsComponent() metricsCmp.init() - innerRing, err = innerring.New(ctx, log, cfg, intErr) + innerRing, err = innerring.New(ctx, log, cfg, intErr, metrics) exitErr(err) pprofCmp.start() diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index 60e567c5a..b35200d27 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -554,22 +554,21 @@ func initCfg(appCfg *config.Config) *cfg { key := nodeconfig.Key(appCfg) + relayOnly := nodeconfig.Relay(appCfg) + + netState := newNetworkState() + netState.metrics = c.metricsCollector + + c.shared = initShared(appCfg, key, netState, relayOnly) + logPrm, err := c.loggerPrm() fatalOnErr(err) - - logPrm.MetricsNamespace = "frostfs_node" - + logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook() log, err := logger.NewLogger(logPrm) fatalOnErr(err) c.internals = initInternals(appCfg, log) - relayOnly := nodeconfig.Relay(appCfg) - - netState := newNetworkState() - - c.shared = initShared(appCfg, key, netState, relayOnly) - c.cfgAccounting = cfgAccounting{ scriptHash: contractsconfig.Balance(appCfg), } @@ -586,9 +585,6 @@ func initCfg(appCfg *config.Config) *cfg { user.IDFromKey(&c.ownerIDFromKey, key.PrivateKey.PublicKey) - c.metricsCollector = metrics.NewNodeMetrics() - netState.metrics = c.metricsCollector - c.onShutdown(c.clientCache.CloseAll) // clean up connections c.onShutdown(c.bgClientCache.CloseAll) // clean up connections c.onShutdown(c.putClientCache.CloseAll) // clean up connections @@ -630,14 +626,15 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt } return shared{ - key: key, - binPublicKey: key.PublicKey().Bytes(), - localAddr: netAddr, - respSvc: response.NewService(netState), - clientCache: cache.NewSDKClientCache(cacheOpts), - bgClientCache: cache.NewSDKClientCache(cacheOpts), - putClientCache: cache.NewSDKClientCache(cacheOpts), - persistate: persistate, + key: key, + binPublicKey: key.PublicKey().Bytes(), + localAddr: netAddr, + respSvc: response.NewService(netState), + clientCache: cache.NewSDKClientCache(cacheOpts), + bgClientCache: cache.NewSDKClientCache(cacheOpts), + putClientCache: cache.NewSDKClientCache(cacheOpts), + persistate: persistate, + metricsCollector: metrics.NewNodeMetrics(), } } diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 1567e40d3..6f2463c74 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -325,11 +325,12 @@ func (s *Server) registerStarter(f func() error) { } // New creates instance of inner ring sever structure. -func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error) (*Server, error) { +func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error, + metrics *metrics.InnerRingServiceMetrics) (*Server, error) { var err error server := &Server{ log: log, - irMetrics: metrics.NewInnerRingMetrics(), + irMetrics: metrics, } server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED) diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index d93b3c432..3e438a01b 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -4,6 +4,7 @@ import ( "strconv" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -14,6 +15,7 @@ type InnerRingServiceMetrics struct { health prometheus.Gauge eventDuration *prometheus.HistogramVec morphCacheMetrics *morphCacheMetrics + logMetrics logger.LogMetrics } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. @@ -44,6 +46,7 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { health: health, eventDuration: eventDuration, morphCacheMetrics: newMorphCacheMetrics(innerRingNamespace), + logMetrics: logger.NewLogMetrics(innerRingNamespace), } } @@ -67,3 +70,7 @@ func (m *InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success func (m *InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { return m.morphCacheMetrics } + +func (m *InnerRingServiceMetrics) LogMetrics() logger.LogMetrics { + return m.logMetrics +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 614300aad..0dd86d90e 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -2,6 +2,7 @@ package metrics import ( morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -22,6 +23,7 @@ type NodeMetrics struct { policer *policerMetrics morphClient *morphClientMetrics morphCache *morphCacheMetrics + log logger.LogMetrics } func NewNodeMetrics() *NodeMetrics { @@ -46,6 +48,7 @@ func NewNodeMetrics() *NodeMetrics { policer: newPolicerMetrics(), morphClient: newMorphClientMetrics(), morphCache: newMorphCacheMetrics(namespace), + log: logger.NewLogMetrics(namespace), } } @@ -109,3 +112,7 @@ func (m *NodeMetrics) MorphClientMetrics() morphmetrics.Register { func (m *NodeMetrics) MorphCacheMetrics() MorphCacheMetrics { return m.morphCache } + +func (m *NodeMetrics) LogMetrics() logger.LogMetrics { + return m.log +} diff --git a/pkg/util/logger/logger.go b/pkg/util/logger/logger.go index cd4297767..f9a5fe38f 100644 --- a/pkg/util/logger/logger.go +++ b/pkg/util/logger/logger.go @@ -31,8 +31,8 @@ type Prm struct { // support runtime rereading level zapcore.Level - // MetricsNamespace is the namespace string used for log counter metrics - MetricsNamespace string + // SamplingHook hook for the zap.Logger + SamplingHook func(e zapcore.Entry, sd zapcore.SamplingDecision) // do not support runtime rereading } @@ -82,14 +82,12 @@ func NewLogger(prm *Prm) (*Logger, error) { lvl := zap.NewAtomicLevelAt(prm.level) - m := NewLogMetrics(prm.MetricsNamespace) - c := zap.NewProductionConfig() c.Level = lvl c.Encoding = "console" c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - c.Sampling.Hook = func(e zapcore.Entry, sd zapcore.SamplingDecision) { - m.Inc(e.Level, sd == zapcore.LogDropped) + if prm.SamplingHook != nil { + c.Sampling.Hook = prm.SamplingHook } lZap, err := c.Build( diff --git a/pkg/util/logger/metrics.go b/pkg/util/logger/metrics.go index 1fa0669a2..7e62e6383 100644 --- a/pkg/util/logger/metrics.go +++ b/pkg/util/logger/metrics.go @@ -14,12 +14,17 @@ const ( logDroppedLabel = "dropped" ) -type LogMetrics struct { +type LogMetrics interface { + Inc(level zapcore.Level, dropped bool) + GetSamplingHook() func(e zapcore.Entry, sd zapcore.SamplingDecision) +} + +type logMetrics struct { logCount *prometheus.CounterVec } -func NewLogMetrics(namespace string) *LogMetrics { - return &LogMetrics{ +func NewLogMetrics(namespace string) LogMetrics { + return &logMetrics{ logCount: metrics.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: logSubsystem, @@ -29,9 +34,15 @@ func NewLogMetrics(namespace string) *LogMetrics { } } -func (m *LogMetrics) Inc(level zapcore.Level, dropped bool) { +func (m *logMetrics) Inc(level zapcore.Level, dropped bool) { m.logCount.With(prometheus.Labels{ logLevelLabel: level.String(), logDroppedLabel: strconv.FormatBool(dropped), }).Inc() } + +func (m *logMetrics) GetSamplingHook() func(zapcore.Entry, zapcore.SamplingDecision) { + return func(e zapcore.Entry, sd zapcore.SamplingDecision) { + m.Inc(e.Level, sd == zapcore.LogDropped) + } +} diff --git a/scripts/export-metrics/main.go b/scripts/export-metrics/main.go index 45b155260..f29eca37c 100644 --- a/scripts/export-metrics/main.go +++ b/scripts/export-metrics/main.go @@ -7,7 +7,6 @@ import ( "os" local_metrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" ) @@ -28,11 +27,9 @@ func main() { switch { case *node != "": _ = local_metrics.NewNodeMetrics() - _ = logger.NewLogMetrics("frostfs_node") filename = *node case *ir != "": _ = local_metrics.NewInnerRingMetrics() - _ = logger.NewLogMetrics("frostfs_ir") filename = *ir default: