[#680] metrics: Initialize log metrics together with services

Signed-off-by: Anton Nikiforov <an.nikiforov@yadro.com>
This commit is contained in:
Anton Nikiforov 2023-10-08 12:32:00 +03:00
parent bf082348d4
commit 3a997d1207
8 changed files with 58 additions and 38 deletions

View file

@ -10,6 +10,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/misc"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring"
irMetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"github.com/spf13/viper"
"go.uber.org/zap"
@ -61,12 +62,13 @@ func main() {
cfg, err = newConfig()
exitErr(err)
logPrm.MetricsNamespace = "frostfs_ir"
metrics := irMetrics.NewInnerRingMetrics()
err = logPrm.SetLevelString(
cfg.GetString("logger.level"),
)
exitErr(err)
logPrm.SamplingHook = metrics.LogMetrics().GetSamplingHook()
log, err = logger.NewLogger(logPrm)
exitErr(err)
@ -78,7 +80,7 @@ func main() {
metricsCmp = newMetricsComponent()
metricsCmp.init()
innerRing, err = innerring.New(ctx, log, cfg, intErr)
innerRing, err = innerring.New(ctx, log, cfg, intErr, metrics)
exitErr(err)
pprofCmp.start()

View file

@ -554,22 +554,21 @@ func initCfg(appCfg *config.Config) *cfg {
key := nodeconfig.Key(appCfg)
relayOnly := nodeconfig.Relay(appCfg)
netState := newNetworkState()
netState.metrics = c.metricsCollector
c.shared = initShared(appCfg, key, netState, relayOnly)
logPrm, err := c.loggerPrm()
fatalOnErr(err)
logPrm.MetricsNamespace = "frostfs_node"
logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook()
log, err := logger.NewLogger(logPrm)
fatalOnErr(err)
c.internals = initInternals(appCfg, log)
relayOnly := nodeconfig.Relay(appCfg)
netState := newNetworkState()
c.shared = initShared(appCfg, key, netState, relayOnly)
c.cfgAccounting = cfgAccounting{
scriptHash: contractsconfig.Balance(appCfg),
}
@ -586,9 +585,6 @@ func initCfg(appCfg *config.Config) *cfg {
user.IDFromKey(&c.ownerIDFromKey, key.PrivateKey.PublicKey)
c.metricsCollector = metrics.NewNodeMetrics()
netState.metrics = c.metricsCollector
c.onShutdown(c.clientCache.CloseAll) // clean up connections
c.onShutdown(c.bgClientCache.CloseAll) // clean up connections
c.onShutdown(c.putClientCache.CloseAll) // clean up connections
@ -638,6 +634,7 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt
bgClientCache: cache.NewSDKClientCache(cacheOpts),
putClientCache: cache.NewSDKClientCache(cacheOpts),
persistate: persistate,
metricsCollector: metrics.NewNodeMetrics(),
}
}

View file

@ -325,11 +325,12 @@ func (s *Server) registerStarter(f func() error) {
}
// New creates instance of inner ring sever structure.
func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error) (*Server, error) {
func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error,
metrics *metrics.InnerRingServiceMetrics) (*Server, error) {
var err error
server := &Server{
log: log,
irMetrics: metrics.NewInnerRingMetrics(),
irMetrics: metrics,
}
server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED)

View file

@ -4,6 +4,7 @@ import (
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
"github.com/prometheus/client_golang/prometheus"
)
@ -14,6 +15,7 @@ type InnerRingServiceMetrics struct {
health prometheus.Gauge
eventDuration *prometheus.HistogramVec
morphCacheMetrics *morphCacheMetrics
logMetrics logger.LogMetrics
}
// NewInnerRingMetrics returns new instance of metrics collectors for inner ring.
@ -44,6 +46,7 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics {
health: health,
eventDuration: eventDuration,
morphCacheMetrics: newMorphCacheMetrics(innerRingNamespace),
logMetrics: logger.NewLogMetrics(innerRingNamespace),
}
}
@ -67,3 +70,7 @@ func (m *InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success
func (m *InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics {
return m.morphCacheMetrics
}
func (m *InnerRingServiceMetrics) LogMetrics() logger.LogMetrics {
return m.logMetrics
}

View file

@ -2,6 +2,7 @@ package metrics
import (
morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
"github.com/prometheus/client_golang/prometheus"
)
@ -22,6 +23,7 @@ type NodeMetrics struct {
policer *policerMetrics
morphClient *morphClientMetrics
morphCache *morphCacheMetrics
log logger.LogMetrics
}
func NewNodeMetrics() *NodeMetrics {
@ -46,6 +48,7 @@ func NewNodeMetrics() *NodeMetrics {
policer: newPolicerMetrics(),
morphClient: newMorphClientMetrics(),
morphCache: newMorphCacheMetrics(namespace),
log: logger.NewLogMetrics(namespace),
}
}
@ -109,3 +112,7 @@ func (m *NodeMetrics) MorphClientMetrics() morphmetrics.Register {
func (m *NodeMetrics) MorphCacheMetrics() MorphCacheMetrics {
return m.morphCache
}
func (m *NodeMetrics) LogMetrics() logger.LogMetrics {
return m.log
}

View file

@ -31,8 +31,8 @@ type Prm struct {
// support runtime rereading
level zapcore.Level
// MetricsNamespace is the namespace string used for log counter metrics
MetricsNamespace string
// SamplingHook hook for the zap.Logger
SamplingHook func(e zapcore.Entry, sd zapcore.SamplingDecision)
// do not support runtime rereading
}
@ -82,14 +82,12 @@ func NewLogger(prm *Prm) (*Logger, error) {
lvl := zap.NewAtomicLevelAt(prm.level)
m := NewLogMetrics(prm.MetricsNamespace)
c := zap.NewProductionConfig()
c.Level = lvl
c.Encoding = "console"
c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
c.Sampling.Hook = func(e zapcore.Entry, sd zapcore.SamplingDecision) {
m.Inc(e.Level, sd == zapcore.LogDropped)
if prm.SamplingHook != nil {
c.Sampling.Hook = prm.SamplingHook
}
lZap, err := c.Build(

View file

@ -14,12 +14,17 @@ const (
logDroppedLabel = "dropped"
)
type LogMetrics struct {
type LogMetrics interface {
Inc(level zapcore.Level, dropped bool)
GetSamplingHook() func(e zapcore.Entry, sd zapcore.SamplingDecision)
}
type logMetrics struct {
logCount *prometheus.CounterVec
}
func NewLogMetrics(namespace string) *LogMetrics {
return &LogMetrics{
func NewLogMetrics(namespace string) LogMetrics {
return &logMetrics{
logCount: metrics.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: logSubsystem,
@ -29,9 +34,15 @@ func NewLogMetrics(namespace string) *LogMetrics {
}
}
func (m *LogMetrics) Inc(level zapcore.Level, dropped bool) {
func (m *logMetrics) Inc(level zapcore.Level, dropped bool) {
m.logCount.With(prometheus.Labels{
logLevelLabel: level.String(),
logDroppedLabel: strconv.FormatBool(dropped),
}).Inc()
}
func (m *logMetrics) GetSamplingHook() func(zapcore.Entry, zapcore.SamplingDecision) {
return func(e zapcore.Entry, sd zapcore.SamplingDecision) {
m.Inc(e.Level, sd == zapcore.LogDropped)
}
}

View file

@ -7,7 +7,6 @@ import (
"os"
local_metrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
)
@ -28,11 +27,9 @@ func main() {
switch {
case *node != "":
_ = local_metrics.NewNodeMetrics()
_ = logger.NewLogMetrics("frostfs_node")
filename = *node
case *ir != "":
_ = local_metrics.NewInnerRingMetrics()
_ = logger.NewLogMetrics("frostfs_ir")
filename = *ir
default: