TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 103 Pull requests 5 Projects Releases 2 Activity Actions

metrics: Export log and morph with script #718

Merged
fyrchik merged 4 commits from acid-ant/frostfs-node:bugfix/680-export-metrics into master 2024-09-04 19:51:03 +00:00
Conversation 7 Commits 4 Files changed 12 +78 -41
12 changed files with 78 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Makefile
View file

@ -70,6 +70,12 @@ dep:
CGO_ENABLED=0 \
go mod tidy -v && echo OK
# Build export-metrics
export-metrics: dep
@printf "⇒ Build export-metrics\n"
CGO_ENABLED=0 \
go build -v -trimpath -o bin/export-metrics ./scripts/export-metrics
# Regenerate proto files:
protoc:
@GOPRIVATE=github.com/TrueCloudLab go mod vendor

8
cmd/frostfs-ir/main.go
View file

@ -10,6 +10,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/misc"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/innerring"
irMetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"github.com/spf13/viper"
"go.uber.org/zap"
@ -61,12 +62,13 @@ func main() {
cfg, err = newConfig()
exitErr(err)
logPrm.MetricsNamespace = "frostfs_ir"
metrics := irMetrics.NewInnerRingMetrics()
err = logPrm.SetLevelString(
cfg.GetString("logger.level"),
)
exitErr(err)
logPrm.SamplingHook = metrics.LogMetrics().GetSamplingHook()
log, err = logger.NewLogger(logPrm)
exitErr(err)
@ -78,7 +80,7 @@ func main() {
metricsCmp = newMetricsComponent()
metricsCmp.init()
innerRing, err = innerring.New(ctx, log, cfg, intErr)
innerRing, err = innerring.New(ctx, log, cfg, intErr, metrics)
exitErr(err)
pprofCmp.start()

37
cmd/frostfs-node/config.go
View file

@ -554,22 +554,21 @@ func initCfg(appCfg *config.Config) *cfg {
key := nodeconfig.Key(appCfg)
relayOnly := nodeconfig.Relay(appCfg)
netState := newNetworkState()
netState.metrics = c.metricsCollector
c.shared = initShared(appCfg, key, netState, relayOnly)
logPrm, err := c.loggerPrm()
fatalOnErr(err)
logPrm.MetricsNamespace = "frostfs_node"
logPrm.SamplingHook = c.metricsCollector.LogMetrics().GetSamplingHook()
log, err := logger.NewLogger(logPrm)
fatalOnErr(err)
c.internals = initInternals(appCfg, log)
relayOnly := nodeconfig.Relay(appCfg)
netState := newNetworkState()
c.shared = initShared(appCfg, key, netState, relayOnly)
c.cfgAccounting = cfgAccounting{
scriptHash: contractsconfig.Balance(appCfg),
}
@ -586,9 +585,6 @@ func initCfg(appCfg *config.Config) *cfg {
user.IDFromKey(&c.ownerIDFromKey, key.PrivateKey.PublicKey)
c.metricsCollector = metrics.NewNodeMetrics()
netState.metrics = c.metricsCollector
c.onShutdown(c.clientCache.CloseAll) // clean up connections
c.onShutdown(c.bgClientCache.CloseAll) // clean up connections
c.onShutdown(c.putClientCache.CloseAll) // clean up connections
@ -630,14 +626,15 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt
}
return shared{
key: key,
binPublicKey: key.PublicKey().Bytes(),
localAddr: netAddr,
respSvc: response.NewService(netState),
clientCache: cache.NewSDKClientCache(cacheOpts),
bgClientCache: cache.NewSDKClientCache(cacheOpts),
putClientCache: cache.NewSDKClientCache(cacheOpts),
persistate: persistate,
key: key,
binPublicKey: key.PublicKey().Bytes(),
localAddr: netAddr,
respSvc: response.NewService(netState),
clientCache: cache.NewSDKClientCache(cacheOpts),
bgClientCache: cache.NewSDKClientCache(cacheOpts),
putClientCache: cache.NewSDKClientCache(cacheOpts),
persistate: persistate,
metricsCollector: metrics.NewNodeMetrics(),
}
}

5
cmd/frostfs-node/morph.go
View file

@ -9,7 +9,6 @@ import (
morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph"
"git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client"
nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event"
@ -42,13 +41,13 @@ func initMorphComponents(ctx context.Context, c *cfg) {
c.key,
client.WithDialTimeout(morphconfig.DialTimeout(c.appCfg)),
client.WithLogger(c.log),
client.WithMetrics(metrics.NewMorphClientMetrics()),
client.WithMetrics(c.metricsCollector.MorphClientMetrics()),
client.WithEndpoints(addresses...),
client.WithConnLostCallback(func() {
c.internalErr <- errors.New("morph connection has been lost")
}),
client.WithSwitchInterval(morphconfig.SwitchInterval(c.appCfg)),
client.WithMorphCacheMetrics(metrics.NewNodeMorphCacheMetrics()),
client.WithMorphCacheMetrics(c.metricsCollector.MorphCacheMetrics()),
)
if err != nil {
c.log.Info(logs.FrostFSNodeFailedToCreateNeoRPCClient,

1
cmd/frostfs-node/object.go
View file

@ -260,6 +260,7 @@ func addPolicer(c *cfg, keyStorage *util.KeyStorage, clientConstructor *cache.Cl
}),
policer.WithMaxCapacity(c.cfgObject.pool.replicatorPoolSize),
policer.WithPool(c.cfgObject.pool.replication),
policer.WithMetrics(c.metricsCollector.PolicerMetrics()),
)
c.workers = append(c.workers, worker{

5
pkg/innerring/innerring.go
View file

@ -325,11 +325,12 @@ func (s *Server) registerStarter(f func() error) {
}
// New creates instance of inner ring sever structure.
func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error) (*Server, error) {
func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error,
metrics *metrics.InnerRingServiceMetrics) (*Server, error) {
var err error
server := &Server{
log: log,
irMetrics: metrics.NewInnerRingMetrics(),
irMetrics: metrics,
}
server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED)

7
pkg/metrics/innerring.go
View file

@ -4,6 +4,7 @@ import (
"strconv"
"time"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
"github.com/prometheus/client_golang/prometheus"
)
@ -14,6 +15,7 @@ type InnerRingServiceMetrics struct {
health prometheus.Gauge
eventDuration *prometheus.HistogramVec
morphCacheMetrics *morphCacheMetrics
logMetrics logger.LogMetrics
}
// NewInnerRingMetrics returns new instance of metrics collectors for inner ring.
@ -44,6 +46,7 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics {
health: health,
eventDuration: eventDuration,
morphCacheMetrics: newMorphCacheMetrics(innerRingNamespace),
logMetrics: logger.NewLogMetrics(innerRingNamespace),
}
}
@ -67,3 +70,7 @@ func (m *InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success
func (m *InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics {
return m.morphCacheMetrics
}
func (m *InnerRingServiceMetrics) LogMetrics() logger.LogMetrics {
return m.logMetrics
}

3
pkg/metrics/morph.go
View file

@ -4,7 +4,6 @@ import (
"strconv"
"time"
morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
"github.com/prometheus/client_golang/prometheus"
)
@ -16,7 +15,7 @@ type morphClientMetrics struct {
invokeDuration *prometheus.HistogramVec
}
func NewMorphClientMetrics() morphmetrics.Register {
func newMorphClientMetrics() *morphClientMetrics {
return &morphClientMetrics{
switchCount: metrics.NewCounter(prometheus.CounterOpts{
Namespace: namespace,

4
pkg/metrics/morphcache.go
View file

@ -18,10 +18,6 @@ type morphCacheMetrics struct {
var _ MorphCacheMetrics = (*morphCacheMetrics)(nil)
func NewNodeMorphCacheMetrics() MorphCacheMetrics {
return newMorphCacheMetrics(namespace)
}
func newMorphCacheMetrics(ns string) *morphCacheMetrics {
return &morphCacheMetrics{
methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{

20
pkg/metrics/node.go
View file

@ -1,6 +1,8 @@
package metrics
import (
morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics"
"git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger"
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
"github.com/prometheus/client_golang/prometheus"
)
@ -19,6 +21,9 @@ type NodeMetrics struct {
pilorama *piloramaMetrics
grpc *grpcServerMetrics
policer *policerMetrics
morphClient *morphClientMetrics
morphCache *morphCacheMetrics
log logger.LogMetrics
}
func NewNodeMetrics() *NodeMetrics {
@ -41,6 +46,9 @@ func NewNodeMetrics() *NodeMetrics {
pilorama: newPiloramaMetrics(),
grpc: newGrpcServerMetrics(),
policer: newPolicerMetrics(),
morphClient: newMorphClientMetrics(),
morphCache: newMorphCacheMetrics(namespace),
log: logger.NewLogMetrics(namespace),
}
}
@ -96,3 +104,15 @@ func (m *NodeMetrics) GrpcServerMetrics() GrpcServerMetrics {
func (m *NodeMetrics) PolicerMetrics() PolicerMetrics {
return m.policer
}
func (m *NodeMetrics) MorphClientMetrics() morphmetrics.Register {
return m.morphClient
}
func (m *NodeMetrics) MorphCacheMetrics() MorphCacheMetrics {
return m.morphCache
}
func (m *NodeMetrics) LogMetrics() logger.LogMetrics {
return m.log
}

10
pkg/util/logger/logger.go
View file

@ -31,8 +31,8 @@ type Prm struct {
// support runtime rereading
level zapcore.Level
// MetricsNamespace is the namespace string used for log counter metrics
MetricsNamespace string
// SamplingHook hook for the zap.Logger
SamplingHook func(e zapcore.Entry, sd zapcore.SamplingDecision)
// do not support runtime rereading
}
@ -82,14 +82,12 @@ func NewLogger(prm *Prm) (*Logger, error) {
lvl := zap.NewAtomicLevelAt(prm.level)
m := newLogMetrics(prm.MetricsNamespace)
c := zap.NewProductionConfig()
c.Level = lvl
c.Encoding = "console"
c.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
c.Sampling.Hook = func(e zapcore.Entry, sd zapcore.SamplingDecision) {
m.Inc(e.Level, sd == zapcore.LogDropped)
if prm.SamplingHook != nil {
c.Sampling.Hook = prm.SamplingHook
}
lZap, err := c.Build(

13
pkg/util/logger/metrics.go
View file

@ -14,11 +14,16 @@ const (
logDroppedLabel = "dropped"
)
type LogMetrics interface {
Inc(level zapcore.Level, dropped bool)
GetSamplingHook() func(e zapcore.Entry, sd zapcore.SamplingDecision)
}
type logMetrics struct {
logCount *prometheus.CounterVec
}
func newLogMetrics(namespace string) *logMetrics {
func NewLogMetrics(namespace string) LogMetrics {
return &logMetrics{
logCount: metrics.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
@ -35,3 +40,9 @@ func (m *logMetrics) Inc(level zapcore.Level, dropped bool) {
logDroppedLabel: strconv.FormatBool(dropped),
}).Inc()
}
func (m *logMetrics) GetSamplingHook() func(zapcore.Entry, zapcore.SamplingDecision) {
return func(e zapcore.Entry, sd zapcore.SamplingDecision) {
m.Inc(e.Level, sd == zapcore.LogDropped)
}
}