From 4f26aba3f213bffd69d7ee13f37494d0ac7d0500 Mon Sep 17 00:00:00 2001 From: Dmitrii Stepanov Date: Tue, 15 Oct 2024 14:46:00 +0300 Subject: [PATCH] [#1422] multinet: Add metrics Signed-off-by: Dmitrii Stepanov --- cmd/frostfs-node/config.go | 11 +++++++---- internal/metrics/consts.go | 2 ++ internal/metrics/innerring.go | 6 ++++++ internal/metrics/multinet.go | 33 +++++++++++++++++++++++++++++++++ internal/metrics/node.go | 6 ++++++ internal/net/config.go | 3 +++ internal/net/event_handler.go | 33 +++++++++++++++++++++++++++++++++ pkg/innerring/initialization.go | 1 + pkg/innerring/innerring.go | 6 ++++-- 9 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 internal/metrics/multinet.go create mode 100644 internal/net/event_handler.go diff --git a/cmd/frostfs-node/config.go b/cmd/frostfs-node/config.go index d44597857..9d2b77210 100644 --- a/cmd/frostfs-node/config.go +++ b/cmd/frostfs-node/config.go @@ -764,7 +764,9 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt persistate, err := state.NewPersistentStorage(nodeconfig.PersistentState(appCfg).Path()) fatalOnErr(err) - ds, err := internalNet.NewDialerSource(internalNetConfig(appCfg)) + nodeMetrics := metrics.NewNodeMetrics() + + ds, err := internalNet.NewDialerSource(internalNetConfig(appCfg, nodeMetrics.MultinetMetrics())) fatalOnErr(err) cacheOpts := cache.ClientCacheOpts{ @@ -785,17 +787,18 @@ func initShared(appCfg *config.Config, key *keys.PrivateKey, netState *networkSt bgClientCache: cache.NewSDKClientCache(cacheOpts), putClientCache: cache.NewSDKClientCache(cacheOpts), persistate: persistate, - metricsCollector: metrics.NewNodeMetrics(), + metricsCollector: nodeMetrics, dialerSource: ds, } } -func internalNetConfig(appCfg *config.Config) internalNet.Config { +func internalNetConfig(appCfg *config.Config, m metrics.MultinetMetrics) internalNet.Config { result := internalNet.Config{ Enabled: multinet.Enabled(appCfg), Balancer: multinet.Balancer(appCfg), Restrict: multinet.Restrict(appCfg), FallbackDelay: multinet.FallbackDelay(appCfg), + Metrics: m, } sn := multinet.Subnets(appCfg) for _, s := range sn { @@ -1362,7 +1365,7 @@ func (c *cfg) reloadConfig(ctx context.Context) { } } - if err := c.dialerSource.Update(internalNetConfig(c.appCfg)); err != nil { + if err := c.dialerSource.Update(internalNetConfig(c.appCfg, c.metricsCollector.MultinetMetrics())); err != nil { c.log.Error(logs.FailedToUpdateMultinetConfiguration, zap.Error(err)) return } diff --git a/internal/metrics/consts.go b/internal/metrics/consts.go index 3aa51c0f0..cb165de69 100644 --- a/internal/metrics/consts.go +++ b/internal/metrics/consts.go @@ -22,6 +22,7 @@ const ( grpcServerSubsystem = "grpc_server" policerSubsystem = "policer" commonCacheSubsystem = "common_cache" + multinetSubsystem = "multinet" successLabel = "success" shardIDLabel = "shard_id" @@ -41,6 +42,7 @@ const ( endpointLabel = "endpoint" hitLabel = "hit" cacheLabel = "cache" + sourceIPLabel = "source_ip" readWriteMode = "READ_WRITE" readOnlyMode = "READ_ONLY" diff --git a/internal/metrics/innerring.go b/internal/metrics/innerring.go index f3f529d05..d0cb8131f 100644 --- a/internal/metrics/innerring.go +++ b/internal/metrics/innerring.go @@ -17,6 +17,7 @@ type InnerRingServiceMetrics struct { eventDuration *prometheus.HistogramVec morphCacheMetrics *morphCacheMetrics logMetrics logger.LogMetrics + multinet *multinetMetrics // nolint: unused appInfo *ApplicationInfo } @@ -51,6 +52,7 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { morphCacheMetrics: newMorphCacheMetrics(innerRingNamespace), appInfo: NewApplicationInfo(misc.Version), logMetrics: logger.NewLogMetrics(innerRingNamespace), + multinet: newMultinetMetrics(innerRingNamespace), } } @@ -78,3 +80,7 @@ func (m *InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { func (m *InnerRingServiceMetrics) LogMetrics() logger.LogMetrics { return m.logMetrics } + +func (m *InnerRingServiceMetrics) Multinet() MultinetMetrics { + return m.multinet +} diff --git a/internal/metrics/multinet.go b/internal/metrics/multinet.go new file mode 100644 index 000000000..e103faab2 --- /dev/null +++ b/internal/metrics/multinet.go @@ -0,0 +1,33 @@ +package metrics + +import ( + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +type multinetMetrics struct { + dials *prometheus.GaugeVec +} + +type MultinetMetrics interface { + Dial(sourceIP, status string) +} + +func newMultinetMetrics(ns string) *multinetMetrics { + return &multinetMetrics{ + dials: metrics.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: ns, + Subsystem: multinetSubsystem, + Name: "dial_count", + Help: "Dials count performed by multinet", + }, []string{sourceIPLabel, statusLabel}), + } +} + +func (m *multinetMetrics) Dial(sourceIP, status string) { + m.dials.With(prometheus.Labels{ + sourceIPLabel: sourceIP, + statusLabel: status, + }).Inc() +} diff --git a/internal/metrics/node.go b/internal/metrics/node.go index 711387875..4ea3c7c24 100644 --- a/internal/metrics/node.go +++ b/internal/metrics/node.go @@ -25,6 +25,7 @@ type NodeMetrics struct { morphClient *morphClientMetrics morphCache *morphCacheMetrics log logger.LogMetrics + multinet *multinetMetrics // nolint: unused appInfo *ApplicationInfo } @@ -53,6 +54,7 @@ func NewNodeMetrics() *NodeMetrics { morphCache: newMorphCacheMetrics(namespace), log: logger.NewLogMetrics(namespace), appInfo: NewApplicationInfo(misc.Version), + multinet: newMultinetMetrics(namespace), } } @@ -120,3 +122,7 @@ func (m *NodeMetrics) MorphCacheMetrics() MorphCacheMetrics { func (m *NodeMetrics) LogMetrics() logger.LogMetrics { return m.log } + +func (m *NodeMetrics) MultinetMetrics() MultinetMetrics { + return m.multinet +} diff --git a/internal/net/config.go b/internal/net/config.go index 10450db23..b84ac3b35 100644 --- a/internal/net/config.go +++ b/internal/net/config.go @@ -7,6 +7,7 @@ import ( "slices" "time" + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" "git.frostfs.info/TrueCloudLab/multinet" ) @@ -23,6 +24,7 @@ type Config struct { Balancer string Restrict bool FallbackDelay time.Duration + Metrics metrics.MultinetMetrics } func (c Config) toMultinetConfig() (multinet.Config, error) { @@ -52,6 +54,7 @@ func (c Config) toMultinetConfig() (multinet.Config, error) { Restrict: c.Restrict, FallbackDelay: c.FallbackDelay, Dialer: newDefaulDialer(), + EventHandler: newEventHandler(c.Metrics), }, nil } diff --git a/internal/net/event_handler.go b/internal/net/event_handler.go new file mode 100644 index 000000000..a7544652e --- /dev/null +++ b/internal/net/event_handler.go @@ -0,0 +1,33 @@ +package net + +import ( + "net" + + "git.frostfs.info/TrueCloudLab/frostfs-node/internal/metrics" + "git.frostfs.info/TrueCloudLab/multinet" +) + +var _ multinet.EventHandler = (*metricsEventHandler)(nil) + +type metricsEventHandler struct { + m metrics.MultinetMetrics +} + +func (m *metricsEventHandler) DialPerformed(sourceIP net.Addr, _ string, _ string, err error) { + sourceIPString := "undefined" + if sourceIP != nil { + sourceIPString = sourceIP.Network() + "://" + sourceIP.String() + } + statusString := "OK" + if err != nil { + statusString = "FAILED" + } + m.m.Dial(sourceIPString, statusString) +} + +func newEventHandler(m metrics.MultinetMetrics) multinet.EventHandler { + if m == nil { + return nil + } + return &metricsEventHandler{m: m} +} diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index c4aaeda56..cb0654b6e 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -463,6 +463,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- name: morphPrefix, from: fromSideChainBlock, morphCacheMetric: s.irMetrics.MorphCacheMetrics(), + multinetMetrics: s.irMetrics.Multinet(), } // create morph client diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index a4a52edec..b94312645 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -117,6 +117,7 @@ type ( sgn *transaction.Signer from uint32 // block height morphCacheMetric metrics.MorphCacheMetrics + multinetMetrics metrics.MultinetMetrics } ) @@ -487,7 +488,7 @@ func createClient(ctx context.Context, p *chainParams, errChan chan<- error) (*c return nil, fmt.Errorf("%s chain client endpoints not provided", p.name) } - nc := parseMultinetConfig(p.cfg) + nc := parseMultinetConfig(p.cfg, p.multinetMetrics) ds, err := internalNet.NewDialerSource(nc) if err != nil { return nil, fmt.Errorf("dialer source: %w", err) @@ -550,12 +551,13 @@ func parseWalletAddressesFromStrings(wallets []string) ([]util.Uint160, error) { return extraWallets, nil } -func parseMultinetConfig(cfg *viper.Viper) internalNet.Config { +func parseMultinetConfig(cfg *viper.Viper, m metrics.MultinetMetrics) internalNet.Config { nc := internalNet.Config{ Enabled: cfg.GetBool("multinet.enabled"), Balancer: cfg.GetString("multinet.balancer"), Restrict: cfg.GetBool("multinet.restrict"), FallbackDelay: cfg.GetDuration("multinet.fallback_delay"), + Metrics: m, } for i := 0; ; i++ { mask := cfg.GetString(fmt.Sprintf("multinet.subnets.%d.mask", i))