From 4887f489a195418340992eca63d609b293c26e6f Mon Sep 17 00:00:00 2001 From: Alejandro Lopez Date: Thu, 8 Jun 2023 16:37:46 +0300 Subject: [PATCH] [#17] Add morph client metrics Signed-off-by: Alejandro Lopez --- cmd/frostfs-node/morph.go | 2 + pkg/innerring/initialization.go | 14 ++--- pkg/innerring/innerring.go | 6 +-- pkg/innerring/state.go | 8 +-- pkg/metrics/morph.go | 82 ++++++++++++++++++++++++++++++ pkg/morph/client/client.go | 30 ++++++++++- pkg/morph/client/constructor.go | 20 ++++++++ pkg/morph/client/notary.go | 7 +++ pkg/morph/metrics/metrics.go | 17 +++++++ pkg/morph/subscriber/subscriber.go | 5 ++ pkg/services/tree/metrics.go | 6 +++ 11 files changed, 182 insertions(+), 15 deletions(-) create mode 100644 pkg/metrics/morph.go create mode 100644 pkg/morph/metrics/metrics.go diff --git a/cmd/frostfs-node/morph.go b/cmd/frostfs-node/morph.go index 2e086f994..ae50b8a8a 100644 --- a/cmd/frostfs-node/morph.go +++ b/cmd/frostfs-node/morph.go @@ -9,6 +9,7 @@ import ( morphconfig "git.frostfs.info/TrueCloudLab/frostfs-node/cmd/frostfs-node/config/morph" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/core/netmap" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client" nmClient "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/client/netmap" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/event" @@ -43,6 +44,7 @@ func initMorphComponents(ctx context.Context, c *cfg) { c.key, client.WithDialTimeout(morphconfig.DialTimeout(c.appCfg)), client.WithLogger(c.log), + client.WithMetrics(metrics.NewMorphClientMetrics()), client.WithEndpoints(addresses...), client.WithConnLostCallback(func() { c.internalErr <- errors.New("morph connection has been lost") diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index 05e503f2f..52ffb10a1 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -52,7 +52,7 @@ func (s *Server) initNetmapProcessor(cfg *viper.Viper, s.netmapProcessor, err = netmap.New(&netmap.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.netmap"), NetmapClient: netmap.NewNetmapClient(s.netmapClient), EpochTimer: s, @@ -163,7 +163,7 @@ func (s *Server) createAlphaSync(cfg *viper.Viper, frostfsCli *frostfsClient.Cli // create governance processor governanceProcessor, err := governance.New(&governance.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, FrostFSClient: frostfsCli, NetmapClient: s.netmapClient, AlphabetState: s, @@ -233,7 +233,7 @@ func (s *Server) initAlphabetProcessor(cfg *viper.Viper) error { s.alphabetProcessor, err = alphabet.New(&alphabet.Params{ ParsedWallets: parsedWallets, Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.alphabet"), AlphabetContracts: s.contracts.alphabet, NetmapClient: s.netmapClient, @@ -258,7 +258,7 @@ func (s *Server) initContainerProcessor(cfg *viper.Viper, cnrClient *container.C // container processor containerProcessor, err := cont.New(&cont.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.container"), AlphabetState: s, ContainerClient: cnrClient, @@ -277,7 +277,7 @@ func (s *Server) initBalanceProcessor(cfg *viper.Viper, frostfsCli *frostfsClien // create balance processor balanceProcessor, err := balance.New(&balance.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.balance"), FrostFSClient: frostfsCli, BalanceSC: s.contracts.balance, @@ -298,7 +298,7 @@ func (s *Server) initFrostFSMainnetProcessor(cfg *viper.Viper, frostfsIDClient * frostfsProcessor, err := frostfs.New(&frostfs.Params{ Log: s.log, - Metrics: s.metrics, + Metrics: s.irMetrics, PoolSize: cfg.GetInt("workers.frostfs"), FrostFSContract: s.contracts.frostfs, FrostFSIDClient: frostfsIDClient, @@ -474,7 +474,7 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- key: s.key, name: morphPrefix, from: fromSideChainBlock, - morphCacheMetric: s.metrics.MorphCacheMetrics(), + morphCacheMetric: s.irMetrics.MorphCacheMetrics(), } // create morph client diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index 335d3d179..1567e40d3 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -58,7 +58,7 @@ type ( persistate *state.PersistentStorage // metrics - metrics *metrics.InnerRingServiceMetrics + irMetrics *metrics.InnerRingServiceMetrics // notary configuration feeConfig *config.FeeConfig @@ -328,8 +328,8 @@ func (s *Server) registerStarter(f func() error) { func New(ctx context.Context, log *logger.Logger, cfg *viper.Viper, errChan chan<- error) (*Server, error) { var err error server := &Server{ - log: log, - metrics: metrics.NewInnerRingMetrics(), + log: log, + irMetrics: metrics.NewInnerRingMetrics(), } server.setHealthStatus(control.HealthStatus_HEALTH_STATUS_UNDEFINED) diff --git a/pkg/innerring/state.go b/pkg/innerring/state.go index 27f265ae2..c5adb71eb 100644 --- a/pkg/innerring/state.go +++ b/pkg/innerring/state.go @@ -29,8 +29,8 @@ func (s *Server) EpochCounter() uint64 { // epoch counter. func (s *Server) SetEpochCounter(val uint64) { s.epochCounter.Store(val) - if s.metrics != nil { - s.metrics.SetEpoch(val) + if s.irMetrics != nil { + s.irMetrics.SetEpoch(val) } } @@ -154,8 +154,8 @@ func (s *Server) ResetEpochTimer(h uint32) error { func (s *Server) setHealthStatus(hs control.HealthStatus) { s.healthStatus.Store(int32(hs)) - if s.metrics != nil { - s.metrics.SetHealth(int32(hs)) + if s.irMetrics != nil { + s.irMetrics.SetHealth(int32(hs)) } } diff --git a/pkg/metrics/morph.go b/pkg/metrics/morph.go new file mode 100644 index 000000000..cc851d3db --- /dev/null +++ b/pkg/metrics/morph.go @@ -0,0 +1,82 @@ +package metrics + +import ( + "strconv" + "time" + + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + morphSubsystem = "morph" + morphNotificationTypeLabel = "notification_type" + morphInvokeTypeLabel = "invoke_type" + morphContractLabel = "contract" + morphMethodLabel = "method" + morphSuccessLabel = "success" +) + +type morphClientMetrics struct { + switchCount prometheus.Counter + lastBlock prometheus.Gauge + notificationCount *prometheus.CounterVec + invokeDuration *prometheus.HistogramVec +} + +func NewMorphClientMetrics() morphmetrics.Register { + return &morphClientMetrics{ + switchCount: metrics.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "switch_count", + Help: "Number of endpoint switches", + }), + lastBlock: metrics.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "last_block", + Help: "Index of the last received block", + }), + notificationCount: metrics.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "notification_count", + Help: "Number of notifications received by notification type", + }, []string{morphNotificationTypeLabel}), + invokeDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: morphSubsystem, + Name: "invoke_duration_seconds", + Help: "Cummulative duration of contract invocations", + }, []string{morphInvokeTypeLabel, morphContractLabel, morphMethodLabel, morphSuccessLabel}), + } +} + +func (m *morphClientMetrics) IncSwitchCount() { + m.switchCount.Inc() +} + +func (m *morphClientMetrics) SetLastBlock(index uint32) { + m.lastBlock.Set(float64(index)) +} + +func (m *morphClientMetrics) IncNotificationCount(typ string) { + m.notificationCount.With( + prometheus.Labels{ + morphNotificationTypeLabel: typ, + }, + ).Inc() +} + +func (m *morphClientMetrics) ObserveInvoke(typ string, contract string, method string, success bool, d time.Duration) { + m.invokeDuration.With( + prometheus.Labels{ + morphInvokeTypeLabel: typ, + morphContractLabel: contract, + morphMethodLabel: method, + morphSuccessLabel: strconv.FormatBool(success), + }, + ).Observe(d.Seconds()) +} diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index f03ee1dbf..606f3bd66 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -11,6 +11,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" @@ -48,7 +49,8 @@ import ( type Client struct { cache cache - logger *logger.Logger // logging component + logger *logger.Logger // logging component + metrics morphmetrics.Register client *rpcclient.WSClient // neo-go websocket client rpcActor *actor.Actor // neo-go RPC actor @@ -172,6 +174,12 @@ func wrapFrostFSError(err error) error { // Invoke invokes contract method by sending transaction into blockchain. // Supported args types: int64, string, util.Uint160, []byte and bool. func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, args ...any) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("Invoke", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -189,6 +197,7 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, zap.Uint32("vub", vub), zap.Stringer("tx_hash", txHash.Reverse())) + success = true return nil } @@ -196,6 +205,12 @@ func (c *Client) Invoke(contract util.Uint160, fee fixedn.Fixed8, method string, // If cb returns an error, the session is closed and this error is returned as-is. // If the remove neo-go node does not support sessions, `unwrap.ErrNoSessionID` is returned. func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, contract util.Uint160, method string, args ...interface{}) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("TestInvokeIterator", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -228,12 +243,20 @@ func (c *Client) TestInvokeIterator(cb func(stackitem.Item) error, contract util } items, err = c.rpcActor.TraverseIterator(sid, &r, 0) } + + success = err == nil return err } // TestInvoke invokes contract method locally in neo-go node. This method should // be used to read data from smart-contract. func (c *Client) TestInvoke(contract util.Uint160, method string, args ...any) (res []stackitem.Item, err error) { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("TestInvoke", contract.String(), method, success, time.Since(start)) + }() + c.switchLock.RLock() defer c.switchLock.RUnlock() @@ -250,6 +273,7 @@ func (c *Client) TestInvoke(contract util.Uint160, method string, args ...any) ( return nil, wrapFrostFSError(¬HaltStateError{state: val.State, exception: val.FaultException}) } + success = true return val.Stack, nil } @@ -512,6 +536,10 @@ func (c *Client) NotificationChannel() <-chan rpcclient.Notification { return c.client.Notifications //lint:ignore SA1019 waits for neo-go v0.102.0 https://github.com/nspcc-dev/neo-go/pull/2980 } +func (c *Client) Metrics() morphmetrics.Register { + return c.metrics +} + func (c *Client) setActor(act *actor.Actor) { c.rpcActor = act c.gasToken = nep17.New(act, gas.Hash) diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 8b5fb3ff0..6cc66c5cc 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -8,6 +8,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" + morphmetrics "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/morph/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -32,6 +33,8 @@ type cfg struct { logger *logger.Logger // logging component + metrics morphmetrics.Register + waitInterval time.Duration signer *transaction.Signer @@ -60,6 +63,7 @@ func defaultConfig() *cfg { return &cfg{ dialTimeout: defaultDialTimeout, logger: &logger.Logger{Logger: zap.L()}, + metrics: morphmetrics.NoopRegister{}, waitInterval: defaultWaitInterval, signer: &transaction.Signer{ Scopes: transaction.Global, @@ -80,6 +84,7 @@ func defaultConfig() *cfg { // - signer with the global scope; // - wait interval: 500ms; // - logger: &logger.Logger{Logger: zap.L()}. +// - metrics: metrics.NoopRegister // // If desired option satisfies the default value, it can be omitted. // If multiple options of the same config value are supplied, @@ -109,6 +114,7 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er cli := &Client{ cache: newClientCache(cfg.morphCacheMetrics), logger: cfg.logger, + metrics: cfg.metrics, acc: acc, accAddr: accAddr, cfg: *cfg, @@ -235,6 +241,20 @@ func WithLogger(logger *logger.Logger) Option { } } +// WithMetrics returns a client constructor option +// that specifies the component for reporting metrics. +// +// Ignores nil value. +// +// If option not provided, NoopMetrics is used. +func WithMetrics(metrics morphmetrics.Register) Option { + return func(c *cfg) { + if metrics != nil { + c.metrics = metrics + } + } +} + // WithSigner returns a client constructor option // that specifies the signer and the scope of the transaction. // diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 3a3ff0b46..680ba6a68 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -444,6 +444,12 @@ func (c *Client) notaryInvokeAsCommittee(method string, nonce, vub uint32, args } func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint160, nonce uint32, vub *uint32, method string, args ...any) error { + start := time.Now() + success := false + defer func() { + c.metrics.ObserveInvoke("notaryInvoke", contract.String(), method, success, time.Since(start)) + }() + alphabetList, err := c.notary.alphabetSource() if err != nil { return err @@ -485,6 +491,7 @@ func (c *Client) notaryInvoke(committee, invokedByAlpha bool, contract util.Uint zap.String("tx_hash", mainH.StringLE()), zap.String("fallback_hash", fbH.StringLE())) + success = true return nil } diff --git a/pkg/morph/metrics/metrics.go b/pkg/morph/metrics/metrics.go new file mode 100644 index 000000000..9e41a0b86 --- /dev/null +++ b/pkg/morph/metrics/metrics.go @@ -0,0 +1,17 @@ +package metrics + +import "time" + +type Register interface { + IncSwitchCount() + SetLastBlock(uint32) + IncNotificationCount(notificationType string) + ObserveInvoke(typ string, contract string, method string, success bool, d time.Duration) +} + +type NoopRegister struct{} + +func (NoopRegister) IncSwitchCount() {} +func (NoopRegister) SetLastBlock(uint32) {} +func (NoopRegister) IncNotificationCount(string) {} +func (NoopRegister) ObserveInvoke(string, string, string, bool, time.Duration) {} diff --git a/pkg/morph/subscriber/subscriber.go b/pkg/morph/subscriber/subscriber.go index 608872dec..4076111f0 100644 --- a/pkg/morph/subscriber/subscriber.go +++ b/pkg/morph/subscriber/subscriber.go @@ -200,18 +200,22 @@ routeloop: break routeloop case ev, ok := <-curr.NotifyChan: if ok { + s.client.Metrics().IncNotificationCount("notify") s.notifyChan <- ev } else { connLost = true } case ev, ok := <-curr.BlockChan: if ok { + s.client.Metrics().IncNotificationCount("block") + s.client.Metrics().SetLastBlock(ev.Index) s.blockChan <- ev } else { connLost = true } case ev, ok := <-curr.NotaryChan: if ok { + s.client.Metrics().IncNotificationCount("notary") s.notaryChan <- ev } else { connLost = true @@ -262,6 +266,7 @@ func (s *subscriber) switchEndpoint(ctx context.Context, finishCh chan<- bool) ( s.current = chs s.Unlock() + s.client.Metrics().IncSwitchCount() return true, cliCh } diff --git a/pkg/services/tree/metrics.go b/pkg/services/tree/metrics.go index 53708c4f1..0f0e4ee57 100644 --- a/pkg/services/tree/metrics.go +++ b/pkg/services/tree/metrics.go @@ -2,6 +2,12 @@ package tree import "time" +type MetricsRegister interface { + AddReplicateTaskDuration(time.Duration, bool) + AddReplicateWaitDuration(time.Duration, bool) + AddSyncDuration(time.Duration, bool) +} + type defaultMetricsRegister struct{} func (defaultMetricsRegister) AddReplicateTaskDuration(time.Duration, bool) {}