From 90e9247b69955a04a1e3aa97b042d52bdfb5b03b Mon Sep 17 00:00:00 2001 From: Airat Arifullin Date: Fri, 26 May 2023 12:15:50 +0300 Subject: [PATCH] [#371] ir: Add morph cache metrics Signed-off-by: Airat Arifullin a.arifullin@yadro.com --- pkg/innerring/initialization.go | 11 ++--- pkg/innerring/innerring.go | 14 +++--- pkg/metrics/innerring.go | 18 +++++--- pkg/metrics/morphcache.go | 77 +++++++++++++++++++++++++++++++++ pkg/metrics/node.go | 3 +- pkg/metrics/treeservice.go | 8 ++++ pkg/morph/client/client.go | 3 ++ pkg/morph/client/constructor.go | 14 +++++- pkg/morph/client/nns.go | 18 ++++++++ pkg/morph/client/notary.go | 9 ++++ pkg/services/tree/metrics.go | 6 --- 11 files changed, 154 insertions(+), 27 deletions(-) create mode 100644 pkg/metrics/morphcache.go diff --git a/pkg/innerring/initialization.go b/pkg/innerring/initialization.go index c49d2250..05e503f2 100644 --- a/pkg/innerring/initialization.go +++ b/pkg/innerring/initialization.go @@ -469,11 +469,12 @@ func (s *Server) initMorph(ctx context.Context, cfg *viper.Viper, errChan chan<- } morphChain := &chainParams{ - log: s.log, - cfg: cfg, - key: s.key, - name: morphPrefix, - from: fromSideChainBlock, + log: s.log, + cfg: cfg, + key: s.key, + name: morphPrefix, + from: fromSideChainBlock, + morphCacheMetric: s.metrics.MorphCacheMetrics(), } // create morph client diff --git a/pkg/innerring/innerring.go b/pkg/innerring/innerring.go index cbcb4699..335d3d17 100644 --- a/pkg/innerring/innerring.go +++ b/pkg/innerring/innerring.go @@ -102,12 +102,13 @@ type ( } chainParams struct { - log *logger.Logger - cfg *viper.Viper - key *keys.PrivateKey - name string - sgn *transaction.Signer - from uint32 // block height + log *logger.Logger + cfg *viper.Viper + key *keys.PrivateKey + name string + sgn *transaction.Signer + from uint32 // block height + morphCacheMetric metrics.MorphCacheMetrics } ) @@ -465,6 +466,7 @@ func createClient(ctx context.Context, p *chainParams, errChan chan<- error) (*c errChan <- fmt.Errorf("%s chain connection has been lost", p.name) }), client.WithSwitchInterval(p.cfg.GetDuration(p.name+".switch_interval")), + client.WithMorphCacheMetrics(p.morphCacheMetric), ) } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 9d8b76bf..d756a539 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -16,9 +16,10 @@ const ( // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { - epoch prometheus.Gauge - health prometheus.Gauge - eventDuration *prometheus.HistogramVec + epoch prometheus.Gauge + health prometheus.Gauge + eventDuration *prometheus.HistogramVec + morphCacheMetrics *morphCacheMetrics } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. @@ -45,9 +46,10 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics { ) return &InnerRingServiceMetrics{ - epoch: epoch, - health: health, - eventDuration: eventDuration, + epoch: epoch, + health: health, + eventDuration: eventDuration, + morphCacheMetrics: newMorphCacheMetrics(), } } @@ -67,3 +69,7 @@ func (m InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success b innerRingLabelSuccess: strconv.FormatBool(success), }).Observe(d.Seconds()) } + +func (m InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics { + return m.morphCacheMetrics +} diff --git a/pkg/metrics/morphcache.go b/pkg/metrics/morphcache.go new file mode 100644 index 00000000..7f6f2d7d --- /dev/null +++ b/pkg/metrics/morphcache.go @@ -0,0 +1,77 @@ +package metrics + +import ( + "fmt" + "time" + + "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + mcSubsystem = "morphcache" + mcSuccess = "success" +) + +type MorphCacheMetrics interface { + AddNNSContractHashDuration(success bool, d time.Duration) + + AddGroupKeyDuration(success bool, d time.Duration) + + AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) +} + +type morphCacheMetrics struct { + // Duration of processing get nns contract hash request + nnsContractHashDuration *prometheus.HistogramVec + + // Duration of processing get group key request + groupKeyDuration *prometheus.HistogramVec + + // Duration of processing get tx height request + txHeightDuration *prometheus.HistogramVec +} + +var _ MorphCacheMetrics = (*morphCacheMetrics)(nil) + +func newMorphCacheMetrics() *morphCacheMetrics { + return &morphCacheMetrics{ + nnsContractHashDuration: newMCMethodDurationCounter("nns_contract_hash"), + groupKeyDuration: newMCMethodDurationCounter("group_key"), + txHeightDuration: newMCMethodDurationCounter("tx_height"), + } +} + +func (m *morphCacheMetrics) AddNNSContractHashDuration(success bool, d time.Duration) { + m.nnsContractHashDuration.With( + prometheus.Labels{ + mcSuccess: fmt.Sprintf("%v", success), + }, + ).Observe(float64(d)) +} + +func (m *morphCacheMetrics) AddGroupKeyDuration(success bool, d time.Duration) { + m.groupKeyDuration.With( + prometheus.Labels{ + mcSuccess: fmt.Sprintf("%v", success), + }, + ).Observe(float64(d)) +} + +func (m *morphCacheMetrics) AddTxHeightDuration(hash util.Uint256, success bool, d time.Duration) { + m.txHeightDuration.With( + prometheus.Labels{ + mcSuccess: fmt.Sprintf("%v", success), + }, + ).Observe(float64(d)) +} + +func newMCMethodDurationCounter(method string) *prometheus.HistogramVec { + return metrics.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: mcSubsystem, + Name: fmt.Sprintf("%s_req_duration_seconds", method), + Help: fmt.Sprintf("Accumulated %s request process duration", method), + }, []string{mcSuccess}) +} diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 8819ba15..16a4c2b5 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -1,7 +1,6 @@ package metrics import ( - "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/services/tree" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "github.com/prometheus/client_golang/prometheus" ) @@ -67,7 +66,7 @@ func (m *NodeMetrics) WriteCache() WriteCacheMetrics { return m.writeCacheMetrics } -func (m *NodeMetrics) TreeService() tree.MetricsRegister { +func (m *NodeMetrics) TreeService() TreeMetricsRegister { return m.treeService } diff --git a/pkg/metrics/treeservice.go b/pkg/metrics/treeservice.go index 903ef349..48a450c7 100644 --- a/pkg/metrics/treeservice.go +++ b/pkg/metrics/treeservice.go @@ -10,12 +10,20 @@ import ( const treeServiceLabelSuccess = "success" +type TreeMetricsRegister interface { + AddReplicateTaskDuration(time.Duration, bool) + AddReplicateWaitDuration(time.Duration, bool) + AddSyncDuration(time.Duration, bool) +} + type treeServiceMetrics struct { replicateTaskDuration *prometheus.HistogramVec replicateWaitDuration *prometheus.HistogramVec syncOpDuration *prometheus.HistogramVec } +var _ TreeMetricsRegister = (*treeServiceMetrics)(nil) + func newTreeServiceMetrics() *treeServiceMetrics { const treeServiceSubsystem = "treeservice" return &treeServiceMetrics{ diff --git a/pkg/morph/client/client.go b/pkg/morph/client/client.go index 6a7a5b51..f03ee1db 100644 --- a/pkg/morph/client/client.go +++ b/pkg/morph/client/client.go @@ -10,6 +10,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/native/noderoles" @@ -88,6 +89,8 @@ type cache struct { nnsHash *util.Uint160 gKey *keys.PublicKey txHeights *lru.Cache[util.Uint256, uint32] + + metrics metrics.MorphCacheMetrics } func (c *cache) nns() *util.Uint160 { diff --git a/pkg/morph/client/constructor.go b/pkg/morph/client/constructor.go index 1d55db71..8b5fb3ff 100644 --- a/pkg/morph/client/constructor.go +++ b/pkg/morph/client/constructor.go @@ -7,6 +7,7 @@ import ( "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" + "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/metrics" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/logger" lru "github.com/hashicorp/golang-lru/v2" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -42,6 +43,8 @@ type cfg struct { inactiveModeCb Callback switchInterval time.Duration + + morphCacheMetrics metrics.MorphCacheMetrics } const ( @@ -104,7 +107,7 @@ func New(ctx context.Context, key *keys.PrivateKey, opts ...Option) (*Client, er } cli := &Client{ - cache: newClientCache(), + cache: newClientCache(cfg.morphCacheMetrics), logger: cfg.logger, acc: acc, accAddr: accAddr, @@ -195,10 +198,11 @@ func newActor(ws *rpcclient.WSClient, acc *wallet.Account, cfg cfg) (*actor.Acto }}) } -func newClientCache() cache { +func newClientCache(morphCacheMetrics metrics.MorphCacheMetrics) cache { c, _ := lru.New[util.Uint256, uint32](100) // returns error only if size is negative return cache{ txHeights: c, + metrics: morphCacheMetrics, } } @@ -282,3 +286,9 @@ func WithSwitchInterval(i time.Duration) Option { c.switchInterval = i } } + +func WithMorphCacheMetrics(morphCacheMetrics metrics.MorphCacheMetrics) Option { + return func(c *cfg) { + c.morphCacheMetrics = morphCacheMetrics + } +} diff --git a/pkg/morph/client/nns.go b/pkg/morph/client/nns.go index 2f7079df..d8d798b0 100644 --- a/pkg/morph/client/nns.go +++ b/pkg/morph/client/nns.go @@ -5,6 +5,7 @@ import ( "fmt" "math/big" "strconv" + "time" "git.frostfs.info/TrueCloudLab/frostfs-contract/nns" "github.com/nspcc-dev/neo-go/pkg/core/transaction" @@ -81,6 +82,13 @@ func (c *Client) NNSHash() (util.Uint160, error) { return util.Uint160{}, ErrConnectionLost } + success := false + startedAt := time.Now() + + defer func() { + c.cache.metrics.AddNNSContractHashDuration(success, time.Since(startedAt)) + }() + nnsHash := c.cache.nns() if nnsHash == nil { @@ -92,6 +100,7 @@ func (c *Client) NNSHash() (util.Uint160, error) { c.cache.setNNSHash(cs.Hash) nnsHash = &cs.Hash } + success = true return *nnsHash, nil } @@ -221,7 +230,14 @@ func (c *Client) SetGroupSignerScope() error { // contractGroupKey returns public key designating FrostFS contract group. func (c *Client) contractGroupKey() (*keys.PublicKey, error) { + success := false + startedAt := time.Now() + defer func() { + c.cache.metrics.AddGroupKeyDuration(success, time.Since(startedAt)) + }() + if gKey := c.cache.groupKey(); gKey != nil { + success = true return gKey, nil } @@ -251,5 +267,7 @@ func (c *Client) contractGroupKey() (*keys.PublicKey, error) { } c.cache.setGroupKey(pub) + + success = true return pub, nil } diff --git a/pkg/morph/client/notary.go b/pkg/morph/client/notary.go index 1ed1ca91..3a3ff0b4 100644 --- a/pkg/morph/client/notary.go +++ b/pkg/morph/client/notary.go @@ -8,6 +8,7 @@ import ( "math" "math/big" "strings" + "time" "git.frostfs.info/TrueCloudLab/frostfs-node/internal/logs" "git.frostfs.info/TrueCloudLab/frostfs-node/pkg/util/rand" @@ -789,7 +790,14 @@ func (c *Client) calculateNonceAndVUB(hash util.Uint256, roundBlockHeight bool) } func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { + success := false + startedAt := time.Now() + defer func() { + c.cache.metrics.AddTxHeightDuration(h, success, time.Since(startedAt)) + }() + if rh, ok := c.cache.txHeights.Get(h); ok { + success = true return rh, nil } height, err := c.client.GetTransactionHeight(h) @@ -797,5 +805,6 @@ func (c *Client) getTransactionHeight(h util.Uint256) (uint32, error) { return 0, err } c.cache.txHeights.Add(h, height) + success = true return height, nil } diff --git a/pkg/services/tree/metrics.go b/pkg/services/tree/metrics.go index 0f0e4ee5..53708c4f 100644 --- a/pkg/services/tree/metrics.go +++ b/pkg/services/tree/metrics.go @@ -2,12 +2,6 @@ package tree import "time" -type MetricsRegister interface { - AddReplicateTaskDuration(time.Duration, bool) - AddReplicateWaitDuration(time.Duration, bool) - AddSyncDuration(time.Duration, bool) -} - type defaultMetricsRegister struct{} func (defaultMetricsRegister) AddReplicateTaskDuration(time.Duration, bool) {}