From bd25db5d4a4b61d390501455ab5e93cb1a065e93 Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Tue, 29 Nov 2022 14:51:19 +0300 Subject: [PATCH] [#1984] metrics: Use separate metrics for success/failed requests Signed-off-by: Evgenii Stratonikov --- CHANGELOG.md | 3 + pkg/metrics/object.go | 146 ++++++++++++++++----------------- pkg/services/object/metrics.go | 86 ++++++++++--------- 3 files changed, 119 insertions(+), 116 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 033b91ed3..992be304f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Changelog for NeoFS Node - `replicator.pool_size` config field to tune replicator pool size (#2049) - Fix NNS hash parsing in morph client (#2063) - `neofs-cli neofs-cli acl basic/extended print` commands (#2012) +- `neofs_node_object_*_req_count_success` prometheus metrics for tracking successfully executed requests (#1984) ### Changed - `object lock` command reads CID and OID the same way other commands do (#1971) @@ -59,6 +60,8 @@ Pass CID and OID parameters via the `--cid` and `--oid` flags, not as the comman Replicator pool size can now be fine-tuned with `replicator.pool_size` config field. The default value is taken from `object.put.pool_size_remote` as in earlier versions. +Added `neofs_node_object_*_req_count_success` metrics for tracking successfully executed requests. + ## [0.34.0] - 2022-10-31 - Marado (마라도, 馬羅島) ### Added diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index d343934cb..f8cecd17c 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -1,6 +1,7 @@ package metrics import ( + "fmt" "time" "github.com/prometheus/client_golang/prometheus" @@ -9,14 +10,19 @@ import ( const objectSubsystem = "object" type ( + methodCount struct { + success prometheus.Counter + total prometheus.Counter + } + objectServiceMetrics struct { - getCounter prometheus.Counter - putCounter prometheus.Counter - headCounter prometheus.Counter - searchCounter prometheus.Counter - deleteCounter prometheus.Counter - rangeCounter prometheus.Counter - rangeHashCounter prometheus.Counter + getCounter methodCount + putCounter methodCount + headCounter methodCount + searchCounter methodCount + deleteCounter methodCount + rangeCounter methodCount + rangeHashCounter methodCount getDuration prometheus.Counter putDuration prometheus.Counter @@ -38,56 +44,44 @@ const ( counterTypeLabelKey = "type" ) +func newMethodCallCounter(name string) methodCount { + return methodCount{ + success: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: objectSubsystem, + Name: fmt.Sprintf("%s_req_count", name), + Help: fmt.Sprintf("The number of successful %s requests processed", name), + }), + total: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: objectSubsystem, + Name: fmt.Sprintf("%s_req_count_success", name), + Help: fmt.Sprintf("Total number of %s requests processed", name), + }), + } +} + +func (m methodCount) mustRegister() { + prometheus.MustRegister(m.success) + prometheus.MustRegister(m.total) +} + +func (m methodCount) Inc(success bool) { + m.total.Inc() + if success { + m.success.Inc() + } +} + func newObjectServiceMetrics() objectServiceMetrics { var ( // Request counter metrics. - getCounter = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "get_req_count", - Help: "Number of get request processed", - }) - - putCounter = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "put_req_count", - Help: "Number of put request processed", - }) - - headCounter = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "head_req_count", - Help: "Number of head request processed", - }) - - searchCounter = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "search_req_count", - Help: "Number of search request processed", - }) - - deleteCounter = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "delete_req_count", - Help: "Number of delete request processed", - }) - - rangeCounter = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "range_req_count", - Help: "Number of range request processed", - }) - - rangeHashCounter = prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: objectSubsystem, - Name: "range_hash_req_count", - Help: "Number of range hash request processed", - }) + getCounter = newMethodCallCounter("get") + putCounter = newMethodCallCounter("put") + headCounter = newMethodCallCounter("head") + searchCounter = newMethodCallCounter("search") + deleteCounter = newMethodCallCounter("delete") + rangeCounter = newMethodCallCounter("range") + rangeHashCounter = newMethodCallCounter("range_hash") ) var ( // Request duration metrics. @@ -188,13 +182,13 @@ func newObjectServiceMetrics() objectServiceMetrics { } func (m objectServiceMetrics) register() { - prometheus.MustRegister(m.getCounter) - prometheus.MustRegister(m.putCounter) - prometheus.MustRegister(m.headCounter) - prometheus.MustRegister(m.searchCounter) - prometheus.MustRegister(m.deleteCounter) - prometheus.MustRegister(m.rangeCounter) - prometheus.MustRegister(m.rangeHashCounter) + m.getCounter.mustRegister() + m.putCounter.mustRegister() + m.headCounter.mustRegister() + m.searchCounter.mustRegister() + m.deleteCounter.mustRegister() + m.rangeCounter.mustRegister() + m.rangeHashCounter.mustRegister() prometheus.MustRegister(m.getDuration) prometheus.MustRegister(m.putDuration) @@ -210,32 +204,32 @@ func (m objectServiceMetrics) register() { prometheus.MustRegister(m.shardMetrics) } -func (m objectServiceMetrics) IncGetReqCounter() { - m.getCounter.Inc() +func (m objectServiceMetrics) IncGetReqCounter(success bool) { + m.getCounter.Inc(success) } -func (m objectServiceMetrics) IncPutReqCounter() { - m.putCounter.Inc() +func (m objectServiceMetrics) IncPutReqCounter(success bool) { + m.putCounter.Inc(success) } -func (m objectServiceMetrics) IncHeadReqCounter() { - m.headCounter.Inc() +func (m objectServiceMetrics) IncHeadReqCounter(success bool) { + m.headCounter.Inc(success) } -func (m objectServiceMetrics) IncSearchReqCounter() { - m.searchCounter.Inc() +func (m objectServiceMetrics) IncSearchReqCounter(success bool) { + m.searchCounter.Inc(success) } -func (m objectServiceMetrics) IncDeleteReqCounter() { - m.deleteCounter.Inc() +func (m objectServiceMetrics) IncDeleteReqCounter(success bool) { + m.deleteCounter.Inc(success) } -func (m objectServiceMetrics) IncRangeReqCounter() { - m.rangeCounter.Inc() +func (m objectServiceMetrics) IncRangeReqCounter(success bool) { + m.rangeCounter.Inc(success) } -func (m objectServiceMetrics) IncRangeHashReqCounter() { - m.rangeHashCounter.Inc() +func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) { + m.rangeHashCounter.Inc(success) } func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) { diff --git a/pkg/services/object/metrics.go b/pkg/services/object/metrics.go index effa024d9..09c2af428 100644 --- a/pkg/services/object/metrics.go +++ b/pkg/services/object/metrics.go @@ -27,13 +27,13 @@ type ( } MetricRegister interface { - IncGetReqCounter() - IncPutReqCounter() - IncHeadReqCounter() - IncSearchReqCounter() - IncDeleteReqCounter() - IncRangeReqCounter() - IncRangeHashReqCounter() + IncGetReqCounter(success bool) + IncPutReqCounter(success bool) + IncHeadReqCounter(success bool) + IncSearchReqCounter(success bool) + IncDeleteReqCounter(success bool) + IncRangeReqCounter(success bool) + IncRangeHashReqCounter(success bool) AddGetReqDuration(time.Duration) AddPutReqDuration(time.Duration) @@ -55,18 +55,19 @@ func NewMetricCollector(next ServiceServer, register MetricRegister) *MetricColl } } -func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) error { +func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (err error) { t := time.Now() defer func() { - m.metrics.IncGetReqCounter() + m.metrics.IncGetReqCounter(err == nil) m.metrics.AddGetReqDuration(time.Since(t)) }() - return m.next.Get(req, &getStreamMetric{ + err = m.next.Get(req, &getStreamMetric{ ServerStream: stream, stream: stream, metrics: m.metrics, }) + return } func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) { @@ -86,52 +87,57 @@ func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) { func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) { t := time.Now() - defer func() { - m.metrics.IncHeadReqCounter() - m.metrics.AddHeadReqDuration(time.Since(t)) - }() - return m.next.Head(ctx, request) + res, err := m.next.Head(ctx, request) + + m.metrics.IncHeadReqCounter(err == nil) + m.metrics.AddHeadReqDuration(time.Since(t)) + + return res, err } func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream) error { t := time.Now() - defer func() { - m.metrics.IncSearchReqCounter() - m.metrics.AddSearchReqDuration(time.Since(t)) - }() - return m.next.Search(req, stream) + err := m.next.Search(req, stream) + + m.metrics.IncSearchReqCounter(err == nil) + m.metrics.AddSearchReqDuration(time.Since(t)) + + return err } func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteRequest) (*object.DeleteResponse, error) { t := time.Now() - defer func() { - m.metrics.IncDeleteReqCounter() - m.metrics.AddDeleteReqDuration(time.Since(t)) - }() - return m.next.Delete(ctx, request) + res, err := m.next.Delete(ctx, request) + + m.metrics.IncDeleteReqCounter(err == nil) + m.metrics.AddDeleteReqDuration(time.Since(t)) + + return res, err } func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { t := time.Now() - defer func() { - m.metrics.IncRangeReqCounter() - m.metrics.AddRangeReqDuration(time.Since(t)) - }() - return m.next.GetRange(req, stream) + err := m.next.GetRange(req, stream) + + m.metrics.IncRangeReqCounter(err == nil) + m.metrics.AddRangeReqDuration(time.Since(t)) + + return err } func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { t := time.Now() - defer func() { - m.metrics.IncRangeHashReqCounter() - m.metrics.AddRangeHashReqDuration(time.Since(t)) - }() - return m.next.GetRangeHash(ctx, request) + res, err := m.next.GetRangeHash(ctx, request) + + m.metrics.IncRangeHashReqCounter(err == nil) + m.metrics.AddRangeHashReqDuration(time.Since(t)) + + return res, err } func (s getStreamMetric) Send(resp *object.GetResponse) error { @@ -153,10 +159,10 @@ func (s putStreamMetric) Send(req *object.PutRequest) error { } func (s putStreamMetric) CloseAndRecv() (*object.PutResponse, error) { - defer func() { - s.metrics.IncPutReqCounter() - s.metrics.AddPutReqDuration(time.Since(s.start)) - }() + res, err := s.stream.CloseAndRecv() - return s.stream.CloseAndRecv() + s.metrics.IncPutReqCounter(err == nil) + s.metrics.AddPutReqDuration(time.Since(s.start)) + + return res, err }