[#1984] metrics: Use separate metrics for success/failed requests

Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>

CHANGELOG.md

@@ -12,6 +12,7 @@ Changelog for NeoFS Node
 - `replicator.pool_size` config field to tune replicator pool size (#2049)
 - Fix NNS hash parsing in morph client (#2063)
 - `neofs-cli neofs-cli acl basic/extended print` commands (#2012)
+- `neofs_node_object_*_req_count_success` prometheus metrics for tracking successfully executed requests (#1984)
 
 ### Changed
 - `object lock` command reads CID and OID the same way other commands do (#1971)
@@ -59,6 +60,8 @@ Pass CID and OID parameters via the `--cid` and `--oid` flags, not as the comman
 Replicator pool size can now be fine-tuned with `replicator.pool_size` config field.
 The default value is taken from `object.put.pool_size_remote` as in earlier versions.
 
+Added `neofs_node_object_*_req_count_success` metrics for tracking successfully executed requests.
+
 ## [0.34.0] - 2022-10-31 - Marado (마라도, 馬羅島)
 
 ### Added

pkg/metrics/object.go

@@ -1,6 +1,7 @@
 package metrics
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -9,14 +10,19 @@ import (
 const objectSubsystem = "object"
 
 type (
+	methodCount struct {
+		success prometheus.Counter
+		total   prometheus.Counter
+	}
+
 	objectServiceMetrics struct {
-		getCounter       prometheus.Counter
+		getCounter       methodCount
-		putCounter       prometheus.Counter
+		putCounter       methodCount
-		headCounter      prometheus.Counter
+		headCounter      methodCount
-		searchCounter    prometheus.Counter
+		searchCounter    methodCount
-		deleteCounter    prometheus.Counter
+		deleteCounter    methodCount
-		rangeCounter     prometheus.Counter
+		rangeCounter     methodCount
-		rangeHashCounter prometheus.Counter
+		rangeHashCounter methodCount
 
 		getDuration       prometheus.Counter
 		putDuration       prometheus.Counter
@@ -38,56 +44,44 @@ const (
 	counterTypeLabelKey = "type"
 )
 
+func newMethodCallCounter(name string) methodCount {
+	return methodCount{
+		success: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: objectSubsystem,
+			Name:      fmt.Sprintf("%s_req_count", name),
+			Help:      fmt.Sprintf("The number of successful %s requests processed", name),
+		}),
+		total: prometheus.NewCounter(prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: objectSubsystem,
+			Name:      fmt.Sprintf("%s_req_count_success", name),
+			Help:      fmt.Sprintf("Total number of %s requests processed", name),
+		}),
+	}
+}
+
+func (m methodCount) mustRegister() {
+	prometheus.MustRegister(m.success)
+	prometheus.MustRegister(m.total)
+}
+
+func (m methodCount) Inc(success bool) {
+	m.total.Inc()
+	if success {
+		m.success.Inc()
+	}
+}
+
 func newObjectServiceMetrics() objectServiceMetrics {
 	var ( // Request counter metrics.
-		getCounter = prometheus.NewCounter(prometheus.CounterOpts{
+		getCounter       = newMethodCallCounter("get")
-			Namespace: namespace,
+		putCounter       = newMethodCallCounter("put")
-			Subsystem: objectSubsystem,
+		headCounter      = newMethodCallCounter("head")
-			Name:      "get_req_count",
+		searchCounter    = newMethodCallCounter("search")
-			Help:      "Number of get request processed",
+		deleteCounter    = newMethodCallCounter("delete")
-		})
+		rangeCounter     = newMethodCallCounter("range")
-
+		rangeHashCounter = newMethodCallCounter("range_hash")
-		putCounter = prometheus.NewCounter(prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: objectSubsystem,
-			Name:      "put_req_count",
-			Help:      "Number of put request processed",
-		})
-
-		headCounter = prometheus.NewCounter(prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: objectSubsystem,
-			Name:      "head_req_count",
-			Help:      "Number of head request processed",
-		})
-
-		searchCounter = prometheus.NewCounter(prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: objectSubsystem,
-			Name:      "search_req_count",
-			Help:      "Number of search request processed",
-		})
-
-		deleteCounter = prometheus.NewCounter(prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: objectSubsystem,
-			Name:      "delete_req_count",
-			Help:      "Number of delete request processed",
-		})
-
-		rangeCounter = prometheus.NewCounter(prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: objectSubsystem,
-			Name:      "range_req_count",
-			Help:      "Number of range request processed",
-		})
-
-		rangeHashCounter = prometheus.NewCounter(prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: objectSubsystem,
-			Name:      "range_hash_req_count",
-			Help:      "Number of range hash request processed",
-		})
 	)
 
 	var ( // Request duration metrics.
@@ -188,13 +182,13 @@ func newObjectServiceMetrics() objectServiceMetrics {
 }
 
 func (m objectServiceMetrics) register() {
-	prometheus.MustRegister(m.getCounter)
+	m.getCounter.mustRegister()
-	prometheus.MustRegister(m.putCounter)
+	m.putCounter.mustRegister()
-	prometheus.MustRegister(m.headCounter)
+	m.headCounter.mustRegister()
-	prometheus.MustRegister(m.searchCounter)
+	m.searchCounter.mustRegister()
-	prometheus.MustRegister(m.deleteCounter)
+	m.deleteCounter.mustRegister()
-	prometheus.MustRegister(m.rangeCounter)
+	m.rangeCounter.mustRegister()
-	prometheus.MustRegister(m.rangeHashCounter)
+	m.rangeHashCounter.mustRegister()
 
 	prometheus.MustRegister(m.getDuration)
 	prometheus.MustRegister(m.putDuration)
@@ -210,32 +204,32 @@ func (m objectServiceMetrics) register() {
 	prometheus.MustRegister(m.shardMetrics)
 }
 
-func (m objectServiceMetrics) IncGetReqCounter() {
+func (m objectServiceMetrics) IncGetReqCounter(success bool) {
-	m.getCounter.Inc()
+	m.getCounter.Inc(success)
 }
 
-func (m objectServiceMetrics) IncPutReqCounter() {
+func (m objectServiceMetrics) IncPutReqCounter(success bool) {
-	m.putCounter.Inc()
+	m.putCounter.Inc(success)
 }
 
-func (m objectServiceMetrics) IncHeadReqCounter() {
+func (m objectServiceMetrics) IncHeadReqCounter(success bool) {
-	m.headCounter.Inc()
+	m.headCounter.Inc(success)
 }
 
-func (m objectServiceMetrics) IncSearchReqCounter() {
+func (m objectServiceMetrics) IncSearchReqCounter(success bool) {
-	m.searchCounter.Inc()
+	m.searchCounter.Inc(success)
 }
 
-func (m objectServiceMetrics) IncDeleteReqCounter() {
+func (m objectServiceMetrics) IncDeleteReqCounter(success bool) {
-	m.deleteCounter.Inc()
+	m.deleteCounter.Inc(success)
 }
 
-func (m objectServiceMetrics) IncRangeReqCounter() {
+func (m objectServiceMetrics) IncRangeReqCounter(success bool) {
-	m.rangeCounter.Inc()
+	m.rangeCounter.Inc(success)
 }
 
-func (m objectServiceMetrics) IncRangeHashReqCounter() {
+func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) {
-	m.rangeHashCounter.Inc()
+	m.rangeHashCounter.Inc(success)
 }
 
 func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) {

pkg/services/object/metrics.go

@@ -27,13 +27,13 @@ type (
 	}
 
 	MetricRegister interface {
-		IncGetReqCounter()
+		IncGetReqCounter(success bool)
-		IncPutReqCounter()
+		IncPutReqCounter(success bool)
-		IncHeadReqCounter()
+		IncHeadReqCounter(success bool)
-		IncSearchReqCounter()
+		IncSearchReqCounter(success bool)
-		IncDeleteReqCounter()
+		IncDeleteReqCounter(success bool)
-		IncRangeReqCounter()
+		IncRangeReqCounter(success bool)
-		IncRangeHashReqCounter()
+		IncRangeHashReqCounter(success bool)
 
 		AddGetReqDuration(time.Duration)
 		AddPutReqDuration(time.Duration)
@@ -55,18 +55,19 @@ func NewMetricCollector(next ServiceServer, register MetricRegister) *MetricColl
 	}
 }
 
-func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) error {
+func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (err error) {
 	t := time.Now()
 	defer func() {
-		m.metrics.IncGetReqCounter()
+		m.metrics.IncGetReqCounter(err == nil)
 		m.metrics.AddGetReqDuration(time.Since(t))
 	}()
 
-	return m.next.Get(req, &getStreamMetric{
+	err = m.next.Get(req, &getStreamMetric{
 		ServerStream: stream,
 		stream:       stream,
 		metrics:      m.metrics,
 	})
+	return
 }
 
 func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) {
@@ -86,52 +87,57 @@ func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) {
 
 func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) {
 	t := time.Now()
-	defer func() {
-		m.metrics.IncHeadReqCounter()
-		m.metrics.AddHeadReqDuration(time.Since(t))
-	}()
 
-	return m.next.Head(ctx, request)
+	res, err := m.next.Head(ctx, request)
+
+	m.metrics.IncHeadReqCounter(err == nil)
+	m.metrics.AddHeadReqDuration(time.Since(t))
+
+	return res, err
 }
 
 func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream) error {
 	t := time.Now()
-	defer func() {
-		m.metrics.IncSearchReqCounter()
-		m.metrics.AddSearchReqDuration(time.Since(t))
-	}()
 
-	return m.next.Search(req, stream)
+	err := m.next.Search(req, stream)
+
+	m.metrics.IncSearchReqCounter(err == nil)
+	m.metrics.AddSearchReqDuration(time.Since(t))
+
+	return err
 }
 
 func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteRequest) (*object.DeleteResponse, error) {
 	t := time.Now()
-	defer func() {
-		m.metrics.IncDeleteReqCounter()
-		m.metrics.AddDeleteReqDuration(time.Since(t))
-	}()
 
-	return m.next.Delete(ctx, request)
+	res, err := m.next.Delete(ctx, request)
+
+	m.metrics.IncDeleteReqCounter(err == nil)
+	m.metrics.AddDeleteReqDuration(time.Since(t))
+
+	return res, err
 }
 
 func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error {
 	t := time.Now()
-	defer func() {
-		m.metrics.IncRangeReqCounter()
-		m.metrics.AddRangeReqDuration(time.Since(t))
-	}()
 
-	return m.next.GetRange(req, stream)
+	err := m.next.GetRange(req, stream)
+
+	m.metrics.IncRangeReqCounter(err == nil)
+	m.metrics.AddRangeReqDuration(time.Since(t))
+
+	return err
 }
 
 func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) {
 	t := time.Now()
-	defer func() {
-		m.metrics.IncRangeHashReqCounter()
-		m.metrics.AddRangeHashReqDuration(time.Since(t))
-	}()
 
-	return m.next.GetRangeHash(ctx, request)
+	res, err := m.next.GetRangeHash(ctx, request)
+
+	m.metrics.IncRangeHashReqCounter(err == nil)
+	m.metrics.AddRangeHashReqDuration(time.Since(t))
+
+	return res, err
 }
 
 func (s getStreamMetric) Send(resp *object.GetResponse) error {
@@ -153,10 +159,10 @@ func (s putStreamMetric) Send(req *object.PutRequest) error {
 }
 
 func (s putStreamMetric) CloseAndRecv() (*object.PutResponse, error) {
-	defer func() {
+	res, err := s.stream.CloseAndRecv()
-		s.metrics.IncPutReqCounter()
-		s.metrics.AddPutReqDuration(time.Since(s.start))
-	}()
 
-	return s.stream.CloseAndRecv()
+	s.metrics.IncPutReqCounter(err == nil)
+	s.metrics.AddPutReqDuration(time.Since(s.start))
+
+	return res, err
 }