[#1984] metrics: Use separate metrics for success/failed requests

Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
This commit is contained in:
Evgenii Stratonikov 2022-11-29 14:51:19 +03:00 committed by fyrchik
parent e21c472dc7
commit bd25db5d4a
3 changed files with 119 additions and 116 deletions

View file

@ -12,6 +12,7 @@ Changelog for NeoFS Node
- `replicator.pool_size` config field to tune replicator pool size (#2049) - `replicator.pool_size` config field to tune replicator pool size (#2049)
- Fix NNS hash parsing in morph client (#2063) - Fix NNS hash parsing in morph client (#2063)
- `neofs-cli neofs-cli acl basic/extended print` commands (#2012) - `neofs-cli neofs-cli acl basic/extended print` commands (#2012)
- `neofs_node_object_*_req_count_success` prometheus metrics for tracking successfully executed requests (#1984)
### Changed ### Changed
- `object lock` command reads CID and OID the same way other commands do (#1971) - `object lock` command reads CID and OID the same way other commands do (#1971)
@ -59,6 +60,8 @@ Pass CID and OID parameters via the `--cid` and `--oid` flags, not as the comman
Replicator pool size can now be fine-tuned with `replicator.pool_size` config field. Replicator pool size can now be fine-tuned with `replicator.pool_size` config field.
The default value is taken from `object.put.pool_size_remote` as in earlier versions. The default value is taken from `object.put.pool_size_remote` as in earlier versions.
Added `neofs_node_object_*_req_count_success` metrics for tracking successfully executed requests.
## [0.34.0] - 2022-10-31 - Marado (마라도, 馬羅島) ## [0.34.0] - 2022-10-31 - Marado (마라도, 馬羅島)
### Added ### Added

View file

@ -1,6 +1,7 @@
package metrics package metrics
import ( import (
"fmt"
"time" "time"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
@ -9,14 +10,19 @@ import (
const objectSubsystem = "object" const objectSubsystem = "object"
type ( type (
methodCount struct {
success prometheus.Counter
total prometheus.Counter
}
objectServiceMetrics struct { objectServiceMetrics struct {
getCounter prometheus.Counter getCounter methodCount
putCounter prometheus.Counter putCounter methodCount
headCounter prometheus.Counter headCounter methodCount
searchCounter prometheus.Counter searchCounter methodCount
deleteCounter prometheus.Counter deleteCounter methodCount
rangeCounter prometheus.Counter rangeCounter methodCount
rangeHashCounter prometheus.Counter rangeHashCounter methodCount
getDuration prometheus.Counter getDuration prometheus.Counter
putDuration prometheus.Counter putDuration prometheus.Counter
@ -38,56 +44,44 @@ const (
counterTypeLabelKey = "type" counterTypeLabelKey = "type"
) )
func newMethodCallCounter(name string) methodCount {
return methodCount{
success: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: fmt.Sprintf("%s_req_count", name),
Help: fmt.Sprintf("The number of successful %s requests processed", name),
}),
total: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: fmt.Sprintf("%s_req_count_success", name),
Help: fmt.Sprintf("Total number of %s requests processed", name),
}),
}
}
func (m methodCount) mustRegister() {
prometheus.MustRegister(m.success)
prometheus.MustRegister(m.total)
}
func (m methodCount) Inc(success bool) {
m.total.Inc()
if success {
m.success.Inc()
}
}
func newObjectServiceMetrics() objectServiceMetrics { func newObjectServiceMetrics() objectServiceMetrics {
var ( // Request counter metrics. var ( // Request counter metrics.
getCounter = prometheus.NewCounter(prometheus.CounterOpts{ getCounter = newMethodCallCounter("get")
Namespace: namespace, putCounter = newMethodCallCounter("put")
Subsystem: objectSubsystem, headCounter = newMethodCallCounter("head")
Name: "get_req_count", searchCounter = newMethodCallCounter("search")
Help: "Number of get request processed", deleteCounter = newMethodCallCounter("delete")
}) rangeCounter = newMethodCallCounter("range")
rangeHashCounter = newMethodCallCounter("range_hash")
putCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "put_req_count",
Help: "Number of put request processed",
})
headCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "head_req_count",
Help: "Number of head request processed",
})
searchCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "search_req_count",
Help: "Number of search request processed",
})
deleteCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "delete_req_count",
Help: "Number of delete request processed",
})
rangeCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "range_req_count",
Help: "Number of range request processed",
})
rangeHashCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "range_hash_req_count",
Help: "Number of range hash request processed",
})
) )
var ( // Request duration metrics. var ( // Request duration metrics.
@ -188,13 +182,13 @@ func newObjectServiceMetrics() objectServiceMetrics {
} }
func (m objectServiceMetrics) register() { func (m objectServiceMetrics) register() {
prometheus.MustRegister(m.getCounter) m.getCounter.mustRegister()
prometheus.MustRegister(m.putCounter) m.putCounter.mustRegister()
prometheus.MustRegister(m.headCounter) m.headCounter.mustRegister()
prometheus.MustRegister(m.searchCounter) m.searchCounter.mustRegister()
prometheus.MustRegister(m.deleteCounter) m.deleteCounter.mustRegister()
prometheus.MustRegister(m.rangeCounter) m.rangeCounter.mustRegister()
prometheus.MustRegister(m.rangeHashCounter) m.rangeHashCounter.mustRegister()
prometheus.MustRegister(m.getDuration) prometheus.MustRegister(m.getDuration)
prometheus.MustRegister(m.putDuration) prometheus.MustRegister(m.putDuration)
@ -210,32 +204,32 @@ func (m objectServiceMetrics) register() {
prometheus.MustRegister(m.shardMetrics) prometheus.MustRegister(m.shardMetrics)
} }
func (m objectServiceMetrics) IncGetReqCounter() { func (m objectServiceMetrics) IncGetReqCounter(success bool) {
m.getCounter.Inc() m.getCounter.Inc(success)
} }
func (m objectServiceMetrics) IncPutReqCounter() { func (m objectServiceMetrics) IncPutReqCounter(success bool) {
m.putCounter.Inc() m.putCounter.Inc(success)
} }
func (m objectServiceMetrics) IncHeadReqCounter() { func (m objectServiceMetrics) IncHeadReqCounter(success bool) {
m.headCounter.Inc() m.headCounter.Inc(success)
} }
func (m objectServiceMetrics) IncSearchReqCounter() { func (m objectServiceMetrics) IncSearchReqCounter(success bool) {
m.searchCounter.Inc() m.searchCounter.Inc(success)
} }
func (m objectServiceMetrics) IncDeleteReqCounter() { func (m objectServiceMetrics) IncDeleteReqCounter(success bool) {
m.deleteCounter.Inc() m.deleteCounter.Inc(success)
} }
func (m objectServiceMetrics) IncRangeReqCounter() { func (m objectServiceMetrics) IncRangeReqCounter(success bool) {
m.rangeCounter.Inc() m.rangeCounter.Inc(success)
} }
func (m objectServiceMetrics) IncRangeHashReqCounter() { func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) {
m.rangeHashCounter.Inc() m.rangeHashCounter.Inc(success)
} }
func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) { func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) {

View file

@ -27,13 +27,13 @@ type (
} }
MetricRegister interface { MetricRegister interface {
IncGetReqCounter() IncGetReqCounter(success bool)
IncPutReqCounter() IncPutReqCounter(success bool)
IncHeadReqCounter() IncHeadReqCounter(success bool)
IncSearchReqCounter() IncSearchReqCounter(success bool)
IncDeleteReqCounter() IncDeleteReqCounter(success bool)
IncRangeReqCounter() IncRangeReqCounter(success bool)
IncRangeHashReqCounter() IncRangeHashReqCounter(success bool)
AddGetReqDuration(time.Duration) AddGetReqDuration(time.Duration)
AddPutReqDuration(time.Duration) AddPutReqDuration(time.Duration)
@ -55,18 +55,19 @@ func NewMetricCollector(next ServiceServer, register MetricRegister) *MetricColl
} }
} }
func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) error { func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (err error) {
t := time.Now() t := time.Now()
defer func() { defer func() {
m.metrics.IncGetReqCounter() m.metrics.IncGetReqCounter(err == nil)
m.metrics.AddGetReqDuration(time.Since(t)) m.metrics.AddGetReqDuration(time.Since(t))
}() }()
return m.next.Get(req, &getStreamMetric{ err = m.next.Get(req, &getStreamMetric{
ServerStream: stream, ServerStream: stream,
stream: stream, stream: stream,
metrics: m.metrics, metrics: m.metrics,
}) })
return
} }
func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) { func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) {
@ -86,52 +87,57 @@ func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) {
func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) { func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) {
t := time.Now() t := time.Now()
defer func() {
m.metrics.IncHeadReqCounter()
m.metrics.AddHeadReqDuration(time.Since(t))
}()
return m.next.Head(ctx, request) res, err := m.next.Head(ctx, request)
m.metrics.IncHeadReqCounter(err == nil)
m.metrics.AddHeadReqDuration(time.Since(t))
return res, err
} }
func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream) error { func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream) error {
t := time.Now() t := time.Now()
defer func() {
m.metrics.IncSearchReqCounter()
m.metrics.AddSearchReqDuration(time.Since(t))
}()
return m.next.Search(req, stream) err := m.next.Search(req, stream)
m.metrics.IncSearchReqCounter(err == nil)
m.metrics.AddSearchReqDuration(time.Since(t))
return err
} }
func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteRequest) (*object.DeleteResponse, error) { func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteRequest) (*object.DeleteResponse, error) {
t := time.Now() t := time.Now()
defer func() {
m.metrics.IncDeleteReqCounter()
m.metrics.AddDeleteReqDuration(time.Since(t))
}()
return m.next.Delete(ctx, request) res, err := m.next.Delete(ctx, request)
m.metrics.IncDeleteReqCounter(err == nil)
m.metrics.AddDeleteReqDuration(time.Since(t))
return res, err
} }
func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error { func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error {
t := time.Now() t := time.Now()
defer func() {
m.metrics.IncRangeReqCounter()
m.metrics.AddRangeReqDuration(time.Since(t))
}()
return m.next.GetRange(req, stream) err := m.next.GetRange(req, stream)
m.metrics.IncRangeReqCounter(err == nil)
m.metrics.AddRangeReqDuration(time.Since(t))
return err
} }
func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) { func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) {
t := time.Now() t := time.Now()
defer func() {
m.metrics.IncRangeHashReqCounter()
m.metrics.AddRangeHashReqDuration(time.Since(t))
}()
return m.next.GetRangeHash(ctx, request) res, err := m.next.GetRangeHash(ctx, request)
m.metrics.IncRangeHashReqCounter(err == nil)
m.metrics.AddRangeHashReqDuration(time.Since(t))
return res, err
} }
func (s getStreamMetric) Send(resp *object.GetResponse) error { func (s getStreamMetric) Send(resp *object.GetResponse) error {
@ -153,10 +159,10 @@ func (s putStreamMetric) Send(req *object.PutRequest) error {
} }
func (s putStreamMetric) CloseAndRecv() (*object.PutResponse, error) { func (s putStreamMetric) CloseAndRecv() (*object.PutResponse, error) {
defer func() { res, err := s.stream.CloseAndRecv()
s.metrics.IncPutReqCounter()
s.metrics.AddPutReqDuration(time.Since(s.start))
}()
return s.stream.CloseAndRecv() s.metrics.IncPutReqCounter(err == nil)
s.metrics.AddPutReqDuration(time.Since(s.start))
return res, err
} }