TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 101 Pull requests 5 Projects Releases 1 Activity Actions

[#1984] metrics: Use separate metrics for success/failed requests

Browse source 
Signed-off-by: Evgenii Stratonikov <evgeniy@morphbits.ru>
This commit is contained in:
Evgenii Stratonikov 2022-11-29 14:51:19 +03:00 committed by fyrchik
parent e21c472dc7
commit bd25db5d4a
3 changed files with 119 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -12,6 +12,7 @@ Changelog for NeoFS Node
- `replicator.pool_size` config field to tune replicator pool size (#2049)
- Fix NNS hash parsing in morph client (#2063)
- `neofs-cli neofs-cli acl basic/extended print` commands (#2012)
- `neofs_node_object_*_req_count_success` prometheus metrics for tracking successfully executed requests (#1984)
### Changed
- `object lock` command reads CID and OID the same way other commands do (#1971)
@ -59,6 +60,8 @@ Pass CID and OID parameters via the `--cid` and `--oid` flags, not as the comman
Replicator pool size can now be fine-tuned with `replicator.pool_size` config field.
The default value is taken from `object.put.pool_size_remote` as in earlier versions.
Added `neofs_node_object_*_req_count_success` metrics for tracking successfully executed requests.
## [0.34.0] - 2022-10-31 - Marado (마라도, 馬羅島)
### Added

146
pkg/metrics/object.go
View file

@ -1,6 +1,7 @@
package metrics
import (
"fmt"
"time"
"github.com/prometheus/client_golang/prometheus"
@ -9,14 +10,19 @@ import (
const objectSubsystem = "object"
type (
methodCount struct {
success prometheus.Counter
total prometheus.Counter
}
objectServiceMetrics struct {
getCounter prometheus.Counter
putCounter prometheus.Counter
headCounter prometheus.Counter
searchCounter prometheus.Counter
deleteCounter prometheus.Counter
rangeCounter prometheus.Counter
rangeHashCounter prometheus.Counter
getCounter methodCount
putCounter methodCount
headCounter methodCount
searchCounter methodCount
deleteCounter methodCount
rangeCounter methodCount
rangeHashCounter methodCount
getDuration prometheus.Counter
putDuration prometheus.Counter
@ -38,56 +44,44 @@ const (
counterTypeLabelKey = "type"
)
func newMethodCallCounter(name string) methodCount {
return methodCount{
success: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: fmt.Sprintf("%s_req_count", name),
Help: fmt.Sprintf("The number of successful %s requests processed", name),
}),
total: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: fmt.Sprintf("%s_req_count_success", name),
Help: fmt.Sprintf("Total number of %s requests processed", name),
}),
}
}
func (m methodCount) mustRegister() {
prometheus.MustRegister(m.success)
prometheus.MustRegister(m.total)
}
func (m methodCount) Inc(success bool) {
m.total.Inc()
if success {
m.success.Inc()
}
}
func newObjectServiceMetrics() objectServiceMetrics {
var ( // Request counter metrics.
getCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "get_req_count",
Help: "Number of get request processed",
})
putCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "put_req_count",
Help: "Number of put request processed",
})
headCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "head_req_count",
Help: "Number of head request processed",
})
searchCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "search_req_count",
Help: "Number of search request processed",
})
deleteCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "delete_req_count",
Help: "Number of delete request processed",
})
rangeCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "range_req_count",
Help: "Number of range request processed",
})
rangeHashCounter = prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: "range_hash_req_count",
Help: "Number of range hash request processed",
})
getCounter = newMethodCallCounter("get")
putCounter = newMethodCallCounter("put")
headCounter = newMethodCallCounter("head")
searchCounter = newMethodCallCounter("search")
deleteCounter = newMethodCallCounter("delete")
rangeCounter = newMethodCallCounter("range")
rangeHashCounter = newMethodCallCounter("range_hash")
)
var ( // Request duration metrics.
@ -188,13 +182,13 @@ func newObjectServiceMetrics() objectServiceMetrics {
}
func (m objectServiceMetrics) register() {
prometheus.MustRegister(m.getCounter)
prometheus.MustRegister(m.putCounter)
prometheus.MustRegister(m.headCounter)
prometheus.MustRegister(m.searchCounter)
prometheus.MustRegister(m.deleteCounter)
prometheus.MustRegister(m.rangeCounter)
prometheus.MustRegister(m.rangeHashCounter)
m.getCounter.mustRegister()
m.putCounter.mustRegister()
m.headCounter.mustRegister()
m.searchCounter.mustRegister()
m.deleteCounter.mustRegister()
m.rangeCounter.mustRegister()
m.rangeHashCounter.mustRegister()
prometheus.MustRegister(m.getDuration)
prometheus.MustRegister(m.putDuration)
@ -210,32 +204,32 @@ func (m objectServiceMetrics) register() {
prometheus.MustRegister(m.shardMetrics)
}
func (m objectServiceMetrics) IncGetReqCounter() {
m.getCounter.Inc()
func (m objectServiceMetrics) IncGetReqCounter(success bool) {
m.getCounter.Inc(success)
}
func (m objectServiceMetrics) IncPutReqCounter() {
m.putCounter.Inc()
func (m objectServiceMetrics) IncPutReqCounter(success bool) {
m.putCounter.Inc(success)
}
func (m objectServiceMetrics) IncHeadReqCounter() {
m.headCounter.Inc()
func (m objectServiceMetrics) IncHeadReqCounter(success bool) {
m.headCounter.Inc(success)
}
func (m objectServiceMetrics) IncSearchReqCounter() {
m.searchCounter.Inc()
func (m objectServiceMetrics) IncSearchReqCounter(success bool) {
m.searchCounter.Inc(success)
}
func (m objectServiceMetrics) IncDeleteReqCounter() {
m.deleteCounter.Inc()
func (m objectServiceMetrics) IncDeleteReqCounter(success bool) {
m.deleteCounter.Inc(success)
}
func (m objectServiceMetrics) IncRangeReqCounter() {
m.rangeCounter.Inc()
func (m objectServiceMetrics) IncRangeReqCounter(success bool) {
m.rangeCounter.Inc(success)
}
func (m objectServiceMetrics) IncRangeHashReqCounter() {
m.rangeHashCounter.Inc()
func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) {
m.rangeHashCounter.Inc(success)
}
func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) {

86
pkg/services/object/metrics.go
View file

@ -27,13 +27,13 @@ type (
}
MetricRegister interface {
IncGetReqCounter()
IncPutReqCounter()
IncHeadReqCounter()
IncSearchReqCounter()
IncDeleteReqCounter()
IncRangeReqCounter()
IncRangeHashReqCounter()
IncGetReqCounter(success bool)
IncPutReqCounter(success bool)
IncHeadReqCounter(success bool)
IncSearchReqCounter(success bool)
IncDeleteReqCounter(success bool)
IncRangeReqCounter(success bool)
IncRangeHashReqCounter(success bool)
AddGetReqDuration(time.Duration)
AddPutReqDuration(time.Duration)
@ -55,18 +55,19 @@ func NewMetricCollector(next ServiceServer, register MetricRegister) *MetricColl
}
}
func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) error {
func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (err error) {
t := time.Now()
defer func() {
m.metrics.IncGetReqCounter()
m.metrics.IncGetReqCounter(err == nil)
m.metrics.AddGetReqDuration(time.Since(t))
}()
return m.next.Get(req, &getStreamMetric{
err = m.next.Get(req, &getStreamMetric{
ServerStream: stream,
stream: stream,
metrics: m.metrics,
})
return
}
func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) {
@ -86,52 +87,57 @@ func (m MetricCollector) Put(ctx context.Context) (PutObjectStream, error) {
func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest) (*object.HeadResponse, error) {
t := time.Now()
defer func() {
m.metrics.IncHeadReqCounter()
m.metrics.AddHeadReqDuration(time.Since(t))
}()
return m.next.Head(ctx, request)
res, err := m.next.Head(ctx, request)
m.metrics.IncHeadReqCounter(err == nil)
m.metrics.AddHeadReqDuration(time.Since(t))
return res, err
}
func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream) error {
t := time.Now()
defer func() {
m.metrics.IncSearchReqCounter()
m.metrics.AddSearchReqDuration(time.Since(t))
}()
return m.next.Search(req, stream)
err := m.next.Search(req, stream)
m.metrics.IncSearchReqCounter(err == nil)
m.metrics.AddSearchReqDuration(time.Since(t))
return err
}
func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteRequest) (*object.DeleteResponse, error) {
t := time.Now()
defer func() {
m.metrics.IncDeleteReqCounter()
m.metrics.AddDeleteReqDuration(time.Since(t))
}()
return m.next.Delete(ctx, request)
res, err := m.next.Delete(ctx, request)
m.metrics.IncDeleteReqCounter(err == nil)
m.metrics.AddDeleteReqDuration(time.Since(t))
return res, err
}
func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectRangeStream) error {
t := time.Now()
defer func() {
m.metrics.IncRangeReqCounter()
m.metrics.AddRangeReqDuration(time.Since(t))
}()
return m.next.GetRange(req, stream)
err := m.next.GetRange(req, stream)
m.metrics.IncRangeReqCounter(err == nil)
m.metrics.AddRangeReqDuration(time.Since(t))
return err
}
func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRangeHashRequest) (*object.GetRangeHashResponse, error) {
t := time.Now()
defer func() {
m.metrics.IncRangeHashReqCounter()
m.metrics.AddRangeHashReqDuration(time.Since(t))
}()
return m.next.GetRangeHash(ctx, request)
res, err := m.next.GetRangeHash(ctx, request)
m.metrics.IncRangeHashReqCounter(err == nil)
m.metrics.AddRangeHashReqDuration(time.Since(t))
return res, err
}
func (s getStreamMetric) Send(resp *object.GetResponse) error {
@ -153,10 +159,10 @@ func (s putStreamMetric) Send(req *object.PutRequest) error {
}
func (s putStreamMetric) CloseAndRecv() (*object.PutResponse, error) {
defer func() {
s.metrics.IncPutReqCounter()
s.metrics.AddPutReqDuration(time.Since(s.start))
}()
res, err := s.stream.CloseAndRecv()
return s.stream.CloseAndRecv()
s.metrics.IncPutReqCounter(err == nil)
s.metrics.AddPutReqDuration(time.Since(s.start))
return res, err
}