2021-03-16 08:14:56 +00:00
|
|
|
package metrics
|
|
|
|
|
|
|
|
import (
|
2022-11-29 11:51:19 +00:00
|
|
|
"fmt"
|
2023-04-05 09:43:32 +00:00
|
|
|
"strings"
|
2021-03-16 08:14:56 +00:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
)
|
|
|
|
|
|
|
|
const objectSubsystem = "object"
|
|
|
|
|
|
|
|
type (
|
2022-11-29 11:51:19 +00:00
|
|
|
methodCount struct {
|
|
|
|
success prometheus.Counter
|
|
|
|
total prometheus.Counter
|
|
|
|
}
|
|
|
|
|
2021-03-16 08:14:56 +00:00
|
|
|
objectServiceMetrics struct {
|
2022-11-29 11:51:19 +00:00
|
|
|
getCounter methodCount
|
|
|
|
putCounter methodCount
|
|
|
|
headCounter methodCount
|
|
|
|
searchCounter methodCount
|
|
|
|
deleteCounter methodCount
|
|
|
|
rangeCounter methodCount
|
|
|
|
rangeHashCounter methodCount
|
2021-03-16 08:14:56 +00:00
|
|
|
|
|
|
|
getDuration prometheus.Counter
|
|
|
|
putDuration prometheus.Counter
|
|
|
|
headDuration prometheus.Counter
|
|
|
|
searchDuration prometheus.Counter
|
|
|
|
deleteDuration prometheus.Counter
|
|
|
|
rangeDuration prometheus.Counter
|
|
|
|
rangeHashDuration prometheus.Counter
|
|
|
|
|
|
|
|
putPayload prometheus.Counter
|
|
|
|
getPayload prometheus.Counter
|
2022-08-19 16:41:29 +00:00
|
|
|
|
2022-12-09 13:52:13 +00:00
|
|
|
shardMetrics *prometheus.GaugeVec
|
|
|
|
shardsReadonly *prometheus.GaugeVec
|
2021-03-16 08:14:56 +00:00
|
|
|
}
|
|
|
|
)
|
|
|
|
|
2022-09-09 09:47:07 +00:00
|
|
|
const (
|
|
|
|
shardIDLabelKey = "shard"
|
|
|
|
counterTypeLabelKey = "type"
|
2022-12-01 11:59:22 +00:00
|
|
|
containerIDLabelKey = "cid"
|
2022-09-09 09:47:07 +00:00
|
|
|
)
|
2022-08-19 16:41:29 +00:00
|
|
|
|
2023-04-05 09:43:32 +00:00
|
|
|
func newObjectMethodCallCounter(name string) methodCount {
|
2022-11-29 11:51:19 +00:00
|
|
|
return methodCount{
|
|
|
|
success: prometheus.NewCounter(prometheus.CounterOpts{
|
2021-03-16 08:14:56 +00:00
|
|
|
Namespace: namespace,
|
|
|
|
Subsystem: objectSubsystem,
|
2023-02-07 09:45:51 +00:00
|
|
|
Name: fmt.Sprintf("%s_req_count_success", name),
|
2022-11-29 11:51:19 +00:00
|
|
|
Help: fmt.Sprintf("The number of successful %s requests processed", name),
|
|
|
|
}),
|
|
|
|
total: prometheus.NewCounter(prometheus.CounterOpts{
|
2021-03-16 08:14:56 +00:00
|
|
|
Namespace: namespace,
|
|
|
|
Subsystem: objectSubsystem,
|
2023-02-07 09:45:51 +00:00
|
|
|
Name: fmt.Sprintf("%s_req_count", name),
|
2022-11-29 11:51:19 +00:00
|
|
|
Help: fmt.Sprintf("Total number of %s requests processed", name),
|
|
|
|
}),
|
|
|
|
}
|
|
|
|
}
|
2021-03-16 08:14:56 +00:00
|
|
|
|
2022-11-29 11:51:19 +00:00
|
|
|
func (m methodCount) mustRegister() {
|
|
|
|
prometheus.MustRegister(m.success)
|
|
|
|
prometheus.MustRegister(m.total)
|
|
|
|
}
|
2021-03-16 08:14:56 +00:00
|
|
|
|
2022-11-29 11:51:19 +00:00
|
|
|
func (m methodCount) Inc(success bool) {
|
|
|
|
m.total.Inc()
|
|
|
|
if success {
|
|
|
|
m.success.Inc()
|
|
|
|
}
|
|
|
|
}
|
2021-03-16 08:14:56 +00:00
|
|
|
|
2022-11-29 11:51:19 +00:00
|
|
|
func newObjectServiceMetrics() objectServiceMetrics {
|
2023-04-05 09:43:32 +00:00
|
|
|
return objectServiceMetrics{
|
|
|
|
getCounter: newObjectMethodCallCounter("get"),
|
|
|
|
putCounter: newObjectMethodCallCounter("put"),
|
|
|
|
headCounter: newObjectMethodCallCounter("head"),
|
|
|
|
searchCounter: newObjectMethodCallCounter("search"),
|
|
|
|
deleteCounter: newObjectMethodCallCounter("delete"),
|
|
|
|
rangeCounter: newObjectMethodCallCounter("range"),
|
|
|
|
rangeHashCounter: newObjectMethodCallCounter("range_hash"),
|
|
|
|
getDuration: newObjectMethodDurationCounter("get"),
|
|
|
|
putDuration: newObjectMethodDurationCounter("put"),
|
|
|
|
headDuration: newObjectMethodDurationCounter("head"),
|
|
|
|
searchDuration: newObjectMethodDurationCounter("search"),
|
|
|
|
deleteDuration: newObjectMethodDurationCounter("delete"),
|
|
|
|
rangeDuration: newObjectMethodDurationCounter("range"),
|
|
|
|
rangeHashDuration: newObjectMethodDurationCounter("range_hash"),
|
|
|
|
putPayload: newObjectMethodPayloadCounter("put"),
|
|
|
|
getPayload: newObjectMethodPayloadCounter("get"),
|
|
|
|
shardMetrics: newObjectGaugeVector("counter", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}),
|
|
|
|
shardsReadonly: newObjectGaugeVector("readonly", "Shard state", []string{shardIDLabelKey}),
|
|
|
|
}
|
|
|
|
}
|
2022-08-19 16:41:29 +00:00
|
|
|
|
2023-04-05 09:43:32 +00:00
|
|
|
func newObjectMethodPayloadCounter(method string) prometheus.Counter {
|
|
|
|
return prometheus.NewCounter(prometheus.CounterOpts{
|
|
|
|
Namespace: namespace,
|
|
|
|
Subsystem: objectSubsystem,
|
|
|
|
Name: fmt.Sprintf("%s_payload", method),
|
|
|
|
Help: fmt.Sprintf("Accumulated payload size at object %s method", strings.ReplaceAll(method, "_", " ")),
|
|
|
|
})
|
|
|
|
}
|
2022-12-09 13:52:13 +00:00
|
|
|
|
2023-04-05 09:43:32 +00:00
|
|
|
func newObjectMethodDurationCounter(method string) prometheus.Counter {
|
|
|
|
return prometheus.NewCounter(prometheus.CounterOpts{
|
|
|
|
Namespace: namespace,
|
|
|
|
Subsystem: objectSubsystem,
|
|
|
|
Name: fmt.Sprintf("%s_req_duration", method),
|
|
|
|
Help: fmt.Sprintf("Accumulated %s request process duration", strings.ReplaceAll(method, "_", " ")),
|
|
|
|
})
|
|
|
|
}
|
2021-03-16 08:14:56 +00:00
|
|
|
|
2023-04-05 09:43:32 +00:00
|
|
|
func newObjectGaugeVector(name, help string, labels []string) *prometheus.GaugeVec {
|
|
|
|
return prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
|
|
Namespace: namespace,
|
|
|
|
Subsystem: objectSubsystem,
|
|
|
|
Name: name,
|
|
|
|
Help: help,
|
|
|
|
}, labels)
|
2021-03-16 08:14:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (m objectServiceMetrics) register() {
|
2022-11-29 11:51:19 +00:00
|
|
|
m.getCounter.mustRegister()
|
|
|
|
m.putCounter.mustRegister()
|
|
|
|
m.headCounter.mustRegister()
|
|
|
|
m.searchCounter.mustRegister()
|
|
|
|
m.deleteCounter.mustRegister()
|
|
|
|
m.rangeCounter.mustRegister()
|
|
|
|
m.rangeHashCounter.mustRegister()
|
2021-03-16 08:14:56 +00:00
|
|
|
|
|
|
|
prometheus.MustRegister(m.getDuration)
|
|
|
|
prometheus.MustRegister(m.putDuration)
|
|
|
|
prometheus.MustRegister(m.headDuration)
|
|
|
|
prometheus.MustRegister(m.searchDuration)
|
|
|
|
prometheus.MustRegister(m.deleteDuration)
|
|
|
|
prometheus.MustRegister(m.rangeDuration)
|
|
|
|
prometheus.MustRegister(m.rangeHashDuration)
|
|
|
|
|
|
|
|
prometheus.MustRegister(m.putPayload)
|
|
|
|
prometheus.MustRegister(m.getPayload)
|
2022-08-19 16:41:29 +00:00
|
|
|
|
|
|
|
prometheus.MustRegister(m.shardMetrics)
|
2022-12-09 13:52:13 +00:00
|
|
|
prometheus.MustRegister(m.shardsReadonly)
|
2021-03-16 08:14:56 +00:00
|
|
|
}
|
|
|
|
|
2022-11-29 11:51:19 +00:00
|
|
|
func (m objectServiceMetrics) IncGetReqCounter(success bool) {
|
|
|
|
m.getCounter.Inc(success)
|
2021-03-16 08:14:56 +00:00
|
|
|
}
|
|
|
|
|
2022-11-29 11:51:19 +00:00
|
|
|
func (m objectServiceMetrics) IncPutReqCounter(success bool) {
|
|
|
|
m.putCounter.Inc(success)
|
2021-03-16 08:14:56 +00:00
|
|
|
}
|
|
|
|
|
2022-11-29 11:51:19 +00:00
|
|
|
func (m objectServiceMetrics) IncHeadReqCounter(success bool) {
|
|
|
|
m.headCounter.Inc(success)
|
2021-03-16 08:14:56 +00:00
|
|
|
}
|
|
|
|
|
2022-11-29 11:51:19 +00:00
|
|
|
func (m objectServiceMetrics) IncSearchReqCounter(success bool) {
|
|
|
|
m.searchCounter.Inc(success)
|
2021-03-16 08:14:56 +00:00
|
|
|
}
|
|
|
|
|
2022-11-29 11:51:19 +00:00
|
|
|
func (m objectServiceMetrics) IncDeleteReqCounter(success bool) {
|
|
|
|
m.deleteCounter.Inc(success)
|
2021-03-16 08:14:56 +00:00
|
|
|
}
|
|
|
|
|
2022-11-29 11:51:19 +00:00
|
|
|
func (m objectServiceMetrics) IncRangeReqCounter(success bool) {
|
|
|
|
m.rangeCounter.Inc(success)
|
2021-03-16 08:14:56 +00:00
|
|
|
}
|
|
|
|
|
2022-11-29 11:51:19 +00:00
|
|
|
func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) {
|
|
|
|
m.rangeHashCounter.Inc(success)
|
2021-03-16 08:14:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) {
|
|
|
|
m.getDuration.Add(float64(d))
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m objectServiceMetrics) AddPutReqDuration(d time.Duration) {
|
|
|
|
m.putDuration.Add(float64(d))
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m objectServiceMetrics) AddHeadReqDuration(d time.Duration) {
|
|
|
|
m.headDuration.Add(float64(d))
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m objectServiceMetrics) AddSearchReqDuration(d time.Duration) {
|
|
|
|
m.searchDuration.Add(float64(d))
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m objectServiceMetrics) AddDeleteReqDuration(d time.Duration) {
|
|
|
|
m.deleteDuration.Add(float64(d))
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m objectServiceMetrics) AddRangeReqDuration(d time.Duration) {
|
|
|
|
m.rangeDuration.Add(float64(d))
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m objectServiceMetrics) AddRangeHashReqDuration(d time.Duration) {
|
|
|
|
m.rangeHashDuration.Add(float64(d))
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m objectServiceMetrics) AddPutPayload(ln int) {
|
|
|
|
m.putPayload.Add(float64(ln))
|
|
|
|
}
|
|
|
|
|
|
|
|
func (m objectServiceMetrics) AddGetPayload(ln int) {
|
|
|
|
m.getPayload.Add(float64(ln))
|
|
|
|
}
|
2022-08-19 16:41:29 +00:00
|
|
|
|
2022-09-09 09:47:07 +00:00
|
|
|
func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, delta int) {
|
|
|
|
m.shardMetrics.With(
|
|
|
|
prometheus.Labels{
|
|
|
|
shardIDLabelKey: shardID,
|
|
|
|
counterTypeLabelKey: objectType,
|
|
|
|
},
|
|
|
|
).Add(float64(delta))
|
2022-08-19 16:41:29 +00:00
|
|
|
}
|
2022-08-19 17:24:05 +00:00
|
|
|
|
2022-09-09 09:47:07 +00:00
|
|
|
func (m objectServiceMetrics) SetObjectCounter(shardID, objectType string, v uint64) {
|
|
|
|
m.shardMetrics.With(
|
|
|
|
prometheus.Labels{
|
|
|
|
shardIDLabelKey: shardID,
|
|
|
|
counterTypeLabelKey: objectType,
|
|
|
|
},
|
|
|
|
).Set(float64(v))
|
2022-08-19 17:24:05 +00:00
|
|
|
}
|
2022-12-09 13:52:13 +00:00
|
|
|
|
|
|
|
func (m objectServiceMetrics) SetReadonly(shardID string, readonly bool) {
|
|
|
|
var flag float64
|
|
|
|
if readonly {
|
|
|
|
flag = 1
|
|
|
|
}
|
|
|
|
m.shardsReadonly.With(
|
|
|
|
prometheus.Labels{
|
|
|
|
shardIDLabelKey: shardID,
|
|
|
|
},
|
|
|
|
).Set(flag)
|
|
|
|
}
|