[#424] metrics: Drop embedded metrics
It was not obvious where metrics are used. Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
parent
1b364d8cf4
commit
c348ae35b0
10 changed files with 173 additions and 285 deletions
|
@ -679,7 +679,7 @@ func (c *cfg) engineOpts() []engine.Option {
|
||||||
)
|
)
|
||||||
|
|
||||||
if c.metricsCollector != nil {
|
if c.metricsCollector != nil {
|
||||||
opts = append(opts, engine.WithMetrics(c.metricsCollector))
|
opts = append(opts, engine.WithMetrics(c.metricsCollector.Engine()))
|
||||||
}
|
}
|
||||||
|
|
||||||
return opts
|
return opts
|
||||||
|
|
|
@ -80,7 +80,7 @@ func (c *cfg) setHealthStatus(st control.HealthStatus) {
|
||||||
c.healthStatus.Store(int32(st))
|
c.healthStatus.Store(int32(st))
|
||||||
|
|
||||||
if c.metricsCollector != nil {
|
if c.metricsCollector != nil {
|
||||||
c.metricsCollector.SetHealth(int32(st))
|
c.metricsCollector.State().SetHealth(int32(st))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -195,7 +195,7 @@ func initObjectService(c *cfg) {
|
||||||
)
|
)
|
||||||
|
|
||||||
c.shared.metricsSvc = objectService.NewMetricCollector(
|
c.shared.metricsSvc = objectService.NewMetricCollector(
|
||||||
signSvc, c.metricsCollector, metricsconfig.Enabled(c.appCfg))
|
signSvc, c.metricsCollector.ObjectService(), metricsconfig.Enabled(c.appCfg))
|
||||||
server := objectTransportGRPC.New(c.shared.metricsSvc)
|
server := objectTransportGRPC.New(c.shared.metricsSvc)
|
||||||
|
|
||||||
for _, srv := range c.cfgGRPC.servers {
|
for _, srv := range c.cfgGRPC.servers {
|
||||||
|
@ -272,7 +272,7 @@ func createReplicator(c *cfg, keyStorage *util.KeyStorage, cache *cache.ClientCa
|
||||||
replicator.WithRemoteSender(
|
replicator.WithRemoteSender(
|
||||||
putsvc.NewRemoteSender(keyStorage, cache),
|
putsvc.NewRemoteSender(keyStorage, cache),
|
||||||
),
|
),
|
||||||
replicator.WithMetrics(c.metricsCollector),
|
replicator.WithMetrics(c.metricsCollector.Replicator()),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,23 +7,40 @@ import (
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type EngineMetrics interface {
|
||||||
engineMetrics struct {
|
AddMethodDuration(method string, d time.Duration)
|
||||||
methodDuration *prometheus.HistogramVec
|
AddToContainerSize(cnrID string, size int64)
|
||||||
|
IncErrorCounter(shardID string)
|
||||||
|
ClearErrorCounter(shardID string)
|
||||||
|
DeleteShardMetrics(shardID string)
|
||||||
|
AddToObjectCounter(shardID, objectType string, delta int)
|
||||||
|
SetObjectCounter(shardID, objectType string, v uint64)
|
||||||
|
AddToPayloadCounter(shardID string, size int64)
|
||||||
|
SetReadonly(shardID string, readonly bool)
|
||||||
|
|
||||||
containerSize *prometheus.GaugeVec
|
WriteCache() WriteCacheMetrics
|
||||||
payloadSize *prometheus.GaugeVec
|
GC() GCMetrics
|
||||||
errorCounter *prometheus.GaugeVec
|
}
|
||||||
}
|
|
||||||
)
|
type engineMetrics struct {
|
||||||
|
methodDuration *prometheus.HistogramVec
|
||||||
|
objectCounter *prometheus.GaugeVec
|
||||||
|
containerSize *prometheus.GaugeVec
|
||||||
|
payloadSize *prometheus.GaugeVec
|
||||||
|
errorCounter *prometheus.GaugeVec
|
||||||
|
shardsReadonly *prometheus.GaugeVec
|
||||||
|
|
||||||
|
gc *gcMetrics
|
||||||
|
writeCache *writeCacheMetrics
|
||||||
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
engineSubsystem = "engine"
|
engineSubsystem = "engine"
|
||||||
engineMethod = "method"
|
engineMethod = "method"
|
||||||
)
|
)
|
||||||
|
|
||||||
func newEngineMetrics() engineMetrics {
|
func newEngineMetrics() *engineMetrics {
|
||||||
return engineMetrics{
|
return &engineMetrics{
|
||||||
containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}),
|
containerSize: newEngineGaugeVector("container_size_bytes", "Accumulated size of all objects in a container", []string{containerIDLabelKey}),
|
||||||
payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}),
|
payloadSize: newEngineGaugeVector("payload_size_bytes", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}),
|
||||||
errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}),
|
errorCounter: newEngineGaugeVector("error_counter", "Shard's error counter", []string{shardIDLabelKey}),
|
||||||
|
@ -33,6 +50,10 @@ func newEngineMetrics() engineMetrics {
|
||||||
Name: "request_duration_seconds",
|
Name: "request_duration_seconds",
|
||||||
Help: "Duration of Engine requests",
|
Help: "Duration of Engine requests",
|
||||||
}, []string{engineMethod}),
|
}, []string{engineMethod}),
|
||||||
|
objectCounter: newEngineGaugeVector("object_counter", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}),
|
||||||
|
shardsReadonly: newEngineGaugeVector("mode", "Shard mode", []string{shardIDLabelKey}),
|
||||||
|
gc: newGCMetrics(),
|
||||||
|
writeCache: newWriteCacheMetrics(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -51,23 +72,63 @@ func (m *engineMetrics) AddMethodDuration(method string, d time.Duration) {
|
||||||
}).Observe(d.Seconds())
|
}).Observe(d.Seconds())
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m engineMetrics) AddToContainerSize(cnrID string, size int64) {
|
func (m *engineMetrics) AddToContainerSize(cnrID string, size int64) {
|
||||||
m.containerSize.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size))
|
m.containerSize.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m engineMetrics) AddToPayloadCounter(shardID string, size int64) {
|
func (m *engineMetrics) AddToPayloadCounter(shardID string, size int64) {
|
||||||
m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size))
|
m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m engineMetrics) IncErrorCounter(shardID string) {
|
func (m *engineMetrics) IncErrorCounter(shardID string) {
|
||||||
m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Inc()
|
m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m engineMetrics) ClearErrorCounter(shardID string) {
|
func (m *engineMetrics) ClearErrorCounter(shardID string) {
|
||||||
m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0)
|
m.errorCounter.With(prometheus.Labels{shardIDLabelKey: shardID}).Set(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m engineMetrics) DeleteShardMetrics(shardID string) {
|
func (m *engineMetrics) DeleteShardMetrics(shardID string) {
|
||||||
m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
m.errorCounter.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
||||||
m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
m.payloadSize.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
||||||
|
m.objectCounter.DeletePartialMatch(prometheus.Labels{shardIDLabelKey: shardID})
|
||||||
|
m.shardsReadonly.Delete(prometheus.Labels{shardIDLabelKey: shardID})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *engineMetrics) AddToObjectCounter(shardID, objectType string, delta int) {
|
||||||
|
m.objectCounter.With(
|
||||||
|
prometheus.Labels{
|
||||||
|
shardIDLabelKey: shardID,
|
||||||
|
counterTypeLabelKey: objectType,
|
||||||
|
},
|
||||||
|
).Add(float64(delta))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *engineMetrics) SetObjectCounter(shardID, objectType string, v uint64) {
|
||||||
|
m.objectCounter.With(
|
||||||
|
prometheus.Labels{
|
||||||
|
shardIDLabelKey: shardID,
|
||||||
|
counterTypeLabelKey: objectType,
|
||||||
|
},
|
||||||
|
).Set(float64(v))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *engineMetrics) SetReadonly(shardID string, readonly bool) {
|
||||||
|
var flag float64
|
||||||
|
if readonly {
|
||||||
|
flag = 1
|
||||||
|
}
|
||||||
|
m.shardsReadonly.With(
|
||||||
|
prometheus.Labels{
|
||||||
|
shardIDLabelKey: shardID,
|
||||||
|
},
|
||||||
|
).Set(flag)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *engineMetrics) WriteCache() WriteCacheMetrics {
|
||||||
|
return m.writeCache
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *engineMetrics) GC() GCMetrics {
|
||||||
|
return m.gc
|
||||||
}
|
}
|
||||||
|
|
|
@ -54,22 +54,22 @@ func NewInnerRingMetrics() *InnerRingServiceMetrics {
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetEpoch updates epoch metrics.
|
// SetEpoch updates epoch metrics.
|
||||||
func (m InnerRingServiceMetrics) SetEpoch(epoch uint64) {
|
func (m *InnerRingServiceMetrics) SetEpoch(epoch uint64) {
|
||||||
m.epoch.Set(float64(epoch))
|
m.epoch.Set(float64(epoch))
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetHealth updates health metrics.
|
// SetHealth updates health metrics.
|
||||||
func (m InnerRingServiceMetrics) SetHealth(s int32) {
|
func (m *InnerRingServiceMetrics) SetHealth(s int32) {
|
||||||
m.health.Set(float64(s))
|
m.health.Set(float64(s))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) {
|
func (m *InnerRingServiceMetrics) AddEvent(d time.Duration, typ string, success bool) {
|
||||||
m.eventDuration.With(prometheus.Labels{
|
m.eventDuration.With(prometheus.Labels{
|
||||||
innerRingLabelType: typ,
|
innerRingLabelType: typ,
|
||||||
innerRingLabelSuccess: strconv.FormatBool(success),
|
innerRingLabelSuccess: strconv.FormatBool(success),
|
||||||
}).Observe(d.Seconds())
|
}).Observe(d.Seconds())
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics {
|
func (m *InnerRingServiceMetrics) MorphCacheMetrics() MorphCacheMetrics {
|
||||||
return m.morphCacheMetrics
|
return m.morphCacheMetrics
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,15 +8,12 @@ import (
|
||||||
const namespace = "frostfs_node"
|
const namespace = "frostfs_node"
|
||||||
|
|
||||||
type NodeMetrics struct {
|
type NodeMetrics struct {
|
||||||
objectServiceMetrics
|
engine *engineMetrics
|
||||||
engineMetrics
|
state *stateMetrics
|
||||||
stateMetrics
|
replicator *replicatorMetrics
|
||||||
replicatorMetrics
|
objectService *objectServiceMetrics
|
||||||
|
treeService *treeServiceMetrics
|
||||||
writeCacheMetrics *writeCacheMetrics
|
epoch prometheus.Gauge
|
||||||
treeService *treeServiceMetrics
|
|
||||||
epoch prometheus.Gauge
|
|
||||||
gc *gcMetrics
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewNodeMetrics() *NodeMetrics {
|
func NewNodeMetrics() *NodeMetrics {
|
||||||
|
@ -37,19 +34,13 @@ func NewNodeMetrics() *NodeMetrics {
|
||||||
Help: "Current epoch as seen by inner-ring node.",
|
Help: "Current epoch as seen by inner-ring node.",
|
||||||
})
|
})
|
||||||
|
|
||||||
writeCacheMetrics := newWriteCacheMetrics()
|
|
||||||
|
|
||||||
gc := newGCMetrics()
|
|
||||||
|
|
||||||
return &NodeMetrics{
|
return &NodeMetrics{
|
||||||
objectServiceMetrics: objectService,
|
objectService: objectService,
|
||||||
engineMetrics: engine,
|
engine: engine,
|
||||||
stateMetrics: state,
|
state: state,
|
||||||
replicatorMetrics: replicator,
|
replicator: replicator,
|
||||||
treeService: treeService,
|
treeService: treeService,
|
||||||
epoch: epoch,
|
epoch: epoch,
|
||||||
writeCacheMetrics: writeCacheMetrics,
|
|
||||||
gc: gc,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -58,18 +49,22 @@ func (m *NodeMetrics) SetEpoch(epoch uint64) {
|
||||||
m.epoch.Set(float64(epoch))
|
m.epoch.Set(float64(epoch))
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteCache returns WriteCache metrics.
|
|
||||||
func (m *NodeMetrics) WriteCache() WriteCacheMetrics {
|
|
||||||
if m == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return m.writeCacheMetrics
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m *NodeMetrics) TreeService() TreeMetricsRegister {
|
func (m *NodeMetrics) TreeService() TreeMetricsRegister {
|
||||||
return m.treeService
|
return m.treeService
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *NodeMetrics) GC() GCMetrics {
|
func (m *NodeMetrics) Replicator() ReplicatorMetrics {
|
||||||
return m.gc
|
return m.replicator
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *NodeMetrics) ObjectService() ObjectServiceMetrics {
|
||||||
|
return m.objectService
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *NodeMetrics) Engine() EngineMetrics {
|
||||||
|
return m.engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *NodeMetrics) State() StateMetrics {
|
||||||
|
return m.state
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,8 +1,7 @@
|
||||||
package metrics
|
package metrics
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
|
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
|
||||||
|
@ -11,207 +10,50 @@ import (
|
||||||
|
|
||||||
const objectSubsystem = "object"
|
const objectSubsystem = "object"
|
||||||
|
|
||||||
type (
|
type ObjectServiceMetrics interface {
|
||||||
methodCount struct {
|
AddRequestDuration(method string, d time.Duration, success bool)
|
||||||
success prometheus.Counter
|
AddPayloadSize(method string, size int)
|
||||||
total prometheus.Counter
|
}
|
||||||
}
|
|
||||||
|
|
||||||
objectServiceMetrics struct {
|
type objectServiceMetrics struct {
|
||||||
getCounter methodCount
|
methodDuration *prometheus.HistogramVec
|
||||||
putCounter methodCount
|
payloadCounter *prometheus.CounterVec
|
||||||
headCounter methodCount
|
}
|
||||||
searchCounter methodCount
|
|
||||||
deleteCounter methodCount
|
|
||||||
rangeCounter methodCount
|
|
||||||
rangeHashCounter methodCount
|
|
||||||
|
|
||||||
getDuration prometheus.Counter
|
|
||||||
putDuration prometheus.Counter
|
|
||||||
headDuration prometheus.Counter
|
|
||||||
searchDuration prometheus.Counter
|
|
||||||
deleteDuration prometheus.Counter
|
|
||||||
rangeDuration prometheus.Counter
|
|
||||||
rangeHashDuration prometheus.Counter
|
|
||||||
|
|
||||||
putPayload prometheus.Counter
|
|
||||||
getPayload prometheus.Counter
|
|
||||||
|
|
||||||
shardMetrics *prometheus.GaugeVec
|
|
||||||
shardsReadonly *prometheus.GaugeVec
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
shardIDLabelKey = "shard"
|
shardIDLabelKey = "shard"
|
||||||
counterTypeLabelKey = "type"
|
counterTypeLabelKey = "type"
|
||||||
containerIDLabelKey = "cid"
|
containerIDLabelKey = "cid"
|
||||||
|
methodLabelKey = "method"
|
||||||
|
successLabelKey = "success"
|
||||||
)
|
)
|
||||||
|
|
||||||
func newObjectMethodCallCounter(name string) methodCount {
|
func newObjectServiceMetrics() *objectServiceMetrics {
|
||||||
return methodCount{
|
return &objectServiceMetrics{
|
||||||
success: metrics.NewCounter(prometheus.CounterOpts{
|
methodDuration: metrics.NewHistogramVec(prometheus.HistogramOpts{
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
Subsystem: objectSubsystem,
|
Subsystem: objectSubsystem,
|
||||||
Name: fmt.Sprintf("%s_req_count_success", name),
|
Name: "request_duration_seconds",
|
||||||
Help: fmt.Sprintf("The number of successful %s requests processed", name),
|
Help: "Object Service request process duration",
|
||||||
}),
|
}, []string{methodLabelKey, successLabelKey}),
|
||||||
total: metrics.NewCounter(prometheus.CounterOpts{
|
payloadCounter: metrics.NewCounterVec(prometheus.CounterOpts{
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
Subsystem: objectSubsystem,
|
Subsystem: objectSubsystem,
|
||||||
Name: fmt.Sprintf("%s_req_count", name),
|
Name: "request_payload_bytes",
|
||||||
Help: fmt.Sprintf("Total number of %s requests processed", name),
|
Help: "Object Service request payload",
|
||||||
}),
|
}, []string{methodLabelKey}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m methodCount) Inc(success bool) {
|
func (m *objectServiceMetrics) AddRequestDuration(method string, d time.Duration, success bool) {
|
||||||
m.total.Inc()
|
m.methodDuration.With(prometheus.Labels{
|
||||||
if success {
|
methodLabelKey: method,
|
||||||
m.success.Inc()
|
successLabelKey: strconv.FormatBool(success),
|
||||||
}
|
}).Observe(d.Seconds())
|
||||||
}
|
}
|
||||||
|
|
||||||
func newObjectServiceMetrics() objectServiceMetrics {
|
func (m *objectServiceMetrics) AddPayloadSize(method string, size int) {
|
||||||
return objectServiceMetrics{
|
m.payloadCounter.With(prometheus.Labels{
|
||||||
getCounter: newObjectMethodCallCounter("get"),
|
methodLabelKey: method,
|
||||||
putCounter: newObjectMethodCallCounter("put"),
|
}).Add(float64(size))
|
||||||
headCounter: newObjectMethodCallCounter("head"),
|
|
||||||
searchCounter: newObjectMethodCallCounter("search"),
|
|
||||||
deleteCounter: newObjectMethodCallCounter("delete"),
|
|
||||||
rangeCounter: newObjectMethodCallCounter("range"),
|
|
||||||
rangeHashCounter: newObjectMethodCallCounter("range_hash"),
|
|
||||||
getDuration: newObjectMethodDurationCounter("get"),
|
|
||||||
putDuration: newObjectMethodDurationCounter("put"),
|
|
||||||
headDuration: newObjectMethodDurationCounter("head"),
|
|
||||||
searchDuration: newObjectMethodDurationCounter("search"),
|
|
||||||
deleteDuration: newObjectMethodDurationCounter("delete"),
|
|
||||||
rangeDuration: newObjectMethodDurationCounter("range"),
|
|
||||||
rangeHashDuration: newObjectMethodDurationCounter("range_hash"),
|
|
||||||
putPayload: newObjectMethodPayloadCounter("put"),
|
|
||||||
getPayload: newObjectMethodPayloadCounter("get"),
|
|
||||||
shardMetrics: newObjectGaugeVector("counter", "Objects counters per shards", []string{shardIDLabelKey, counterTypeLabelKey}),
|
|
||||||
shardsReadonly: newObjectGaugeVector("readonly", "Shard state", []string{shardIDLabelKey}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newObjectMethodPayloadCounter(method string) prometheus.Counter {
|
|
||||||
return metrics.NewCounter(prometheus.CounterOpts{
|
|
||||||
Namespace: namespace,
|
|
||||||
Subsystem: objectSubsystem,
|
|
||||||
Name: fmt.Sprintf("%s_payload", method),
|
|
||||||
Help: fmt.Sprintf("Accumulated payload size at object %s method", strings.ReplaceAll(method, "_", " ")),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func newObjectMethodDurationCounter(method string) prometheus.Counter {
|
|
||||||
return metrics.NewCounter(prometheus.CounterOpts{
|
|
||||||
Namespace: namespace,
|
|
||||||
Subsystem: objectSubsystem,
|
|
||||||
Name: fmt.Sprintf("%s_req_duration", method),
|
|
||||||
Help: fmt.Sprintf("Accumulated %s request process duration", strings.ReplaceAll(method, "_", " ")),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func newObjectGaugeVector(name, help string, labels []string) *prometheus.GaugeVec {
|
|
||||||
return metrics.NewGaugeVec(prometheus.GaugeOpts{
|
|
||||||
Namespace: namespace,
|
|
||||||
Subsystem: objectSubsystem,
|
|
||||||
Name: name,
|
|
||||||
Help: help,
|
|
||||||
}, labels)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) IncGetReqCounter(success bool) {
|
|
||||||
m.getCounter.Inc(success)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) IncPutReqCounter(success bool) {
|
|
||||||
m.putCounter.Inc(success)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) IncHeadReqCounter(success bool) {
|
|
||||||
m.headCounter.Inc(success)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) IncSearchReqCounter(success bool) {
|
|
||||||
m.searchCounter.Inc(success)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) IncDeleteReqCounter(success bool) {
|
|
||||||
m.deleteCounter.Inc(success)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) IncRangeReqCounter(success bool) {
|
|
||||||
m.rangeCounter.Inc(success)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) {
|
|
||||||
m.rangeHashCounter.Inc(success)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) {
|
|
||||||
m.getDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) AddPutReqDuration(d time.Duration) {
|
|
||||||
m.putDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) AddHeadReqDuration(d time.Duration) {
|
|
||||||
m.headDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) AddSearchReqDuration(d time.Duration) {
|
|
||||||
m.searchDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) AddDeleteReqDuration(d time.Duration) {
|
|
||||||
m.deleteDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) AddRangeReqDuration(d time.Duration) {
|
|
||||||
m.rangeDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) AddRangeHashReqDuration(d time.Duration) {
|
|
||||||
m.rangeHashDuration.Add(float64(d))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) AddPutPayload(ln int) {
|
|
||||||
m.putPayload.Add(float64(ln))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) AddGetPayload(ln int) {
|
|
||||||
m.getPayload.Add(float64(ln))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, delta int) {
|
|
||||||
m.shardMetrics.With(
|
|
||||||
prometheus.Labels{
|
|
||||||
shardIDLabelKey: shardID,
|
|
||||||
counterTypeLabelKey: objectType,
|
|
||||||
},
|
|
||||||
).Add(float64(delta))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) SetObjectCounter(shardID, objectType string, v uint64) {
|
|
||||||
m.shardMetrics.With(
|
|
||||||
prometheus.Labels{
|
|
||||||
shardIDLabelKey: shardID,
|
|
||||||
counterTypeLabelKey: objectType,
|
|
||||||
},
|
|
||||||
).Set(float64(v))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (m objectServiceMetrics) SetReadonly(shardID string, readonly bool) {
|
|
||||||
var flag float64
|
|
||||||
if readonly {
|
|
||||||
flag = 1
|
|
||||||
}
|
|
||||||
m.shardsReadonly.With(
|
|
||||||
prometheus.Labels{
|
|
||||||
shardIDLabelKey: shardID,
|
|
||||||
},
|
|
||||||
).Set(flag)
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,30 +7,39 @@ import (
|
||||||
|
|
||||||
const replicatorSubsystem = "replicator"
|
const replicatorSubsystem = "replicator"
|
||||||
|
|
||||||
|
//TODO
|
||||||
|
|
||||||
|
type ReplicatorMetrics interface {
|
||||||
|
IncInFlightRequest()
|
||||||
|
DecInFlightRequest()
|
||||||
|
IncProcessedObjects()
|
||||||
|
AddPayloadSize(size int64)
|
||||||
|
}
|
||||||
|
|
||||||
type replicatorMetrics struct {
|
type replicatorMetrics struct {
|
||||||
inFlightRequests prometheus.Gauge
|
inFlightRequests prometheus.Gauge
|
||||||
processedObjects prometheus.Counter
|
processedObjects prometheus.Counter
|
||||||
totalReplicatedPayloadSize prometheus.Counter
|
totalReplicatedPayloadSize prometheus.Counter
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m replicatorMetrics) IncInFlightRequest() {
|
func (m *replicatorMetrics) IncInFlightRequest() {
|
||||||
m.inFlightRequests.Inc()
|
m.inFlightRequests.Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m replicatorMetrics) DecInFlightRequest() {
|
func (m *replicatorMetrics) DecInFlightRequest() {
|
||||||
m.inFlightRequests.Dec()
|
m.inFlightRequests.Dec()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m replicatorMetrics) IncProcessedObjects() {
|
func (m *replicatorMetrics) IncProcessedObjects() {
|
||||||
m.processedObjects.Inc()
|
m.processedObjects.Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m replicatorMetrics) AddPayloadSize(size int64) {
|
func (m *replicatorMetrics) AddPayloadSize(size int64) {
|
||||||
m.totalReplicatedPayloadSize.Add(float64(size))
|
m.totalReplicatedPayloadSize.Add(float64(size))
|
||||||
}
|
}
|
||||||
|
|
||||||
func newReplicatorMetrics() replicatorMetrics {
|
func newReplicatorMetrics() *replicatorMetrics {
|
||||||
return replicatorMetrics{
|
return &replicatorMetrics{
|
||||||
inFlightRequests: newReplicatorGauge("in_flight_requests", "Number of in-flight requests"),
|
inFlightRequests: newReplicatorGauge("in_flight_requests", "Number of in-flight requests"),
|
||||||
processedObjects: newReplicatorCounter("processed_objects", "Number of objects processed since the node startup"),
|
processedObjects: newReplicatorCounter("processed_objects", "Number of objects processed since the node startup"),
|
||||||
totalReplicatedPayloadSize: newReplicatorCounter("total_replicated_payload_size", "Total size of payloads replicated"),
|
totalReplicatedPayloadSize: newReplicatorCounter("total_replicated_payload_size", "Total size of payloads replicated"),
|
||||||
|
|
|
@ -7,12 +7,16 @@ import (
|
||||||
|
|
||||||
const stateSubsystem = "state"
|
const stateSubsystem = "state"
|
||||||
|
|
||||||
|
type StateMetrics interface {
|
||||||
|
SetHealth(s int32)
|
||||||
|
}
|
||||||
|
|
||||||
type stateMetrics struct {
|
type stateMetrics struct {
|
||||||
healthCheck prometheus.Gauge
|
healthCheck prometheus.Gauge
|
||||||
}
|
}
|
||||||
|
|
||||||
func newStateMetrics() stateMetrics {
|
func newStateMetrics() *stateMetrics {
|
||||||
return stateMetrics{
|
return &stateMetrics{
|
||||||
healthCheck: metrics.NewGauge(prometheus.GaugeOpts{
|
healthCheck: metrics.NewGauge(prometheus.GaugeOpts{
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
Subsystem: stateSubsystem,
|
Subsystem: stateSubsystem,
|
||||||
|
@ -22,6 +26,6 @@ func newStateMetrics() stateMetrics {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m stateMetrics) SetHealth(s int32) {
|
func (m *stateMetrics) SetHealth(s int32) {
|
||||||
m.healthCheck.Set(float64(s))
|
m.healthCheck.Set(float64(s))
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,24 +28,8 @@ type (
|
||||||
}
|
}
|
||||||
|
|
||||||
MetricRegister interface {
|
MetricRegister interface {
|
||||||
IncGetReqCounter(success bool)
|
AddRequestDuration(string, time.Duration, bool)
|
||||||
IncPutReqCounter(success bool)
|
AddPayloadSize(string, int)
|
||||||
IncHeadReqCounter(success bool)
|
|
||||||
IncSearchReqCounter(success bool)
|
|
||||||
IncDeleteReqCounter(success bool)
|
|
||||||
IncRangeReqCounter(success bool)
|
|
||||||
IncRangeHashReqCounter(success bool)
|
|
||||||
|
|
||||||
AddGetReqDuration(time.Duration)
|
|
||||||
AddPutReqDuration(time.Duration)
|
|
||||||
AddHeadReqDuration(time.Duration)
|
|
||||||
AddSearchReqDuration(time.Duration)
|
|
||||||
AddDeleteReqDuration(time.Duration)
|
|
||||||
AddRangeReqDuration(time.Duration)
|
|
||||||
AddRangeHashReqDuration(time.Duration)
|
|
||||||
|
|
||||||
AddPutPayload(int)
|
|
||||||
AddGetPayload(int)
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -61,8 +45,7 @@ func (m MetricCollector) Get(req *object.GetRequest, stream GetObjectStream) (er
|
||||||
if m.enabled {
|
if m.enabled {
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
defer func() {
|
defer func() {
|
||||||
m.metrics.IncGetReqCounter(err == nil)
|
m.metrics.AddRequestDuration("Get", time.Since(t), err == nil)
|
||||||
m.metrics.AddGetReqDuration(time.Since(t))
|
|
||||||
}()
|
}()
|
||||||
err = m.next.Get(req, &getStreamMetric{
|
err = m.next.Get(req, &getStreamMetric{
|
||||||
ServerStream: stream,
|
ServerStream: stream,
|
||||||
|
@ -99,8 +82,7 @@ func (m MetricCollector) Head(ctx context.Context, request *object.HeadRequest)
|
||||||
|
|
||||||
res, err := m.next.Head(ctx, request)
|
res, err := m.next.Head(ctx, request)
|
||||||
|
|
||||||
m.metrics.IncHeadReqCounter(err == nil)
|
m.metrics.AddRequestDuration("Head", time.Since(t), err == nil)
|
||||||
m.metrics.AddHeadReqDuration(time.Since(t))
|
|
||||||
|
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
|
@ -113,8 +95,7 @@ func (m MetricCollector) Search(req *object.SearchRequest, stream SearchStream)
|
||||||
|
|
||||||
err := m.next.Search(req, stream)
|
err := m.next.Search(req, stream)
|
||||||
|
|
||||||
m.metrics.IncSearchReqCounter(err == nil)
|
m.metrics.AddRequestDuration("Search", time.Since(t), err == nil)
|
||||||
m.metrics.AddSearchReqDuration(time.Since(t))
|
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -127,8 +108,7 @@ func (m MetricCollector) Delete(ctx context.Context, request *object.DeleteReque
|
||||||
|
|
||||||
res, err := m.next.Delete(ctx, request)
|
res, err := m.next.Delete(ctx, request)
|
||||||
|
|
||||||
m.metrics.IncDeleteReqCounter(err == nil)
|
m.metrics.AddRequestDuration("Delete", time.Since(t), err == nil)
|
||||||
m.metrics.AddDeleteReqDuration(time.Since(t))
|
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
return m.next.Delete(ctx, request)
|
return m.next.Delete(ctx, request)
|
||||||
|
@ -140,8 +120,7 @@ func (m MetricCollector) GetRange(req *object.GetRangeRequest, stream GetObjectR
|
||||||
|
|
||||||
err := m.next.GetRange(req, stream)
|
err := m.next.GetRange(req, stream)
|
||||||
|
|
||||||
m.metrics.IncRangeReqCounter(err == nil)
|
m.metrics.AddRequestDuration("GetRange", time.Since(t), err == nil)
|
||||||
m.metrics.AddRangeReqDuration(time.Since(t))
|
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -154,8 +133,7 @@ func (m MetricCollector) GetRangeHash(ctx context.Context, request *object.GetRa
|
||||||
|
|
||||||
res, err := m.next.GetRangeHash(ctx, request)
|
res, err := m.next.GetRangeHash(ctx, request)
|
||||||
|
|
||||||
m.metrics.IncRangeHashReqCounter(err == nil)
|
m.metrics.AddRequestDuration("GetRangeHash", time.Since(t), err == nil)
|
||||||
m.metrics.AddRangeHashReqDuration(time.Since(t))
|
|
||||||
|
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
|
@ -173,7 +151,7 @@ func (m *MetricCollector) Disable() {
|
||||||
func (s getStreamMetric) Send(resp *object.GetResponse) error {
|
func (s getStreamMetric) Send(resp *object.GetResponse) error {
|
||||||
chunk, ok := resp.GetBody().GetObjectPart().(*object.GetObjectPartChunk)
|
chunk, ok := resp.GetBody().GetObjectPart().(*object.GetObjectPartChunk)
|
||||||
if ok {
|
if ok {
|
||||||
s.metrics.AddGetPayload(len(chunk.GetChunk()))
|
s.metrics.AddPayloadSize("Get", len(chunk.GetChunk()))
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.stream.Send(resp)
|
return s.stream.Send(resp)
|
||||||
|
@ -182,7 +160,7 @@ func (s getStreamMetric) Send(resp *object.GetResponse) error {
|
||||||
func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error {
|
func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error {
|
||||||
chunk, ok := req.GetBody().GetObjectPart().(*object.PutObjectPartChunk)
|
chunk, ok := req.GetBody().GetObjectPart().(*object.PutObjectPartChunk)
|
||||||
if ok {
|
if ok {
|
||||||
s.metrics.AddPutPayload(len(chunk.GetChunk()))
|
s.metrics.AddPayloadSize("Put", len(chunk.GetChunk()))
|
||||||
}
|
}
|
||||||
|
|
||||||
return s.stream.Send(ctx, req)
|
return s.stream.Send(ctx, req)
|
||||||
|
@ -191,8 +169,7 @@ func (s putStreamMetric) Send(ctx context.Context, req *object.PutRequest) error
|
||||||
func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) {
|
func (s putStreamMetric) CloseAndRecv(ctx context.Context) (*object.PutResponse, error) {
|
||||||
res, err := s.stream.CloseAndRecv(ctx)
|
res, err := s.stream.CloseAndRecv(ctx)
|
||||||
|
|
||||||
s.metrics.IncPutReqCounter(err == nil)
|
s.metrics.AddRequestDuration("Put", time.Since(s.start), err == nil)
|
||||||
s.metrics.AddPutReqDuration(time.Since(s.start))
|
|
||||||
|
|
||||||
return res, err
|
return res, err
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue