forked from TrueCloudLab/frostfs-node
[#312] metrics: Add writecache metrcis
Signed-off-by: Dmitrii Stepanov <d.stepanov@yadro.com>
This commit is contained in:
parent
d212d908b5
commit
2ce43935f9
14 changed files with 415 additions and 32 deletions
|
|
@ -48,6 +48,18 @@ func newGaugeVec(opts prometheus.GaugeOpts, labelNames []string) metric[*prometh
|
|||
}
|
||||
}
|
||||
|
||||
func newGaugeFunc(opts prometheus.GaugeOpts, f func() float64) metric[prometheus.GaugeFunc] {
|
||||
return metric[prometheus.GaugeFunc]{
|
||||
value: prometheus.NewGaugeFunc(opts, f),
|
||||
desc: Description{
|
||||
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
|
||||
Type: dto.MetricType_GAUGE.String(),
|
||||
Help: opts.Help,
|
||||
ConstantLabels: opts.ConstLabels,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] {
|
||||
return metric[prometheus.Counter]{
|
||||
value: prometheus.NewCounter(opts),
|
||||
|
|
@ -60,6 +72,32 @@ func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] {
|
|||
}
|
||||
}
|
||||
|
||||
func newCounterVec(opts prometheus.CounterOpts, labels []string) metric[*prometheus.CounterVec] {
|
||||
return metric[*prometheus.CounterVec]{
|
||||
value: prometheus.NewCounterVec(opts, labels),
|
||||
desc: Description{
|
||||
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
|
||||
Type: dto.MetricType_COUNTER.String(),
|
||||
Help: opts.Help,
|
||||
ConstantLabels: opts.ConstLabels,
|
||||
VariableLabels: labels,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newHistogramVec(opts prometheus.HistogramOpts, labelNames []string) metric[*prometheus.HistogramVec] {
|
||||
return metric[*prometheus.HistogramVec]{
|
||||
value: prometheus.NewHistogramVec(opts, labelNames),
|
||||
desc: Description{
|
||||
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
|
||||
Type: dto.MetricType_HISTOGRAM.String(),
|
||||
Help: opts.Help,
|
||||
ConstantLabels: opts.ConstLabels,
|
||||
VariableLabels: labelNames,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// DescribeAll returns descriptions for all registered metrics.
|
||||
func DescribeAll() ([]Description, error) {
|
||||
registeredDescriptionsMtx.Lock()
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ type NodeMetrics struct {
|
|||
stateMetrics
|
||||
replicatorMetrics
|
||||
epoch metric[prometheus.Gauge]
|
||||
|
||||
writeCacheMetrics *writeCacheMetrics
|
||||
}
|
||||
|
||||
func NewNodeMetrics() *NodeMetrics {
|
||||
|
|
@ -33,12 +35,16 @@ func NewNodeMetrics() *NodeMetrics {
|
|||
})
|
||||
mustRegister(epoch)
|
||||
|
||||
writeCacheMetrics := newWriteCacheMetrics()
|
||||
writeCacheMetrics.register()
|
||||
|
||||
return &NodeMetrics{
|
||||
objectServiceMetrics: objectService,
|
||||
engineMetrics: engine,
|
||||
stateMetrics: state,
|
||||
replicatorMetrics: replicator,
|
||||
epoch: epoch,
|
||||
writeCacheMetrics: writeCacheMetrics,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -46,3 +52,11 @@ func NewNodeMetrics() *NodeMetrics {
|
|||
func (m *NodeMetrics) SetEpoch(epoch uint64) {
|
||||
m.epoch.value.Set(float64(epoch))
|
||||
}
|
||||
|
||||
// WriteCache returns WriteCache metrics.
|
||||
func (m *NodeMetrics) WriteCache() WriteCacheMetrics {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
return m.writeCacheMetrics
|
||||
}
|
||||
|
|
|
|||
252
pkg/metrics/writecache.go
Normal file
252
pkg/metrics/writecache.go
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
package metrics
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
const (
|
||||
wcSubsystem = "writecache"
|
||||
wcShardID = "shard_id"
|
||||
wcSuccess = "success"
|
||||
wcStorage = "storage"
|
||||
wcMode = "mode"
|
||||
)
|
||||
|
||||
type shardIDMode struct {
|
||||
shardID, mode string
|
||||
}
|
||||
|
||||
type WriteCacheMetrics interface {
|
||||
AddGetDuration(shardID string, success bool, d time.Duration)
|
||||
IncGetCounter(shardID string, success bool, storageType string)
|
||||
|
||||
AddDeleteDuration(shardID string, success bool, d time.Duration)
|
||||
IncDeleteCounter(shardID string, success bool, storageType string)
|
||||
|
||||
AddPutDuration(shardID string, success bool, d time.Duration)
|
||||
IncPutCounter(shardID string, success bool, storageType string)
|
||||
|
||||
IncActualCount(shardID string, storageType string)
|
||||
DecActualCount(shardID string, storageType string)
|
||||
SetActualCount(shardID string, count uint64, storageType string)
|
||||
|
||||
SetEstimateSize(shardID string, size uint64, storageType string)
|
||||
SetMode(shardID string, mode string)
|
||||
|
||||
IncFlushCounter(shardID string, success bool, storageType string)
|
||||
IncEvictCounter(shardID string, storageType string)
|
||||
}
|
||||
|
||||
type writeCacheMetrics struct {
|
||||
getDuration metric[*prometheus.HistogramVec]
|
||||
getCounter metric[*prometheus.CounterVec]
|
||||
|
||||
putDuration metric[*prometheus.HistogramVec]
|
||||
putCounter metric[*prometheus.CounterVec]
|
||||
|
||||
deleteDuration metric[*prometheus.HistogramVec]
|
||||
deleteCounter metric[*prometheus.CounterVec]
|
||||
|
||||
flushCounter metric[*prometheus.CounterVec]
|
||||
evictCounter metric[*prometheus.CounterVec]
|
||||
|
||||
actualCount metric[*prometheus.GaugeVec]
|
||||
|
||||
estimatedSize metric[*prometheus.GaugeVec]
|
||||
|
||||
modeMetrics map[shardIDMode]metric[prometheus.GaugeFunc]
|
||||
modeValues map[string]string
|
||||
modeMtx sync.RWMutex
|
||||
}
|
||||
|
||||
func newWriteCacheMetrics() *writeCacheMetrics {
|
||||
return &writeCacheMetrics{
|
||||
getDuration: newWCMethodDurationCounter("get"),
|
||||
getCounter: newWCMethodCounterVec("get"),
|
||||
putDuration: newWCMethodDurationCounter("put"),
|
||||
putCounter: newWCMethodCounterVec("put"),
|
||||
deleteDuration: newWCMethodDurationCounter("delete"),
|
||||
deleteCounter: newWCMethodCounterVec("delete"),
|
||||
flushCounter: newWCOperationCounterVec("flush", []string{wcShardID, wcStorage, wcSuccess}),
|
||||
evictCounter: newWCOperationCounterVec("evict", []string{wcShardID, wcStorage}),
|
||||
actualCount: newWCGaugeVec("actual_objects_count", "Actual objects count in writecache", []string{wcShardID, wcStorage}),
|
||||
estimatedSize: newWCGaugeVec("estimated_size_bytes", "Estimated writecache size", []string{wcShardID, wcStorage}),
|
||||
modeMtx: sync.RWMutex{},
|
||||
modeMetrics: make(map[shardIDMode]metric[prometheus.GaugeFunc]),
|
||||
modeValues: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) AddGetDuration(shardID string, success bool, d time.Duration) {
|
||||
setWriteCacheDuration(m.getDuration.value, shardID, success, d)
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) IncGetCounter(shardID string, success bool, storageType string) {
|
||||
incWriteCacheCounter(m.getCounter.value, shardID, success, storageType)
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) AddDeleteDuration(shardID string, success bool, d time.Duration) {
|
||||
setWriteCacheDuration(m.deleteDuration.value, shardID, success, d)
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) IncDeleteCounter(shardID string, success bool, storageType string) {
|
||||
incWriteCacheCounter(m.deleteCounter.value, shardID, success, storageType)
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) AddPutDuration(shardID string, success bool, d time.Duration) {
|
||||
setWriteCacheDuration(m.putDuration.value, shardID, success, d)
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) IncPutCounter(shardID string, success bool, storageType string) {
|
||||
incWriteCacheCounter(m.putCounter.value, shardID, success, storageType)
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) IncActualCount(shardID string, storageType string) {
|
||||
m.actualCount.value.With(prometheus.Labels{
|
||||
wcShardID: shardID,
|
||||
wcStorage: storageType,
|
||||
}).Inc()
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) DecActualCount(shardID string, storageType string) {
|
||||
m.actualCount.value.With(prometheus.Labels{
|
||||
wcShardID: shardID,
|
||||
wcStorage: storageType,
|
||||
}).Dec()
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) SetActualCount(shardID string, count uint64, storageType string) {
|
||||
m.actualCount.value.With(prometheus.Labels{
|
||||
wcShardID: shardID,
|
||||
wcStorage: storageType,
|
||||
}).Set(float64(count))
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) SetEstimateSize(shardID string, size uint64, storageType string) {
|
||||
m.estimatedSize.value.With(prometheus.Labels{
|
||||
wcShardID: shardID,
|
||||
wcStorage: storageType,
|
||||
}).Set(float64(size))
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) SetMode(shardID string, mode string) {
|
||||
m.modeMtx.Lock()
|
||||
defer m.modeMtx.Unlock()
|
||||
|
||||
m.modeValues[shardID] = mode
|
||||
key := shardIDMode{
|
||||
shardID: shardID,
|
||||
mode: mode,
|
||||
}
|
||||
if _, found := m.modeMetrics[key]; found {
|
||||
return
|
||||
}
|
||||
|
||||
metric := newGaugeFunc(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: wcSubsystem,
|
||||
Name: "writecache_mode",
|
||||
Help: "Writecache mode value",
|
||||
ConstLabels: prometheus.Labels{
|
||||
wcShardID: shardID,
|
||||
wcMode: mode,
|
||||
},
|
||||
}, func() float64 {
|
||||
m.modeMtx.RLock()
|
||||
defer m.modeMtx.RUnlock()
|
||||
|
||||
value := m.modeValues[shardID]
|
||||
if value == mode {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
mustRegister(metric)
|
||||
m.modeMetrics[key] = metric
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) IncFlushCounter(shardID string, success bool, storageType string) {
|
||||
m.flushCounter.value.With(prometheus.Labels{
|
||||
wcShardID: shardID,
|
||||
wcSuccess: fmt.Sprintf("%v", success),
|
||||
wcStorage: storageType,
|
||||
}).Inc()
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) IncEvictCounter(shardID string, storageType string) {
|
||||
m.evictCounter.value.With(prometheus.Labels{
|
||||
wcShardID: shardID,
|
||||
wcStorage: storageType,
|
||||
}).Inc()
|
||||
}
|
||||
|
||||
func (m *writeCacheMetrics) register() {
|
||||
mustRegister(m.getDuration)
|
||||
mustRegister(m.getCounter)
|
||||
mustRegister(m.putDuration)
|
||||
mustRegister(m.putCounter)
|
||||
mustRegister(m.deleteDuration)
|
||||
mustRegister(m.deleteCounter)
|
||||
mustRegister(m.actualCount)
|
||||
mustRegister(m.estimatedSize)
|
||||
mustRegister(m.flushCounter)
|
||||
mustRegister(m.evictCounter)
|
||||
}
|
||||
|
||||
func setWriteCacheDuration(m *prometheus.HistogramVec, shardID string, success bool, d time.Duration) {
|
||||
m.With(
|
||||
prometheus.Labels{
|
||||
wcShardID: shardID,
|
||||
wcSuccess: fmt.Sprintf("%v", success),
|
||||
},
|
||||
).Observe(float64(d))
|
||||
}
|
||||
|
||||
func incWriteCacheCounter(m *prometheus.CounterVec, shardID string, success bool, storageType string) {
|
||||
m.With(prometheus.Labels{
|
||||
wcShardID: shardID,
|
||||
wcSuccess: fmt.Sprintf("%v", success),
|
||||
wcStorage: storageType,
|
||||
}).Inc()
|
||||
}
|
||||
|
||||
func newWCMethodDurationCounter(method string) metric[*prometheus.HistogramVec] {
|
||||
return newHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: wcSubsystem,
|
||||
Name: fmt.Sprintf("%s_req_duration_seconds", method),
|
||||
Help: fmt.Sprintf("Accumulated %s request process duration", method),
|
||||
}, []string{wcShardID, wcSuccess})
|
||||
}
|
||||
|
||||
func newWCMethodCounterVec(method string) metric[*prometheus.CounterVec] {
|
||||
return newCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: wcSubsystem,
|
||||
Name: fmt.Sprintf("%s_req_count", method),
|
||||
Help: fmt.Sprintf("The number of %s requests processed", method),
|
||||
}, []string{wcShardID, wcSuccess, wcStorage})
|
||||
}
|
||||
|
||||
func newWCOperationCounterVec(operation string, labels []string) metric[*prometheus.CounterVec] {
|
||||
return newCounterVec(prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: wcSubsystem,
|
||||
Name: fmt.Sprintf("%s_operation_count", operation),
|
||||
Help: fmt.Sprintf("The number of %s operations processed", operation),
|
||||
}, labels)
|
||||
}
|
||||
|
||||
func newWCGaugeVec(name, help string, labels []string) metric[*prometheus.GaugeVec] {
|
||||
return newGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: wcSubsystem,
|
||||
Name: name,
|
||||
Help: help,
|
||||
}, labels)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue