TrueCloudLab/frostfs-node
20
1
Fork 27
Code Issues 93 Pull requests 4 Projects Releases 1 Activity Actions

[#164] metrics: Allow to export metrics description

Browse source 
Signed-off-by: Evgenii Stratonikov <e.stratonikov@yadro.com>
This commit is contained in:
Evgenii Stratonikov 2023-04-05 13:39:02 +03:00 committed by Evgenii Stratonikov
parent 4ade5339da
commit 6fef2726b8
9 changed files with 229 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -24,6 +24,7 @@ require (
github.com/panjf2000/ants/v2 v2.4.0
github.com/paulmach/orb v0.2.2
github.com/prometheus/client_golang v1.15.0
github.com/prometheus/client_model v0.3.0
github.com/spf13/cast v1.5.0
github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5
@ -85,7 +86,6 @@ require (
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.7 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect

71
pkg/metrics/desc.go Normal file
View file

@ -0,0 +1,71 @@
package metrics
import (
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
)
type metric[T prometheus.Collector] struct {
value T
desc Description
}
// Descriptions contains metric description suitable for further processing.
// The only reason for it to exist is `prometheus.Desc` disallowing field access directly.
// https://github.com/prometheus/client_golang/pull/326
// https://github.com/prometheus/client_golang/issues/516
// https://github.com/prometheus/client_golang/issues/222
type Description struct {
Name string `json:"name"`
Help string `json:"help"`
Type string `json:"type"`
ConstantLabels prometheus.Labels `json:"constant_labels,omitempty"`
VariableLabels []string `json:"variable_labels,omitempty"`
}
func newGauge(opts prometheus.GaugeOpts) metric[prometheus.Gauge] {
return metric[prometheus.Gauge]{
value: prometheus.NewGauge(opts),
desc: Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_GAUGE.String(),
Help: opts.Help,
ConstantLabels: opts.ConstLabels,
},
}
}
func newGaugeVec(opts prometheus.GaugeOpts, labelNames []string) metric[*prometheus.GaugeVec] {
return metric[*prometheus.GaugeVec]{
value: prometheus.NewGaugeVec(opts, labelNames),
desc: Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_GAUGE.String(),
Help: opts.Help,
ConstantLabels: opts.ConstLabels,
VariableLabels: labelNames,
},
}
}
func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] {
return metric[prometheus.Counter]{
value: prometheus.NewCounter(opts),
desc: Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_COUNTER.String(),
Help: opts.Help,
ConstantLabels: opts.ConstLabels,
},
}
}
// DescribeAll returns descriptions for all registered metrics.
func DescribeAll() ([]Description, error) {
registeredDescriptionsMtx.Lock()
defer registeredDescriptionsMtx.Unlock()
ds := make([]Description, len(registeredDescriptions))
copy(ds, registeredDescriptions)
return ds, nil
}

65
pkg/metrics/desc_test.go Normal file
View file

@ -0,0 +1,65 @@
package metrics
import (
"strings"
"testing"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
)
func TestDescribeAll(t *testing.T) {
const (
namespace = "my_ns"
subsystem = "mysub"
)
mustRegister(newCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "my_counter",
}))
labels := []string{"label1", "label2"}
mustRegister(newGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "my_gauge",
}, labels))
constLabels := prometheus.Labels{
"const1": "abc",
"const2": "xyz",
}
mustRegister(newCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "with_const_labels",
ConstLabels: constLabels,
}))
descriptions, err := DescribeAll()
require.NoError(t, err)
seen := make(map[string]bool)
for i := range descriptions {
if !strings.HasPrefix(descriptions[i].Name, namespace) {
continue
}
require.False(t, seen[descriptions[i].Name], "metric %s was seen twice", descriptions[i].Name)
seen[descriptions[i].Name] = true
switch descriptions[i].Name {
case prometheus.BuildFQName(namespace, subsystem, "my_counter"):
require.True(t, len(descriptions[i].VariableLabels) == 0)
case prometheus.BuildFQName(namespace, subsystem, "my_gauge"):
require.Equal(t, labels, descriptions[i].VariableLabels)
case prometheus.BuildFQName(namespace, subsystem, "with_const_labels"):
require.Equal(t, len(constLabels), len(descriptions[i].ConstantLabels))
require.Equal(t, constLabels, descriptions[i].ConstantLabels)
default:
require.FailNow(t, "unexpected metric name: %s", descriptions[i].Name)
}
}
require.Equal(t, 3, len(seen), "not all registered metrics were iterated over")
}

66
pkg/metrics/engine.go
View file

@ -10,19 +10,19 @@ import (
type (
engineMetrics struct {
listContainersDuration prometheus.Counter
estimateContainerSizeDuration prometheus.Counter
deleteDuration prometheus.Counter
existsDuration prometheus.Counter
getDuration prometheus.Counter
headDuration prometheus.Counter
inhumeDuration prometheus.Counter
putDuration prometheus.Counter
rangeDuration prometheus.Counter
searchDuration prometheus.Counter
listObjectsDuration prometheus.Counter
containerSize prometheus.GaugeVec
payloadSize prometheus.GaugeVec
listContainersDuration metric[prometheus.Counter]
estimateContainerSizeDuration metric[prometheus.Counter]
deleteDuration metric[prometheus.Counter]
existsDuration metric[prometheus.Counter]
getDuration metric[prometheus.Counter]
headDuration metric[prometheus.Counter]
inhumeDuration metric[prometheus.Counter]
putDuration metric[prometheus.Counter]
rangeDuration metric[prometheus.Counter]
searchDuration metric[prometheus.Counter]
listObjectsDuration metric[prometheus.Counter]
containerSize metric[*prometheus.GaugeVec]
payloadSize metric[*prometheus.GaugeVec]
}
)
@ -41,13 +41,13 @@ func newEngineMetrics() engineMetrics {
rangeDuration: newEngineMethodDurationCounter("range"),
searchDuration: newEngineMethodDurationCounter("search"),
listObjectsDuration: newEngineMethodDurationCounter("list_objects"),
containerSize: *newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}),
payloadSize: *newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}),
containerSize: newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}),
payloadSize: newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}),
}
}
func newEngineCounter(name, help string) prometheus.Counter {
return prometheus.NewCounter(prometheus.CounterOpts{
func newEngineCounter(name, help string) metric[prometheus.Counter] {
return newCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: engineSubsystem,
Name: name,
@ -55,15 +55,15 @@ func newEngineCounter(name, help string) prometheus.Counter {
})
}
func newEngineMethodDurationCounter(method string) prometheus.Counter {
func newEngineMethodDurationCounter(method string) metric[prometheus.Counter] {
return newEngineCounter(
fmt.Sprintf("%s_duration", method),
fmt.Sprintf("Accumulated duration of engine %s operations", strings.ReplaceAll(method, "_", " ")),
)
}
func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeVec {
return prometheus.NewGaugeVec(prometheus.GaugeOpts{
func newEngineGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] {
return newGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: engineSubsystem,
Name: name,
@ -88,53 +88,53 @@ func (m engineMetrics) register() {
}
func (m engineMetrics) AddListContainersDuration(d time.Duration) {
m.listObjectsDuration.Add(float64(d))
m.listObjectsDuration.value.Add(float64(d))
}
func (m engineMetrics) AddEstimateContainerSizeDuration(d time.Duration) {
m.estimateContainerSizeDuration.Add(float64(d))
m.estimateContainerSizeDuration.value.Add(float64(d))
}
func (m engineMetrics) AddDeleteDuration(d time.Duration) {
m.deleteDuration.Add(float64(d))
m.deleteDuration.value.Add(float64(d))
}
func (m engineMetrics) AddExistsDuration(d time.Duration) {
m.existsDuration.Add(float64(d))
m.existsDuration.value.Add(float64(d))
}
func (m engineMetrics) AddGetDuration(d time.Duration) {
m.getDuration.Add(float64(d))
m.getDuration.value.Add(float64(d))
}
func (m engineMetrics) AddHeadDuration(d time.Duration) {
m.headDuration.Add(float64(d))
m.headDuration.value.Add(float64(d))
}
func (m engineMetrics) AddInhumeDuration(d time.Duration) {
m.inhumeDuration.Add(float64(d))
m.inhumeDuration.value.Add(float64(d))
}
func (m engineMetrics) AddPutDuration(d time.Duration) {
m.putDuration.Add(float64(d))
m.putDuration.value.Add(float64(d))
}
func (m engineMetrics) AddRangeDuration(d time.Duration) {
m.rangeDuration.Add(float64(d))
m.rangeDuration.value.Add(float64(d))
}
func (m engineMetrics) AddSearchDuration(d time.Duration) {
m.searchDuration.Add(float64(d))
m.searchDuration.value.Add(float64(d))
}
func (m engineMetrics) AddListObjectsDuration(d time.Duration) {
m.listObjectsDuration.Add(float64(d))
m.listObjectsDuration.value.Add(float64(d))
}
func (m engineMetrics) AddToContainerSize(cnrID string, size int64) {
m.containerSize.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size))
m.containerSize.value.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size))
}
func (m engineMetrics) AddToPayloadCounter(shardID string, size int64) {
m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size))
m.payloadSize.value.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size))
}

12
pkg/metrics/innerring.go
View file

@ -6,20 +6,20 @@ const innerRingSubsystem = "ir"
// InnerRingServiceMetrics contains metrics collected by inner ring.
type InnerRingServiceMetrics struct {
epoch prometheus.Gauge
health prometheus.Gauge
epoch metric[prometheus.Gauge]
health metric[prometheus.Gauge]
}
// NewInnerRingMetrics returns new instance of metrics collectors for inner ring.
func NewInnerRingMetrics() InnerRingServiceMetrics {
var (
epoch = prometheus.NewGauge(prometheus.GaugeOpts{
epoch = newGauge(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: innerRingSubsystem,
Name: "epoch",
Help: "Current epoch as seen by inner-ring node.",
})
health = prometheus.NewGauge(prometheus.GaugeOpts{
health = newGauge(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: innerRingSubsystem,
Name: "health",
@ -38,10 +38,10 @@ func NewInnerRingMetrics() InnerRingServiceMetrics {
// SetEpoch updates epoch metrics.
func (m InnerRingServiceMetrics) SetEpoch(epoch uint64) {
m.epoch.Set(float64(epoch))
m.epoch.value.Set(float64(epoch))
}
// SetHealth updates health metrics.
func (m InnerRingServiceMetrics) SetHealth(s int32) {
m.health.Set(float64(s))
m.health.value.Set(float64(s))
}

6
pkg/metrics/node.go
View file

@ -8,7 +8,7 @@ type NodeMetrics struct {
objectServiceMetrics
engineMetrics
stateMetrics
epoch prometheus.Gauge
epoch metric[prometheus.Gauge]
}
func NewNodeMetrics() *NodeMetrics {
@ -21,7 +21,7 @@ func NewNodeMetrics() *NodeMetrics {
state := newStateMetrics()
state.register()
epoch := prometheus.NewGauge(prometheus.GaugeOpts{
epoch := newGauge(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: innerRingSubsystem,
Name: "epoch",
@ -39,5 +39,5 @@ func NewNodeMetrics() *NodeMetrics {
// SetEpoch updates epoch metric.
func (m *NodeMetrics) SetEpoch(epoch uint64) {
m.epoch.Set(float64(epoch))
m.epoch.value.Set(float64(epoch))
}

70
pkg/metrics/object.go
View file

@ -12,8 +12,8 @@ const objectSubsystem = "object"
type (
methodCount struct {
success prometheus.Counter
total prometheus.Counter
success metric[prometheus.Counter]
total metric[prometheus.Counter]
}
objectServiceMetrics struct {
@ -25,19 +25,19 @@ type (
rangeCounter methodCount
rangeHashCounter methodCount
getDuration prometheus.Counter
putDuration prometheus.Counter
headDuration prometheus.Counter
searchDuration prometheus.Counter
deleteDuration prometheus.Counter
rangeDuration prometheus.Counter
rangeHashDuration prometheus.Counter
getDuration metric[prometheus.Counter]
putDuration metric[prometheus.Counter]
headDuration metric[prometheus.Counter]
searchDuration metric[prometheus.Counter]
deleteDuration metric[prometheus.Counter]
rangeDuration metric[prometheus.Counter]
rangeHashDuration metric[prometheus.Counter]
putPayload prometheus.Counter
getPayload prometheus.Counter
putPayload metric[prometheus.Counter]
getPayload metric[prometheus.Counter]
shardMetrics *prometheus.GaugeVec
shardsReadonly *prometheus.GaugeVec
shardMetrics metric[*prometheus.GaugeVec]
shardsReadonly metric[*prometheus.GaugeVec]
}
)
@ -49,13 +49,13 @@ const (
func newObjectMethodCallCounter(name string) methodCount {
return methodCount{
success: prometheus.NewCounter(prometheus.CounterOpts{
success: newCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: fmt.Sprintf("%s_req_count_success", name),
Help: fmt.Sprintf("The number of successful %s requests processed", name),
}),
total: prometheus.NewCounter(prometheus.CounterOpts{
total: newCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: fmt.Sprintf("%s_req_count", name),
@ -70,9 +70,9 @@ func (m methodCount) mustRegister() {
}
func (m methodCount) Inc(success bool) {
m.total.Inc()
m.total.value.Inc()
if success {
m.success.Inc()
m.success.value.Inc()
}
}
@ -99,8 +99,8 @@ func newObjectServiceMetrics() objectServiceMetrics {
}
}
func newObjectMethodPayloadCounter(method string) prometheus.Counter {
return prometheus.NewCounter(prometheus.CounterOpts{
func newObjectMethodPayloadCounter(method string) metric[prometheus.Counter] {
return newCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: fmt.Sprintf("%s_payload", method),
@ -108,8 +108,8 @@ func newObjectMethodPayloadCounter(method string) prometheus.Counter {
})
}
func newObjectMethodDurationCounter(method string) prometheus.Counter {
return prometheus.NewCounter(prometheus.CounterOpts{
func newObjectMethodDurationCounter(method string) metric[prometheus.Counter] {
return newCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: fmt.Sprintf("%s_req_duration", method),
@ -117,8 +117,8 @@ func newObjectMethodDurationCounter(method string) prometheus.Counter {
})
}
func newObjectGaugeVector(name, help string, labels []string) *prometheus.GaugeVec {
return prometheus.NewGaugeVec(prometheus.GaugeOpts{
func newObjectGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] {
return newGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: objectSubsystem,
Name: name,
@ -179,43 +179,43 @@ func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) {
}
func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) {
m.getDuration.Add(float64(d))
m.getDuration.value.Add(float64(d))
}
func (m objectServiceMetrics) AddPutReqDuration(d time.Duration) {
m.putDuration.Add(float64(d))
m.putDuration.value.Add(float64(d))
}
func (m objectServiceMetrics) AddHeadReqDuration(d time.Duration) {
m.headDuration.Add(float64(d))
m.headDuration.value.Add(float64(d))
}
func (m objectServiceMetrics) AddSearchReqDuration(d time.Duration) {
m.searchDuration.Add(float64(d))
m.searchDuration.value.Add(float64(d))
}
func (m objectServiceMetrics) AddDeleteReqDuration(d time.Duration) {
m.deleteDuration.Add(float64(d))
m.deleteDuration.value.Add(float64(d))
}
func (m objectServiceMetrics) AddRangeReqDuration(d time.Duration) {
m.rangeDuration.Add(float64(d))
m.rangeDuration.value.Add(float64(d))
}
func (m objectServiceMetrics) AddRangeHashReqDuration(d time.Duration) {
m.rangeHashDuration.Add(float64(d))
m.rangeHashDuration.value.Add(float64(d))
}
func (m objectServiceMetrics) AddPutPayload(ln int) {
m.putPayload.Add(float64(ln))
m.putPayload.value.Add(float64(ln))
}
func (m objectServiceMetrics) AddGetPayload(ln int) {
m.getPayload.Add(float64(ln))
m.getPayload.value.Add(float64(ln))
}
func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, delta int) {
m.shardMetrics.With(
m.shardMetrics.value.With(
prometheus.Labels{
shardIDLabelKey: shardID,
counterTypeLabelKey: objectType,
@ -224,7 +224,7 @@ func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, del
}
func (m objectServiceMetrics) SetObjectCounter(shardID, objectType string, v uint64) {
m.shardMetrics.With(
m.shardMetrics.value.With(
prometheus.Labels{
shardIDLabelKey: shardID,
counterTypeLabelKey: objectType,
@ -237,7 +237,7 @@ func (m objectServiceMetrics) SetReadonly(shardID string, readonly bool) {
if readonly {
flag = 1
}
m.shardsReadonly.With(
m.shardsReadonly.value.With(
prometheus.Labels{
shardIDLabelKey: shardID,
},

22
pkg/metrics/registry.go
View file

@ -19,22 +19,24 @@ func Handler() http.Handler {
var (
registry = prometheus.NewRegistry()
// registeredCollectorsMtx protects collectors slice.
// registeredDescriptionsMtx protects collectors slice.
// It should not be acessed concurrently, but we can easily forget this in future, thus this mutex.
registeredCollectorsMtx sync.Mutex
registeredCollectors []prometheus.Collector
registeredDescriptionsMtx sync.Mutex
registeredDescriptions []Description
)
func init() {
mustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
mustRegister(collectors.NewGoCollector())
registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
registry.MustRegister(collectors.NewGoCollector())
}
func mustRegister(cs ...prometheus.Collector) {
func mustRegister[T prometheus.Collector](cs ...metric[T]) {
for i := range cs {
registry.MustRegister(cs[i])
registry.MustRegister(cs[i].value)
}
registeredCollectorsMtx.Lock()
registeredCollectors = append(registeredCollectors, cs...)
registeredCollectorsMtx.Unlock()
registeredDescriptionsMtx.Lock()
for i := range cs {
registeredDescriptions = append(registeredDescriptions, cs[i].desc)
}
registeredDescriptionsMtx.Unlock()
}

6
pkg/metrics/state.go
View file

@ -5,12 +5,12 @@ import "github.com/prometheus/client_golang/prometheus"
const stateSubsystem = "state"
type stateMetrics struct {
healthCheck prometheus.Gauge
healthCheck metric[prometheus.Gauge]
}
func newStateMetrics() stateMetrics {
return stateMetrics{
healthCheck: prometheus.NewGauge(prometheus.GaugeOpts{
healthCheck: newGauge(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: stateSubsystem,
Name: "health",
@ -24,5 +24,5 @@ func (m stateMetrics) register() {
}
func (m stateMetrics) SetHealth(s int32) {
m.healthCheck.Set(float64(s))
m.healthCheck.value.Set(float64(s))
}