From 6fef2726b865de241ddcb7eb2bdbbb818fb439de Mon Sep 17 00:00:00 2001 From: Evgenii Stratonikov Date: Wed, 5 Apr 2023 13:39:02 +0300 Subject: [PATCH] [#164] metrics: Allow to export metrics description Signed-off-by: Evgenii Stratonikov --- go.mod | 2 +- pkg/metrics/desc.go | 71 ++++++++++++++++++++++++++++++++++++++++ pkg/metrics/desc_test.go | 65 ++++++++++++++++++++++++++++++++++++ pkg/metrics/engine.go | 66 ++++++++++++++++++------------------- pkg/metrics/innerring.go | 12 +++---- pkg/metrics/node.go | 6 ++-- pkg/metrics/object.go | 70 +++++++++++++++++++-------------------- pkg/metrics/registry.go | 22 +++++++------ pkg/metrics/state.go | 6 ++-- 9 files changed, 229 insertions(+), 91 deletions(-) create mode 100644 pkg/metrics/desc.go create mode 100644 pkg/metrics/desc_test.go diff --git a/go.mod b/go.mod index 494bac00a..8cbb4ba38 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/panjf2000/ants/v2 v2.4.0 github.com/paulmach/orb v0.2.2 github.com/prometheus/client_golang v1.15.0 + github.com/prometheus/client_model v0.3.0 github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 @@ -85,7 +86,6 @@ require ( github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/pkg/metrics/desc.go b/pkg/metrics/desc.go new file mode 100644 index 000000000..74d2d4e6e --- /dev/null +++ b/pkg/metrics/desc.go @@ -0,0 +1,71 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +type metric[T prometheus.Collector] struct { + value T + desc Description +} + +// Descriptions contains metric description suitable for further processing. +// The only reason for it to exist is `prometheus.Desc` disallowing field access directly. +// https://github.com/prometheus/client_golang/pull/326 +// https://github.com/prometheus/client_golang/issues/516 +// https://github.com/prometheus/client_golang/issues/222 +type Description struct { + Name string `json:"name"` + Help string `json:"help"` + Type string `json:"type"` + ConstantLabels prometheus.Labels `json:"constant_labels,omitempty"` + VariableLabels []string `json:"variable_labels,omitempty"` +} + +func newGauge(opts prometheus.GaugeOpts) metric[prometheus.Gauge] { + return metric[prometheus.Gauge]{ + value: prometheus.NewGauge(opts), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_GAUGE.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + }, + } +} + +func newGaugeVec(opts prometheus.GaugeOpts, labelNames []string) metric[*prometheus.GaugeVec] { + return metric[*prometheus.GaugeVec]{ + value: prometheus.NewGaugeVec(opts, labelNames), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_GAUGE.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + VariableLabels: labelNames, + }, + } +} + +func newCounter(opts prometheus.CounterOpts) metric[prometheus.Counter] { + return metric[prometheus.Counter]{ + value: prometheus.NewCounter(opts), + desc: Description{ + Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), + Type: dto.MetricType_COUNTER.String(), + Help: opts.Help, + ConstantLabels: opts.ConstLabels, + }, + } +} + +// DescribeAll returns descriptions for all registered metrics. +func DescribeAll() ([]Description, error) { + registeredDescriptionsMtx.Lock() + defer registeredDescriptionsMtx.Unlock() + + ds := make([]Description, len(registeredDescriptions)) + copy(ds, registeredDescriptions) + return ds, nil +} diff --git a/pkg/metrics/desc_test.go b/pkg/metrics/desc_test.go new file mode 100644 index 000000000..28b5e2132 --- /dev/null +++ b/pkg/metrics/desc_test.go @@ -0,0 +1,65 @@ +package metrics + +import ( + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" +) + +func TestDescribeAll(t *testing.T) { + const ( + namespace = "my_ns" + subsystem = "mysub" + ) + mustRegister(newCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "my_counter", + })) + + labels := []string{"label1", "label2"} + mustRegister(newGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "my_gauge", + }, labels)) + + constLabels := prometheus.Labels{ + "const1": "abc", + "const2": "xyz", + } + mustRegister(newCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "with_const_labels", + ConstLabels: constLabels, + })) + + descriptions, err := DescribeAll() + require.NoError(t, err) + + seen := make(map[string]bool) + for i := range descriptions { + if !strings.HasPrefix(descriptions[i].Name, namespace) { + continue + } + + require.False(t, seen[descriptions[i].Name], "metric %s was seen twice", descriptions[i].Name) + seen[descriptions[i].Name] = true + + switch descriptions[i].Name { + case prometheus.BuildFQName(namespace, subsystem, "my_counter"): + require.True(t, len(descriptions[i].VariableLabels) == 0) + case prometheus.BuildFQName(namespace, subsystem, "my_gauge"): + require.Equal(t, labels, descriptions[i].VariableLabels) + case prometheus.BuildFQName(namespace, subsystem, "with_const_labels"): + require.Equal(t, len(constLabels), len(descriptions[i].ConstantLabels)) + require.Equal(t, constLabels, descriptions[i].ConstantLabels) + default: + require.FailNow(t, "unexpected metric name: %s", descriptions[i].Name) + } + } + require.Equal(t, 3, len(seen), "not all registered metrics were iterated over") +} diff --git a/pkg/metrics/engine.go b/pkg/metrics/engine.go index 73606fd23..28fc1e028 100644 --- a/pkg/metrics/engine.go +++ b/pkg/metrics/engine.go @@ -10,19 +10,19 @@ import ( type ( engineMetrics struct { - listContainersDuration prometheus.Counter - estimateContainerSizeDuration prometheus.Counter - deleteDuration prometheus.Counter - existsDuration prometheus.Counter - getDuration prometheus.Counter - headDuration prometheus.Counter - inhumeDuration prometheus.Counter - putDuration prometheus.Counter - rangeDuration prometheus.Counter - searchDuration prometheus.Counter - listObjectsDuration prometheus.Counter - containerSize prometheus.GaugeVec - payloadSize prometheus.GaugeVec + listContainersDuration metric[prometheus.Counter] + estimateContainerSizeDuration metric[prometheus.Counter] + deleteDuration metric[prometheus.Counter] + existsDuration metric[prometheus.Counter] + getDuration metric[prometheus.Counter] + headDuration metric[prometheus.Counter] + inhumeDuration metric[prometheus.Counter] + putDuration metric[prometheus.Counter] + rangeDuration metric[prometheus.Counter] + searchDuration metric[prometheus.Counter] + listObjectsDuration metric[prometheus.Counter] + containerSize metric[*prometheus.GaugeVec] + payloadSize metric[*prometheus.GaugeVec] } ) @@ -41,13 +41,13 @@ func newEngineMetrics() engineMetrics { rangeDuration: newEngineMethodDurationCounter("range"), searchDuration: newEngineMethodDurationCounter("search"), listObjectsDuration: newEngineMethodDurationCounter("list_objects"), - containerSize: *newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), - payloadSize: *newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), + containerSize: newEngineGaugeVector("container_size", "Accumulated size of all objects in a container", []string{containerIDLabelKey}), + payloadSize: newEngineGaugeVector("payload_size", "Accumulated size of all objects in a shard", []string{shardIDLabelKey}), } } -func newEngineCounter(name, help string) prometheus.Counter { - return prometheus.NewCounter(prometheus.CounterOpts{ +func newEngineCounter(name, help string) metric[prometheus.Counter] { + return newCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: name, @@ -55,15 +55,15 @@ func newEngineCounter(name, help string) prometheus.Counter { }) } -func newEngineMethodDurationCounter(method string) prometheus.Counter { +func newEngineMethodDurationCounter(method string) metric[prometheus.Counter] { return newEngineCounter( fmt.Sprintf("%s_duration", method), fmt.Sprintf("Accumulated duration of engine %s operations", strings.ReplaceAll(method, "_", " ")), ) } -func newEngineGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { - return prometheus.NewGaugeVec(prometheus.GaugeOpts{ +func newEngineGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] { + return newGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: engineSubsystem, Name: name, @@ -88,53 +88,53 @@ func (m engineMetrics) register() { } func (m engineMetrics) AddListContainersDuration(d time.Duration) { - m.listObjectsDuration.Add(float64(d)) + m.listObjectsDuration.value.Add(float64(d)) } func (m engineMetrics) AddEstimateContainerSizeDuration(d time.Duration) { - m.estimateContainerSizeDuration.Add(float64(d)) + m.estimateContainerSizeDuration.value.Add(float64(d)) } func (m engineMetrics) AddDeleteDuration(d time.Duration) { - m.deleteDuration.Add(float64(d)) + m.deleteDuration.value.Add(float64(d)) } func (m engineMetrics) AddExistsDuration(d time.Duration) { - m.existsDuration.Add(float64(d)) + m.existsDuration.value.Add(float64(d)) } func (m engineMetrics) AddGetDuration(d time.Duration) { - m.getDuration.Add(float64(d)) + m.getDuration.value.Add(float64(d)) } func (m engineMetrics) AddHeadDuration(d time.Duration) { - m.headDuration.Add(float64(d)) + m.headDuration.value.Add(float64(d)) } func (m engineMetrics) AddInhumeDuration(d time.Duration) { - m.inhumeDuration.Add(float64(d)) + m.inhumeDuration.value.Add(float64(d)) } func (m engineMetrics) AddPutDuration(d time.Duration) { - m.putDuration.Add(float64(d)) + m.putDuration.value.Add(float64(d)) } func (m engineMetrics) AddRangeDuration(d time.Duration) { - m.rangeDuration.Add(float64(d)) + m.rangeDuration.value.Add(float64(d)) } func (m engineMetrics) AddSearchDuration(d time.Duration) { - m.searchDuration.Add(float64(d)) + m.searchDuration.value.Add(float64(d)) } func (m engineMetrics) AddListObjectsDuration(d time.Duration) { - m.listObjectsDuration.Add(float64(d)) + m.listObjectsDuration.value.Add(float64(d)) } func (m engineMetrics) AddToContainerSize(cnrID string, size int64) { - m.containerSize.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) + m.containerSize.value.With(prometheus.Labels{containerIDLabelKey: cnrID}).Add(float64(size)) } func (m engineMetrics) AddToPayloadCounter(shardID string, size int64) { - m.payloadSize.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) + m.payloadSize.value.With(prometheus.Labels{shardIDLabelKey: shardID}).Add(float64(size)) } diff --git a/pkg/metrics/innerring.go b/pkg/metrics/innerring.go index 4171b7423..05b76f9c9 100644 --- a/pkg/metrics/innerring.go +++ b/pkg/metrics/innerring.go @@ -6,20 +6,20 @@ const innerRingSubsystem = "ir" // InnerRingServiceMetrics contains metrics collected by inner ring. type InnerRingServiceMetrics struct { - epoch prometheus.Gauge - health prometheus.Gauge + epoch metric[prometheus.Gauge] + health metric[prometheus.Gauge] } // NewInnerRingMetrics returns new instance of metrics collectors for inner ring. func NewInnerRingMetrics() InnerRingServiceMetrics { var ( - epoch = prometheus.NewGauge(prometheus.GaugeOpts{ + epoch = newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "epoch", Help: "Current epoch as seen by inner-ring node.", }) - health = prometheus.NewGauge(prometheus.GaugeOpts{ + health = newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "health", @@ -38,10 +38,10 @@ func NewInnerRingMetrics() InnerRingServiceMetrics { // SetEpoch updates epoch metrics. func (m InnerRingServiceMetrics) SetEpoch(epoch uint64) { - m.epoch.Set(float64(epoch)) + m.epoch.value.Set(float64(epoch)) } // SetHealth updates health metrics. func (m InnerRingServiceMetrics) SetHealth(s int32) { - m.health.Set(float64(s)) + m.health.value.Set(float64(s)) } diff --git a/pkg/metrics/node.go b/pkg/metrics/node.go index 4ebd2afa1..0f9c6183d 100644 --- a/pkg/metrics/node.go +++ b/pkg/metrics/node.go @@ -8,7 +8,7 @@ type NodeMetrics struct { objectServiceMetrics engineMetrics stateMetrics - epoch prometheus.Gauge + epoch metric[prometheus.Gauge] } func NewNodeMetrics() *NodeMetrics { @@ -21,7 +21,7 @@ func NewNodeMetrics() *NodeMetrics { state := newStateMetrics() state.register() - epoch := prometheus.NewGauge(prometheus.GaugeOpts{ + epoch := newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: innerRingSubsystem, Name: "epoch", @@ -39,5 +39,5 @@ func NewNodeMetrics() *NodeMetrics { // SetEpoch updates epoch metric. func (m *NodeMetrics) SetEpoch(epoch uint64) { - m.epoch.Set(float64(epoch)) + m.epoch.value.Set(float64(epoch)) } diff --git a/pkg/metrics/object.go b/pkg/metrics/object.go index ccc7c129c..5ec575749 100644 --- a/pkg/metrics/object.go +++ b/pkg/metrics/object.go @@ -12,8 +12,8 @@ const objectSubsystem = "object" type ( methodCount struct { - success prometheus.Counter - total prometheus.Counter + success metric[prometheus.Counter] + total metric[prometheus.Counter] } objectServiceMetrics struct { @@ -25,19 +25,19 @@ type ( rangeCounter methodCount rangeHashCounter methodCount - getDuration prometheus.Counter - putDuration prometheus.Counter - headDuration prometheus.Counter - searchDuration prometheus.Counter - deleteDuration prometheus.Counter - rangeDuration prometheus.Counter - rangeHashDuration prometheus.Counter + getDuration metric[prometheus.Counter] + putDuration metric[prometheus.Counter] + headDuration metric[prometheus.Counter] + searchDuration metric[prometheus.Counter] + deleteDuration metric[prometheus.Counter] + rangeDuration metric[prometheus.Counter] + rangeHashDuration metric[prometheus.Counter] - putPayload prometheus.Counter - getPayload prometheus.Counter + putPayload metric[prometheus.Counter] + getPayload metric[prometheus.Counter] - shardMetrics *prometheus.GaugeVec - shardsReadonly *prometheus.GaugeVec + shardMetrics metric[*prometheus.GaugeVec] + shardsReadonly metric[*prometheus.GaugeVec] } ) @@ -49,13 +49,13 @@ const ( func newObjectMethodCallCounter(name string) methodCount { return methodCount{ - success: prometheus.NewCounter(prometheus.CounterOpts{ + success: newCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_count_success", name), Help: fmt.Sprintf("The number of successful %s requests processed", name), }), - total: prometheus.NewCounter(prometheus.CounterOpts{ + total: newCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_count", name), @@ -70,9 +70,9 @@ func (m methodCount) mustRegister() { } func (m methodCount) Inc(success bool) { - m.total.Inc() + m.total.value.Inc() if success { - m.success.Inc() + m.success.value.Inc() } } @@ -99,8 +99,8 @@ func newObjectServiceMetrics() objectServiceMetrics { } } -func newObjectMethodPayloadCounter(method string) prometheus.Counter { - return prometheus.NewCounter(prometheus.CounterOpts{ +func newObjectMethodPayloadCounter(method string) metric[prometheus.Counter] { + return newCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_payload", method), @@ -108,8 +108,8 @@ func newObjectMethodPayloadCounter(method string) prometheus.Counter { }) } -func newObjectMethodDurationCounter(method string) prometheus.Counter { - return prometheus.NewCounter(prometheus.CounterOpts{ +func newObjectMethodDurationCounter(method string) metric[prometheus.Counter] { + return newCounter(prometheus.CounterOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: fmt.Sprintf("%s_req_duration", method), @@ -117,8 +117,8 @@ func newObjectMethodDurationCounter(method string) prometheus.Counter { }) } -func newObjectGaugeVector(name, help string, labels []string) *prometheus.GaugeVec { - return prometheus.NewGaugeVec(prometheus.GaugeOpts{ +func newObjectGaugeVector(name, help string, labels []string) metric[*prometheus.GaugeVec] { + return newGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: objectSubsystem, Name: name, @@ -179,43 +179,43 @@ func (m objectServiceMetrics) IncRangeHashReqCounter(success bool) { } func (m objectServiceMetrics) AddGetReqDuration(d time.Duration) { - m.getDuration.Add(float64(d)) + m.getDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddPutReqDuration(d time.Duration) { - m.putDuration.Add(float64(d)) + m.putDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddHeadReqDuration(d time.Duration) { - m.headDuration.Add(float64(d)) + m.headDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddSearchReqDuration(d time.Duration) { - m.searchDuration.Add(float64(d)) + m.searchDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddDeleteReqDuration(d time.Duration) { - m.deleteDuration.Add(float64(d)) + m.deleteDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddRangeReqDuration(d time.Duration) { - m.rangeDuration.Add(float64(d)) + m.rangeDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddRangeHashReqDuration(d time.Duration) { - m.rangeHashDuration.Add(float64(d)) + m.rangeHashDuration.value.Add(float64(d)) } func (m objectServiceMetrics) AddPutPayload(ln int) { - m.putPayload.Add(float64(ln)) + m.putPayload.value.Add(float64(ln)) } func (m objectServiceMetrics) AddGetPayload(ln int) { - m.getPayload.Add(float64(ln)) + m.getPayload.value.Add(float64(ln)) } func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, delta int) { - m.shardMetrics.With( + m.shardMetrics.value.With( prometheus.Labels{ shardIDLabelKey: shardID, counterTypeLabelKey: objectType, @@ -224,7 +224,7 @@ func (m objectServiceMetrics) AddToObjectCounter(shardID, objectType string, del } func (m objectServiceMetrics) SetObjectCounter(shardID, objectType string, v uint64) { - m.shardMetrics.With( + m.shardMetrics.value.With( prometheus.Labels{ shardIDLabelKey: shardID, counterTypeLabelKey: objectType, @@ -237,7 +237,7 @@ func (m objectServiceMetrics) SetReadonly(shardID string, readonly bool) { if readonly { flag = 1 } - m.shardsReadonly.With( + m.shardsReadonly.value.With( prometheus.Labels{ shardIDLabelKey: shardID, }, diff --git a/pkg/metrics/registry.go b/pkg/metrics/registry.go index c21c80449..eef613d04 100644 --- a/pkg/metrics/registry.go +++ b/pkg/metrics/registry.go @@ -19,22 +19,24 @@ func Handler() http.Handler { var ( registry = prometheus.NewRegistry() - // registeredCollectorsMtx protects collectors slice. + // registeredDescriptionsMtx protects collectors slice. // It should not be acessed concurrently, but we can easily forget this in future, thus this mutex. - registeredCollectorsMtx sync.Mutex - registeredCollectors []prometheus.Collector + registeredDescriptionsMtx sync.Mutex + registeredDescriptions []Description ) func init() { - mustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) - mustRegister(collectors.NewGoCollector()) + registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + registry.MustRegister(collectors.NewGoCollector()) } -func mustRegister(cs ...prometheus.Collector) { +func mustRegister[T prometheus.Collector](cs ...metric[T]) { for i := range cs { - registry.MustRegister(cs[i]) + registry.MustRegister(cs[i].value) } - registeredCollectorsMtx.Lock() - registeredCollectors = append(registeredCollectors, cs...) - registeredCollectorsMtx.Unlock() + registeredDescriptionsMtx.Lock() + for i := range cs { + registeredDescriptions = append(registeredDescriptions, cs[i].desc) + } + registeredDescriptionsMtx.Unlock() } diff --git a/pkg/metrics/state.go b/pkg/metrics/state.go index 9c96743ff..dce0402cd 100644 --- a/pkg/metrics/state.go +++ b/pkg/metrics/state.go @@ -5,12 +5,12 @@ import "github.com/prometheus/client_golang/prometheus" const stateSubsystem = "state" type stateMetrics struct { - healthCheck prometheus.Gauge + healthCheck metric[prometheus.Gauge] } func newStateMetrics() stateMetrics { return stateMetrics{ - healthCheck: prometheus.NewGauge(prometheus.GaugeOpts{ + healthCheck: newGauge(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: stateSubsystem, Name: "health", @@ -24,5 +24,5 @@ func (m stateMetrics) register() { } func (m stateMetrics) SetHealth(s int32) { - m.healthCheck.Set(float64(s)) + m.healthCheck.value.Set(float64(s)) }