[#80] Add type to metrics description

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
This commit is contained in:
Denis Kirillov 2023-04-10 11:40:58 +03:00
parent 644524e8a5
commit c154f934e4
9 changed files with 123 additions and 74 deletions

3
go.mod
View file

@ -15,6 +15,7 @@ require (
github.com/nspcc-dev/neo-go v0.101.0 github.com/nspcc-dev/neo-go v0.101.0
github.com/panjf2000/ants/v2 v2.5.0 github.com/panjf2000/ants/v2 v2.5.0
github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_golang v1.13.0
github.com/prometheus/client_model v0.2.0
github.com/spf13/pflag v1.0.5 github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.15.0 github.com/spf13/viper v1.15.0
github.com/stretchr/testify v1.8.1 github.com/stretchr/testify v1.8.1
@ -31,6 +32,7 @@ require (
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
@ -56,7 +58,6 @@ require (
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect

View file

@ -4,6 +4,7 @@ import (
"net/http" "net/http"
"sync" "sync"
dto "github.com/prometheus/client_model/go"
"go.uber.org/zap" "go.uber.org/zap"
) )
@ -78,3 +79,7 @@ func (m *AppMetrics) Statistic() *APIStatMetrics {
return m.gate.Stats return m.gate.Stats
} }
func (m *AppMetrics) Gather() ([]*dto.MetricFamily, error) {
return m.gate.Gather()
}

View file

@ -203,11 +203,15 @@ func newBillingMetrics() *billingMetrics {
} }
} }
func (b *billingMetrics) register() { func (b *billingMetrics) Gatherer() prometheus.Gatherer {
return b.registry
}
func (b *billingMetrics) Register() {
b.registry.MustRegister(b) b.registry.MustRegister(b)
} }
func (b *billingMetrics) unregister() { func (b *billingMetrics) Unregister() {
b.registry.Unregister(b) b.registry.Unregister(b)
} }

View file

@ -4,17 +4,20 @@ import (
"encoding/json" "encoding/json"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
) )
var appMetricsDesc = map[string]map[string]Description{ var appMetricsDesc = map[string]map[string]Description{
poolSubsystem: { poolSubsystem: {
overallErrorsMetric: Description{ overallErrorsMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: poolSubsystem, Subsystem: poolSubsystem,
Name: overallErrorsMetric, Name: overallErrorsMetric,
Help: "Total number of errors in pool", Help: "Total number of errors in pool",
}, },
overallNodeErrorsMetric: Description{ overallNodeErrorsMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: poolSubsystem, Subsystem: poolSubsystem,
Name: overallNodeErrorsMetric, Name: overallNodeErrorsMetric,
@ -22,6 +25,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"node"}, VariableLabels: []string{"node"},
}, },
overallNodeRequestsMetric: Description{ overallNodeRequestsMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: poolSubsystem, Subsystem: poolSubsystem,
Name: overallNodeRequestsMetric, Name: overallNodeRequestsMetric,
@ -29,6 +33,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"node"}, VariableLabels: []string{"node"},
}, },
currentErrorMetric: Description{ currentErrorMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: poolSubsystem, Subsystem: poolSubsystem,
Name: currentErrorMetric, Name: currentErrorMetric,
@ -36,6 +41,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"node"}, VariableLabels: []string{"node"},
}, },
avgRequestDurationMetric: Description{ avgRequestDurationMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: poolSubsystem, Subsystem: poolSubsystem,
Name: avgRequestDurationMetric, Name: avgRequestDurationMetric,
@ -45,6 +51,7 @@ var appMetricsDesc = map[string]map[string]Description{
}, },
billingSubsystem: { billingSubsystem: {
userRequestsMetric: Description{ userRequestsMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: billingSubsystem, Subsystem: billingSubsystem,
Name: userRequestsMetric, Name: userRequestsMetric,
@ -52,6 +59,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"user", "bucket", "cid", "operation"}, VariableLabels: []string{"user", "bucket", "cid", "operation"},
}, },
userTrafficMetric: Description{ userTrafficMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: billingSubsystem, Subsystem: billingSubsystem,
Name: userTrafficMetric, Name: userTrafficMetric,
@ -61,12 +69,14 @@ var appMetricsDesc = map[string]map[string]Description{
}, },
stateSubsystem: { stateSubsystem: {
healthMetric: Description{ healthMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: stateSubsystem, Subsystem: stateSubsystem,
Name: healthMetric, Name: healthMetric,
Help: "Current S3 gateway state", Help: "Current S3 gateway state",
}, },
versionInfoMetric: Description{ versionInfoMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: stateSubsystem, Subsystem: stateSubsystem,
Name: versionInfoMetric, Name: versionInfoMetric,
@ -76,6 +86,7 @@ var appMetricsDesc = map[string]map[string]Description{
}, },
statisticSubsystem: { statisticSubsystem: {
requestsSecondsMetric: Description{ requestsSecondsMetric: Description{
Type: dto.MetricType_HISTOGRAM,
Namespace: namespace, Namespace: namespace,
Subsystem: statisticSubsystem, Subsystem: statisticSubsystem,
Name: requestsSecondsMetric, Name: requestsSecondsMetric,
@ -83,6 +94,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"api"}, VariableLabels: []string{"api"},
}, },
requestsCurrentMetric: Description{ requestsCurrentMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: statisticSubsystem, Subsystem: statisticSubsystem,
Name: requestsCurrentMetric, Name: requestsCurrentMetric,
@ -90,6 +102,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"api"}, VariableLabels: []string{"api"},
}, },
requestsTotalMetric: Description{ requestsTotalMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: statisticSubsystem, Subsystem: statisticSubsystem,
Name: requestsTotalMetric, Name: requestsTotalMetric,
@ -97,6 +110,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"api"}, VariableLabels: []string{"api"},
}, },
errorsTotalMetric: Description{ errorsTotalMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: statisticSubsystem, Subsystem: statisticSubsystem,
Name: errorsTotalMetric, Name: errorsTotalMetric,
@ -104,12 +118,14 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"api"}, VariableLabels: []string{"api"},
}, },
txBytesTotalMetric: Description{ txBytesTotalMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: statisticSubsystem, Subsystem: statisticSubsystem,
Name: txBytesTotalMetric, Name: txBytesTotalMetric,
Help: "Total number of bytes sent by current FrostFS S3 Gate instance", Help: "Total number of bytes sent by current FrostFS S3 Gate instance",
}, },
rxBytesTotalMetric: Description{ rxBytesTotalMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace, Namespace: namespace,
Subsystem: statisticSubsystem, Subsystem: statisticSubsystem,
Name: rxBytesTotalMetric, Name: rxBytesTotalMetric,
@ -119,6 +135,7 @@ var appMetricsDesc = map[string]map[string]Description{
} }
type Description struct { type Description struct {
Type dto.MetricType
Namespace string Namespace string
Subsystem string Subsystem string
Name string Name string
@ -134,11 +151,13 @@ type KeyValue struct {
func (d *Description) MarshalJSON() ([]byte, error) { func (d *Description) MarshalJSON() ([]byte, error) {
return json.Marshal(&struct { return json.Marshal(&struct {
Type string `json:"type"`
FQName string `json:"name"` FQName string `json:"name"`
Help string `json:"help"` Help string `json:"help"`
ConstantLabels []KeyValue `json:"constant_labels"` ConstantLabels []KeyValue `json:"constant_labels"`
VariableLabels []string `json:"variable_labels"` VariableLabels []string `json:"variable_labels"`
}{ }{
Type: d.Type.String(),
FQName: d.BuildFQName(), FQName: d.BuildFQName(),
Help: d.Help, Help: d.Help,
ConstantLabels: d.ConstantLabels, ConstantLabels: d.ConstantLabels,
@ -188,15 +207,39 @@ func newDesc(description Description) *prometheus.Desc {
description.ConstLabelsMap()) description.ConstLabelsMap())
} }
func newGauge(description Description) prometheus.Gauge { func mustNewGauge(description Description) prometheus.Gauge {
if description.Type != dto.MetricType_GAUGE {
panic("invalid metric type")
}
return prometheus.NewGauge( return prometheus.NewGauge(
prometheus.GaugeOpts(newOpts(description)), prometheus.GaugeOpts(newOpts(description)),
) )
} }
func newGaugeVec(description Description) *prometheus.GaugeVec { func mustNewGaugeVec(description Description) *prometheus.GaugeVec {
if description.Type != dto.MetricType_GAUGE {
panic("invalid metric type")
}
return prometheus.NewGaugeVec( return prometheus.NewGaugeVec(
prometheus.GaugeOpts(newOpts(description)), prometheus.GaugeOpts(newOpts(description)),
description.VariableLabels, description.VariableLabels,
) )
} }
func mustNewHistogramVec(description Description, buckets []float64) *prometheus.HistogramVec {
if description.Type != dto.MetricType_HISTOGRAM {
panic("invalid metric type")
}
return prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: description.Namespace,
Subsystem: description.Subsystem,
Name: description.Name,
Help: description.Name,
ConstLabels: description.ConstLabelsMap(),
Buckets: buckets,
},
description.VariableLabels,
)
}

View file

@ -8,12 +8,24 @@ import (
"os" "os"
"testing" "testing"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
) )
type mock struct {
}
func (m mock) Statistic() pool.Statistic {
return pool.Statistic{}
}
var metricsPath = flag.String("out", "", "File to export s3 gateway metrics to.") var metricsPath = flag.String("out", "", "File to export s3 gateway metrics to.")
func TestDescribeAll(t *testing.T) { func TestDescribeAll(t *testing.T) {
// to check correct metrics type mapping
_ = NewAppMetrics(zaptest.NewLogger(t), mock{}, true)
flag.Parse() flag.Parse()
require.NotEmpty(t, metricsPath, "flag 'out' must be provided to dump metrics description") require.NotEmpty(t, metricsPath, "flag 'out' must be provided to dump metrics description")

View file

@ -6,6 +6,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
dto "github.com/prometheus/client_model/go"
) )
const namespace = "frostfs_s3_gw" const namespace = "frostfs_s3_gw"
@ -15,43 +16,51 @@ type StatisticScraper interface {
} }
type GateMetrics struct { type GateMetrics struct {
registry *prometheus.Registry
State *StateMetrics State *StateMetrics
Pool poolMetricsCollector Pool *poolMetricsCollector
Billing *billingMetrics Billing *billingMetrics
Stats *APIStatMetrics Stats *APIStatMetrics
} }
func NewGateMetrics(scraper StatisticScraper) *GateMetrics { func NewGateMetrics(scraper StatisticScraper) *GateMetrics {
registry := prometheus.NewRegistry()
stateMetric := newStateMetrics() stateMetric := newStateMetrics()
stateMetric.register() registry.MustRegister(stateMetric)
poolMetric := newPoolMetricsCollector(scraper) poolMetric := newPoolMetricsCollector(scraper)
poolMetric.register() registry.MustRegister(poolMetric)
billingMetric := newBillingMetrics() billingMetric := newBillingMetrics()
billingMetric.register() billingMetric.Register()
statsMetric := newAPIStatMetrics() statsMetric := newAPIStatMetrics()
statsMetric.register() registry.MustRegister(statsMetric)
return &GateMetrics{ return &GateMetrics{
registry: registry,
State: stateMetric, State: stateMetric,
Pool: *poolMetric, Pool: poolMetric,
Billing: billingMetric, Billing: billingMetric,
Stats: statsMetric, Stats: statsMetric,
} }
} }
func (g *GateMetrics) Unregister() { func (g *GateMetrics) Unregister() {
g.State.unregister() g.registry.Unregister(g.State)
prometheus.Unregister(&g.Pool) g.registry.Unregister(g.Pool)
g.Billing.unregister() g.Billing.Unregister()
g.Stats.unregister() g.registry.Unregister(g.Stats)
} }
func (g *GateMetrics) Handler() http.Handler { func (g *GateMetrics) Handler() http.Handler {
handler := http.NewServeMux() handler := http.NewServeMux()
handler.Handle("/", promhttp.Handler()) handler.Handle("/", promhttp.Handler())
handler.Handle("/metrics/billing", promhttp.HandlerFor(g.Billing.registry, promhttp.HandlerOpts{})) handler.Handle("/metrics/billing", promhttp.HandlerFor(g.Billing.Gatherer(), promhttp.HandlerOpts{}))
return handler return handler
} }
func (g *GateMetrics) Gather() ([]*dto.MetricFamily, error) {
return prometheus.Gatherers([]prometheus.Gatherer{g.Billing.Gatherer()}).Gather()
}

View file

@ -47,11 +47,11 @@ type poolMetricsCollector struct {
func newPoolMetricsCollector(scraper StatisticScraper) *poolMetricsCollector { func newPoolMetricsCollector(scraper StatisticScraper) *poolMetricsCollector {
return &poolMetricsCollector{ return &poolMetricsCollector{
poolStatScraper: scraper, poolStatScraper: scraper,
overallErrors: newGauge(appMetricsDesc[poolSubsystem][overallErrorsMetric]), overallErrors: mustNewGauge(appMetricsDesc[poolSubsystem][overallErrorsMetric]),
overallNodeErrors: newGaugeVec(appMetricsDesc[poolSubsystem][overallNodeErrorsMetric]), overallNodeErrors: mustNewGaugeVec(appMetricsDesc[poolSubsystem][overallNodeErrorsMetric]),
overallNodeRequests: newGaugeVec(appMetricsDesc[poolSubsystem][overallNodeRequestsMetric]), overallNodeRequests: mustNewGaugeVec(appMetricsDesc[poolSubsystem][overallNodeRequestsMetric]),
currentErrors: newGaugeVec(appMetricsDesc[poolSubsystem][currentErrorMetric]), currentErrors: mustNewGaugeVec(appMetricsDesc[poolSubsystem][currentErrorMetric]),
requestDuration: newGaugeVec(appMetricsDesc[poolSubsystem][avgRequestDurationMetric]), requestDuration: mustNewGaugeVec(appMetricsDesc[poolSubsystem][avgRequestDurationMetric]),
} }
} }
@ -72,10 +72,6 @@ func (m *poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) {
m.requestDuration.Describe(descs) m.requestDuration.Describe(descs)
} }
func (m *poolMetricsCollector) register() {
prometheus.MustRegister(m)
}
func (m *poolMetricsCollector) updateStatistic() { func (m *poolMetricsCollector) updateStatistic() {
stat := m.poolStatScraper.Statistic() stat := m.poolStatScraper.Statistic()

View file

@ -28,25 +28,11 @@ type StateMetrics struct {
func newStateMetrics() *StateMetrics { func newStateMetrics() *StateMetrics {
return &StateMetrics{ return &StateMetrics{
healthCheck: newGauge(appMetricsDesc[stateSubsystem][healthMetric]), healthCheck: mustNewGauge(appMetricsDesc[stateSubsystem][healthMetric]),
versionInfo: newGaugeVec(appMetricsDesc[stateSubsystem][versionInfoMetric]), versionInfo: mustNewGaugeVec(appMetricsDesc[stateSubsystem][versionInfoMetric]),
} }
} }
func (m *StateMetrics) register() {
if m == nil {
return
}
prometheus.MustRegister(m.healthCheck)
}
func (m *StateMetrics) unregister() {
if m == nil {
return
}
prometheus.Unregister(m.healthCheck)
}
func (m *StateMetrics) SetHealth(s HealthStatus) { func (m *StateMetrics) SetHealth(s HealthStatus) {
if m == nil { if m == nil {
return return
@ -60,3 +46,21 @@ func (m *StateMetrics) SetVersion(ver string) {
} }
m.versionInfo.WithLabelValues(ver).Set(1) m.versionInfo.WithLabelValues(ver).Set(1)
} }
func (m *StateMetrics) Describe(desc chan<- *prometheus.Desc) {
if m == nil {
return
}
m.healthCheck.Describe(desc)
m.versionInfo.Describe(desc)
}
func (m *StateMetrics) Collect(ch chan<- prometheus.Metric) {
if m == nil {
return
}
m.healthCheck.Collect(ch)
m.versionInfo.Collect(ch)
}

View file

@ -56,36 +56,11 @@ func newAPIStatMetrics() *APIStatMetrics {
return &APIStatMetrics{ return &APIStatMetrics{
stats: newHTTPStats(), stats: newHTTPStats(),
httpRequestsDuration: prometheus.NewHistogramVec( httpRequestsDuration: mustNewHistogramVec(histogramDesc,
prometheus.HistogramOpts{ []float64{.05, .1, .25, .5, 1, 2.5, 5, 10}),
Namespace: histogramDesc.Namespace,
Subsystem: histogramDesc.Subsystem,
Name: histogramDesc.Name,
Help: histogramDesc.Name,
ConstLabels: histogramDesc.ConstLabelsMap(),
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10},
},
histogramDesc.VariableLabels,
),
} }
} }
func (a *APIStatMetrics) register() {
if a == nil {
return
}
prometheus.MustRegister(a.stats)
prometheus.MustRegister(a.httpRequestsDuration)
}
func (a *APIStatMetrics) unregister() {
if a == nil {
return
}
prometheus.Unregister(a.stats)
prometheus.Unregister(a.httpRequestsDuration)
}
func (a *APIStatMetrics) CurrentS3RequestsInc(api string) { func (a *APIStatMetrics) CurrentS3RequestsInc(api string) {
if a == nil { if a == nil {
return return