diff --git a/go.mod b/go.mod index 60cd8d7d..f2571f7e 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/nspcc-dev/neo-go v0.101.0 github.com/panjf2000/ants/v2 v2.5.0 github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_model v0.2.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 @@ -31,6 +32,7 @@ require ( git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect + github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -56,7 +58,6 @@ require ( github.com/nspcc-dev/rfc6979 v0.2.0 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect diff --git a/metrics/app.go b/metrics/app.go index 4779d92b..d04c0839 100644 --- a/metrics/app.go +++ b/metrics/app.go @@ -4,6 +4,7 @@ import ( "net/http" "sync" + dto "github.com/prometheus/client_model/go" "go.uber.org/zap" ) @@ -78,3 +79,7 @@ func (m *AppMetrics) Statistic() *APIStatMetrics { return m.gate.Stats } + +func (m *AppMetrics) Gather() ([]*dto.MetricFamily, error) { + return m.gate.Gather() +} diff --git a/metrics/billing.go b/metrics/billing.go index 7bd36393..609fb921 100644 --- a/metrics/billing.go +++ b/metrics/billing.go @@ -203,11 +203,15 @@ func newBillingMetrics() *billingMetrics { } } -func (b *billingMetrics) register() { +func (b *billingMetrics) Gatherer() prometheus.Gatherer { + return b.registry +} + +func (b *billingMetrics) Register() { b.registry.MustRegister(b) } -func (b *billingMetrics) unregister() { +func (b *billingMetrics) Unregister() { b.registry.Unregister(b) } diff --git a/metrics/desc.go b/metrics/desc.go index c3cc9f6a..cc88730b 100644 --- a/metrics/desc.go +++ b/metrics/desc.go @@ -4,17 +4,20 @@ import ( "encoding/json" "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" ) var appMetricsDesc = map[string]map[string]Description{ poolSubsystem: { overallErrorsMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: poolSubsystem, Name: overallErrorsMetric, Help: "Total number of errors in pool", }, overallNodeErrorsMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: poolSubsystem, Name: overallNodeErrorsMetric, @@ -22,6 +25,7 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"node"}, }, overallNodeRequestsMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: poolSubsystem, Name: overallNodeRequestsMetric, @@ -29,6 +33,7 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"node"}, }, currentErrorMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: poolSubsystem, Name: currentErrorMetric, @@ -36,6 +41,7 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"node"}, }, avgRequestDurationMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: poolSubsystem, Name: avgRequestDurationMetric, @@ -45,6 +51,7 @@ var appMetricsDesc = map[string]map[string]Description{ }, billingSubsystem: { userRequestsMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: billingSubsystem, Name: userRequestsMetric, @@ -52,6 +59,7 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"user", "bucket", "cid", "operation"}, }, userTrafficMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: billingSubsystem, Name: userTrafficMetric, @@ -61,12 +69,14 @@ var appMetricsDesc = map[string]map[string]Description{ }, stateSubsystem: { healthMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: stateSubsystem, Name: healthMetric, Help: "Current S3 gateway state", }, versionInfoMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: stateSubsystem, Name: versionInfoMetric, @@ -76,6 +86,7 @@ var appMetricsDesc = map[string]map[string]Description{ }, statisticSubsystem: { requestsSecondsMetric: Description{ + Type: dto.MetricType_HISTOGRAM, Namespace: namespace, Subsystem: statisticSubsystem, Name: requestsSecondsMetric, @@ -83,6 +94,7 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"api"}, }, requestsCurrentMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: statisticSubsystem, Name: requestsCurrentMetric, @@ -90,6 +102,7 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"api"}, }, requestsTotalMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: statisticSubsystem, Name: requestsTotalMetric, @@ -97,6 +110,7 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"api"}, }, errorsTotalMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: statisticSubsystem, Name: errorsTotalMetric, @@ -104,12 +118,14 @@ var appMetricsDesc = map[string]map[string]Description{ VariableLabels: []string{"api"}, }, txBytesTotalMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: statisticSubsystem, Name: txBytesTotalMetric, Help: "Total number of bytes sent by current FrostFS S3 Gate instance", }, rxBytesTotalMetric: Description{ + Type: dto.MetricType_GAUGE, Namespace: namespace, Subsystem: statisticSubsystem, Name: rxBytesTotalMetric, @@ -119,6 +135,7 @@ var appMetricsDesc = map[string]map[string]Description{ } type Description struct { + Type dto.MetricType Namespace string Subsystem string Name string @@ -134,11 +151,13 @@ type KeyValue struct { func (d *Description) MarshalJSON() ([]byte, error) { return json.Marshal(&struct { + Type string `json:"type"` FQName string `json:"name"` Help string `json:"help"` ConstantLabels []KeyValue `json:"constant_labels"` VariableLabels []string `json:"variable_labels"` }{ + Type: d.Type.String(), FQName: d.BuildFQName(), Help: d.Help, ConstantLabels: d.ConstantLabels, @@ -188,15 +207,39 @@ func newDesc(description Description) *prometheus.Desc { description.ConstLabelsMap()) } -func newGauge(description Description) prometheus.Gauge { +func mustNewGauge(description Description) prometheus.Gauge { + if description.Type != dto.MetricType_GAUGE { + panic("invalid metric type") + } return prometheus.NewGauge( prometheus.GaugeOpts(newOpts(description)), ) } -func newGaugeVec(description Description) *prometheus.GaugeVec { +func mustNewGaugeVec(description Description) *prometheus.GaugeVec { + if description.Type != dto.MetricType_GAUGE { + panic("invalid metric type") + } return prometheus.NewGaugeVec( prometheus.GaugeOpts(newOpts(description)), description.VariableLabels, ) } + +func mustNewHistogramVec(description Description, buckets []float64) *prometheus.HistogramVec { + if description.Type != dto.MetricType_HISTOGRAM { + panic("invalid metric type") + } + + return prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: description.Namespace, + Subsystem: description.Subsystem, + Name: description.Name, + Help: description.Name, + ConstLabels: description.ConstLabelsMap(), + Buckets: buckets, + }, + description.VariableLabels, + ) +} diff --git a/metrics/desc_test.go b/metrics/desc_test.go index 0b799145..9ff251b9 100644 --- a/metrics/desc_test.go +++ b/metrics/desc_test.go @@ -8,12 +8,24 @@ import ( "os" "testing" + "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" ) +type mock struct { +} + +func (m mock) Statistic() pool.Statistic { + return pool.Statistic{} +} + var metricsPath = flag.String("out", "", "File to export s3 gateway metrics to.") func TestDescribeAll(t *testing.T) { + // to check correct metrics type mapping + _ = NewAppMetrics(zaptest.NewLogger(t), mock{}, true) + flag.Parse() require.NotEmpty(t, metricsPath, "flag 'out' must be provided to dump metrics description") diff --git a/metrics/gate.go b/metrics/gate.go index ca8c70ef..c03df3b5 100644 --- a/metrics/gate.go +++ b/metrics/gate.go @@ -6,6 +6,7 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + dto "github.com/prometheus/client_model/go" ) const namespace = "frostfs_s3_gw" @@ -15,43 +16,51 @@ type StatisticScraper interface { } type GateMetrics struct { - State *StateMetrics - Pool poolMetricsCollector - Billing *billingMetrics - Stats *APIStatMetrics + registry *prometheus.Registry + State *StateMetrics + Pool *poolMetricsCollector + Billing *billingMetrics + Stats *APIStatMetrics } func NewGateMetrics(scraper StatisticScraper) *GateMetrics { + registry := prometheus.NewRegistry() + stateMetric := newStateMetrics() - stateMetric.register() + registry.MustRegister(stateMetric) poolMetric := newPoolMetricsCollector(scraper) - poolMetric.register() + registry.MustRegister(poolMetric) billingMetric := newBillingMetrics() - billingMetric.register() + billingMetric.Register() statsMetric := newAPIStatMetrics() - statsMetric.register() + registry.MustRegister(statsMetric) return &GateMetrics{ - State: stateMetric, - Pool: *poolMetric, - Billing: billingMetric, - Stats: statsMetric, + registry: registry, + State: stateMetric, + Pool: poolMetric, + Billing: billingMetric, + Stats: statsMetric, } } func (g *GateMetrics) Unregister() { - g.State.unregister() - prometheus.Unregister(&g.Pool) - g.Billing.unregister() - g.Stats.unregister() + g.registry.Unregister(g.State) + g.registry.Unregister(g.Pool) + g.Billing.Unregister() + g.registry.Unregister(g.Stats) } func (g *GateMetrics) Handler() http.Handler { handler := http.NewServeMux() handler.Handle("/", promhttp.Handler()) - handler.Handle("/metrics/billing", promhttp.HandlerFor(g.Billing.registry, promhttp.HandlerOpts{})) + handler.Handle("/metrics/billing", promhttp.HandlerFor(g.Billing.Gatherer(), promhttp.HandlerOpts{})) return handler } + +func (g *GateMetrics) Gather() ([]*dto.MetricFamily, error) { + return prometheus.Gatherers([]prometheus.Gatherer{g.Billing.Gatherer()}).Gather() +} diff --git a/metrics/pool.go b/metrics/pool.go index b71f4793..41173008 100644 --- a/metrics/pool.go +++ b/metrics/pool.go @@ -47,11 +47,11 @@ type poolMetricsCollector struct { func newPoolMetricsCollector(scraper StatisticScraper) *poolMetricsCollector { return &poolMetricsCollector{ poolStatScraper: scraper, - overallErrors: newGauge(appMetricsDesc[poolSubsystem][overallErrorsMetric]), - overallNodeErrors: newGaugeVec(appMetricsDesc[poolSubsystem][overallNodeErrorsMetric]), - overallNodeRequests: newGaugeVec(appMetricsDesc[poolSubsystem][overallNodeRequestsMetric]), - currentErrors: newGaugeVec(appMetricsDesc[poolSubsystem][currentErrorMetric]), - requestDuration: newGaugeVec(appMetricsDesc[poolSubsystem][avgRequestDurationMetric]), + overallErrors: mustNewGauge(appMetricsDesc[poolSubsystem][overallErrorsMetric]), + overallNodeErrors: mustNewGaugeVec(appMetricsDesc[poolSubsystem][overallNodeErrorsMetric]), + overallNodeRequests: mustNewGaugeVec(appMetricsDesc[poolSubsystem][overallNodeRequestsMetric]), + currentErrors: mustNewGaugeVec(appMetricsDesc[poolSubsystem][currentErrorMetric]), + requestDuration: mustNewGaugeVec(appMetricsDesc[poolSubsystem][avgRequestDurationMetric]), } } @@ -72,10 +72,6 @@ func (m *poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) { m.requestDuration.Describe(descs) } -func (m *poolMetricsCollector) register() { - prometheus.MustRegister(m) -} - func (m *poolMetricsCollector) updateStatistic() { stat := m.poolStatScraper.Statistic() diff --git a/metrics/state.go b/metrics/state.go index fe71aaaf..62d9fac6 100644 --- a/metrics/state.go +++ b/metrics/state.go @@ -28,25 +28,11 @@ type StateMetrics struct { func newStateMetrics() *StateMetrics { return &StateMetrics{ - healthCheck: newGauge(appMetricsDesc[stateSubsystem][healthMetric]), - versionInfo: newGaugeVec(appMetricsDesc[stateSubsystem][versionInfoMetric]), + healthCheck: mustNewGauge(appMetricsDesc[stateSubsystem][healthMetric]), + versionInfo: mustNewGaugeVec(appMetricsDesc[stateSubsystem][versionInfoMetric]), } } -func (m *StateMetrics) register() { - if m == nil { - return - } - prometheus.MustRegister(m.healthCheck) -} - -func (m *StateMetrics) unregister() { - if m == nil { - return - } - prometheus.Unregister(m.healthCheck) -} - func (m *StateMetrics) SetHealth(s HealthStatus) { if m == nil { return @@ -60,3 +46,21 @@ func (m *StateMetrics) SetVersion(ver string) { } m.versionInfo.WithLabelValues(ver).Set(1) } + +func (m *StateMetrics) Describe(desc chan<- *prometheus.Desc) { + if m == nil { + return + } + + m.healthCheck.Describe(desc) + m.versionInfo.Describe(desc) +} + +func (m *StateMetrics) Collect(ch chan<- prometheus.Metric) { + if m == nil { + return + } + + m.healthCheck.Collect(ch) + m.versionInfo.Collect(ch) +} diff --git a/metrics/stats.go b/metrics/stats.go index 2666029b..aaa218d0 100644 --- a/metrics/stats.go +++ b/metrics/stats.go @@ -56,36 +56,11 @@ func newAPIStatMetrics() *APIStatMetrics { return &APIStatMetrics{ stats: newHTTPStats(), - httpRequestsDuration: prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Namespace: histogramDesc.Namespace, - Subsystem: histogramDesc.Subsystem, - Name: histogramDesc.Name, - Help: histogramDesc.Name, - ConstLabels: histogramDesc.ConstLabelsMap(), - Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10}, - }, - histogramDesc.VariableLabels, - ), + httpRequestsDuration: mustNewHistogramVec(histogramDesc, + []float64{.05, .1, .25, .5, 1, 2.5, 5, 10}), } } -func (a *APIStatMetrics) register() { - if a == nil { - return - } - prometheus.MustRegister(a.stats) - prometheus.MustRegister(a.httpRequestsDuration) -} - -func (a *APIStatMetrics) unregister() { - if a == nil { - return - } - prometheus.Unregister(a.stats) - prometheus.Unregister(a.httpRequestsDuration) -} - func (a *APIStatMetrics) CurrentS3RequestsInc(api string) { if a == nil { return