TrueCloudLab
/
frostfs-s3-gw
16
2
Fork 13
Code Issues 24 Pull Requests 8 Actions Packages Releases 3 Activity

[#80] Add type to metrics description 

Signed-off-by: Denis Kirillov <d.kirillov@yadro.com>
pull/82/head
Denis Kirillov 2023-04-10 11:40:58 +03:00
parent 644524e8a5
commit c154f934e4
9 changed files with 123 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@ -15,6 +15,7 @@ require (
github.com/nspcc-dev/neo-go v0.101.0
github.com/panjf2000/ants/v2 v2.5.0
github.com/prometheus/client_golang v1.13.0
github.com/prometheus/client_model v0.2.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.15.0
github.com/stretchr/testify v1.8.1
@ -31,6 +32,7 @@ require (
git.frostfs.info/TrueCloudLab/rfc6979 v0.4.0 // indirect
git.frostfs.info/TrueCloudLab/tzhash v1.8.0 // indirect
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
@ -56,7 +58,6 @@ require (
github.com/nspcc-dev/rfc6979 v0.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect

5
metrics/app.go
View File

@ -4,6 +4,7 @@ import (
"net/http"
"sync"
dto "github.com/prometheus/client_model/go"
"go.uber.org/zap"
)
@ -78,3 +79,7 @@ func (m *AppMetrics) Statistic() *APIStatMetrics {
return m.gate.Stats
}
func (m *AppMetrics) Gather() ([]*dto.MetricFamily, error) {
return m.gate.Gather()
}

8
metrics/billing.go
View File

@ -203,11 +203,15 @@ func newBillingMetrics() *billingMetrics {
}
}
func (b *billingMetrics) register() {
func (b *billingMetrics) Gatherer() prometheus.Gatherer {
return b.registry
}
func (b *billingMetrics) Register() {
b.registry.MustRegister(b)
}
func (b *billingMetrics) unregister() {
func (b *billingMetrics) Unregister() {
b.registry.Unregister(b)
}

47
metrics/desc.go
View File

@ -4,17 +4,20 @@ import (
"encoding/json"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
)
var appMetricsDesc = map[string]map[string]Description{
poolSubsystem: {
overallErrorsMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: poolSubsystem,
Name: overallErrorsMetric,
Help: "Total number of errors in pool",
},
overallNodeErrorsMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: poolSubsystem,
Name: overallNodeErrorsMetric,
@ -22,6 +25,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"node"},
},
overallNodeRequestsMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: poolSubsystem,
Name: overallNodeRequestsMetric,
@ -29,6 +33,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"node"},
},
currentErrorMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: poolSubsystem,
Name: currentErrorMetric,
@ -36,6 +41,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"node"},
},
avgRequestDurationMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: poolSubsystem,
Name: avgRequestDurationMetric,
@ -45,6 +51,7 @@ var appMetricsDesc = map[string]map[string]Description{
},
billingSubsystem: {
userRequestsMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: billingSubsystem,
Name: userRequestsMetric,
@ -52,6 +59,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"user", "bucket", "cid", "operation"},
},
userTrafficMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: billingSubsystem,
Name: userTrafficMetric,
@ -61,12 +69,14 @@ var appMetricsDesc = map[string]map[string]Description{
},
stateSubsystem: {
healthMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: stateSubsystem,
Name: healthMetric,
Help: "Current S3 gateway state",
},
versionInfoMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: stateSubsystem,
Name: versionInfoMetric,
@ -76,6 +86,7 @@ var appMetricsDesc = map[string]map[string]Description{
},
statisticSubsystem: {
requestsSecondsMetric: Description{
Type: dto.MetricType_HISTOGRAM,
Namespace: namespace,
Subsystem: statisticSubsystem,
Name: requestsSecondsMetric,
@ -83,6 +94,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"api"},
},
requestsCurrentMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: statisticSubsystem,
Name: requestsCurrentMetric,
@ -90,6 +102,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"api"},
},
requestsTotalMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: statisticSubsystem,
Name: requestsTotalMetric,
@ -97,6 +110,7 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"api"},
},
errorsTotalMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: statisticSubsystem,
Name: errorsTotalMetric,
@ -104,12 +118,14 @@ var appMetricsDesc = map[string]map[string]Description{
VariableLabels: []string{"api"},
},
txBytesTotalMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: statisticSubsystem,
Name: txBytesTotalMetric,
Help: "Total number of bytes sent by current FrostFS S3 Gate instance",
},
rxBytesTotalMetric: Description{
Type: dto.MetricType_GAUGE,
Namespace: namespace,
Subsystem: statisticSubsystem,
Name: rxBytesTotalMetric,
@ -119,6 +135,7 @@ var appMetricsDesc = map[string]map[string]Description{
}
type Description struct {
Type dto.MetricType
Namespace string
Subsystem string
Name string
@ -134,11 +151,13 @@ type KeyValue struct {
func (d *Description) MarshalJSON() ([]byte, error) {
return json.Marshal(&struct {
Type string `json:"type"`
FQName string `json:"name"`
Help string `json:"help"`
ConstantLabels []KeyValue `json:"constant_labels"`
VariableLabels []string `json:"variable_labels"`
}{
Type: d.Type.String(),
FQName: d.BuildFQName(),
Help: d.Help,
ConstantLabels: d.ConstantLabels,
@ -188,15 +207,39 @@ func newDesc(description Description) *prometheus.Desc {
description.ConstLabelsMap())
}
func newGauge(description Description) prometheus.Gauge {
func mustNewGauge(description Description) prometheus.Gauge {
if description.Type != dto.MetricType_GAUGE {
panic("invalid metric type")
}
return prometheus.NewGauge(
prometheus.GaugeOpts(newOpts(description)),
)
}
func newGaugeVec(description Description) *prometheus.GaugeVec {
func mustNewGaugeVec(description Description) *prometheus.GaugeVec {
if description.Type != dto.MetricType_GAUGE {
panic("invalid metric type")
}
return prometheus.NewGaugeVec(
prometheus.GaugeOpts(newOpts(description)),
description.VariableLabels,
)
}
func mustNewHistogramVec(description Description, buckets []float64) *prometheus.HistogramVec {
if description.Type != dto.MetricType_HISTOGRAM {
panic("invalid metric type")
}
return prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: description.Namespace,
Subsystem: description.Subsystem,
Name: description.Name,
Help: description.Name,
ConstLabels: description.ConstLabelsMap(),
Buckets: buckets,
},
description.VariableLabels,
)
}

12
metrics/desc_test.go
View File

@ -8,12 +8,24 @@ import (
"os"
"testing"
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
)
type mock struct {
}
func (m mock) Statistic() pool.Statistic {
return pool.Statistic{}
}
var metricsPath = flag.String("out", "", "File to export s3 gateway metrics to.")
func TestDescribeAll(t *testing.T) {
// to check correct metrics type mapping
_ = NewAppMetrics(zaptest.NewLogger(t), mock{}, true)
flag.Parse()
require.NotEmpty(t, metricsPath, "flag 'out' must be provided to dump metrics description")

43
metrics/gate.go
View File

@ -6,6 +6,7 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-sdk-go/pool"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
dto "github.com/prometheus/client_model/go"
)
const namespace = "frostfs_s3_gw"
@ -15,43 +16,51 @@ type StatisticScraper interface {
}
type GateMetrics struct {
State *StateMetrics
Pool poolMetricsCollector
Billing *billingMetrics
Stats *APIStatMetrics
registry *prometheus.Registry
State *StateMetrics
Pool *poolMetricsCollector
Billing *billingMetrics
Stats *APIStatMetrics
}
func NewGateMetrics(scraper StatisticScraper) *GateMetrics {
registry := prometheus.NewRegistry()
stateMetric := newStateMetrics()
stateMetric.register()
registry.MustRegister(stateMetric)
poolMetric := newPoolMetricsCollector(scraper)
poolMetric.register()
registry.MustRegister(poolMetric)
billingMetric := newBillingMetrics()
billingMetric.register()
billingMetric.Register()
statsMetric := newAPIStatMetrics()
statsMetric.register()
registry.MustRegister(statsMetric)
return &GateMetrics{
State: stateMetric,
Pool: *poolMetric,
Billing: billingMetric,
Stats: statsMetric,
registry: registry,
State: stateMetric,
Pool: poolMetric,
Billing: billingMetric,
Stats: statsMetric,
}
}
func (g *GateMetrics) Unregister() {
g.State.unregister()
prometheus.Unregister(&g.Pool)
g.Billing.unregister()
g.Stats.unregister()
g.registry.Unregister(g.State)
g.registry.Unregister(g.Pool)
g.Billing.Unregister()
g.registry.Unregister(g.Stats)
}
func (g *GateMetrics) Handler() http.Handler {
handler := http.NewServeMux()
handler.Handle("/", promhttp.Handler())
handler.Handle("/metrics/billing", promhttp.HandlerFor(g.Billing.registry, promhttp.HandlerOpts{}))
handler.Handle("/metrics/billing", promhttp.HandlerFor(g.Billing.Gatherer(), promhttp.HandlerOpts{}))
return handler
}
func (g *GateMetrics) Gather() ([]*dto.MetricFamily, error) {
return prometheus.Gatherers([]prometheus.Gatherer{g.Billing.Gatherer()}).Gather()
}

14
metrics/pool.go
View File

@ -47,11 +47,11 @@ type poolMetricsCollector struct {
func newPoolMetricsCollector(scraper StatisticScraper) *poolMetricsCollector {
return &poolMetricsCollector{
poolStatScraper: scraper,
overallErrors: newGauge(appMetricsDesc[poolSubsystem][overallErrorsMetric]),
overallNodeErrors: newGaugeVec(appMetricsDesc[poolSubsystem][overallNodeErrorsMetric]),
overallNodeRequests: newGaugeVec(appMetricsDesc[poolSubsystem][overallNodeRequestsMetric]),
currentErrors: newGaugeVec(appMetricsDesc[poolSubsystem][currentErrorMetric]),
requestDuration: newGaugeVec(appMetricsDesc[poolSubsystem][avgRequestDurationMetric]),
overallErrors: mustNewGauge(appMetricsDesc[poolSubsystem][overallErrorsMetric]),
overallNodeErrors: mustNewGaugeVec(appMetricsDesc[poolSubsystem][overallNodeErrorsMetric]),
overallNodeRequests: mustNewGaugeVec(appMetricsDesc[poolSubsystem][overallNodeRequestsMetric]),
currentErrors: mustNewGaugeVec(appMetricsDesc[poolSubsystem][currentErrorMetric]),
requestDuration: mustNewGaugeVec(appMetricsDesc[poolSubsystem][avgRequestDurationMetric]),
}
}
@ -72,10 +72,6 @@ func (m *poolMetricsCollector) Describe(descs chan<- *prometheus.Desc) {
m.requestDuration.Describe(descs)
}
func (m *poolMetricsCollector) register() {
prometheus.MustRegister(m)
}
func (m *poolMetricsCollector) updateStatistic() {
stat := m.poolStatScraper.Statistic()

36
metrics/state.go
View File

@ -28,25 +28,11 @@ type StateMetrics struct {
func newStateMetrics() *StateMetrics {
return &StateMetrics{
healthCheck: newGauge(appMetricsDesc[stateSubsystem][healthMetric]),
versionInfo: newGaugeVec(appMetricsDesc[stateSubsystem][versionInfoMetric]),
healthCheck: mustNewGauge(appMetricsDesc[stateSubsystem][healthMetric]),
versionInfo: mustNewGaugeVec(appMetricsDesc[stateSubsystem][versionInfoMetric]),
}
}
func (m *StateMetrics) register() {
if m == nil {
return
}
prometheus.MustRegister(m.healthCheck)
}
func (m *StateMetrics) unregister() {
if m == nil {
return
}
prometheus.Unregister(m.healthCheck)
}
func (m *StateMetrics) SetHealth(s HealthStatus) {
if m == nil {
return
@ -60,3 +46,21 @@ func (m *StateMetrics) SetVersion(ver string) {
}
m.versionInfo.WithLabelValues(ver).Set(1)
}
func (m *StateMetrics) Describe(desc chan<- *prometheus.Desc) {
if m == nil {
return
}
m.healthCheck.Describe(desc)
m.versionInfo.Describe(desc)
}
func (m *StateMetrics) Collect(ch chan<- prometheus.Metric) {
if m == nil {
return
}
m.healthCheck.Collect(ch)
m.versionInfo.Collect(ch)
}

29
metrics/stats.go
View File

@ -56,36 +56,11 @@ func newAPIStatMetrics() *APIStatMetrics {
return &APIStatMetrics{
stats: newHTTPStats(),
httpRequestsDuration: prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: histogramDesc.Namespace,
Subsystem: histogramDesc.Subsystem,
Name: histogramDesc.Name,
Help: histogramDesc.Name,
ConstLabels: histogramDesc.ConstLabelsMap(),
Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10},
},
histogramDesc.VariableLabels,
),
httpRequestsDuration: mustNewHistogramVec(histogramDesc,
[]float64{.05, .1, .25, .5, 1, 2.5, 5, 10}),
}
}
func (a *APIStatMetrics) register() {
if a == nil {
return
}
prometheus.MustRegister(a.stats)
prometheus.MustRegister(a.httpRequestsDuration)
}
func (a *APIStatMetrics) unregister() {
if a == nil {
return
}
prometheus.Unregister(a.stats)
prometheus.Unregister(a.httpRequestsDuration)
}
func (a *APIStatMetrics) CurrentS3RequestsInc(api string) {
if a == nil {
return