Export gRPC client and server metrics #6

Merged
fyrchik merged 1 commit from acid-ant/frostfs-observability:feature/4-export-grpc-metrics into master 2023-10-19 15:39:51 +00:00
4 changed files with 88 additions and 12 deletions

View file

@ -21,7 +21,7 @@ type Description struct {
// NewGauge returns new registered prometheus.Gauge. // NewGauge returns new registered prometheus.Gauge.
func NewGauge(opts prometheus.GaugeOpts) prometheus.Gauge { func NewGauge(opts prometheus.GaugeOpts) prometheus.Gauge {
value := prometheus.NewGauge(opts) value := prometheus.NewGauge(opts)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_GAUGE.String(), Type: dto.MetricType_GAUGE.String(),
Help: opts.Help, Help: opts.Help,
@ -33,7 +33,7 @@ func NewGauge(opts prometheus.GaugeOpts) prometheus.Gauge {
// NewGaugeVec returns new registered *prometheus.GaugeVec. // NewGaugeVec returns new registered *prometheus.GaugeVec.
func NewGaugeVec(opts prometheus.GaugeOpts, labelNames []string) *prometheus.GaugeVec { func NewGaugeVec(opts prometheus.GaugeOpts, labelNames []string) *prometheus.GaugeVec {
value := prometheus.NewGaugeVec(opts, labelNames) value := prometheus.NewGaugeVec(opts, labelNames)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_GAUGE.String(), Type: dto.MetricType_GAUGE.String(),
Help: opts.Help, Help: opts.Help,
@ -47,7 +47,7 @@ func NewGaugeVec(opts prometheus.GaugeOpts, labelNames []string) *prometheus.Gau
// NewGaugeFunc returns new registered prometheus.GaugeFunc. // NewGaugeFunc returns new registered prometheus.GaugeFunc.
func NewGaugeFunc(opts prometheus.GaugeOpts, f func() float64) prometheus.GaugeFunc { func NewGaugeFunc(opts prometheus.GaugeOpts, f func() float64) prometheus.GaugeFunc {
value := prometheus.NewGaugeFunc(opts, f) value := prometheus.NewGaugeFunc(opts, f)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_GAUGE.String(), Type: dto.MetricType_GAUGE.String(),
Help: opts.Help, Help: opts.Help,
@ -59,7 +59,7 @@ func NewGaugeFunc(opts prometheus.GaugeOpts, f func() float64) prometheus.GaugeF
// NewCounter returns new registered prometheus.Counter. // NewCounter returns new registered prometheus.Counter.
func NewCounter(opts prometheus.CounterOpts) prometheus.Counter { func NewCounter(opts prometheus.CounterOpts) prometheus.Counter {
value := prometheus.NewCounter(opts) value := prometheus.NewCounter(opts)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_COUNTER.String(), Type: dto.MetricType_COUNTER.String(),
Help: opts.Help, Help: opts.Help,
@ -71,7 +71,7 @@ func NewCounter(opts prometheus.CounterOpts) prometheus.Counter {
// NewCounterVec returns new registered *prometheus.CounterVec. // NewCounterVec returns new registered *prometheus.CounterVec.
func NewCounterVec(opts prometheus.CounterOpts, labels []string) *prometheus.CounterVec { func NewCounterVec(opts prometheus.CounterOpts, labels []string) *prometheus.CounterVec {
value := prometheus.NewCounterVec(opts, labels) value := prometheus.NewCounterVec(opts, labels)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_COUNTER.String(), Type: dto.MetricType_COUNTER.String(),
Help: opts.Help, Help: opts.Help,
@ -84,7 +84,7 @@ func NewCounterVec(opts prometheus.CounterOpts, labels []string) *prometheus.Cou
// NewHistogramVec returns new registered *prometheus.HistogramVec. // NewHistogramVec returns new registered *prometheus.HistogramVec.
func NewHistogramVec(opts prometheus.HistogramOpts, labelNames []string) *prometheus.HistogramVec { func NewHistogramVec(opts prometheus.HistogramOpts, labelNames []string) *prometheus.HistogramVec {
value := prometheus.NewHistogramVec(opts, labelNames) value := prometheus.NewHistogramVec(opts, labelNames)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_HISTOGRAM.String(), Type: dto.MetricType_HISTOGRAM.String(),
Help: opts.Help, Help: opts.Help,

View file

@ -4,10 +4,11 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"google.golang.org/grpc" "google.golang.org/grpc"
) )
var clientMetrics *grpcprom.ClientMetrics = grpcprom.NewClientMetrics( var clientMetrics = grpcprom.NewClientMetrics(
grpcprom.WithClientHandlingTimeHistogram( grpcprom.WithClientHandlingTimeHistogram(
grpcprom.WithHistogramBuckets(prometheus.DefBuckets), grpcprom.WithHistogramBuckets(prometheus.DefBuckets),
), ),
@ -17,7 +18,47 @@ var clientMetrics *grpcprom.ClientMetrics = grpcprom.NewClientMetrics(
) )
func init() { func init() {
metrics.Register(clientMetrics) // Description copied from repository of grpc-ecosystem
// https://github.com/grpc-ecosystem/go-grpc-middleware/blob/71d7422112b1d7fadd4b8bf12a6f33ba6d22e98e/providers/prometheus/client_metrics.go#L31
descs := []metrics.Description{
{
Name: "grpc_client_started_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPCs started on the client.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_client_handled_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPCs completed by the client, regardless of success or failure.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"},
},
{
Name: "grpc_client_msg_received_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPC stream messages received by the client.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_client_msg_sent_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of gRPC stream messages sent by the client.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_client_handling_seconds",
Type: dto.MetricType_HISTOGRAM.String(),
Help: "Histogram of response latency (seconds) of the gRPC until it is finished by the application.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_client_msg_recv_handling_seconds",
Type: dto.MetricType_HISTOGRAM.String(),
Help: "Histogram of response latency (seconds) of the gRPC single message receive.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
}
metrics.MustRegister(clientMetrics, descs...)
} }
// NewUnaryClientInterceptor returns client interceptor to collect metrics from unary RPCs. // NewUnaryClientInterceptor returns client interceptor to collect metrics from unary RPCs.

View file

@ -4,17 +4,52 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"google.golang.org/grpc" "google.golang.org/grpc"
) )
var serverMetrics *grpcprom.ServerMetrics = grpcprom.NewServerMetrics( var serverMetrics = grpcprom.NewServerMetrics(
grpcprom.WithServerHandlingTimeHistogram( grpcprom.WithServerHandlingTimeHistogram(
grpcprom.WithHistogramBuckets(prometheus.DefBuckets), grpcprom.WithHistogramBuckets(prometheus.DefBuckets),
), ),
) )
func init() { func init() {
metrics.Register(serverMetrics) // Description copied from grpc-ecosystem:
// https://github.com/grpc-ecosystem/go-grpc-middleware/blob/71d7422112b1d7fadd4b8bf12a6f33ba6d22e98e/providers/prometheus/server_metrics.go#L26
descs := []metrics.Description{
{
Name: "grpc_server_started_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPCs started on the server.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_server_handled_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPCs completed on the server, regardless of success or failure.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"},
},
{
Name: "grpc_server_msg_received_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPC stream messages received on the server.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_server_msg_sent_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of gRPC stream messages sent by the server.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_server_handling_seconds",
Type: dto.MetricType_HISTOGRAM.String(),
Help: "Histogram of response latency (seconds) of gRPC that had been application-level handled by the server.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
}
metrics.MustRegister(serverMetrics, descs...)
} }
// NewUnaryServerInterceptor returns server interceptor to collect metrics from unary RPCs. // NewUnaryServerInterceptor returns server interceptor to collect metrics from unary RPCs.

View file

@ -28,11 +28,11 @@ func Register(customCollectors ...prometheus.Collector) {
registry.MustRegister(customCollectors...) registry.MustRegister(customCollectors...)
} }
func mustRegister(c prometheus.Collector, desc Description) { func MustRegister(c prometheus.Collector, descs ...Description) {
registry.MustRegister(c) registry.MustRegister(c)
registeredDescriptionsMtx.Lock() registeredDescriptionsMtx.Lock()
defer registeredDescriptionsMtx.Unlock() defer registeredDescriptionsMtx.Unlock()
registeredDescriptions = append(registeredDescriptions, desc) registeredDescriptions = append(registeredDescriptions, descs...)
} }
// Handler returns an http.Handler for the local registry. // Handler returns an http.Handler for the local registry.