From 796009980943794a1255af980bf9a797572efd9c Mon Sep 17 00:00:00 2001 From: Anton Nikiforov Date: Tue, 17 Oct 2023 18:48:36 +0300 Subject: [PATCH] [#4] Export gRPC client and server metrics Signed-off-by: Anton Nikiforov --- metrics/desc.go | 12 +++++------ metrics/grpc/client.go | 45 ++++++++++++++++++++++++++++++++++++++++-- metrics/grpc/server.go | 39 ++++++++++++++++++++++++++++++++++-- metrics/registry.go | 4 ++-- 4 files changed, 88 insertions(+), 12 deletions(-) diff --git a/metrics/desc.go b/metrics/desc.go index 554c065..e9eb92b 100644 --- a/metrics/desc.go +++ b/metrics/desc.go @@ -21,7 +21,7 @@ type Description struct { // NewGauge returns new registered prometheus.Gauge. func NewGauge(opts prometheus.GaugeOpts) prometheus.Gauge { value := prometheus.NewGauge(opts) - mustRegister(value, Description{ + MustRegister(value, Description{ Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Type: dto.MetricType_GAUGE.String(), Help: opts.Help, @@ -33,7 +33,7 @@ func NewGauge(opts prometheus.GaugeOpts) prometheus.Gauge { // NewGaugeVec returns new registered *prometheus.GaugeVec. func NewGaugeVec(opts prometheus.GaugeOpts, labelNames []string) *prometheus.GaugeVec { value := prometheus.NewGaugeVec(opts, labelNames) - mustRegister(value, Description{ + MustRegister(value, Description{ Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Type: dto.MetricType_GAUGE.String(), Help: opts.Help, @@ -47,7 +47,7 @@ func NewGaugeVec(opts prometheus.GaugeOpts, labelNames []string) *prometheus.Gau // NewGaugeFunc returns new registered prometheus.GaugeFunc. func NewGaugeFunc(opts prometheus.GaugeOpts, f func() float64) prometheus.GaugeFunc { value := prometheus.NewGaugeFunc(opts, f) - mustRegister(value, Description{ + MustRegister(value, Description{ Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Type: dto.MetricType_GAUGE.String(), Help: opts.Help, @@ -59,7 +59,7 @@ func NewGaugeFunc(opts prometheus.GaugeOpts, f func() float64) prometheus.GaugeF // NewCounter returns new registered prometheus.Counter. func NewCounter(opts prometheus.CounterOpts) prometheus.Counter { value := prometheus.NewCounter(opts) - mustRegister(value, Description{ + MustRegister(value, Description{ Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Type: dto.MetricType_COUNTER.String(), Help: opts.Help, @@ -71,7 +71,7 @@ func NewCounter(opts prometheus.CounterOpts) prometheus.Counter { // NewCounterVec returns new registered *prometheus.CounterVec. func NewCounterVec(opts prometheus.CounterOpts, labels []string) *prometheus.CounterVec { value := prometheus.NewCounterVec(opts, labels) - mustRegister(value, Description{ + MustRegister(value, Description{ Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Type: dto.MetricType_COUNTER.String(), Help: opts.Help, @@ -84,7 +84,7 @@ func NewCounterVec(opts prometheus.CounterOpts, labels []string) *prometheus.Cou // NewHistogramVec returns new registered *prometheus.HistogramVec. func NewHistogramVec(opts prometheus.HistogramOpts, labelNames []string) *prometheus.HistogramVec { value := prometheus.NewHistogramVec(opts, labelNames) - mustRegister(value, Description{ + MustRegister(value, Description{ Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Type: dto.MetricType_HISTOGRAM.String(), Help: opts.Help, diff --git a/metrics/grpc/client.go b/metrics/grpc/client.go index 4305d53..bbc7691 100644 --- a/metrics/grpc/client.go +++ b/metrics/grpc/client.go @@ -4,10 +4,11 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "google.golang.org/grpc" ) -var clientMetrics *grpcprom.ClientMetrics = grpcprom.NewClientMetrics( +var clientMetrics = grpcprom.NewClientMetrics( grpcprom.WithClientHandlingTimeHistogram( grpcprom.WithHistogramBuckets(prometheus.DefBuckets), ), @@ -17,7 +18,47 @@ var clientMetrics *grpcprom.ClientMetrics = grpcprom.NewClientMetrics( ) func init() { - metrics.Register(clientMetrics) + // Description copied from repository of grpc-ecosystem + // https://github.com/grpc-ecosystem/go-grpc-middleware/blob/71d7422112b1d7fadd4b8bf12a6f33ba6d22e98e/providers/prometheus/client_metrics.go#L31 + descs := []metrics.Description{ + { + Name: "grpc_client_started_total", + Type: dto.MetricType_COUNTER.String(), + Help: "Total number of RPCs started on the client.", + VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"}, + }, + { + Name: "grpc_client_handled_total", + Type: dto.MetricType_COUNTER.String(), + Help: "Total number of RPCs completed by the client, regardless of success or failure.", + VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"}, + }, + { + Name: "grpc_client_msg_received_total", + Type: dto.MetricType_COUNTER.String(), + Help: "Total number of RPC stream messages received by the client.", + VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"}, + }, + { + Name: "grpc_client_msg_sent_total", + Type: dto.MetricType_COUNTER.String(), + Help: "Total number of gRPC stream messages sent by the client.", + VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"}, + }, + { + Name: "grpc_client_handling_seconds", + Type: dto.MetricType_HISTOGRAM.String(), + Help: "Histogram of response latency (seconds) of the gRPC until it is finished by the application.", + VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"}, + }, + { + Name: "grpc_client_msg_recv_handling_seconds", + Type: dto.MetricType_HISTOGRAM.String(), + Help: "Histogram of response latency (seconds) of the gRPC single message receive.", + VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"}, + }, + } + metrics.MustRegister(clientMetrics, descs...) } // NewUnaryClientInterceptor returns client interceptor to collect metrics from unary RPCs. diff --git a/metrics/grpc/server.go b/metrics/grpc/server.go index b47c488..15112b3 100644 --- a/metrics/grpc/server.go +++ b/metrics/grpc/server.go @@ -4,17 +4,52 @@ import ( "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "google.golang.org/grpc" ) -var serverMetrics *grpcprom.ServerMetrics = grpcprom.NewServerMetrics( +var serverMetrics = grpcprom.NewServerMetrics( grpcprom.WithServerHandlingTimeHistogram( grpcprom.WithHistogramBuckets(prometheus.DefBuckets), ), ) func init() { - metrics.Register(serverMetrics) + // Description copied from grpc-ecosystem: + // https://github.com/grpc-ecosystem/go-grpc-middleware/blob/71d7422112b1d7fadd4b8bf12a6f33ba6d22e98e/providers/prometheus/server_metrics.go#L26 + descs := []metrics.Description{ + { + Name: "grpc_server_started_total", + Type: dto.MetricType_COUNTER.String(), + Help: "Total number of RPCs started on the server.", + VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"}, + }, + { + Name: "grpc_server_handled_total", + Type: dto.MetricType_COUNTER.String(), + Help: "Total number of RPCs completed on the server, regardless of success or failure.", + VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"}, + }, + { + Name: "grpc_server_msg_received_total", + Type: dto.MetricType_COUNTER.String(), + Help: "Total number of RPC stream messages received on the server.", + VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"}, + }, + { + Name: "grpc_server_msg_sent_total", + Type: dto.MetricType_COUNTER.String(), + Help: "Total number of gRPC stream messages sent by the server.", + VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"}, + }, + { + Name: "grpc_server_handling_seconds", + Type: dto.MetricType_HISTOGRAM.String(), + Help: "Histogram of response latency (seconds) of gRPC that had been application-level handled by the server.", + VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"}, + }, + } + metrics.MustRegister(serverMetrics, descs...) } // NewUnaryServerInterceptor returns server interceptor to collect metrics from unary RPCs. diff --git a/metrics/registry.go b/metrics/registry.go index 85f0aa8..2e90a23 100644 --- a/metrics/registry.go +++ b/metrics/registry.go @@ -28,11 +28,11 @@ func Register(customCollectors ...prometheus.Collector) { registry.MustRegister(customCollectors...) } -func mustRegister(c prometheus.Collector, desc Description) { +func MustRegister(c prometheus.Collector, descs ...Description) { registry.MustRegister(c) registeredDescriptionsMtx.Lock() defer registeredDescriptionsMtx.Unlock() - registeredDescriptions = append(registeredDescriptions, desc) + registeredDescriptions = append(registeredDescriptions, descs...) } // Handler returns an http.Handler for the local registry.