TrueCloudLab/frostfs-observability
22
0
Fork 10
generated from TrueCloudLab/basic
Code Issues 4 Pull requests 1 Projects Releases Packages Wiki Activity Actions

Export gRPC client and server metrics #6

Merged
fyrchik merged 1 commit from acid-ant/frostfs-observability:feature/4-export-grpc-metrics into master 2023-10-19 15:39:51 +00:00
Conversation 0 Commits 1 Files changed 4 +88 -12
4 changed files with 88 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 7960099809 - Show all commits

12
metrics/desc.go
View file

@ -21,7 +21,7 @@ type Description struct {
// NewGauge returns new registered prometheus.Gauge. // NewGauge returns new registered prometheus.Gauge.
func NewGauge(opts prometheus.GaugeOpts) prometheus.Gauge { func NewGauge(opts prometheus.GaugeOpts) prometheus.Gauge {
value := prometheus.NewGauge(opts) value := prometheus.NewGauge(opts)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_GAUGE.String(), Type: dto.MetricType_GAUGE.String(),
Help: opts.Help, Help: opts.Help,
@ -33,7 +33,7 @@ func NewGauge(opts prometheus.GaugeOpts) prometheus.Gauge {
// NewGaugeVec returns new registered *prometheus.GaugeVec. // NewGaugeVec returns new registered *prometheus.GaugeVec.
func NewGaugeVec(opts prometheus.GaugeOpts, labelNames []string) *prometheus.GaugeVec { func NewGaugeVec(opts prometheus.GaugeOpts, labelNames []string) *prometheus.GaugeVec {
value := prometheus.NewGaugeVec(opts, labelNames) value := prometheus.NewGaugeVec(opts, labelNames)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_GAUGE.String(), Type: dto.MetricType_GAUGE.String(),
Help: opts.Help, Help: opts.Help,
@ -47,7 +47,7 @@ func NewGaugeVec(opts prometheus.GaugeOpts, labelNames []string) *prometheus.Gau
// NewGaugeFunc returns new registered prometheus.GaugeFunc. // NewGaugeFunc returns new registered prometheus.GaugeFunc.
func NewGaugeFunc(opts prometheus.GaugeOpts, f func() float64) prometheus.GaugeFunc { func NewGaugeFunc(opts prometheus.GaugeOpts, f func() float64) prometheus.GaugeFunc {
value := prometheus.NewGaugeFunc(opts, f) value := prometheus.NewGaugeFunc(opts, f)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_GAUGE.String(), Type: dto.MetricType_GAUGE.String(),
Help: opts.Help, Help: opts.Help,
@ -59,7 +59,7 @@ func NewGaugeFunc(opts prometheus.GaugeOpts, f func() float64) prometheus.GaugeF
// NewCounter returns new registered prometheus.Counter. // NewCounter returns new registered prometheus.Counter.
func NewCounter(opts prometheus.CounterOpts) prometheus.Counter { func NewCounter(opts prometheus.CounterOpts) prometheus.Counter {
value := prometheus.NewCounter(opts) value := prometheus.NewCounter(opts)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_COUNTER.String(), Type: dto.MetricType_COUNTER.String(),
Help: opts.Help, Help: opts.Help,
@ -71,7 +71,7 @@ func NewCounter(opts prometheus.CounterOpts) prometheus.Counter {
// NewCounterVec returns new registered *prometheus.CounterVec. // NewCounterVec returns new registered *prometheus.CounterVec.
func NewCounterVec(opts prometheus.CounterOpts, labels []string) *prometheus.CounterVec { func NewCounterVec(opts prometheus.CounterOpts, labels []string) *prometheus.CounterVec {
value := prometheus.NewCounterVec(opts, labels) value := prometheus.NewCounterVec(opts, labels)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_COUNTER.String(), Type: dto.MetricType_COUNTER.String(),
Help: opts.Help, Help: opts.Help,
@ -84,7 +84,7 @@ func NewCounterVec(opts prometheus.CounterOpts, labels []string) *prometheus.Cou
// NewHistogramVec returns new registered *prometheus.HistogramVec. // NewHistogramVec returns new registered *prometheus.HistogramVec.
func NewHistogramVec(opts prometheus.HistogramOpts, labelNames []string) *prometheus.HistogramVec { func NewHistogramVec(opts prometheus.HistogramOpts, labelNames []string) *prometheus.HistogramVec {
value := prometheus.NewHistogramVec(opts, labelNames) value := prometheus.NewHistogramVec(opts, labelNames)
mustRegister(value, Description{ MustRegister(value, Description{
Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), Name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
Type: dto.MetricType_HISTOGRAM.String(), Type: dto.MetricType_HISTOGRAM.String(),
Help: opts.Help, Help: opts.Help,

45
metrics/grpc/client.go
View file

@ -4,10 +4,11 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"google.golang.org/grpc" "google.golang.org/grpc"
) )
var clientMetrics *grpcprom.ClientMetrics = grpcprom.NewClientMetrics( var clientMetrics = grpcprom.NewClientMetrics(
grpcprom.WithClientHandlingTimeHistogram( grpcprom.WithClientHandlingTimeHistogram(
grpcprom.WithHistogramBuckets(prometheus.DefBuckets), grpcprom.WithHistogramBuckets(prometheus.DefBuckets),
), ),
@ -17,7 +18,47 @@ var clientMetrics *grpcprom.ClientMetrics = grpcprom.NewClientMetrics(
) )
func init() { func init() {
metrics.Register(clientMetrics) // Description copied from repository of grpc-ecosystem
// https://github.com/grpc-ecosystem/go-grpc-middleware/blob/71d7422112b1d7fadd4b8bf12a6f33ba6d22e98e/providers/prometheus/client_metrics.go#L31
descs := []metrics.Description{
{
Name: "grpc_client_started_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPCs started on the client.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_client_handled_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPCs completed by the client, regardless of success or failure.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"},
},
{
Name: "grpc_client_msg_received_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPC stream messages received by the client.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_client_msg_sent_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of gRPC stream messages sent by the client.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_client_handling_seconds",
Type: dto.MetricType_HISTOGRAM.String(),
Help: "Histogram of response latency (seconds) of the gRPC until it is finished by the application.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_client_msg_recv_handling_seconds",
Type: dto.MetricType_HISTOGRAM.String(),
Help: "Histogram of response latency (seconds) of the gRPC single message receive.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
}
metrics.MustRegister(clientMetrics, descs...)
fyrchik marked this conversation as resolved Outdated
fyrchik commented 2023-10-19 06:42:01 +00:00
Outdated
Review
Copy link

Why export MustRegister instead of using our New* wrappers? Not saying this is a bad thing.

Why export `MustRegister` instead of using our `New*` wrappers? Not saying this is a bad thing.
acid-ant commented 2023-10-19 07:03:32 +00:00
Outdated
Review
Copy link

Because all methods in grpcprom.ServerMetrics are private, there are no possibility to initialize them.

Because all methods in `grpcprom.ServerMetrics` are private, there are no possibility to initialize them.
} }
// NewUnaryClientInterceptor returns client interceptor to collect metrics from unary RPCs. // NewUnaryClientInterceptor returns client interceptor to collect metrics from unary RPCs.

39
metrics/grpc/server.go
View file

@ -4,17 +4,52 @@ import (
"git.frostfs.info/TrueCloudLab/frostfs-observability/metrics" "git.frostfs.info/TrueCloudLab/frostfs-observability/metrics"
grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" grpcprom "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"google.golang.org/grpc" "google.golang.org/grpc"
) )
var serverMetrics *grpcprom.ServerMetrics = grpcprom.NewServerMetrics( var serverMetrics = grpcprom.NewServerMetrics(
fyrchik commented 2023-10-19 06:43:32 +00:00
Outdated
Review
Copy link

How is it different from what was before? We register metric descriptions, but the implementation is still in the imported package?

How is it different from what was before? We register metric descriptions, but the implementation is still in the imported package?
acid-ant commented 2023-10-19 07:42:09 +00:00
Outdated
Review
Copy link

Didn't catch your question. You propose to create our own metrics instead of predefined?

Didn't catch your question. You propose to create our own metrics instead of predefined?
fyrchik commented 2023-10-19 15:39:41 +00:00
Outdated
Review
Copy link

Yes, but ok, let's do it in a separate task. It is not ideal currently -- definitions are decoupled from the actual metrics, so it is easy to miss something.

Yes, but ok, let's do it in a separate task. It is not ideal currently -- definitions are decoupled from the actual metrics, so it is easy to miss something.
👍 1
grpcprom.WithServerHandlingTimeHistogram( grpcprom.WithServerHandlingTimeHistogram(
grpcprom.WithHistogramBuckets(prometheus.DefBuckets), grpcprom.WithHistogramBuckets(prometheus.DefBuckets),
), ),
) )
func init() { func init() {
metrics.Register(serverMetrics) // Description copied from grpc-ecosystem:
// https://github.com/grpc-ecosystem/go-grpc-middleware/blob/71d7422112b1d7fadd4b8bf12a6f33ba6d22e98e/providers/prometheus/server_metrics.go#L26
fyrchik marked this conversation as resolved Outdated
fyrchik commented 2023-10-19 06:40:50 +00:00
Outdated
Review
Copy link

Can we use a permalink here (replace main with a tag)?

Can we use a permalink here (replace `main` with a tag)?
acid-ant commented 2023-10-19 07:50:00 +00:00
Outdated
Review
Copy link

Updated.

Updated.
descs := []metrics.Description{
{
Name: "grpc_server_started_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPCs started on the server.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_server_handled_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPCs completed on the server, regardless of success or failure.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"},
},
{
Name: "grpc_server_msg_received_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of RPC stream messages received on the server.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_server_msg_sent_total",
Type: dto.MetricType_COUNTER.String(),
Help: "Total number of gRPC stream messages sent by the server.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
{
Name: "grpc_server_handling_seconds",
Type: dto.MetricType_HISTOGRAM.String(),
Help: "Histogram of response latency (seconds) of gRPC that had been application-level handled by the server.",
VariableLabels: []string{"grpc_type", "grpc_service", "grpc_method"},
},
}
metrics.MustRegister(serverMetrics, descs...)
} }
// NewUnaryServerInterceptor returns server interceptor to collect metrics from unary RPCs. // NewUnaryServerInterceptor returns server interceptor to collect metrics from unary RPCs.

4
metrics/registry.go
View file

@ -28,11 +28,11 @@ func Register(customCollectors ...prometheus.Collector) {
registry.MustRegister(customCollectors...) registry.MustRegister(customCollectors...)
} }
func mustRegister(c prometheus.Collector, desc Description) { func MustRegister(c prometheus.Collector, descs ...Description) {
registry.MustRegister(c) registry.MustRegister(c)
registeredDescriptionsMtx.Lock() registeredDescriptionsMtx.Lock()
defer registeredDescriptionsMtx.Unlock() defer registeredDescriptionsMtx.Unlock()
registeredDescriptions = append(registeredDescriptions, desc) registeredDescriptions = append(registeredDescriptions, descs...)
} }
// Handler returns an http.Handler for the local registry. // Handler returns an http.Handler for the local registry.