diff --git a/plugin/kubernetes/README.md b/plugin/kubernetes/README.md index 8d66af8da..e60ca2e0e 100644 --- a/plugin/kubernetes/README.md +++ b/plugin/kubernetes/README.md @@ -229,6 +229,11 @@ If monitoring is enabled (via the *prometheus* plugin) then the following metric * `headless_with_selector` * `headless_without_selector` +The following are client level metrics to monitor apiserver request latency & status codes. `verb` identifies the apiserver [request type](https://kubernetes.io/docs/reference/using-api/api-concepts/#single-resource-api) and `url`/`host` denotes the apiserver endpoint. +* `coredns_kubernetes_rest_client_request_duration_seconds{verb, url}` - captures apiserver request latency perceived by client grouped by `verb` and `url`. +* `coredns_kubernetes_rest_client_rate_limiter_duration_seconds{verb, url}` - captures apiserver request latency contributed by client side rate limiter grouped by `verb` & `url`. +* `coredns_kubernetes_rest_client_requests_total{method, code, host}` - captures total apiserver requests grouped by `method`, `status_code` & `host`. + ## Bugs The duration metric only supports the "headless\_with\_selector" service currently. diff --git a/plugin/kubernetes/metrics.go b/plugin/kubernetes/metrics.go new file mode 100644 index 000000000..788982302 --- /dev/null +++ b/plugin/kubernetes/metrics.go @@ -0,0 +1,74 @@ +package kubernetes + +import ( + "context" + "net/url" + "time" + + "github.com/coredns/coredns/plugin" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "k8s.io/client-go/tools/metrics" +) + +var ( + // requestLatency measures K8s rest client requests latency grouped by verb and url. + requestLatency = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: plugin.Namespace, + Subsystem: "kubernetes", + Name: "rest_client_request_duration_seconds", + Help: "Request latency in seconds. Broken down by verb and URL.", + Buckets: prometheus.DefBuckets, + }, + []string{"verb", "url"}, + ) + + // rateLimiterLatency measures K8s rest client rate limiter latency grouped by verb and url. + rateLimiterLatency = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: plugin.Namespace, + Subsystem: "kubernetes", + Name: "rest_client_rate_limiter_duration_seconds", + Help: "Client side rate limiter latency in seconds. Broken down by verb and URL.", + Buckets: prometheus.DefBuckets, + }, + []string{"verb", "url"}, + ) + + // requestResult measures K8s rest client request metrics grouped by status code, method & host. + requestResult = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: plugin.Namespace, + Subsystem: "kubernetes", + Name: "rest_client_requests_total", + Help: "Number of HTTP requests, partitioned by status code, method, and host.", + }, + []string{"code", "method", "host"}, + ) +) + +func init() { + metrics.Register(metrics.RegisterOpts{ + RequestLatency: &latencyAdapter{m: requestLatency}, + RateLimiterLatency: &latencyAdapter{m: rateLimiterLatency}, + RequestResult: &resultAdapter{requestResult}, + }) +} + +type latencyAdapter struct { + m *prometheus.HistogramVec +} + +func (l *latencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) { + l.m.WithLabelValues(verb, u.Host).Observe(latency.Seconds()) +} + +type resultAdapter struct { + m *prometheus.CounterVec +} + +func (r *resultAdapter) Increment(_ context.Context, code, method, host string) { + r.m.WithLabelValues(code, method, host).Inc() +}