plugin/kubernetes: expose client-go internal request metrics (#5991)

plugin/kubernetes/README.md

@@ -229,6 +229,11 @@ If monitoring is enabled (via the *prometheus* plugin) then the following metric
     * `headless_with_selector`
     * `headless_without_selector`
 
+The following are client level metrics to monitor apiserver request latency & status codes. `verb` identifies the apiserver [request type](https://kubernetes.io/docs/reference/using-api/api-concepts/#single-resource-api) and `url`/`host` denotes the apiserver endpoint.
+* `coredns_kubernetes_rest_client_request_duration_seconds{verb, url}` - captures apiserver request latency perceived by client grouped by `verb` and `url`.
+* `coredns_kubernetes_rest_client_rate_limiter_duration_seconds{verb, url}` - captures apiserver request latency contributed by client side rate limiter grouped by `verb` & `url`.
+* `coredns_kubernetes_rest_client_requests_total{method, code, host}` - captures total apiserver requests grouped by `method`, `status_code` & `host`.
+
 ## Bugs
 
 The duration metric only supports the "headless\_with\_selector" service currently.

plugin/kubernetes/metrics.go

@@ -0,0 +1,74 @@
+package kubernetes
+
+import (
+	"context"
+	"net/url"
+	"time"
+
+	"github.com/coredns/coredns/plugin"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"k8s.io/client-go/tools/metrics"
+)
+
+var (
+	// requestLatency measures K8s rest client requests latency grouped by verb and url.
+	requestLatency = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: plugin.Namespace,
+			Subsystem: "kubernetes",
+			Name:      "rest_client_request_duration_seconds",
+			Help:      "Request latency in seconds. Broken down by verb and URL.",
+			Buckets:   prometheus.DefBuckets,
+		},
+		[]string{"verb", "url"},
+	)
+
+	// rateLimiterLatency measures K8s rest client rate limiter latency grouped by verb and url.
+	rateLimiterLatency = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: plugin.Namespace,
+			Subsystem: "kubernetes",
+			Name:      "rest_client_rate_limiter_duration_seconds",
+			Help:      "Client side rate limiter latency in seconds. Broken down by verb and URL.",
+			Buckets:   prometheus.DefBuckets,
+		},
+		[]string{"verb", "url"},
+	)
+
+	// requestResult measures K8s rest client request metrics grouped by status code, method & host.
+	requestResult = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: plugin.Namespace,
+			Subsystem: "kubernetes",
+			Name:      "rest_client_requests_total",
+			Help:      "Number of HTTP requests, partitioned by status code, method, and host.",
+		},
+		[]string{"code", "method", "host"},
+	)
+)
+
+func init() {
+	metrics.Register(metrics.RegisterOpts{
+		RequestLatency:     &latencyAdapter{m: requestLatency},
+		RateLimiterLatency: &latencyAdapter{m: rateLimiterLatency},
+		RequestResult:      &resultAdapter{requestResult},
+	})
+}
+
+type latencyAdapter struct {
+	m *prometheus.HistogramVec
+}
+
+func (l *latencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) {
+	l.m.WithLabelValues(verb, u.Host).Observe(latency.Seconds())
+}
+
+type resultAdapter struct {
+	m *prometheus.CounterVec
+}
+
+func (r *resultAdapter) Increment(_ context.Context, code, method, host string) {
+	r.m.WithLabelValues(code, method, host).Inc()
+}