plugin/metrics: add panic counter (#1778)

Count and export number of panics we see.

Fixes #1294

core/dnsserver/server.go

@@ -208,6 +208,7 @@ func (s *Server) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg)
 			// In case the user doesn't enable error plugin, we still
 			// need to make sure that we stay alive up here
 			if rec := recover(); rec != nil {
+				vars.Panic.Inc()
 				DefaultErrorFunc(ctx, w, r, dns.RcodeServerFailure)
 			}
 		}()

plugin/metrics/README.md

@@ -11,6 +11,7 @@ The default location for the metrics is `localhost:9153`. The metrics path is fi
 The following metrics are exported:
 
 * `coredns_build_info{version, revision, goversion}` - info about CoreDNS itself.
+* `coredns_panic_count_total{}` - total number of panics.
 * `coredns_dns_request_count_total{server, zone, proto, family}` - total query count.
 * `coredns_dns_request_duration_seconds{server, zone}` - duration to process each query.
 * `coredns_dns_request_size_bytes{server, zone, proto}` - size of the request in bytes.

plugin/metrics/metrics.go

@@ -41,6 +41,7 @@ func New(addr string) *Metrics {
 
 	// Add all of our collectors
 	met.MustRegister(buildInfo)
+	met.MustRegister(vars.Panic)
 	met.MustRegister(vars.RequestCount)
 	met.MustRegister(vars.RequestDuration)
 	met.MustRegister(vars.RequestSize)
@@ -137,10 +138,8 @@ func keys(m map[string]bool) []string {
 // we listen on "localhost:0" and need to retrieve the actual address.
 var ListenAddr string
 
-var (
+var buildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
-	buildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 	Namespace: plugin.Namespace,
 	Name:      "build_info",
 	Help:      "A metric with a constant '1' value labeled by version, revision, and goversion from which CoreDNS was built.",
-	}, []string{"version", "revision", "goversion"})
+}, []string{"version", "revision", "goversion"})
-)

plugin/metrics/vars/vars.go

@@ -59,6 +59,12 @@ var (
 		Name:      "response_rcode_count_total",
 		Help:      "Counter of response status codes.",
 	}, []string{"server", "zone", "rcode"})
+
+	Panic = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: plugin.Namespace,
+		Name:      "panic_count_total",
+		Help:      "A metrics that counts the number of panics.",
+	})
 )
 
 const (