* reload: use OnRestart Close the listener on OnRestart for health and metrics so the default setup function can setup the listener when the plugin is "starting up". Lightly test with some SIGUSR1-ing. Also checked the reload plugin with this, seems fine: .com.:1043 .:1043 2018/04/20 15:01:25 [INFO] CoreDNS-1.1.1 2018/04/20 15:01:25 [INFO] linux/amd64, go1.10, CoreDNS-1.1.1 linux/amd64, go1.10, 2018/04/20 15:01:25 [INFO] Running configuration MD5 = aa8b3f03946fb60546ca1f725d482714 2018/04/20 15:02:01 [INFO] Reloading 2018/04/20 15:02:01 [INFO] Running configuration MD5 = b34a96d99e01db4015a892212560155f 2018/04/20 15:02:01 [INFO] Reloading complete ^C2018/04/20 15:02:06 [INFO] SIGINT: Shutting down With this corefile: .com { proxy . 127.0.0.1:53 prometheus :9054 whoami reload } . { proxy . 127.0.0.1:53 prometheus :9054 whoami reload } The prometheus port was 9053, changed that to 54 so reload would pick it up. From a cursory look it seems this also fixes: Fixes #1604 #1618 #1686 #1492 * At least make it test * Use onfinalshutdown * reload: add reload test This test #1604 adn right now fails. * Address review comments * Add bug section explaining things a bit * compile tests * Fix tests * fixes * slightly less crazy * try to make prometheus setup less confusing * Use ephermal port for test * Don't use the listener * These are shared between goroutines, just use the boolean in the main structure. * Fix text in the reload README, * Set addr to TODO once stopping it * Morph fturb's comment into test, to test reload and scrape health and metric endpoint
152 lines
3.7 KiB
Go
152 lines
3.7 KiB
Go
// Package metrics implement a handler and plugin that provides Prometheus metrics.
|
|
package metrics
|
|
|
|
import (
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"runtime"
|
|
"sync"
|
|
|
|
"github.com/coredns/coredns/coremain"
|
|
"github.com/coredns/coredns/plugin"
|
|
"github.com/coredns/coredns/plugin/metrics/vars"
|
|
"github.com/coredns/coredns/plugin/pkg/log"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
)
|
|
|
|
// Metrics holds the prometheus configuration. The metrics' path is fixed to be /metrics
|
|
type Metrics struct {
|
|
Next plugin.Handler
|
|
Addr string
|
|
Reg *prometheus.Registry
|
|
ln net.Listener
|
|
lnSetup bool
|
|
mux *http.ServeMux
|
|
|
|
zoneNames []string
|
|
zoneMap map[string]bool
|
|
zoneMu sync.RWMutex
|
|
}
|
|
|
|
// New returns a new instance of Metrics with the given address
|
|
func New(addr string) *Metrics {
|
|
met := &Metrics{
|
|
Addr: addr,
|
|
Reg: prometheus.NewRegistry(),
|
|
zoneMap: make(map[string]bool),
|
|
}
|
|
// Add the default collectors
|
|
met.MustRegister(prometheus.NewGoCollector())
|
|
met.MustRegister(prometheus.NewProcessCollector(os.Getpid(), ""))
|
|
|
|
// Add all of our collectors
|
|
met.MustRegister(buildInfo)
|
|
met.MustRegister(vars.RequestCount)
|
|
met.MustRegister(vars.RequestDuration)
|
|
met.MustRegister(vars.RequestSize)
|
|
met.MustRegister(vars.RequestDo)
|
|
met.MustRegister(vars.RequestType)
|
|
met.MustRegister(vars.ResponseSize)
|
|
met.MustRegister(vars.ResponseRcode)
|
|
|
|
// Initialize metrics.
|
|
buildInfo.WithLabelValues(coremain.CoreVersion, coremain.GitCommit, runtime.Version()).Set(1)
|
|
|
|
return met
|
|
}
|
|
|
|
// MustRegister wraps m.Reg.MustRegister.
|
|
func (m *Metrics) MustRegister(c prometheus.Collector) { m.Reg.MustRegister(c) }
|
|
|
|
// AddZone adds zone z to m.
|
|
func (m *Metrics) AddZone(z string) {
|
|
m.zoneMu.Lock()
|
|
m.zoneMap[z] = true
|
|
m.zoneNames = keys(m.zoneMap)
|
|
m.zoneMu.Unlock()
|
|
}
|
|
|
|
// RemoveZone remove zone z from m.
|
|
func (m *Metrics) RemoveZone(z string) {
|
|
m.zoneMu.Lock()
|
|
delete(m.zoneMap, z)
|
|
m.zoneNames = keys(m.zoneMap)
|
|
m.zoneMu.Unlock()
|
|
}
|
|
|
|
// ZoneNames returns the zones of m.
|
|
func (m *Metrics) ZoneNames() []string {
|
|
m.zoneMu.RLock()
|
|
s := m.zoneNames
|
|
m.zoneMu.RUnlock()
|
|
return s
|
|
}
|
|
|
|
// OnStartup sets up the metrics on startup.
|
|
func (m *Metrics) OnStartup() error {
|
|
ln, err := net.Listen("tcp", m.Addr)
|
|
if err != nil {
|
|
log.Errorf("Failed to start metrics handler: %s", err)
|
|
return err
|
|
}
|
|
|
|
m.ln = ln
|
|
m.lnSetup = true
|
|
ListenAddr = m.ln.Addr().String() // For tests
|
|
|
|
m.mux = http.NewServeMux()
|
|
m.mux.Handle("/metrics", promhttp.HandlerFor(m.Reg, promhttp.HandlerOpts{}))
|
|
|
|
go func() {
|
|
http.Serve(m.ln, m.mux)
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
// OnRestart stops the listener on reload.
|
|
func (m *Metrics) OnRestart() error {
|
|
if !m.lnSetup {
|
|
return nil
|
|
}
|
|
|
|
uniqAddr.setAddressTodo(m.Addr)
|
|
|
|
m.ln.Close()
|
|
m.lnSetup = false
|
|
return nil
|
|
}
|
|
|
|
// OnFinalShutdown tears down the metrics listener on shutdown and restart.
|
|
func (m *Metrics) OnFinalShutdown() error {
|
|
// We allow prometheus statements in multiple Server Blocks, but only the first
|
|
// will open the listener, for the rest they are all nil; guard against that.
|
|
if !m.lnSetup {
|
|
return nil
|
|
}
|
|
|
|
m.lnSetup = false
|
|
return m.ln.Close()
|
|
}
|
|
|
|
func keys(m map[string]bool) []string {
|
|
sx := []string{}
|
|
for k := range m {
|
|
sx = append(sx, k)
|
|
}
|
|
return sx
|
|
}
|
|
|
|
// ListenAddr is assigned the address of the prometheus listener. Its use is mainly in tests where
|
|
// we listen on "localhost:0" and need to retrieve the actual address.
|
|
var ListenAddr string
|
|
|
|
var (
|
|
buildInfo = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Namespace: plugin.Namespace,
|
|
Name: "build_info",
|
|
Help: "A metric with a constant '1' value labeled by version, revision, and goversion from which CoreDNS was built.",
|
|
}, []string{"version", "revision", "goversion"})
|
|
)
|