diff --git a/core/restart.go b/core/restart.go index 1c053c13c..aa77e152d 100644 --- a/core/restart.go +++ b/core/restart.go @@ -117,6 +117,9 @@ func Restart(newCorefile Input) error { } wpipe.Close() + // Run all shutdown functions for the middleware, if child start fails, restart them all... + executeShutdownCallbacks("SIGUSR1") + // Determine whether child startup succeeded answer, readErr := ioutil.ReadAll(sigrpipe) if answer == nil || len(answer) == 0 { @@ -125,6 +128,9 @@ func Restart(newCorefile Input) error { if readErr != nil { log.Printf("[ERROR] Restart: additionally, error communicating with child process: %v", readErr) } + // re-call all startup functions. + // TODO(miek): this needs to be tested, somehow. + executeStartupCallbacks("SIGUSR1") return errIncompleteRestart } diff --git a/core/setup/health.go b/core/setup/health.go index 608147b13..542cb3260 100644 --- a/core/setup/health.go +++ b/core/setup/health.go @@ -11,8 +11,9 @@ func Health(c *Controller) (middleware.Middleware, error) { return nil, err } - h := health.Health{Addr: addr} - c.Startup = append(c.Startup, h.ListenAndServe) + h := &health.Health{Addr: addr} + c.Startup = append(c.Startup, h.Start) + c.Shutdown = append(c.Shutdown, h.Shutdown) return nil, nil } diff --git a/core/setup/metrics.go b/core/setup/metrics.go index e54f5b011..e88d93c86 100644 --- a/core/setup/metrics.go +++ b/core/setup/metrics.go @@ -12,18 +12,19 @@ const addr = "localhost:9153" var metricsOnce sync.Once func Prometheus(c *Controller) (middleware.Middleware, error) { - met, err := parsePrometheus(c) + m, err := parsePrometheus(c) if err != nil { return nil, err } metricsOnce.Do(func() { - c.Startup = append(c.Startup, met.Start) + c.Startup = append(c.Startup, m.Start) + c.Shutdown = append(c.Shutdown, m.Shutdown) }) return func(next middleware.Handler) middleware.Handler { - met.Next = next - return met + m.Next = next + return m }, nil } diff --git a/core/setup/pprof.go b/core/setup/pprof.go index e202bfc00..cf6b49636 100644 --- a/core/setup/pprof.go +++ b/core/setup/pprof.go @@ -27,6 +27,7 @@ func PProf(c *Controller) (middleware.Middleware, error) { handler := &pprof.Handler{} pprofOnce.Do(func() { c.Startup = append(c.Startup, handler.Start) + c.Shutdown = append(c.Shutdown, handler.Shutdown) }) return func(next middleware.Handler) middleware.Handler { diff --git a/core/sigtrap.go b/core/sigtrap.go index 3b74efb02..f40dd971a 100644 --- a/core/sigtrap.go +++ b/core/sigtrap.go @@ -68,4 +68,26 @@ func executeShutdownCallbacks(signame string) (exitCode int) { return } -var shutdownCallbacksOnce sync.Once +// executeStartupCallbacks executes the startup callbacks as initiated +// by signame. This is used when on restart when the child failed to start and +// all middleware executed their shutdown functions +func executeStartupCallbacks(signame string) (exitCode int) { + startupCallbacksOnce.Do(func() { + serversMu.Lock() + errs := server.StartupCallbacks(servers) + serversMu.Unlock() + + if len(errs) > 0 { + for _, err := range errs { + log.Printf("[ERROR] %s shutdown: %v", signame, err) + } + exitCode = 1 + } + }) + return +} + +var ( + shutdownCallbacksOnce sync.Once + startupCallbacksOnce sync.Once +) diff --git a/middleware/health/health.go b/middleware/health/health.go index 8c1665e57..035c9ca7a 100644 --- a/middleware/health/health.go +++ b/middleware/health/health.go @@ -3,6 +3,7 @@ package health import ( "io" "log" + "net" "net/http" "sync" ) @@ -11,28 +12,45 @@ var once sync.Once type Health struct { Addr string + ln net.Listener + mux *http.ServeMux } func health(w http.ResponseWriter, r *http.Request) { io.WriteString(w, ok) } -func (h Health) ListenAndServe() error { +func (h *Health) Start() error { if h.Addr == "" { h.Addr = defAddr } + once.Do(func() { - http.HandleFunc("/health", health) + if ln, err := net.Listen("tcp", h.Addr); err != nil { + log.Printf("[ERROR] Failed to start health handler: %s", err) + return + } else { + h.ln = ln + } + h.mux = http.NewServeMux() + + h.mux.HandleFunc(path, health) go func() { - if err := http.ListenAndServe(h.Addr, nil); err != nil { - log.Printf("[ERROR] Failed to start health handler: %s", err) - } + http.Serve(h.ln, h.mux) }() }) return nil } +func (h *Health) Shutdown() error { + if h.ln != nil { + return h.ln.Close() + } + return nil +} + const ( ok = "OK" defAddr = ":8080" + path = "/health" ) diff --git a/middleware/metrics/metrics.go b/middleware/metrics/metrics.go index b525042f7..42f1439eb 100644 --- a/middleware/metrics/metrics.go +++ b/middleware/metrics/metrics.go @@ -2,6 +2,7 @@ package metrics import ( "log" + "net" "net/http" "sync" @@ -17,12 +18,12 @@ var ( responseRcode *prometheus.CounterVec ) -const path = "/metrics" - // Metrics holds the prometheus configuration. The metrics' path is fixed to be /metrics type Metrics struct { Next middleware.Handler - Addr string // where to we listen + Addr string + ln net.Listener + mux *http.ServeMux Once sync.Once ZoneNames []string } @@ -31,21 +32,35 @@ func (m *Metrics) Start() error { m.Once.Do(func() { define() + if ln, err := net.Listen("tcp", m.Addr); err != nil { + log.Printf("[ERROR] Failed to start metrics handler: %s", err) + return + } else { + m.ln = ln + } + m.mux = http.NewServeMux() + prometheus.MustRegister(requestCount) prometheus.MustRegister(requestDuration) prometheus.MustRegister(responseSize) prometheus.MustRegister(responseRcode) - http.Handle(path, prometheus.Handler()) + m.mux.Handle(path, prometheus.Handler()) + go func() { - if err := http.ListenAndServe(m.Addr, nil); err != nil { - log.Printf("[ERROR] Failed to start prometheus handler: %s", err) - } + http.Serve(m.ln, m.mux) }() }) return nil } +func (m *Metrics) Shutdown() error { + if m.ln != nil { + return m.ln.Close() + } + return nil +} + func define() { requestCount = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: middleware.Namespace, @@ -80,7 +95,7 @@ func define() { const ( // Dropped indicates we dropped the query before any handling. It has no closing dot, so it can not be a valid zone. - Dropped = "dropped" - + Dropped = "dropped" subsystem = "dns" + path = "/metrics" ) diff --git a/middleware/pprof/README.md b/middleware/pprof/README.md index 7798f5a63..c62605520 100644 --- a/middleware/pprof/README.md +++ b/middleware/pprof/README.md @@ -1,7 +1,7 @@ # pprof pprof publishes runtime profiling data at endpoints under /debug/pprof. You can visit /debug/pprof -on your site for an index of the available endpoints. By default it will listen on localhost:8053. +on your site for an index of the available endpoints. By default it will listen on localhost:6053. > This is a debugging tool. Certain requests (such as collecting execution traces) can be slow. If > you use pprof on a live site, consider restricting access or enabling it only temporarily. diff --git a/middleware/pprof/pprof.go b/middleware/pprof/pprof.go index 677690ff8..2c812f246 100644 --- a/middleware/pprof/pprof.go +++ b/middleware/pprof/pprof.go @@ -2,8 +2,9 @@ package pprof import ( "log" + "net" "net/http" - _ "net/http/pprof" + pp "net/http/pprof" "github.com/miekg/coredns/middleware" @@ -11,10 +12,10 @@ import ( "golang.org/x/net/context" ) -const addr = "localhost:8053" - type Handler struct { Next middleware.Handler + ln net.Listener + mux *http.ServeMux } // ServeDNS passes all other requests up the chain. @@ -23,10 +24,34 @@ func (h *Handler) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg } func (h *Handler) Start() error { + if ln, err := net.Listen("tcp", addr); err != nil { + log.Printf("[ERROR] Failed to start pprof handler: %s", err) + return err + } else { + h.ln = ln + } + + h.mux = http.NewServeMux() + h.mux.HandleFunc(path+"/", pp.Index) + h.mux.HandleFunc(path+"/cmdline", pp.Cmdline) + h.mux.HandleFunc(path+"/profile", pp.Profile) + h.mux.HandleFunc(path+"/symbol", pp.Symbol) + h.mux.HandleFunc(path+"/trace", pp.Trace) + go func() { - if err := http.ListenAndServe(addr, nil); err != nil { - log.Printf("[ERROR] Failed to start pprof handler: %s", err) - } + http.Serve(h.ln, h.mux) }() return nil } + +func (h *Handler) Shutdown() error { + if h.ln != nil { + return h.ln.Close() + } + return nil +} + +const ( + addr = "localhost:6053" + path = "/debug/pprof" +) diff --git a/server/server.go b/server/server.go index 068a54a31..af5dd35e1 100644 --- a/server/server.go +++ b/server/server.go @@ -445,32 +445,6 @@ func (s *Server) RunFirstStartupFuncs() error { return nil } -// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted -// connections. It's used by ListenAndServe and ListenAndServeTLS so -// dead TCP connections (e.g. closing laptop mid-download) eventually -// go away. -// -// Borrowed from the Go standard library. -type tcpKeepAliveListener struct { - *net.TCPListener -} - -// Accept accepts the connection with a keep-alive enabled. -func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { - tc, err := ln.AcceptTCP() - if err != nil { - return - } - tc.SetKeepAlive(true) - tc.SetKeepAlivePeriod(3 * time.Minute) - return tc, nil -} - -// File implements ListenerFile; returns the underlying file of the listener. -func (ln tcpKeepAliveListener) File() (*os.File, error) { - return ln.TCPListener.File() -} - // ShutdownCallbacks executes all the shutdown callbacks // for all the virtualhosts in servers, and returns all the // errors generated during their execution. In other words, @@ -493,6 +467,21 @@ func ShutdownCallbacks(servers []*Server) []error { return errs } +func StartupCallbacks(servers []*Server) []error { + var errs []error + for _, s := range servers { + for _, zone := range s.zones { + for _, startupFunc := range zone.config.Startup { + err := startupFunc() + if err != nil { + errs = append(errs, err) + } + } + } + } + return errs +} + func RcodeNoClientWrite(rcode int) bool { switch rcode { case dns.RcodeServerFailure: