plugin/health: implement dyn health checks (#1214)

Implement health.Healther in erratic and kubernetes plugin. The
kubernetes' healtcheck is only performed on startup - i.e. turn
healthy after the initial loading.

Erratic follow the drop count: every query%drop turns the healthcheck
unhealthy.

Fixes: #985

plugin/erratic/README.md

@@ -25,6 +25,10 @@ erratic {
 * `delay`: delay 1 per **AMOUNT** of queries for **DURATION**, the default for **AMOUNT** is 2 and
   the default for **DURATION** is 100ms.
 
+## Health
+
+This plugin implements dynamic health checking. For every dropped query it turns unhealthy.
+
 ## Examples
 
 ~~~ corefile

plugin/erratic/health.go

@@ -0,0 +1,14 @@
+package erratic
+
+import (
+	"sync/atomic"
+)
+
+// Health implements the health.Healther interface.
+func (e *Erratic) Health() bool {
+	q := atomic.LoadUint64(&e.q)
+	if e.drop > 0 && q%e.drop == 0 {
+		return false
+	}
+	return true
+}

plugin/health/README.md

@@ -16,6 +16,13 @@ a 503. *health* periodically (1s) polls plugin that exports health information.
 plugin signals that it is unhealthy, the server will go unhealthy too. Each plugin that
 supports health checks has a section "Health" in their README.
 
+## Plugins
+
+The following plugins report health to the health plugin:
+
+* erratic
+* kubernetes
+
 ## Examples
 
 Run another health endpoint on http://localhost:8091.

plugin/health/health_test.go

@@ -1,7 +1,14 @@
 package health
 
-// TODO(miek): enable again if plugin gets health check.
-/*
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/coredns/coredns/plugin/erratic"
+)
+
 func TestHealth(t *testing.T) {
 	h := health{Addr: ":0"}
 	h.h = append(h.h, &erratic.Erratic{})
@@ -14,7 +21,7 @@ func TestHealth(t *testing.T) {
 	// Reconstruct the http address based on the port allocated by operating system.
 	address := fmt.Sprintf("http://%s%s", h.ln.Addr().String(), path)
 
-	// Norhing set should be unhealthy
+	// Nothing set should return unhealthy
 	response, err := http.Get(address)
 	if err != nil {
 		t.Fatalf("Unable to query %s: %v", address, err)
@@ -24,8 +31,7 @@ func TestHealth(t *testing.T) {
 	}
 	response.Body.Close()
 
-	// Make healthy
-	h.Poll()
+	h.poll()
 
 	response, err = http.Get(address)
 	if err != nil {
@@ -44,4 +50,3 @@ func TestHealth(t *testing.T) {
 		t.Errorf("Invalid response body: expecting 'OK', got '%s'", string(content))
 	}
 }
-*/

plugin/health/healther.go

@@ -38,5 +38,7 @@ func (h *health) poll() {
 }
 
 // Plugins that implements the Healther interface.
-// TODO(miek): none yet.
-var healthers = map[string]bool{}
+var healthers = map[string]bool{
+	"erratic":    true,
+	"kubernetes": true,
+}

plugin/kubernetes/README.md

@@ -85,6 +85,11 @@ kubernetes [ZONES...] {
   what the response will be. However, if you specify this option, the query will instead be passed
   on down the plugin chain, which can include another plugin to handle the query.
 
+## Health
+
+This plugin implements dynamic health checking. Currently this is limited to reporting healthy when
+the API has synced.
+
 ## Examples
 
 Handle all queries in the `cluster.local` zone. Connect to Kubernetes in-cluster. Also handle all

plugin/kubernetes/health.go

@@ -0,0 +1,4 @@
+package kubernetes
+
+// Health implements the health.Healther interface.
+func (k *Kubernetes) Health() bool { return k.APIConn.HasSynced() }