plugin/cache: Fix prefetching issues (#1363)

* Improve plugin/cache metrics

* Add coredns_cache_prefetch_total metric to track number of prefetches.
* Remove unnecessary Cache.get() call which would incorrectly increment
  cache counters.
* Initialize all counters and gauges at zero.

* Allow prefetching of a single request per ttl

The original implementation didn't allow prefetching queries which are
only requested once during the duration of a TTL. The minimum amount of
queries which had to be seen was therefore capped at 2.

This change also implements a real prefetch test. The existing test was
a noop and always passed regardless of any prefetch implementation.

* Fix prefetching for items with a short TTL

The default prefetch threshold (percentage) is 10% of the lifetime of a
cache item. With the previous implementation, this disabled prefetching
for all items with a TTL < 10s (the resulting percentage would be 0, at
which point a cached item is already discarded).

This change uses a time based threshold calculation and ensures that
a prefetch is triggered at a TTL of 1 at the latest.

* Fix wrong duration reporting of cached responses

The logging and metrics plugins (among others) included the duration of
a cache prefetch in the request latency of client request. This change
fixes this wrong reporting and executes the prefetch request in a
goroutine in the background.

plugin/cache/cache.go

@@ -32,6 +32,27 @@ type Cache struct {
 	prefetch   int
 	duration   time.Duration
 	percentage int
+
+	// Testing.
+	now func() time.Time
+}
+
+// New returns an initialized Cache with default settings. It's up to the
+// caller to set the Next handler.
+func New() *Cache {
+	return &Cache{
+		Zones:      []string{"."},
+		pcap:       defaultCap,
+		pcache:     cache.New(defaultCap),
+		pttl:       maxTTL,
+		ncap:       defaultCap,
+		ncache:     cache.New(defaultCap),
+		nttl:       maxNTTL,
+		prefetch:   0,
+		duration:   1 * time.Minute,
+		percentage: 10,
+		now:        time.Now,
+	}
 }
 
 // Return key under which we store the item, -1 will be returned if we don't store the
@@ -88,7 +109,7 @@ type ResponseWriter struct {
 // WriteMsg implements the dns.ResponseWriter interface.
 func (w *ResponseWriter) WriteMsg(res *dns.Msg) error {
 	do := false
-	mt, opt := response.Typify(res, time.Now().UTC())
+	mt, opt := response.Typify(res, w.now().UTC())
 	if opt != nil {
 		do = opt.Do()
 	}
@@ -140,11 +161,11 @@ func (w *ResponseWriter) set(m *dns.Msg, key int, mt response.Type, duration tim
 
 	switch mt {
 	case response.NoError, response.Delegation:
-		i := newItem(m, duration)
+		i := newItem(m, w.now(), duration)
 		w.pcache.Add(uint32(key), i)
 
 	case response.NameError, response.NoData:
-		i := newItem(m, duration)
+		i := newItem(m, w.now(), duration)
 		w.ncache.Add(uint32(key), i)
 
 	case response.OtherError: