plugin/forward: move Dial goroutine out (#1738)

Rework the TestProxyClose - close the proxy in the *same* goroutine
as where we started it. Close channels as long as we don't get dataraces
(this may need another fix).

Move the Dial goroutine out of the connManager - this simplifies things
*and* makes another goroutine go away and removes the need for connErr
channels - can now just be dns.Conn.

Also:

Revert "plugin/forward: gracefull stop (#1701)"
This reverts commit 135377bf77.

Revert "rework TestProxyClose (#1735)"
This reverts commit 9e8893a0b5.

plugin/forward/persistent.go

@@ -14,13 +14,6 @@ type persistConn struct {
 	used time.Time
 }
 
-// connErr is used to communicate the connection manager.
-type connErr struct {
-	c      *dns.Conn
-	err    error
-	cached bool
-}
-
 // transport hold the persistent cache.
 type transport struct {
 	conns     map[string][]*persistConn //  Buckets for udp, tcp and tcp-tls.
@@ -29,8 +22,8 @@ type transport struct {
 	tlsConfig *tls.Config
 
 	dial  chan string
-	yield chan connErr
-	ret   chan connErr
+	yield chan *dns.Conn
+	ret   chan *dns.Conn
 	stop  chan bool
 }
 
@@ -40,18 +33,11 @@ func newTransport(addr string, tlsConfig *tls.Config) *transport {
 		expire: defaultExpire,
 		addr:   addr,
 		dial:   make(chan string),
-		yield:  make(chan connErr),
-		ret:    make(chan connErr),
+		yield:  make(chan *dns.Conn),
+		ret:    make(chan *dns.Conn),
 		stop:   make(chan bool),
 	}
-	go func() {
-		t.connManager()
-		// if connManager returns it has been stopped.
-		close(t.stop)
-		close(t.yield)
-		close(t.dial)
-		// close(t.ret) // we can still be dialing and wanting to send back the socket on t.ret
-	}()
+	go func() { t.connManager() }()
 	return t
 }
 
@@ -80,7 +66,7 @@ Wait:
 				if time.Since(pc.used) < t.expire {
 					// Found one, remove from pool and return this conn.
 					t.conns[proto] = t.conns[proto][i+1:]
-					t.ret <- connErr{pc.c, nil, true}
+					t.ret <- pc.c
 					continue Wait
 				}
 				// This conn has expired. Close it.
@@ -91,35 +77,27 @@ Wait:
 			t.conns[proto] = t.conns[proto][i:]
 			SocketGauge.WithLabelValues(t.addr).Set(float64(t.len()))
 
-			go func() {
-				if proto != "tcp-tls" {
-					c, err := dns.DialTimeout(proto, t.addr, dialTimeout)
-					t.ret <- connErr{c, err, false}
-					return
-				}
-
-				c, err := dns.DialTimeoutWithTLS("tcp", t.addr, t.tlsConfig, dialTimeout)
-				t.ret <- connErr{c, err, false}
-			}()
+			t.ret <- nil
 
 		case conn := <-t.yield:
 
 			SocketGauge.WithLabelValues(t.addr).Set(float64(t.len() + 1))
 
 			// no proto here, infer from config and conn
-			if _, ok := conn.c.Conn.(*net.UDPConn); ok {
-				t.conns["udp"] = append(t.conns["udp"], &persistConn{conn.c, time.Now()})
+			if _, ok := conn.Conn.(*net.UDPConn); ok {
+				t.conns["udp"] = append(t.conns["udp"], &persistConn{conn, time.Now()})
 				continue Wait
 			}
 
 			if t.tlsConfig == nil {
-				t.conns["tcp"] = append(t.conns["tcp"], &persistConn{conn.c, time.Now()})
+				t.conns["tcp"] = append(t.conns["tcp"], &persistConn{conn, time.Now()})
 				continue Wait
 			}
 
-			t.conns["tcp-tls"] = append(t.conns["tcp-tls"], &persistConn{conn.c, time.Now()})
+			t.conns["tcp-tls"] = append(t.conns["tcp-tls"], &persistConn{conn, time.Now()})
 
 		case <-t.stop:
+			close(t.ret)
 			return
 		}
 	}
@@ -134,16 +112,24 @@ func (t *transport) Dial(proto string) (*dns.Conn, bool, error) {
 
 	t.dial <- proto
 	c := <-t.ret
-	return c.c, c.cached, c.err
+
+	if c != nil {
+		return c, true, nil
+	}
+
+	if proto == "tcp-tls" {
+		conn, err := dns.DialTimeoutWithTLS("tcp", t.addr, t.tlsConfig, dialTimeout)
+		return conn, false, err
+	}
+	conn, err := dns.DialTimeout(proto, t.addr, dialTimeout)
+	return conn, false, err
 }
 
 // Yield return the connection to transport for reuse.
-func (t *transport) Yield(c *dns.Conn) {
-	t.yield <- connErr{c, nil, false}
-}
+func (t *transport) Yield(c *dns.Conn) { t.yield <- c }
 
 // Stop stops the transport's connection manager.
-func (t *transport) Stop() { t.stop <- true }
+func (t *transport) Stop() { close(t.stop) }
 
 // SetExpire sets the connection expire time in transport.
 func (t *transport) SetExpire(expire time.Duration) { t.expire = expire }