network: don't attempt to connect to the same node twice

We can have multiple copies of the same address in the pool and we should only
proceed to connect once per attempt.

pkg/network/discovery.go

@@ -45,6 +45,7 @@ type DefaultDiscovery struct {
 	connectedAddrs   map[string]bool
 	goodAddrs        map[string]capability.Capabilities
 	unconnectedAddrs map[string]int
+	attempted        map[string]bool
 	isDead           bool
 	requestCh        chan int
 	pool             chan string
@@ -60,6 +61,7 @@ func NewDefaultDiscovery(addrs []string, dt time.Duration, ts Transporter) *Defa
 		connectedAddrs:   make(map[string]bool),
 		goodAddrs:        make(map[string]capability.Capabilities),
 		unconnectedAddrs: make(map[string]int),
+		attempted:        make(map[string]bool),
 		requestCh:        make(chan int),
 		pool:             make(chan string, maxPoolSize),
 	}
@@ -189,7 +191,11 @@ func (d *DefaultDiscovery) RegisterConnectedAddr(addr string) {
 }
 
 func (d *DefaultDiscovery) tryAddress(addr string) {
-	if err := d.transport.Dial(addr, d.dialTimeout); err != nil {
+	err := d.transport.Dial(addr, d.dialTimeout)
+	d.lock.Lock()
+	delete(d.attempted, addr)
+	d.lock.Unlock()
+	if err != nil {
 		d.RegisterBadAddr(addr)
 		d.RequestRemote(1)
 	}
@@ -225,14 +231,14 @@ func (d *DefaultDiscovery) run() {
 					requested = r
 				}
 			case addr := <-d.pool:
-				d.lock.RLock()
-				addrIsConnected := d.connectedAddrs[addr]
-				d.lock.RUnlock()
 				updatePoolCountMetric(d.PoolCount())
-				if !addrIsConnected {
+				d.lock.Lock()
+				if !d.connectedAddrs[addr] && !d.attempted[addr] {
+					d.attempted[addr] = true
 					go d.tryAddress(addr)
 					requested--
 				}
+				d.lock.Unlock()
 			default: // Empty pool
 				var added int
 				d.lock.Lock()