middleware/proxy: async health checks (#749)

* Switches out Unhealthy bool for OkUntil timestamp

* Make sure servers are healthy forever if there are no health checks

* Moves health check off into a go routine to avoid blocking conditions

* Improved logging info

* Fixes initial date

* Fixes health checking; alters tests to adapt to async health checking

* Moves future variable into static upstream and populates it in more places

* Restores silencing of stdout during testing

* Restores silencing of stdout during testing

* keeps check url string once built

* Removes debug message

* uses zero value to signal no checking; reduces in-mutex code to a fetch

middleware/proxy/upstream_test.go

@@ -23,9 +23,14 @@ func TestHealthCheck(t *testing.T) {
 		Policy:      &Random{},
 		Spray:       nil,
 		FailTimeout: 10 * time.Second,
+		Future:      60 * time.Second,
 		MaxFails:    1,
 	}
+
 	upstream.healthCheck()
+	// sleep a bit, it's async now
+	time.Sleep(time.Duration(2 * time.Second))
+
 	if upstream.Hosts[0].Down() {
 		t.Error("Expected first host in testpool to not fail healthcheck.")
 	}
@@ -40,15 +45,16 @@ func TestSelect(t *testing.T) {
 		Hosts:       testPool()[:3],
 		Policy:      &Random{},
 		FailTimeout: 10 * time.Second,
+		Future:      60 * time.Second,
 		MaxFails:    1,
 	}
-	upstream.Hosts[0].Unhealthy = true
-	upstream.Hosts[1].Unhealthy = true
-	upstream.Hosts[2].Unhealthy = true
+	upstream.Hosts[0].OkUntil = time.Unix(0, 0)
+	upstream.Hosts[1].OkUntil = time.Unix(0, 0)
+	upstream.Hosts[2].OkUntil = time.Unix(0, 0)
 	if h := upstream.Select(); h != nil {
 		t.Error("Expected select to return nil as all host are down")
 	}
-	upstream.Hosts[2].Unhealthy = false
+	upstream.Hosts[2].OkUntil = time.Time{}
 	if h := upstream.Select(); h == nil {
 		t.Error("Expected select to not return nil")
 	}