Start health checks early.

Do not wait a full tick cycle to execute the first health check.

Additional changes:

- Make request timeout configurable (for testing purposes).
- Support synchronizing on health check goroutine termination through an
  internal wait group (for testing purposes).
- Stop leaking by closing the HTTP response body.
- Extend health check logging and use WARNING level for (continuously)
  failing health checks.

healthcheck/healthcheck.go

@@ -25,10 +25,11 @@ func GetHealthCheck() *HealthCheck {
 
 // BackendHealthCheck HealthCheck configuration for a backend
 type BackendHealthCheck struct {
-	URL          string
-	Interval     time.Duration
-	DisabledURLs []*url.URL
-	lb           loadBalancer
+	URL            string
+	Interval       time.Duration
+	DisabledURLs   []*url.URL
+	requestTimeout time.Duration
+	lb             loadBalancer
 }
 
 var launch = false
@@ -46,12 +47,19 @@ type loadBalancer interface {
 }
 
 func newHealthCheck() *HealthCheck {
-	return &HealthCheck{make(map[string]*BackendHealthCheck), nil}
+	return &HealthCheck{
+		Backends: make(map[string]*BackendHealthCheck),
+	}
 }
 
 // NewBackendHealthCheck Instantiate a new BackendHealthCheck
 func NewBackendHealthCheck(URL string, interval time.Duration, lb loadBalancer) *BackendHealthCheck {
-	return &BackendHealthCheck{URL, interval, nil, lb}
+	return &BackendHealthCheck{
+		URL:            URL,
+		Interval:       interval,
+		requestTimeout: 5 * time.Second,
+		lb:             lb,
+	}
 }
 
 //SetBackendsConfiguration set backends configuration
@@ -62,56 +70,63 @@ func (hc *HealthCheck) SetBackendsConfiguration(parentCtx context.Context, backe
 	}
 	ctx, cancel := context.WithCancel(parentCtx)
 	hc.cancel = cancel
-	hc.execute(ctx)
-}
 
-func (hc *HealthCheck) execute(ctx context.Context) {
 	for backendID, backend := range hc.Backends {
-		currentBackend := backend
 		currentBackendID := backendID
+		currentBackend := backend
 		safe.Go(func() {
-			for {
-				ticker := time.NewTicker(currentBackend.Interval)
-				select {
-				case <-ctx.Done():
-					log.Debugf("Stopping all current Healthcheck goroutines")
-					return
-				case <-ticker.C:
-					log.Debugf("Refreshing Healthcheck for currentBackend %s ", currentBackendID)
-					enabledURLs := currentBackend.lb.Servers()
-					var newDisabledURLs []*url.URL
-					for _, url := range currentBackend.DisabledURLs {
-						if checkHealth(url, currentBackend.URL) {
-							log.Debugf("HealthCheck is up [%s]: Upsert in server list", url.String())
-							currentBackend.lb.UpsertServer(url, roundrobin.Weight(1))
-						} else {
-							newDisabledURLs = append(newDisabledURLs, url)
-						}
-					}
-					currentBackend.DisabledURLs = newDisabledURLs
-
-					for _, url := range enabledURLs {
-						if !checkHealth(url, currentBackend.URL) {
-							log.Debugf("HealthCheck has failed [%s]: Remove from server list", url.String())
-							currentBackend.lb.RemoveServer(url)
-							currentBackend.DisabledURLs = append(currentBackend.DisabledURLs, url)
-						}
-					}
-
-				}
-			}
+			hc.execute(ctx, currentBackendID, currentBackend)
 		})
 	}
 }
 
-func checkHealth(serverURL *url.URL, checkURL string) bool {
-	timeout := time.Duration(5 * time.Second)
-	client := http.Client{
-		Timeout: timeout,
+func (hc *HealthCheck) execute(ctx context.Context, backendID string, backend *BackendHealthCheck) {
+	log.Debugf("Initial healthcheck for currentBackend %s ", backendID)
+	checkBackend(backend)
+	ticker := time.NewTicker(backend.Interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			log.Debugf("Stopping all current Healthcheck goroutines")
+			return
+		case <-ticker.C:
+			log.Debugf("Refreshing healthcheck for currentBackend %s ", backendID)
+			checkBackend(backend)
+		}
 	}
-	resp, err := client.Get(serverURL.String() + checkURL)
-	if err != nil || resp.StatusCode != 200 {
-		return false
-	}
-	return true
+}
+
+func checkBackend(currentBackend *BackendHealthCheck) {
+	enabledURLs := currentBackend.lb.Servers()
+	var newDisabledURLs []*url.URL
+	for _, url := range currentBackend.DisabledURLs {
+		if checkHealth(url, currentBackend) {
+			log.Debugf("HealthCheck is up [%s]: Upsert in server list", url.String())
+			currentBackend.lb.UpsertServer(url, roundrobin.Weight(1))
+		} else {
+			log.Warnf("HealthCheck is still failing [%s]", url.String())
+			newDisabledURLs = append(newDisabledURLs, url)
+		}
+	}
+	currentBackend.DisabledURLs = newDisabledURLs
+
+	for _, url := range enabledURLs {
+		if !checkHealth(url, currentBackend) {
+			log.Warnf("HealthCheck has failed [%s]: Remove from server list", url.String())
+			currentBackend.lb.RemoveServer(url)
+			currentBackend.DisabledURLs = append(currentBackend.DisabledURLs, url)
+		}
+	}
+}
+
+func checkHealth(serverURL *url.URL, backend *BackendHealthCheck) bool {
+	client := http.Client{
+		Timeout: backend.requestTimeout,
+	}
+	resp, err := client.Get(serverURL.String() + backend.URL)
+	if err == nil {
+		defer resp.Body.Close()
+	}
+	return err == nil && resp.StatusCode == 200
 }