extend metrics and rebuild prometheus exporting logic

healthcheck/healthcheck.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/containous/traefik/log"
 	"github.com/containous/traefik/safe"
+	"github.com/go-kit/kit/metrics"
 	"github.com/vulcand/oxy/roundrobin"
 )
 
@@ -19,9 +20,9 @@ var singleton *HealthCheck
 var once sync.Once
 
 // GetHealthCheck returns the health check which is guaranteed to be a singleton.
-func GetHealthCheck() *HealthCheck {
+func GetHealthCheck(metrics metricsRegistry) *HealthCheck {
 	once.Do(func() {
-		singleton = newHealthCheck()
+		singleton = newHealthCheck(metrics)
 	})
 	return singleton
 }
@@ -50,6 +51,7 @@ type BackendHealthCheck struct {
 //HealthCheck struct
 type HealthCheck struct {
 	Backends map[string]*BackendHealthCheck
+	metrics  metricsRegistry
 	cancel   context.CancelFunc
 }
 
@@ -60,12 +62,19 @@ type LoadBalancer interface {
 	Servers() []*url.URL
 }
 
-func newHealthCheck() *HealthCheck {
+func newHealthCheck(metrics metricsRegistry) *HealthCheck {
 	return &HealthCheck{
 		Backends: make(map[string]*BackendHealthCheck),
+		metrics:  metrics,
 	}
 }
 
+// metricsRegistry is a local interface in the healthcheck package, exposing only the required metrics
+// necessary for the healthcheck package. This makes it easier for the tests.
+type metricsRegistry interface {
+	BackendServerUpGauge() metrics.Gauge
+}
+
 // NewBackendHealthCheck Instantiate a new BackendHealthCheck
 func NewBackendHealthCheck(options Options, backendName string) *BackendHealthCheck {
 	return &BackendHealthCheck{
@@ -113,22 +122,30 @@ func (hc *HealthCheck) checkBackend(backend *BackendHealthCheck) {
 	enabledURLs := backend.LB.Servers()
 	var newDisabledURLs []*url.URL
 	for _, url := range backend.disabledURLs {
+		serverUpMetricValue := float64(0)
 		if err := checkHealth(url, backend); err == nil {
 			log.Warnf("Health check up: Returning to server list. Backend: %q URL: %q", backend.name, url.String())
 			backend.LB.UpsertServer(url, roundrobin.Weight(1))
+			serverUpMetricValue = 1
 		} else {
 			log.Warnf("Health check still failing. Backend: %q URL: %q Reason: %s", backend.name, url.String(), err)
 			newDisabledURLs = append(newDisabledURLs, url)
 		}
+		labelValues := []string{"backend", backend.name, "url", url.String()}
+		hc.metrics.BackendServerUpGauge().With(labelValues...).Set(serverUpMetricValue)
 	}
 	backend.disabledURLs = newDisabledURLs
 
 	for _, url := range enabledURLs {
+		serverUpMetricValue := float64(1)
 		if err := checkHealth(url, backend); err != nil {
 			log.Warnf("Health check failed: Remove from server list. Backend: %q URL: %q Reason: %s", backend.name, url.String(), err)
 			backend.LB.RemoveServer(url)
 			backend.disabledURLs = append(backend.disabledURLs, url)
+			serverUpMetricValue = 0
 		}
+		labelValues := []string{"backend", backend.name, "url", url.String()}
+		hc.metrics.BackendServerUpGauge().With(labelValues...).Set(serverUpMetricValue)
 	}
 }
 

healthcheck/healthcheck_test.go

@@ -27,6 +27,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
 		healthSequence         []bool
 		wantNumRemovedServers  int
 		wantNumUpsertedServers int
+		wantGaugeValue         float64
 	}{
 		{
 			desc:                   "healthy server staying healthy",
@@ -34,6 +35,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
 			healthSequence:         []bool{true},
 			wantNumRemovedServers:  0,
 			wantNumUpsertedServers: 0,
+			wantGaugeValue:         1,
 		},
 		{
 			desc:                   "healthy server becoming sick",
@@ -41,6 +43,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
 			healthSequence:         []bool{false},
 			wantNumRemovedServers:  1,
 			wantNumUpsertedServers: 0,
+			wantGaugeValue:         0,
 		},
 		{
 			desc:                   "sick server becoming healthy",
@@ -48,6 +51,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
 			healthSequence:         []bool{true},
 			wantNumRemovedServers:  0,
 			wantNumUpsertedServers: 1,
+			wantGaugeValue:         1,
 		},
 		{
 			desc:                   "sick server staying sick",
@@ -55,6 +59,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
 			healthSequence:         []bool{false},
 			wantNumRemovedServers:  0,
 			wantNumUpsertedServers: 0,
+			wantGaugeValue:         0,
 		},
 		{
 			desc:                   "healthy server toggling to sick and back to healthy",
@@ -62,6 +67,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
 			healthSequence:         []bool{false, true},
 			wantNumRemovedServers:  1,
 			wantNumUpsertedServers: 1,
+			wantGaugeValue:         1,
 		},
 	}
 
@@ -89,8 +95,10 @@ func TestSetBackendsConfiguration(t *testing.T) {
 				backend.disabledURLs = append(backend.disabledURLs, serverURL)
 			}
 
+			collectingMetrics := testhelpers.NewCollectingHealthCheckMetrics()
 			check := HealthCheck{
 				Backends: make(map[string]*BackendHealthCheck),
+				metrics:  collectingMetrics,
 			}
 			wg := sync.WaitGroup{}
 			wg.Add(1)
@@ -118,6 +126,10 @@ func TestSetBackendsConfiguration(t *testing.T) {
 			if lb.numUpsertedServers != test.wantNumUpsertedServers {
 				t.Errorf("got %d upserted servers, wanted %d", lb.numUpsertedServers, test.wantNumUpsertedServers)
 			}
+
+			if collectingMetrics.Gauge.GaugeValue != test.wantGaugeValue {
+				t.Errorf("got %v ServerUp Gauge, want %v", collectingMetrics.Gauge.GaugeValue, test.wantGaugeValue)
+			}
 		})
 	}
 }