homelab/traefik
1
0
Fork 0
Code Activity

Support gRPC healthcheck

Browse source
This commit is contained in:
jjacque 2022-09-20 16:54:08 +02:00 • committed by GitHub
parent df99a9fb57
commit 033fccccc7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 374 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

170
pkg/healthcheck/healthcheck_test.go
View file

@ -14,6 +14,7 @@ import (
"github.com/traefik/traefik/v2/pkg/config/runtime"
"github.com/traefik/traefik/v2/pkg/testhelpers"
"github.com/vulcand/oxy/roundrobin"
healthpb "google.golang.org/grpc/health/grpc_health_v1"
)
const (
@ -21,16 +22,12 @@ const (
healthCheckTimeout = 100 * time.Millisecond
)
type testHandler struct {
done func()
healthSequence []int
}
func TestSetBackendsConfiguration(t *testing.T) {
testCases := []struct {
desc string
startHealthy bool
healthSequence []int
mode string
server StartTestServer
expectedNumRemovedServers int
expectedNumUpsertedServers int
expectedGaugeValue float64
@ -38,7 +35,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
{
desc: "healthy server staying healthy",
startHealthy: true,
healthSequence: []int{http.StatusOK},
server: newHTTPServer(http.StatusOK),
expectedNumRemovedServers: 0,
expectedNumUpsertedServers: 0,
expectedGaugeValue: 1,
@ -46,7 +43,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
{
desc: "healthy server staying healthy (StatusNoContent)",
startHealthy: true,
healthSequence: []int{http.StatusNoContent},
server: newHTTPServer(http.StatusNoContent),
expectedNumRemovedServers: 0,
expectedNumUpsertedServers: 0,
expectedGaugeValue: 1,
@ -54,7 +51,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
{
desc: "healthy server staying healthy (StatusPermanentRedirect)",
startHealthy: true,
healthSequence: []int{http.StatusPermanentRedirect},
server: newHTTPServer(http.StatusPermanentRedirect),
expectedNumRemovedServers: 0,
expectedNumUpsertedServers: 0,
expectedGaugeValue: 1,
@ -62,7 +59,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
{
desc: "healthy server becoming sick",
startHealthy: true,
healthSequence: []int{http.StatusServiceUnavailable},
server: newHTTPServer(http.StatusServiceUnavailable),
expectedNumRemovedServers: 1,
expectedNumUpsertedServers: 0,
expectedGaugeValue: 0,
@ -70,7 +67,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
{
desc: "sick server becoming healthy",
startHealthy: false,
healthSequence: []int{http.StatusOK},
server: newHTTPServer(http.StatusOK),
expectedNumRemovedServers: 0,
expectedNumUpsertedServers: 1,
expectedGaugeValue: 1,
@ -78,7 +75,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
{
desc: "sick server staying sick",
startHealthy: false,
healthSequence: []int{http.StatusServiceUnavailable},
server: newHTTPServer(http.StatusServiceUnavailable),
expectedNumRemovedServers: 0,
expectedNumUpsertedServers: 0,
expectedGaugeValue: 0,
@ -86,7 +83,52 @@ func TestSetBackendsConfiguration(t *testing.T) {
{
desc: "healthy server toggling to sick and back to healthy",
startHealthy: true,
healthSequence: []int{http.StatusServiceUnavailable, http.StatusOK},
server: newHTTPServer(http.StatusServiceUnavailable, http.StatusOK),
expectedNumRemovedServers: 1,
expectedNumUpsertedServers: 1,
expectedGaugeValue: 1,
},
{
desc: "healthy grpc server staying healthy",
mode: "grpc",
startHealthy: true,
server: newGRPCServer(healthpb.HealthCheckResponse_SERVING),
expectedNumRemovedServers: 0,
expectedNumUpsertedServers: 0,
expectedGaugeValue: 1,
},
{
desc: "healthy grpc server becoming sick",
mode: "grpc",
startHealthy: true,
server: newGRPCServer(healthpb.HealthCheckResponse_NOT_SERVING),
expectedNumRemovedServers: 1,
expectedNumUpsertedServers: 0,
expectedGaugeValue: 0,
},
{
desc: "sick grpc server becoming healthy",
mode: "grpc",
startHealthy: false,
server: newGRPCServer(healthpb.HealthCheckResponse_SERVING),
expectedNumRemovedServers: 0,
expectedNumUpsertedServers: 1,
expectedGaugeValue: 1,
},
{
desc: "sick grpc server staying sick",
mode: "grpc",
startHealthy: false,
server: newGRPCServer(healthpb.HealthCheckResponse_NOT_SERVING),
expectedNumRemovedServers: 0,
expectedNumUpsertedServers: 0,
expectedGaugeValue: 0,
},
{
desc: "healthy grpc server toggling to sick and back to healthy",
mode: "grpc",
startHealthy: true,
server: newGRPCServer(healthpb.HealthCheckResponse_NOT_SERVING, healthpb.HealthCheckResponse_SERVING),
expectedNumRemovedServers: 1,
expectedNumUpsertedServers: 1,
expectedGaugeValue: 1,
@ -98,22 +140,24 @@ func TestSetBackendsConfiguration(t *testing.T) {
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
// The context is passed to the health check and canonically canceled by
// the test server once all expected requests have been received.
// The context is passed to the health check and
// canonically canceled by the test server once all expected requests have been received.
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
ts := newTestServer(cancel, test.healthSequence)
defer ts.Close()
t.Cleanup(cancel)
serverURL, timeout := test.server.Start(t, cancel)
lb := &testLoadBalancer{RWMutex: &sync.RWMutex{}}
backend := NewBackendConfig(Options{
options := Options{
Mode: test.mode,
Path: "/path",
Interval: healthCheckInterval,
Timeout: healthCheckTimeout,
LB: lb,
}, "backendName")
}
backend := NewBackendConfig(options, "backendName")
serverURL := testhelpers.MustParseURL(ts.URL)
if test.startHealthy {
lb.servers = append(lb.servers, serverURL)
} else {
@ -121,6 +165,7 @@ func TestSetBackendsConfiguration(t *testing.T) {
}
collectingMetrics := &testhelpers.CollectingGauge{}
check := HealthCheck{
Backends: make(map[string]*BackendConfig),
metrics: metricsHealthcheck{serverUpGauge: collectingMetrics},
@ -134,9 +179,6 @@ func TestSetBackendsConfiguration(t *testing.T) {
wg.Done()
}()
// Make test timeout dependent on number of expected requests, health
// check interval, and a safety margin.
timeout := time.Duration(len(test.healthSequence)*int(healthCheckInterval) + 500)
select {
case <-time.After(timeout):
t.Fatal("test did not complete in time")
@ -453,86 +495,6 @@ func TestBalancers_RemoveServer(t *testing.T) {
assert.Equal(t, 0, len(balancer2.Servers()))
}
type testLoadBalancer struct {
// RWMutex needed due to parallel test execution: Both the system-under-test
// and the test assertions reference the counters.
*sync.RWMutex
numRemovedServers int
numUpsertedServers int
servers []*url.URL
// options is just to make sure that LBStatusUpdater forwards options on Upsert to its BalancerHandler
options []roundrobin.ServerOption
}
func (lb *testLoadBalancer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// noop
}
func (lb *testLoadBalancer) RemoveServer(u *url.URL) error {
lb.Lock()
defer lb.Unlock()
lb.numRemovedServers++
lb.removeServer(u)
return nil
}
func (lb *testLoadBalancer) UpsertServer(u *url.URL, options ...roundrobin.ServerOption) error {
lb.Lock()
defer lb.Unlock()
lb.numUpsertedServers++
lb.servers = append(lb.servers, u)
lb.options = append(lb.options, options...)
return nil
}
func (lb *testLoadBalancer) Servers() []*url.URL {
return lb.servers
}
func (lb *testLoadBalancer) Options() []roundrobin.ServerOption {
return lb.options
}
func (lb *testLoadBalancer) removeServer(u *url.URL) {
var i int
var serverURL *url.URL
found := false
for i, serverURL = range lb.servers {
if *serverURL == *u {
found = true
break
}
}
if !found {
return
}
lb.servers = append(lb.servers[:i], lb.servers[i+1:]...)
}
func newTestServer(done func(), healthSequence []int) *httptest.Server {
handler := &testHandler{
done: done,
healthSequence: healthSequence,
}
return httptest.NewServer(handler)
}
// ServeHTTP returns HTTP response codes following a status sequences.
// It calls the given 'done' function once all request health indicators have been depleted.
func (th *testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if len(th.healthSequence) == 0 {
panic("received unexpected request")
}
w.WriteHeader(th.healthSequence[0])
th.healthSequence = th.healthSequence[1:]
if len(th.healthSequence) == 0 {
th.done()
}
}
func TestLBStatusUpdater(t *testing.T) {
lb := &testLoadBalancer{RWMutex: &sync.RWMutex{}}
svInfo := &runtime.ServiceInfo{}