Add Failover service

Co-authored-by: Kevin Pollet <pollet.kevin@gmail.com>

pkg/config/dynamic/http_config.go

@@ -35,6 +35,7 @@ type Service struct {
 	LoadBalancer *ServersLoadBalancer `json:"loadBalancer,omitempty" toml:"loadBalancer,omitempty" yaml:"loadBalancer,omitempty" export:"true"`
 	Weighted     *WeightedRoundRobin  `json:"weighted,omitempty" toml:"weighted,omitempty" yaml:"weighted,omitempty" label:"-" export:"true"`
 	Mirroring    *Mirroring           `json:"mirroring,omitempty" toml:"mirroring,omitempty" yaml:"mirroring,omitempty" label:"-" export:"true"`
+	Failover     *Failover            `json:"failover,omitempty" toml:"failover,omitempty" yaml:"failover,omitempty" label:"-" export:"true"`
 }
 
 // +k8s:deepcopy-gen=true
@@ -76,6 +77,15 @@ func (m *Mirroring) SetDefaults() {
 
 // +k8s:deepcopy-gen=true
 
+// Failover holds the Failover configuration.
+type Failover struct {
+	Service     string       `json:"service,omitempty" toml:"service,omitempty" yaml:"service,omitempty" export:"true"`
+	Fallback    string       `json:"fallback,omitempty" toml:"fallback,omitempty" yaml:"fallback,omitempty" export:"true"`
+	HealthCheck *HealthCheck `json:"healthCheck,omitempty" toml:"healthCheck,omitempty" yaml:"healthCheck,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
+}
+
+// +k8s:deepcopy-gen=true
+
 // MirrorService holds the MirrorService configuration.
 type MirrorService struct {
 	Name    string `json:"name,omitempty" toml:"name,omitempty" yaml:"name,omitempty" export:"true"`
@@ -98,7 +108,7 @@ type WeightedRoundRobin struct {
 
 // +k8s:deepcopy-gen=true
 
-// WRRService is a reference to a service load-balanced with weighted round robin.
+// WRRService is a reference to a service load-balanced with weighted round-robin.
 type WRRService struct {
 	Name   string `json:"name,omitempty" toml:"name,omitempty" yaml:"name,omitempty" export:"true"`
 	Weight *int   `json:"weight,omitempty" toml:"weight,omitempty" yaml:"weight,omitempty" export:"true"`

pkg/config/dynamic/zz_generated.deepcopy.go

@@ -285,6 +285,27 @@ func (in *ErrorPage) DeepCopy() *ErrorPage {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Failover) DeepCopyInto(out *Failover) {
+	*out = *in
+	if in.HealthCheck != nil {
+		in, out := &in.HealthCheck, &out.HealthCheck
+		*out = new(HealthCheck)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Failover.
+func (in *Failover) DeepCopy() *Failover {
+	if in == nil {
+		return nil
+	}
+	out := new(Failover)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ForwardAuth) DeepCopyInto(out *ForwardAuth) {
 	*out = *in
@@ -1171,6 +1192,11 @@ func (in *Service) DeepCopyInto(out *Service) {
 		*out = new(Mirroring)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.Failover != nil {
+		in, out := &in.Failover, &out.Failover
+		*out = new(Failover)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 

pkg/provider/kv/kv_test.go

@@ -69,6 +69,8 @@ func Test_buildConfiguration(t *testing.T) {
 		"traefik/http/services/Service03/weighted/services/0/weight":                                 "42",
 		"traefik/http/services/Service03/weighted/services/1/name":                                   "foobar",
 		"traefik/http/services/Service03/weighted/services/1/weight":                                 "42",
+		"traefik/http/services/Service04/failover/service":                                           "foobar",
+		"traefik/http/services/Service04/failover/fallback":                                          "foobar",
 		"traefik/http/middlewares/Middleware08/forwardAuth/authResponseHeaders/0":                    "foobar",
 		"traefik/http/middlewares/Middleware08/forwardAuth/authResponseHeaders/1":                    "foobar",
 		"traefik/http/middlewares/Middleware08/forwardAuth/authRequestHeaders/0":                     "foobar",
@@ -688,6 +690,12 @@ func Test_buildConfiguration(t *testing.T) {
 						},
 					},
 				},
+				"Service04": {
+					Failover: &dynamic.Failover{
+						Service:  "foobar",
+						Fallback: "foobar",
+					},
+				},
 			},
 		},
 		TCP: &dynamic.TCPConfiguration{

pkg/server/service/loadbalancer/failover/failover.go

@@ -0,0 +1,140 @@
+package failover
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"sync"
+
+	"github.com/traefik/traefik/v2/pkg/config/dynamic"
+	"github.com/traefik/traefik/v2/pkg/log"
+)
+
+// Failover is an http.Handler that can forward requests to the fallback handler
+// when the main handler status is down.
+type Failover struct {
+	wantsHealthCheck bool
+	handler          http.Handler
+	fallbackHandler  http.Handler
+	// updaters is the list of hooks that are run (to update the Failover
+	// parent(s)), whenever the Failover status changes.
+	updaters []func(bool)
+
+	handlerStatusMu sync.RWMutex
+	handlerStatus   bool
+
+	fallbackStatusMu sync.RWMutex
+	fallbackStatus   bool
+}
+
+// New creates a new Failover handler.
+func New(hc *dynamic.HealthCheck) *Failover {
+	return &Failover{
+		wantsHealthCheck: hc != nil,
+	}
+}
+
+// RegisterStatusUpdater adds fn to the list of hooks that are run when the
+// status of the Failover changes.
+// Not thread safe.
+func (f *Failover) RegisterStatusUpdater(fn func(up bool)) error {
+	if !f.wantsHealthCheck {
+		return errors.New("healthCheck not enabled in config for this failover service")
+	}
+
+	f.updaters = append(f.updaters, fn)
+
+	return nil
+}
+
+func (f *Failover) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	f.handlerStatusMu.RLock()
+	handlerStatus := f.handlerStatus
+	f.handlerStatusMu.RUnlock()
+
+	if handlerStatus {
+		f.handler.ServeHTTP(w, req)
+		return
+	}
+
+	f.fallbackStatusMu.RLock()
+	fallbackStatus := f.fallbackStatus
+	f.fallbackStatusMu.RUnlock()
+
+	if fallbackStatus {
+		f.fallbackHandler.ServeHTTP(w, req)
+		return
+	}
+
+	http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
+}
+
+// SetHandler sets the main http.Handler.
+func (f *Failover) SetHandler(handler http.Handler) {
+	f.handlerStatusMu.Lock()
+	defer f.handlerStatusMu.Unlock()
+
+	f.handler = handler
+	f.handlerStatus = true
+}
+
+// SetHandlerStatus sets the main handler status.
+func (f *Failover) SetHandlerStatus(ctx context.Context, up bool) {
+	f.handlerStatusMu.Lock()
+	defer f.handlerStatusMu.Unlock()
+
+	status := "DOWN"
+	if up {
+		status = "UP"
+	}
+
+	if up == f.handlerStatus {
+		// We're still with the same status, no need to propagate.
+		log.FromContext(ctx).Debugf("Still %s, no need to propagate", status)
+		return
+	}
+
+	log.FromContext(ctx).Debugf("Propagating new %s status", status)
+	f.handlerStatus = up
+
+	for _, fn := range f.updaters {
+		// Failover service status is set to DOWN
+		// when main and fallback handlers have a DOWN status.
+		fn(f.handlerStatus || f.fallbackStatus)
+	}
+}
+
+// SetFallbackHandler sets the fallback http.Handler.
+func (f *Failover) SetFallbackHandler(handler http.Handler) {
+	f.fallbackStatusMu.Lock()
+	defer f.fallbackStatusMu.Unlock()
+
+	f.fallbackHandler = handler
+	f.fallbackStatus = true
+}
+
+// SetFallbackHandlerStatus sets the fallback handler status.
+func (f *Failover) SetFallbackHandlerStatus(ctx context.Context, up bool) {
+	f.fallbackStatusMu.Lock()
+	defer f.fallbackStatusMu.Unlock()
+
+	status := "DOWN"
+	if up {
+		status = "UP"
+	}
+
+	if up == f.fallbackStatus {
+		// We're still with the same status, no need to propagate.
+		log.FromContext(ctx).Debugf("Still %s, no need to propagate", status)
+		return
+	}
+
+	log.FromContext(ctx).Debugf("Propagating new %s status", status)
+	f.fallbackStatus = up
+
+	for _, fn := range f.updaters {
+		// Failover service status is set to DOWN
+		// when main and fallback handlers have a DOWN status.
+		fn(f.handlerStatus || f.fallbackStatus)
+	}
+}

pkg/server/service/loadbalancer/failover/failover_test.go

@@ -0,0 +1,163 @@
+package failover
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/traefik/traefik/v2/pkg/config/dynamic"
+)
+
+type responseRecorder struct {
+	*httptest.ResponseRecorder
+	save     map[string]int
+	sequence []string
+	status   []int
+}
+
+func (r *responseRecorder) WriteHeader(statusCode int) {
+	r.save[r.Header().Get("server")]++
+	r.sequence = append(r.sequence, r.Header().Get("server"))
+	r.status = append(r.status, statusCode)
+	r.ResponseRecorder.WriteHeader(statusCode)
+}
+
+func TestFailover(t *testing.T) {
+	failover := New(&dynamic.HealthCheck{})
+
+	status := true
+	require.NoError(t, failover.RegisterStatusUpdater(func(up bool) {
+		status = up
+	}))
+
+	failover.SetHandler(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Set("server", "handler")
+		rw.WriteHeader(http.StatusOK)
+	}))
+
+	failover.SetFallbackHandler(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Set("server", "fallback")
+		rw.WriteHeader(http.StatusOK)
+	}))
+
+	recorder := &responseRecorder{ResponseRecorder: httptest.NewRecorder(), save: map[string]int{}}
+	failover.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/", nil))
+
+	assert.Equal(t, 1, recorder.save["handler"])
+	assert.Equal(t, 0, recorder.save["fallback"])
+	assert.Equal(t, []int{200}, recorder.status)
+	assert.True(t, status)
+
+	failover.SetHandlerStatus(context.Background(), false)
+
+	recorder = &responseRecorder{ResponseRecorder: httptest.NewRecorder(), save: map[string]int{}}
+	failover.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/", nil))
+
+	assert.Equal(t, 0, recorder.save["handler"])
+	assert.Equal(t, 1, recorder.save["fallback"])
+	assert.Equal(t, []int{200}, recorder.status)
+	assert.True(t, status)
+
+	failover.SetFallbackHandlerStatus(context.Background(), false)
+
+	recorder = &responseRecorder{ResponseRecorder: httptest.NewRecorder(), save: map[string]int{}}
+	failover.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/", nil))
+
+	assert.Equal(t, 0, recorder.save["handler"])
+	assert.Equal(t, 0, recorder.save["fallback"])
+	assert.Equal(t, []int{503}, recorder.status)
+	assert.False(t, status)
+}
+
+func TestFailoverDownThenUp(t *testing.T) {
+	failover := New(nil)
+
+	failover.SetHandler(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Set("server", "handler")
+		rw.WriteHeader(http.StatusOK)
+	}))
+
+	failover.SetFallbackHandler(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Set("server", "fallback")
+		rw.WriteHeader(http.StatusOK)
+	}))
+
+	recorder := &responseRecorder{ResponseRecorder: httptest.NewRecorder(), save: map[string]int{}}
+	failover.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/", nil))
+
+	assert.Equal(t, 1, recorder.save["handler"])
+	assert.Equal(t, 0, recorder.save["fallback"])
+	assert.Equal(t, []int{200}, recorder.status)
+
+	failover.SetHandlerStatus(context.Background(), false)
+
+	recorder = &responseRecorder{ResponseRecorder: httptest.NewRecorder(), save: map[string]int{}}
+	failover.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/", nil))
+
+	assert.Equal(t, 0, recorder.save["handler"])
+	assert.Equal(t, 1, recorder.save["fallback"])
+	assert.Equal(t, []int{200}, recorder.status)
+
+	failover.SetHandlerStatus(context.Background(), true)
+
+	recorder = &responseRecorder{ResponseRecorder: httptest.NewRecorder(), save: map[string]int{}}
+	failover.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/", nil))
+
+	assert.Equal(t, 1, recorder.save["handler"])
+	assert.Equal(t, 0, recorder.save["fallback"])
+	assert.Equal(t, []int{200}, recorder.status)
+}
+
+func TestFailoverPropagate(t *testing.T) {
+	failover := New(&dynamic.HealthCheck{})
+	failover.SetHandler(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Set("server", "handler")
+		rw.WriteHeader(http.StatusOK)
+	}))
+	failover.SetFallbackHandler(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Set("server", "fallback")
+		rw.WriteHeader(http.StatusOK)
+	}))
+
+	topFailover := New(nil)
+	topFailover.SetHandler(failover)
+	topFailover.SetFallbackHandler(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Set("server", "topFailover")
+		rw.WriteHeader(http.StatusOK)
+	}))
+	err := failover.RegisterStatusUpdater(func(up bool) {
+		topFailover.SetHandlerStatus(context.Background(), up)
+	})
+	require.NoError(t, err)
+
+	recorder := &responseRecorder{ResponseRecorder: httptest.NewRecorder(), save: map[string]int{}}
+	topFailover.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/", nil))
+
+	assert.Equal(t, 1, recorder.save["handler"])
+	assert.Equal(t, 0, recorder.save["fallback"])
+	assert.Equal(t, 0, recorder.save["topFailover"])
+	assert.Equal(t, []int{200}, recorder.status)
+
+	failover.SetHandlerStatus(context.Background(), false)
+
+	recorder = &responseRecorder{ResponseRecorder: httptest.NewRecorder(), save: map[string]int{}}
+	topFailover.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/", nil))
+
+	assert.Equal(t, 0, recorder.save["handler"])
+	assert.Equal(t, 1, recorder.save["fallback"])
+	assert.Equal(t, 0, recorder.save["topFailover"])
+	assert.Equal(t, []int{200}, recorder.status)
+
+	failover.SetFallbackHandlerStatus(context.Background(), false)
+
+	recorder = &responseRecorder{ResponseRecorder: httptest.NewRecorder(), save: map[string]int{}}
+	topFailover.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/", nil))
+
+	assert.Equal(t, 0, recorder.save["handler"])
+	assert.Equal(t, 0, recorder.save["fallback"])
+	assert.Equal(t, 1, recorder.save["topFailover"])
+	assert.Equal(t, []int{200}, recorder.status)
+}

pkg/server/service/loadbalancer/wrr/wrr.go

@@ -29,7 +29,7 @@ type stickyCookie struct {
 // (https://en.wikipedia.org/wiki/Earliest_deadline_first_scheduling)
 // Each pick from the schedule has the earliest deadline entry selected.
 // Entries have deadlines set at currentDeadline + 1 / weight,
-// providing weighted round robin behavior with floating point weights and an O(log n) pick time.
+// providing weighted round-robin behavior with floating point weights and an O(log n) pick time.
 type Balancer struct {
 	stickyCookie     *stickyCookie
 	wantsHealthCheck bool
@@ -230,6 +230,7 @@ func (b *Balancer) AddService(name string, handler http.Handler, weight *int) {
 	if weight != nil {
 		w = *weight
 	}
+
 	if w <= 0 { // non-positive weight is meaningless
 		return
 	}

pkg/server/service/service.go

@@ -23,6 +23,7 @@ import (
 	"github.com/traefik/traefik/v2/pkg/safe"
 	"github.com/traefik/traefik/v2/pkg/server/cookie"
 	"github.com/traefik/traefik/v2/pkg/server/provider"
+	"github.com/traefik/traefik/v2/pkg/server/service/loadbalancer/failover"
 	"github.com/traefik/traefik/v2/pkg/server/service/loadbalancer/mirror"
 	"github.com/traefik/traefik/v2/pkg/server/service/loadbalancer/wrr"
 	"github.com/vulcand/oxy/roundrobin"
@@ -116,6 +117,13 @@ func (m *Manager) BuildHTTP(rootCtx context.Context, serviceName string) (http.H
 			conf.AddError(err, true)
 			return nil, err
 		}
+	case conf.Failover != nil:
+		var err error
+		lb, err = m.getFailoverServiceHandler(ctx, serviceName, conf.Failover)
+		if err != nil {
+			conf.AddError(err, true)
+			return nil, err
+		}
 	default:
 		sErr := fmt.Errorf("the service %q does not have any type defined", serviceName)
 		conf.AddError(sErr, true)
@@ -125,6 +133,53 @@ func (m *Manager) BuildHTTP(rootCtx context.Context, serviceName string) (http.H
 	return lb, nil
 }
 
+func (m *Manager) getFailoverServiceHandler(ctx context.Context, serviceName string, config *dynamic.Failover) (http.Handler, error) {
+	f := failover.New(config.HealthCheck)
+
+	serviceHandler, err := m.BuildHTTP(ctx, config.Service)
+	if err != nil {
+		return nil, err
+	}
+
+	f.SetHandler(serviceHandler)
+
+	updater, ok := serviceHandler.(healthcheck.StatusUpdater)
+	if !ok {
+		return nil, fmt.Errorf("child service %v of %v not a healthcheck.StatusUpdater (%T)", config.Service, serviceName, serviceHandler)
+	}
+
+	if err := updater.RegisterStatusUpdater(func(up bool) {
+		f.SetHandlerStatus(ctx, up)
+	}); err != nil {
+		return nil, fmt.Errorf("cannot register %v as updater for %v: %w", config.Service, serviceName, err)
+	}
+
+	fallbackHandler, err := m.BuildHTTP(ctx, config.Fallback)
+	if err != nil {
+		return nil, err
+	}
+
+	f.SetFallbackHandler(fallbackHandler)
+
+	// Do not report the health of the fallback handler.
+	if config.HealthCheck == nil {
+		return f, nil
+	}
+
+	fallbackUpdater, ok := fallbackHandler.(healthcheck.StatusUpdater)
+	if !ok {
+		return nil, fmt.Errorf("child service %v of %v not a healthcheck.StatusUpdater (%T)", config.Fallback, serviceName, fallbackHandler)
+	}
+
+	if err := fallbackUpdater.RegisterStatusUpdater(func(up bool) {
+		f.SetFallbackHandlerStatus(ctx, up)
+	}); err != nil {
+		return nil, fmt.Errorf("cannot register %v as updater for %v: %w", config.Fallback, serviceName, err)
+	}
+
+	return f, nil
+}
+
 func (m *Manager) getMirrorServiceHandler(ctx context.Context, config *dynamic.Mirroring) (http.Handler, error) {
 	serviceHandler, err := m.BuildHTTP(ctx, config.Service)
 	if err != nil {
@@ -164,6 +219,7 @@ func (m *Manager) getWRRServiceHandler(ctx context.Context, serviceName string,
 		}
 
 		balancer.AddService(service.Name, serviceHandler, service.Weight)
+
 		if config.HealthCheck == nil {
 			continue
 		}