avoid retries when any data was written to the backend

middlewares/retry.go

@@ -2,10 +2,10 @@ package middlewares
 
 import (
 	"bufio"
-	"context"
 	"io/ioutil"
 	"net"
 	"net/http"
+	"net/http/httptrace"
 
 	"github.com/containous/traefik/log"
 )
@@ -40,11 +40,24 @@ func (retry *Retry) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 
 	attempts := 1
 	for {
-		netErrorOccurred := false
-		// We pass in a pointer to netErrorOccurred so that we can set it to true on network errors
-		// when proxying the HTTP requests to the backends. This happens in the custom RecordingErrorHandler.
-		newCtx := context.WithValue(r.Context(), defaultNetErrCtxKey, &netErrorOccurred)
-		retryResponseWriter := newRetryResponseWriter(rw, attempts >= retry.attempts, &netErrorOccurred)
+		attemptsExhausted := attempts >= retry.attempts
+		// Websocket requests can't be retried at this point in time.
+		// This is due to the fact that gorilla/websocket doesn't use the request
+		// context and so we don't get httptrace information.
+		// Websocket clients should however retry on their own anyway.
+		shouldRetry := !attemptsExhausted && !isWebsocketRequest(r)
+		retryResponseWriter := newRetryResponseWriter(rw, shouldRetry)
+
+		// Disable retries when the backend already received request data
+		trace := &httptrace.ClientTrace{
+			WroteHeaders: func() {
+				retryResponseWriter.DisableRetries()
+			},
+			WroteRequest: func(httptrace.WroteRequestInfo) {
+				retryResponseWriter.DisableRetries()
+			},
+		}
+		newCtx := httptrace.WithClientTrace(r.Context(), trace)
 
 		retry.next.ServeHTTP(retryResponseWriter, r.WithContext(newCtx))
 		if !retryResponseWriter.ShouldRetry() {
@@ -57,31 +70,6 @@ func (retry *Retry) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// netErrorCtxKey is a custom type that is used as key for the context.
-type netErrorCtxKey string
-
-// defaultNetErrCtxKey is the actual key which value is used to record network errors.
-var defaultNetErrCtxKey netErrorCtxKey = "NetErrCtxKey"
-
-// NetErrorRecorder is an interface to record net errors.
-type NetErrorRecorder interface {
-	// Record can be used to signal the retry middleware that an network error happened
-	// and therefore the request should be retried.
-	Record(ctx context.Context)
-}
-
-// DefaultNetErrorRecorder is the default NetErrorRecorder implementation.
-type DefaultNetErrorRecorder struct{}
-
-// Record is recording network errors by setting the context value for the defaultNetErrCtxKey to true.
-func (DefaultNetErrorRecorder) Record(ctx context.Context) {
-	val := ctx.Value(defaultNetErrCtxKey)
-
-	if netErrorOccurred, isBoolPointer := val.(*bool); isBoolPointer {
-		*netErrorOccurred = true
-	}
-}
-
 // RetryListener is used to inform about retry attempts.
 type RetryListener interface {
 	// Retried will be called when a retry happens, with the request attempt passed to it.
@@ -104,13 +92,13 @@ type retryResponseWriter interface {
 	http.ResponseWriter
 	http.Flusher
 	ShouldRetry() bool
+	DisableRetries()
 }
 
-func newRetryResponseWriter(rw http.ResponseWriter, attemptsExhausted bool, netErrorOccured *bool) retryResponseWriter {
+func newRetryResponseWriter(rw http.ResponseWriter, shouldRetry bool) retryResponseWriter {
 	responseWriter := &retryResponseWriterWithoutCloseNotify{
-		responseWriter:    rw,
-		attemptsExhausted: attemptsExhausted,
-		netErrorOccured:   netErrorOccured,
+		responseWriter: rw,
+		shouldRetry:    shouldRetry,
 	}
 	if _, ok := rw.(http.CloseNotifier); ok {
 		return &retryResponseWriterWithCloseNotify{responseWriter}
@@ -119,13 +107,16 @@ func newRetryResponseWriter(rw http.ResponseWriter, attemptsExhausted bool, netE
 }
 
 type retryResponseWriterWithoutCloseNotify struct {
-	responseWriter    http.ResponseWriter
-	attemptsExhausted bool
-	netErrorOccured   *bool
+	responseWriter http.ResponseWriter
+	shouldRetry    bool
 }
 
 func (rr *retryResponseWriterWithoutCloseNotify) ShouldRetry() bool {
-	return *rr.netErrorOccured && !rr.attemptsExhausted
+	return rr.shouldRetry
+}
+
+func (rr *retryResponseWriterWithoutCloseNotify) DisableRetries() {
+	rr.shouldRetry = false
 }
 
 func (rr *retryResponseWriterWithoutCloseNotify) Header() http.Header {
@@ -143,6 +134,15 @@ func (rr *retryResponseWriterWithoutCloseNotify) Write(buf []byte) (int, error)
 }
 
 func (rr *retryResponseWriterWithoutCloseNotify) WriteHeader(code int) {
+	if rr.ShouldRetry() && code == http.StatusServiceUnavailable {
+		// We get a 503 HTTP Status Code when there is no backend server in the pool
+		// to which the request could be sent.  Also, note that rr.ShouldRetry()
+		// will never return true in case there was a connetion established to
+		// the backend server and so we can be sure that the 503 was produced
+		// inside Traefik already and we don't have to retry in this cases.
+		rr.DisableRetries()
+	}
+
 	if rr.ShouldRetry() {
 		return
 	}