Add multiple entry points support, add entry point redirection

This commit is contained in:
emile 2016-01-29 20:34:17 +01:00
parent bb3b9f61cd
commit c22598c8ff
No known key found for this signature in database
GPG key ID: D808B4C167352E59
9 changed files with 366 additions and 145 deletions

307
server.go
View file

@ -5,6 +5,7 @@ package main
import (
"crypto/tls"
"encoding/json"
"errors"
log "github.com/Sirupsen/logrus"
"github.com/codegangsta/negroni"
@ -16,12 +17,12 @@ import (
"github.com/mailgun/oxy/cbreaker"
"github.com/mailgun/oxy/forward"
"github.com/mailgun/oxy/roundrobin"
"github.com/spf13/viper"
"net/http"
"net/url"
"os"
"os/signal"
"reflect"
"regexp"
"sync"
"syscall"
"time"
@ -31,11 +32,10 @@ var oxyLogger = &OxyLogger{}
// Server is the reverse-proxy/load-balancer engine
type Server struct {
srv *manners.GracefulServer
configurationRouter *mux.Router
serverEntryPoints map[string]serverEntryPoint
configurationChan chan types.ConfigMessage
configurationChanValidated chan types.ConfigMessage
sigs chan os.Signal
configurationValidatedChan chan types.ConfigMessage
signals chan os.Signal
stopChan chan bool
providers []provider.Provider
serverLock sync.Mutex
@ -44,16 +44,22 @@ type Server struct {
loggerMiddleware *middlewares.Logger
}
type serverEntryPoint struct {
httpServer *manners.GracefulServer
httpRouter *mux.Router
}
// NewServer returns an initialized Server.
func NewServer(globalConfiguration GlobalConfiguration) *Server {
server := new(Server)
server.serverEntryPoints = make(map[string]serverEntryPoint)
server.configurationChan = make(chan types.ConfigMessage, 10)
server.configurationChanValidated = make(chan types.ConfigMessage, 10)
server.sigs = make(chan os.Signal, 1)
server.configurationValidatedChan = make(chan types.ConfigMessage, 10)
server.signals = make(chan os.Signal, 1)
server.stopChan = make(chan bool)
server.providers = []provider.Provider{}
signal.Notify(server.sigs, syscall.SIGINT, syscall.SIGTERM)
signal.Notify(server.signals, syscall.SIGINT, syscall.SIGTERM)
server.currentConfigurations = make(configs)
server.globalConfiguration = globalConfiguration
server.loggerMiddleware = middlewares.NewLogger(globalConfiguration.AccessLogsFile)
@ -63,38 +69,27 @@ func NewServer(globalConfiguration GlobalConfiguration) *Server {
// Start starts the server and blocks until server is shutted down.
func (server *Server) Start() {
server.configurationRouter = LoadDefaultConfig(server.globalConfiguration)
go server.listenProviders()
go server.enableRouter()
go server.listenConfigurations()
server.configureProviders()
server.startProviders()
go server.listenSignals()
var er error
server.serverLock.Lock()
server.srv, er = server.prepareServer(server.configurationRouter, server.globalConfiguration, nil, server.loggerMiddleware, metrics)
if er != nil {
log.Fatal("Error preparing server: ", er)
}
go server.startServer(server.srv, server.globalConfiguration)
//TODO change that!
time.Sleep(100 * time.Millisecond)
server.serverLock.Unlock()
<-server.stopChan
}
// Stop stops the server
func (server *Server) Stop() {
server.srv.Close()
for _, serverEntryPoint := range server.serverEntryPoints {
serverEntryPoint.httpServer.Close()
}
server.stopChan <- true
}
// Close destroys the server
func (server *Server) Close() {
close(server.configurationChan)
close(server.configurationChanValidated)
close(server.sigs)
close(server.configurationValidatedChan)
close(server.signals)
close(server.stopChan)
server.loggerMiddleware.Close()
}
@ -104,19 +99,20 @@ func (server *Server) listenProviders() {
lastConfigs := make(map[string]*types.ConfigMessage)
for {
configMsg := <-server.configurationChan
log.Debugf("Configuration receveived from provider %s: %#v", configMsg.ProviderName, configMsg.Configuration)
jsonConf, _ := json.Marshal(configMsg.Configuration)
log.Debugf("Configuration receveived from provider %s: %s", configMsg.ProviderName, string(jsonConf))
lastConfigs[configMsg.ProviderName] = &configMsg
if time.Now().After(lastReceivedConfiguration.Add(time.Duration(server.globalConfiguration.ProvidersThrottleDuration))) {
log.Debugf("Last %s config received more than %s, OK", configMsg.ProviderName, server.globalConfiguration.ProvidersThrottleDuration)
// last config received more than n s ago
server.configurationChanValidated <- configMsg
server.configurationValidatedChan <- configMsg
} else {
log.Debugf("Last %s config received less than %s, waiting...", configMsg.ProviderName, server.globalConfiguration.ProvidersThrottleDuration)
go func() {
<-time.After(server.globalConfiguration.ProvidersThrottleDuration)
if time.Now().After(lastReceivedConfiguration.Add(time.Duration(server.globalConfiguration.ProvidersThrottleDuration))) {
log.Debugf("Waited for %s config, OK", configMsg.ProviderName)
server.configurationChanValidated <- *lastConfigs[configMsg.ProviderName]
server.configurationValidatedChan <- *lastConfigs[configMsg.ProviderName]
}
}()
}
@ -124,9 +120,9 @@ func (server *Server) listenProviders() {
}
}
func (server *Server) enableRouter() {
func (server *Server) listenConfigurations() {
for {
configMsg := <-server.configurationChanValidated
configMsg := <-server.configurationValidatedChan
if configMsg.Configuration == nil {
log.Info("Skipping empty Configuration")
} else if reflect.DeepEqual(server.currentConfigurations[configMsg.ProviderName], configMsg.Configuration) {
@ -139,22 +135,26 @@ func (server *Server) enableRouter() {
}
newConfigurations[configMsg.ProviderName] = configMsg.Configuration
newConfigurationRouter, err := server.loadConfig(newConfigurations, server.globalConfiguration)
newServerEntryPoints, err := server.loadConfig(newConfigurations, server.globalConfiguration)
if err == nil {
server.serverLock.Lock()
server.currentConfigurations = newConfigurations
server.configurationRouter = newConfigurationRouter
oldServer := server.srv
newsrv, err := server.prepareServer(server.configurationRouter, server.globalConfiguration, oldServer, server.loggerMiddleware, metrics)
if err != nil {
log.Fatal("Error preparing server: ", err)
}
go server.startServer(newsrv, server.globalConfiguration)
server.srv = newsrv
time.Sleep(1 * time.Second)
if oldServer != nil {
log.Info("Stopping old server")
oldServer.Close()
for newServerEntryPointName, newServerEntryPoint := range newServerEntryPoints {
currentServerEntryPoint := server.serverEntryPoints[newServerEntryPointName]
server.currentConfigurations = newConfigurations
currentServerEntryPoint.httpRouter = newServerEntryPoint.httpRouter
oldServer := currentServerEntryPoint.httpServer
newsrv, err := server.prepareServer(currentServerEntryPoint.httpRouter, server.globalConfiguration.EntryPoints[newServerEntryPointName], oldServer, server.loggerMiddleware, metrics)
if err != nil {
log.Fatal("Error preparing server: ", err)
}
go server.startServer(newsrv, server.globalConfiguration)
currentServerEntryPoint.httpServer = newsrv
server.serverEntryPoints[newServerEntryPointName] = currentServerEntryPoint
time.Sleep(1 * time.Second)
if oldServer != nil {
log.Info("Stopping old server")
oldServer.Close()
}
}
server.serverLock.Unlock()
} else {
@ -173,10 +173,6 @@ func (server *Server) configureProviders() {
server.providers = append(server.providers, server.globalConfiguration.Marathon)
}
if server.globalConfiguration.File != nil {
if len(server.globalConfiguration.File.Filename) == 0 {
// no filename, setting to global config file
server.globalConfiguration.File.Filename = viper.GetString("configFile")
}
server.providers = append(server.providers, server.globalConfiguration.File)
}
if server.globalConfiguration.Web != nil {
@ -200,7 +196,8 @@ func (server *Server) configureProviders() {
func (server *Server) startProviders() {
// start providers
for _, provider := range server.providers {
log.Infof("Starting provider %v %+v", reflect.TypeOf(provider), provider)
jsonConf, _ := json.Marshal(provider)
log.Infof("Starting provider %v %s", reflect.TypeOf(provider), jsonConf)
currentProvider := provider
go func() {
err := currentProvider.Provide(server.configurationChan)
@ -212,15 +209,18 @@ func (server *Server) startProviders() {
}
func (server *Server) listenSignals() {
sig := <-server.sigs
sig := <-server.signals
log.Infof("I have to go... %+v", sig)
log.Info("Stopping server")
server.Stop()
}
// creates a TLS config that allows terminating HTTPS for multiple domains using SNI
func (server *Server) createTLSConfig(certs []Certificate) (*tls.Config, error) {
if len(certs) == 0 {
func (server *Server) createTLSConfig(tlsOption *TLS) (*tls.Config, error) {
if tlsOption == nil {
return nil, nil
}
if len(tlsOption.Certificates) == 0 {
return nil, nil
}
@ -230,8 +230,8 @@ func (server *Server) createTLSConfig(certs []Certificate) (*tls.Config, error)
}
var err error
config.Certificates = make([]tls.Certificate, len(certs))
for i, v := range certs {
config.Certificates = make([]tls.Certificate, len(tlsOption.Certificates))
for i, v := range tlsOption.Certificates {
config.Certificates[i], err = tls.LoadX509KeyPair(v.CertFile, v.KeyFile)
if err != nil {
return nil, err
@ -244,7 +244,7 @@ func (server *Server) createTLSConfig(certs []Certificate) (*tls.Config, error)
}
func (server *Server) startServer(srv *manners.GracefulServer, globalConfiguration GlobalConfiguration) {
log.Info("Starting server")
log.Info("Starting server on ", srv.Addr)
if srv.TLSConfig != nil {
err := srv.ListenAndServeTLSWithConfig(srv.TLSConfig)
if err != nil {
@ -259,7 +259,7 @@ func (server *Server) startServer(srv *manners.GracefulServer, globalConfigurati
log.Info("Server stopped")
}
func (server *Server) prepareServer(router *mux.Router, globalConfiguration GlobalConfiguration, oldServer *manners.GracefulServer, middlewares ...negroni.Handler) (*manners.GracefulServer, error) {
func (server *Server) prepareServer(router *mux.Router, entryPoint *EntryPoint, oldServer *manners.GracefulServer, middlewares ...negroni.Handler) (*manners.GracefulServer, error) {
log.Info("Preparing server")
// middlewares
var negroni = negroni.New()
@ -267,7 +267,7 @@ func (server *Server) prepareServer(router *mux.Router, globalConfiguration Glob
negroni.Use(middleware)
}
negroni.UseHandler(router)
tlsConfig, err := server.createTLSConfig(globalConfiguration.Certificates)
tlsConfig, err := server.createTLSConfig(entryPoint.TLS)
if err != nil {
log.Fatalf("Error creating TLS config %s", err)
return nil, err
@ -276,13 +276,13 @@ func (server *Server) prepareServer(router *mux.Router, globalConfiguration Glob
if oldServer == nil {
return manners.NewWithServer(
&http.Server{
Addr: globalConfiguration.Port,
Addr: entryPoint.Address,
Handler: negroni,
TLSConfig: tlsConfig,
}), nil
}
gracefulServer, err := oldServer.HijackListener(&http.Server{
Addr: globalConfiguration.Port,
Addr: entryPoint.Address,
Handler: negroni,
TLSConfig: tlsConfig,
}, tlsConfig)
@ -293,80 +293,147 @@ func (server *Server) prepareServer(router *mux.Router, globalConfiguration Glob
return gracefulServer, nil
}
func (server *Server) buildEntryPoints(globalConfiguration GlobalConfiguration) map[string]serverEntryPoint {
serverEntryPoints := make(map[string]serverEntryPoint)
for entryPointName := range globalConfiguration.EntryPoints {
router := server.buildDefaultHTTPRouter()
serverEntryPoints[entryPointName] = serverEntryPoint{
httpRouter: router,
}
}
return serverEntryPoints
}
// LoadConfig returns a new gorilla.mux Route from the specified global configuration and the dynamic
// provider configurations.
func (server *Server) loadConfig(configurations configs, globalConfiguration GlobalConfiguration) (*mux.Router, error) {
router := mux.NewRouter()
router.NotFoundHandler = http.HandlerFunc(notFoundHandler)
func (server *Server) loadConfig(configurations configs, globalConfiguration GlobalConfiguration) (map[string]serverEntryPoint, error) {
serverEntryPoints := server.buildEntryPoints(globalConfiguration)
redirectHandlers := make(map[string]http.Handler)
backends := map[string]http.Handler{}
for _, configuration := range configurations {
for frontendName, frontend := range configuration.Frontends {
log.Infof("Creating frontend %s", frontendName)
log.Debugf("Creating frontend %s", frontendName)
fwd, _ := forward.New(forward.Logger(oxyLogger), forward.PassHostHeader(frontend.PassHostHeader))
newRoute := router.NewRoute().Name(frontendName)
for routeName, route := range frontend.Routes {
log.Infof("Creating route %s %s:%s", routeName, route.Rule, route.Value)
newRouteReflect, err := invoke(newRoute, route.Rule, route.Value)
if err != nil {
return nil, err
}
newRoute = newRouteReflect[0].Interface().(*mux.Route)
// default endpoints if not defined in frontends
if len(frontend.EntryPoints) == 0 {
frontend.EntryPoints = globalConfiguration.DefaultEntryPoints
}
if backends[frontend.Backend] == nil {
log.Infof("Creating backend %s", frontend.Backend)
var lb http.Handler
rr, _ := roundrobin.New(fwd)
if configuration.Backends[frontend.Backend] == nil {
return nil, errors.New("Backend not found: " + frontend.Backend)
for _, entryPointName := range frontend.EntryPoints {
log.Debugf("Wiring frontend %s to entryPoint %s", frontendName, entryPointName)
if _, ok := serverEntryPoints[entryPointName]; !ok {
return nil, errors.New("Undefined entrypoint: " + entryPointName)
}
lbMethod, err := types.NewLoadBalancerMethod(configuration.Backends[frontend.Backend].LoadBalancer)
if err != nil {
configuration.Backends[frontend.Backend].LoadBalancer = &types.LoadBalancer{Method: "wrr"}
}
switch lbMethod {
case types.Drr:
log.Infof("Creating load-balancer drr")
rebalancer, _ := roundrobin.NewRebalancer(rr, roundrobin.RebalancerLogger(oxyLogger))
lb = rebalancer
for serverName, server := range configuration.Backends[frontend.Backend].Servers {
url, err := url.Parse(server.URL)
if err != nil {
return nil, err
}
log.Infof("Creating server %s %s", serverName, url.String())
rebalancer.UpsertServer(url, roundrobin.Weight(server.Weight))
}
case types.Wrr:
log.Infof("Creating load-balancer wrr")
lb = middlewares.NewWebsocketUpgrader(rr)
for serverName, server := range configuration.Backends[frontend.Backend].Servers {
url, err := url.Parse(server.URL)
if err != nil {
return nil, err
}
log.Infof("Creating server %s at %s with weight %d", serverName, url.String(), server.Weight)
rr.UpsertServer(url, roundrobin.Weight(server.Weight))
newRoute := serverEntryPoints[entryPointName].httpRouter.NewRoute().Name(frontendName)
for routeName, route := range frontend.Routes {
log.Debugf("Creating route %s %s:%s", routeName, route.Rule, route.Value)
newRouteReflect, err := invoke(newRoute, route.Rule, route.Value)
if err != nil {
return nil, err
}
newRoute = newRouteReflect[0].Interface().(*mux.Route)
}
var negroni = negroni.New()
if configuration.Backends[frontend.Backend].CircuitBreaker != nil {
log.Infof("Creating circuit breaker %s", configuration.Backends[frontend.Backend].CircuitBreaker.Expression)
negroni.Use(middlewares.NewCircuitBreaker(lb, configuration.Backends[frontend.Backend].CircuitBreaker.Expression, cbreaker.Logger(oxyLogger)))
entryPoint := globalConfiguration.EntryPoints[entryPointName]
if entryPoint.Redirect != nil {
if redirectHandlers[entryPointName] != nil {
newRoute.Handler(redirectHandlers[entryPointName])
} else if handler, err := server.loadEntryPointConfig(entryPointName, entryPoint); err != nil {
return nil, err
} else {
newRoute.Handler(handler)
redirectHandlers[entryPointName] = handler
}
} else {
negroni.UseHandler(lb)
if backends[frontend.Backend] == nil {
log.Debugf("Creating backend %s", frontend.Backend)
var lb http.Handler
rr, _ := roundrobin.New(fwd)
if configuration.Backends[frontend.Backend] == nil {
return nil, errors.New("Undefined backend: " + frontend.Backend)
}
lbMethod, err := types.NewLoadBalancerMethod(configuration.Backends[frontend.Backend].LoadBalancer)
if err != nil {
configuration.Backends[frontend.Backend].LoadBalancer = &types.LoadBalancer{Method: "wrr"}
}
switch lbMethod {
case types.Drr:
log.Debugf("Creating load-balancer drr")
rebalancer, _ := roundrobin.NewRebalancer(rr, roundrobin.RebalancerLogger(oxyLogger))
lb = rebalancer
for serverName, server := range configuration.Backends[frontend.Backend].Servers {
url, err := url.Parse(server.URL)
if err != nil {
return nil, err
}
log.Debugf("Creating server %s at %s with weight %d", serverName, url.String(), server.Weight)
rebalancer.UpsertServer(url, roundrobin.Weight(server.Weight))
}
case types.Wrr:
log.Debugf("Creating load-balancer wrr")
lb = middlewares.NewWebsocketUpgrader(rr)
for serverName, server := range configuration.Backends[frontend.Backend].Servers {
url, err := url.Parse(server.URL)
if err != nil {
return nil, err
}
log.Debugf("Creating server %s at %s with weight %d", serverName, url.String(), server.Weight)
rr.UpsertServer(url, roundrobin.Weight(server.Weight))
}
}
var negroni = negroni.New()
if configuration.Backends[frontend.Backend].CircuitBreaker != nil {
log.Debugf("Creating circuit breaker %s", configuration.Backends[frontend.Backend].CircuitBreaker.Expression)
negroni.Use(middlewares.NewCircuitBreaker(lb, configuration.Backends[frontend.Backend].CircuitBreaker.Expression, cbreaker.Logger(oxyLogger)))
} else {
negroni.UseHandler(lb)
}
backends[frontend.Backend] = negroni
} else {
log.Debugf("Reusing backend %s", frontend.Backend)
}
newRoute.Handler(backends[frontend.Backend])
}
err := newRoute.GetError()
if err != nil {
log.Errorf("Error building route: %s", err)
}
backends[frontend.Backend] = negroni
} else {
log.Infof("Reusing backend %s", frontend.Backend)
}
// stream.New(backends[frontend.Backend], stream.Retry("IsNetworkError() && Attempts() <= " + strconv.Itoa(globalConfiguration.Replay)), stream.Logger(oxyLogger))
newRoute.Handler(backends[frontend.Backend])
err := newRoute.GetError()
if err != nil {
log.Errorf("Error building route: %s", err)
}
}
}
return router, nil
return serverEntryPoints, nil
}
func (server *Server) loadEntryPointConfig(entryPointName string, entryPoint *EntryPoint) (http.Handler, error) {
regex := entryPoint.Redirect.Regex
replacement := entryPoint.Redirect.Replacement
if len(entryPoint.Redirect.EntryPoint) > 0 {
regex = "^(?:https?:\\/\\/)?([\\da-z\\.-]+)(?::\\d+)(.*)$"
if server.globalConfiguration.EntryPoints[entryPoint.Redirect.EntryPoint] == nil {
return nil, errors.New("Unkown entrypoint " + entryPoint.Redirect.EntryPoint)
}
protocol := "http"
if server.globalConfiguration.EntryPoints[entryPoint.Redirect.EntryPoint].TLS != nil {
protocol = "https"
}
r, _ := regexp.Compile("(:\\d+)")
match := r.FindStringSubmatch(server.globalConfiguration.EntryPoints[entryPoint.Redirect.EntryPoint].Address)
if len(match) == 0 {
return nil, errors.New("Bad Address format: " + server.globalConfiguration.EntryPoints[entryPoint.Redirect.EntryPoint].Address)
}
replacement = protocol + "://$1" + match[0] + "$2"
}
rewrite, err := middlewares.NewRewrite(regex, replacement, true)
if err != nil {
return nil, err
}
log.Debugf("Creating entryPoint redirect %s -> %s : %s -> %s", entryPointName, entryPoint.Redirect.EntryPoint, regex, replacement)
negroni := negroni.New()
negroni.Use(rewrite)
return negroni, nil
}
func (server *Server) buildDefaultHTTPRouter() *mux.Router {
router := mux.NewRouter()
router.NotFoundHandler = http.HandlerFunc(notFoundHandler)
return router
}