homelab/traefik
1
0
Fork 0
Code Activity

Dynamic Configuration Refactoring

Browse source
This commit is contained in:
Ludovic Fernandez 2018-11-14 10:18:03 +01:00 • committed by Traefiker Bot
parent d3ae88f108
commit a09dfa3ce1
452 changed files with 21023 additions and 9419 deletions
Download patch file Download diff file Expand all files Collapse all files

86
provider/file/file.go
View file

@ -1,6 +1,7 @@
package file
import (
"context"
"fmt"
"io/ioutil"
"os"
@ -9,6 +10,7 @@ import (
"strings"
"text/template"
"github.com/containous/traefik/config"
"github.com/containous/traefik/log"
"github.com/containous/traefik/provider"
"github.com/containous/traefik/safe"
@ -18,6 +20,8 @@ import (
"gopkg.in/fsnotify.v1"
)
const providerName = "file"
var _ provider.Provider = (*Provider)(nil)
// Provider holds configurations of the provider.
@ -34,7 +38,7 @@ func (p *Provider) Init(constraints types.Constraints) error {
// Provide allows the file provider to provide configurations to traefik
// using the given configuration channel.
func (p *Provider) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool) error {
func (p *Provider) Provide(configurationChan chan<- config.Message, pool *safe.Pool) error {
configuration, err := p.BuildConfiguration()
if err != nil {
@ -63,9 +67,11 @@ func (p *Provider) Provide(configurationChan chan<- types.ConfigMessage, pool *s
// BuildConfiguration loads configuration either from file or a directory specified by 'Filename'/'Directory'
// and returns a 'Configuration' object
func (p *Provider) BuildConfiguration() (*types.Configuration, error) {
func (p *Provider) BuildConfiguration() (*config.Configuration, error) {
ctx := log.With(context.Background(), log.Str(log.ProviderName, providerName))
if len(p.Directory) > 0 {
return p.loadFileConfigFromDirectory(p.Directory, nil)
return p.loadFileConfigFromDirectory(ctx, p.Directory, nil)
}
if len(p.Filename) > 0 {
@ -79,7 +85,7 @@ func (p *Provider) BuildConfiguration() (*types.Configuration, error) {
return nil, errors.New("error using file configuration backend, no filename defined")
}
func (p *Provider) addWatcher(pool *safe.Pool, directory string, configurationChan chan<- types.ConfigMessage, callback func(chan<- types.ConfigMessage, fsnotify.Event)) error {
func (p *Provider) addWatcher(pool *safe.Pool, directory string, configurationChan chan<- config.Message, callback func(chan<- config.Message, fsnotify.Event)) error {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return fmt.Errorf("error creating file watcher: %s", err)
@ -115,14 +121,14 @@ func (p *Provider) addWatcher(pool *safe.Pool, directory string, configurationCh
callback(configurationChan, evt)
}
case err := <-watcher.Errors:
log.Errorf("Watcher event error: %s", err)
log.WithoutContext().WithField(log.ProviderName, providerName).Errorf("Watcher event error: %s", err)
}
}
})
return nil
}
func (p *Provider) watcherCallback(configurationChan chan<- types.ConfigMessage, event fsnotify.Event) {
func (p *Provider) watcherCallback(configurationChan chan<- config.Message, event fsnotify.Event) {
watchItem := p.TraefikFile
if len(p.Directory) > 0 {
watchItem = p.Directory
@ -130,23 +136,24 @@ func (p *Provider) watcherCallback(configurationChan chan<- types.ConfigMessage,
watchItem = p.Filename
}
logger := log.WithoutContext().WithField(log.ProviderName, providerName)
if _, err := os.Stat(watchItem); err != nil {
log.Debugf("Unable to watch %s : %v", watchItem, err)
logger.Errorf("Unable to watch %s : %v", watchItem, err)
return
}
configuration, err := p.BuildConfiguration()
if err != nil {
log.Errorf("Error occurred during watcher callback: %s", err)
logger.Errorf("Error occurred during watcher callback: %s", err)
return
}
sendConfigToChannel(configurationChan, configuration)
}
func sendConfigToChannel(configurationChan chan<- types.ConfigMessage, configuration *types.Configuration) {
configurationChan <- types.ConfigMessage{
func sendConfigToChannel(configurationChan chan<- config.Message, configuration *config.Configuration) {
configurationChan <- config.Message{
ProviderName: "file",
Configuration: configuration,
}
@ -163,13 +170,13 @@ func readFile(filename string) (string, error) {
return "", fmt.Errorf("invalid filename: %s", filename)
}
func (p *Provider) loadFileConfig(filename string, parseTemplate bool) (*types.Configuration, error) {
func (p *Provider) loadFileConfig(filename string, parseTemplate bool) (*config.Configuration, error) {
fileContent, err := readFile(filename)
if err != nil {
return nil, fmt.Errorf("error reading configuration file: %s - %s", filename, err)
}
var configuration *types.Configuration
var configuration *config.Configuration
if parseTemplate {
configuration, err = p.CreateConfiguration(fileContent, template.FuncMap{}, false)
} else {
@ -179,16 +186,20 @@ func (p *Provider) loadFileConfig(filename string, parseTemplate bool) (*types.C
if err != nil {
return nil, err
}
if configuration == nil || configuration.Backends == nil && configuration.Frontends == nil && configuration.TLS == nil {
configuration = &types.Configuration{
Frontends: make(map[string]*types.Frontend),
Backends: make(map[string]*types.Backend),
if configuration == nil || configuration.Routers == nil && configuration.Middlewares == nil && configuration.Services == nil && configuration.TLS == nil {
configuration = &config.Configuration{
Routers: make(map[string]*config.Router),
Middlewares: make(map[string]*config.Middleware),
Services: make(map[string]*config.Service),
}
}
return configuration, err
}
func (p *Provider) loadFileConfigFromDirectory(directory string, configuration *types.Configuration) (*types.Configuration, error) {
func (p *Provider) loadFileConfigFromDirectory(ctx context.Context, directory string, configuration *config.Configuration) (*config.Configuration, error) {
logger := log.FromContext(ctx)
fileList, err := ioutil.ReadDir(directory)
if err != nil {
@ -196,9 +207,10 @@ func (p *Provider) loadFileConfigFromDirectory(directory string, configuration *
}
if configuration == nil {
configuration = &types.Configuration{
Frontends: make(map[string]*types.Frontend),
Backends: make(map[string]*types.Backend),
configuration = &config.Configuration{
Routers: make(map[string]*config.Router),
Middlewares: make(map[string]*config.Middleware),
Services: make(map[string]*config.Service),
}
}
@ -206,7 +218,7 @@ func (p *Provider) loadFileConfigFromDirectory(directory string, configuration *
for _, item := range fileList {
if item.IsDir() {
configuration, err = p.loadFileConfigFromDirectory(filepath.Join(directory, item.Name()), configuration)
configuration, err = p.loadFileConfigFromDirectory(ctx, filepath.Join(directory, item.Name()), configuration)
if err != nil {
return configuration, fmt.Errorf("unable to load content configuration from subdirectory %s: %v", item, err)
}
@ -215,38 +227,46 @@ func (p *Provider) loadFileConfigFromDirectory(directory string, configuration *
continue
}
var c *types.Configuration
var c *config.Configuration
c, err = p.loadFileConfig(path.Join(directory, item.Name()), true)
if err != nil {
return configuration, err
}
for backendName, backend := range c.Backends {
if _, exists := configuration.Backends[backendName]; exists {
log.Warnf("Backend %s already configured, skipping", backendName)
for name, conf := range c.Routers {
if _, exists := configuration.Routers[name]; exists {
logger.WithField(log.RouterName, name).Warn("Router already configured, skipping")
} else {
configuration.Backends[backendName] = backend
configuration.Routers[name] = conf
}
}
for frontendName, frontend := range c.Frontends {
if _, exists := configuration.Frontends[frontendName]; exists {
log.Warnf("Frontend %s already configured, skipping", frontendName)
for name, conf := range c.Middlewares {
if _, exists := configuration.Middlewares[name]; exists {
logger.WithField(log.MiddlewareName, name).Warn("Middleware already configured, skipping")
} else {
configuration.Frontends[frontendName] = frontend
configuration.Middlewares[name] = conf
}
}
for name, conf := range c.Services {
if _, exists := configuration.Services[name]; exists {
logger.WithField(log.ServiceName, name).Warn("Service already configured, skipping")
} else {
configuration.Services[name] = conf
}
}
for _, conf := range c.TLS {
if _, exists := configTLSMaps[conf]; exists {
log.Warnf("TLS Configuration %v already configured, skipping", conf)
logger.Warnf("TLS Configuration %v already configured, skipping", conf)
} else {
configTLSMaps[conf] = struct{}{}
}
}
}
for conf := range configTLSMaps {
configuration.TLS = append(configuration.TLS, conf)
}

390
provider/file/file_test.go
View file

@ -9,193 +9,29 @@ import (
"testing"
"time"
"github.com/containous/traefik/config"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
"github.com/stretchr/testify/assert"
)
// createRandomFile Helper
func createRandomFile(t *testing.T, tempDir string, contents ...string) *os.File {
return createFile(t, tempDir, fmt.Sprintf("temp%d.toml", time.Now().UnixNano()), contents...)
}
// createFile Helper
func createFile(t *testing.T, tempDir string, name string, contents ...string) *os.File {
t.Helper()
fileName := path.Join(tempDir, name)
tempFile, err := os.Create(fileName)
if err != nil {
t.Fatal(err)
}
for _, content := range contents {
_, err := tempFile.WriteString(content)
if err != nil {
t.Fatal(err)
}
}
err = tempFile.Close()
if err != nil {
t.Fatal(err)
}
return tempFile
}
// createTempDir Helper
func createTempDir(t *testing.T, dir string) string {
t.Helper()
d, err := ioutil.TempDir("", dir)
if err != nil {
t.Fatal(err)
}
return d
}
// createFrontendConfiguration Helper
func createFrontendConfiguration(n int) string {
conf := "[frontends]\n"
for i := 1; i <= n; i++ {
conf += fmt.Sprintf(` [frontends."frontend%[1]d"]
backend = "backend%[1]d"
`, i)
}
return conf
}
// createBackendConfiguration Helper
func createBackendConfiguration(n int) string {
conf := "[backends]\n"
for i := 1; i <= n; i++ {
conf += fmt.Sprintf(` [backends.backend%[1]d]
[backends.backend%[1]d.servers.server1]
url = "http://172.17.0.%[1]d:80"
`, i)
}
return conf
}
// createTLS Helper
func createTLS(n int) string {
var conf string
for i := 1; i <= n; i++ {
conf += fmt.Sprintf(`[[TLS]]
EntryPoints = ["https"]
[TLS.Certificate]
CertFile = "integration/fixtures/https/snitest%[1]d.com.cert"
KeyFile = "integration/fixtures/https/snitest%[1]d.com.key"
`, i)
}
return conf
}
type ProvideTestCase struct {
desc string
directoryContent []string
fileContent string
traefikFileContent string
expectedNumFrontend int
expectedNumBackend int
expectedNumTLSConf int
}
func getTestCases() []ProvideTestCase {
return []ProvideTestCase{
{
desc: "simple file",
fileContent: createFrontendConfiguration(2) + createBackendConfiguration(3) + createTLS(4),
expectedNumFrontend: 2,
expectedNumBackend: 3,
expectedNumTLSConf: 4,
},
{
desc: "simple file and a traefik file",
fileContent: createFrontendConfiguration(2) + createBackendConfiguration(3) + createTLS(4),
traefikFileContent: `
debug=true
`,
expectedNumFrontend: 2,
expectedNumBackend: 3,
expectedNumTLSConf: 4,
},
{
desc: "template file",
fileContent: `
[frontends]
{{ range $i, $e := until 20 }}
[frontends.frontend{{ $e }}]
backend = "backend"
{{ end }}
`,
expectedNumFrontend: 20,
},
{
desc: "simple directory",
directoryContent: []string{
createFrontendConfiguration(2),
createBackendConfiguration(3),
createTLS(4),
},
expectedNumFrontend: 2,
expectedNumBackend: 3,
expectedNumTLSConf: 4,
},
{
desc: "template in directory",
directoryContent: []string{
`
[frontends]
{{ range $i, $e := until 20 }}
[frontends.frontend{{ $e }}]
backend = "backend"
{{ end }}
`,
`
[backends]
{{ range $i, $e := until 20 }}
[backends.backend{{ $e }}]
[backends.backend{{ $e }}.servers.server1]
url="http://127.0.0.1"
{{ end }}
`,
},
expectedNumFrontend: 20,
expectedNumBackend: 20,
},
{
desc: "simple traefik file",
traefikFileContent: `
debug=true
[file]
` + createFrontendConfiguration(2) + createBackendConfiguration(3) + createTLS(4),
expectedNumFrontend: 2,
expectedNumBackend: 3,
expectedNumTLSConf: 4,
},
{
desc: "simple traefik file with templating",
traefikFileContent: `
temp="{{ getTag \"test\" }}"
[file]
` + createFrontendConfiguration(2) + createBackendConfiguration(3) + createTLS(4),
expectedNumFrontend: 2,
expectedNumBackend: 3,
expectedNumTLSConf: 4,
},
}
desc string
directoryContent []string
fileContent string
traefikFileContent string
expectedNumRouter int
expectedNumService int
expectedNumTLSConf int
}
func TestProvideWithoutWatch(t *testing.T) {
for _, test := range getTestCases() {
test := test
t.Run(test.desc+" without watch", func(t *testing.T) {
t.Parallel()
provider, clean := createProvider(t, test, false)
defer clean()
configChan := make(chan types.ConfigMessage)
configChan := make(chan config.Message)
provider.DebugLogGeneratedTemplate = true
go func() {
err := provider.Provide(configChan, safe.NewPool(context.Background()))
@ -204,10 +40,10 @@ func TestProvideWithoutWatch(t *testing.T) {
timeout := time.After(time.Second)
select {
case config := <-configChan:
assert.Len(t, config.Configuration.Backends, test.expectedNumBackend)
assert.Len(t, config.Configuration.Frontends, test.expectedNumFrontend)
assert.Len(t, config.Configuration.TLS, test.expectedNumTLSConf)
case conf := <-configChan:
assert.Len(t, conf.Configuration.Services, test.expectedNumService)
assert.Len(t, conf.Configuration.Routers, test.expectedNumRouter)
assert.Len(t, conf.Configuration.TLS, test.expectedNumTLSConf)
case <-timeout:
t.Errorf("timeout while waiting for config")
}
@ -217,13 +53,10 @@ func TestProvideWithoutWatch(t *testing.T) {
func TestProvideWithWatch(t *testing.T) {
for _, test := range getTestCases() {
test := test
t.Run(test.desc+" with watch", func(t *testing.T) {
t.Parallel()
provider, clean := createProvider(t, test, true)
defer clean()
configChan := make(chan types.ConfigMessage)
configChan := make(chan config.Message)
go func() {
err := provider.Provide(configChan, safe.NewPool(context.Background()))
@ -232,10 +65,10 @@ func TestProvideWithWatch(t *testing.T) {
timeout := time.After(time.Second)
select {
case config := <-configChan:
assert.Len(t, config.Configuration.Backends, 0)
assert.Len(t, config.Configuration.Frontends, 0)
assert.Len(t, config.Configuration.TLS, 0)
case conf := <-configChan:
assert.Len(t, conf.Configuration.Services, 0)
assert.Len(t, conf.Configuration.Routers, 0)
assert.Len(t, conf.Configuration.TLS, 0)
case <-timeout:
t.Errorf("timeout while waiting for config")
}
@ -259,17 +92,17 @@ func TestProvideWithWatch(t *testing.T) {
}
timeout = time.After(time.Second * 1)
var numUpdates, numBackends, numFrontends, numTLSConfs int
var numUpdates, numServices, numRouters, numTLSConfs int
for {
select {
case config := <-configChan:
case conf := <-configChan:
numUpdates++
numBackends = len(config.Configuration.Backends)
numFrontends = len(config.Configuration.Frontends)
numTLSConfs = len(config.Configuration.TLS)
t.Logf("received update #%d: backends %d/%d, frontends %d/%d, TLS configs %d/%d", numUpdates, numBackends, test.expectedNumBackend, numFrontends, test.expectedNumFrontend, numTLSConfs, test.expectedNumTLSConf)
numServices = len(conf.Configuration.Services)
numRouters = len(conf.Configuration.Routers)
numTLSConfs = len(conf.Configuration.TLS)
t.Logf("received update #%d: services %d/%d, routers %d/%d, TLS configs %d/%d", numUpdates, numServices, test.expectedNumService, numRouters, test.expectedNumRouter, numTLSConfs, test.expectedNumTLSConf)
if numBackends == test.expectedNumBackend && numFrontends == test.expectedNumFrontend && numTLSConfs == test.expectedNumTLSConf {
if numServices == test.expectedNumService && numRouters == test.expectedNumRouter && numTLSConfs == test.expectedNumTLSConf {
return
}
case <-timeout:
@ -282,7 +115,7 @@ func TestProvideWithWatch(t *testing.T) {
func TestErrorWhenEmptyConfig(t *testing.T) {
provider := &Provider{}
configChan := make(chan types.ConfigMessage)
configChan := make(chan config.Message)
errorChan := make(chan struct{})
go func() {
err := provider.Provide(configChan, safe.NewPool(context.Background()))
@ -300,6 +133,93 @@ func TestErrorWhenEmptyConfig(t *testing.T) {
}
}
func getTestCases() []ProvideTestCase {
return []ProvideTestCase{
{
desc: "simple file",
fileContent: createRoutersConfiguration(3) + createServicesConfiguration(6) + createTLS(5),
expectedNumRouter: 3,
expectedNumService: 6,
expectedNumTLSConf: 5,
},
{
desc: "simple file and a traefik file",
fileContent: createRoutersConfiguration(4) + createServicesConfiguration(8) + createTLS(4),
traefikFileContent: `
debug=true
`,
expectedNumRouter: 4,
expectedNumService: 8,
expectedNumTLSConf: 4,
},
{
desc: "template file",
fileContent: `
[routers]
{{ range $i, $e := until 20 }}
[routers.router{{ $e }}]
service = "application"
{{ end }}
`,
expectedNumRouter: 20,
},
{
desc: "simple directory",
directoryContent: []string{
createRoutersConfiguration(2),
createServicesConfiguration(3),
createTLS(4),
},
expectedNumRouter: 2,
expectedNumService: 3,
expectedNumTLSConf: 4,
},
{
desc: "template in directory",
directoryContent: []string{
`
[routers]
{{ range $i, $e := until 20 }}
[routers.router{{ $e }}]
service = "application"
{{ end }}
`,
`
[services]
{{ range $i, $e := until 20 }}
[services.application-{{ $e }}]
[[services.application-{{ $e }}.servers]]
url="http://127.0.0.1"
weight = 1
{{ end }}
`,
},
expectedNumRouter: 20,
expectedNumService: 20,
},
{
desc: "simple traefik file",
traefikFileContent: `
debug=true
[file]
` + createRoutersConfiguration(2) + createServicesConfiguration(3) + createTLS(4),
expectedNumRouter: 2,
expectedNumService: 3,
expectedNumTLSConf: 4,
},
{
desc: "simple traefik file with templating",
traefikFileContent: `
temp="{{ getTag \"test\" }}"
[file]
` + createRoutersConfiguration(2) + createServicesConfiguration(3) + createTLS(4),
expectedNumRouter: 2,
expectedNumService: 3,
expectedNumTLSConf: 4,
},
}
}
func createProvider(t *testing.T, test ProvideTestCase, watch bool) (*Provider, func()) {
tempDir := createTempDir(t, "testdir")
@ -336,3 +256,83 @@ func createProvider(t *testing.T, test ProvideTestCase, watch bool) (*Provider,
os.Remove(tempDir)
}
}
// createRandomFile Helper
func createRandomFile(t *testing.T, tempDir string, contents ...string) *os.File {
return createFile(t, tempDir, fmt.Sprintf("temp%d.toml", time.Now().UnixNano()), contents...)
}
// createFile Helper
func createFile(t *testing.T, tempDir string, name string, contents ...string) *os.File {
t.Helper()
fileName := path.Join(tempDir, name)
tempFile, err := os.Create(fileName)
if err != nil {
t.Fatal(err)
}
for _, content := range contents {
_, err = tempFile.WriteString(content)
if err != nil {
t.Fatal(err)
}
}
err = tempFile.Close()
if err != nil {
t.Fatal(err)
}
return tempFile
}
// createTempDir Helper
func createTempDir(t *testing.T, dir string) string {
t.Helper()
d, err := ioutil.TempDir("", dir)
if err != nil {
t.Fatal(err)
}
return d
}
// createRoutersConfiguration Helper
func createRoutersConfiguration(n int) string {
conf := "[routers]\n"
for i := 1; i <= n; i++ {
conf += fmt.Sprintf(`
[routers."router%[1]d"]
service = "application-%[1]d"
`, i)
}
return conf
}
// createServicesConfiguration Helper
func createServicesConfiguration(n int) string {
conf := "[services]\n"
for i := 1; i <= n; i++ {
conf += fmt.Sprintf(`
[services.application-%[1]d.loadbalancer]
[[services.application-%[1]d.loadbalancer.servers]]
url = "http://172.17.0.%[1]d:80"
weight = 1
`, i)
}
return conf
}
// createTLS Helper
func createTLS(n int) string {
var conf string
for i := 1; i <= n; i++ {
conf += fmt.Sprintf(`[[TLS]]
EntryPoints = ["https"]
[TLS.Certificate]
CertFile = "integration/fixtures/https/snitest%[1]d.com.cert"
KeyFile = "integration/fixtures/https/snitest%[1]d.com.key"
`, i)
}
return conf
}