homelab/traefik
1
0
Fork 0
Code Activity

Merge current v2.8 into master

Browse source
This commit is contained in:
kevinpollet 2022-07-25 17:31:51 +02:00
commit ab94bbaece
No known key found for this signature in database
GPG key ID: 0C9A5DDD1B292453
71 changed files with 1093 additions and 475 deletions
Download patch file Download diff file Expand all files Collapse all files

2
pkg/metrics/datadog.go
View file

@ -53,7 +53,7 @@ func RegisterDatadog(ctx context.Context, config *types.Datadog) Registry {
}
datadogClient = dogstatsd.New(config.Prefix+".", kitlog.LoggerFunc(func(keyvals ...interface{}) error {
log.WithoutContext().WithField(log.MetricsProviderName, "datadog").Info(keyvals)
log.WithoutContext().WithField(log.MetricsProviderName, "datadog").Info(keyvals...)
return nil
}))

2
pkg/metrics/influxdb.go
View file

@ -136,7 +136,7 @@ func initInfluxDBClient(ctx context.Context, config *types.InfluxDB) *influx.Inf
RetentionPolicy: config.RetentionPolicy,
},
kitlog.LoggerFunc(func(keyvals ...interface{}) error {
log.WithoutContext().WithField(log.MetricsProviderName, "influxdb").Info(keyvals)
log.WithoutContext().WithField(log.MetricsProviderName, "influxdb").Info(keyvals...)
return nil
}))
}

2
pkg/metrics/influxdb2.go
View file

@ -38,7 +38,7 @@ func RegisterInfluxDB2(ctx context.Context, config *types.InfluxDB2) Registry {
config.AdditionalLabels,
influxdb.BatchPointsConfig{},
kitlog.LoggerFunc(func(kv ...interface{}) error {
log.FromContext(ctx).Error(kv)
log.FromContext(ctx).Error(kv...)
return nil
}),
)

351
pkg/metrics/prometheus.go
View file

@ -4,8 +4,6 @@ import (
"context"
"errors"
"net/http"
"sort"
"strings"
"sync"
"time"
@ -15,7 +13,6 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/traefik/traefik/v2/pkg/config/dynamic"
"github.com/traefik/traefik/v2/pkg/log"
"github.com/traefik/traefik/v2/pkg/safe"
"github.com/traefik/traefik/v2/pkg/types"
)
@ -111,37 +108,33 @@ func initStandardRegistry(config *types.Prometheus) Registry {
buckets = config.Buckets
}
safe.Go(func() {
promState.ListenValueUpdates()
})
configReloads := newCounterFrom(promState.collectors, stdprometheus.CounterOpts{
configReloads := newCounterFrom(stdprometheus.CounterOpts{
Name: configReloadsTotalName,
Help: "Config reloads",
}, []string{})
configReloadsFailures := newCounterFrom(promState.collectors, stdprometheus.CounterOpts{
configReloadsFailures := newCounterFrom(stdprometheus.CounterOpts{
Name: configReloadsFailuresTotalName,
Help: "Config failure reloads",
}, []string{})
lastConfigReloadSuccess := newGaugeFrom(promState.collectors, stdprometheus.GaugeOpts{
lastConfigReloadSuccess := newGaugeFrom(stdprometheus.GaugeOpts{
Name: configLastReloadSuccessName,
Help: "Last config reload success",
}, []string{})
lastConfigReloadFailure := newGaugeFrom(promState.collectors, stdprometheus.GaugeOpts{
lastConfigReloadFailure := newGaugeFrom(stdprometheus.GaugeOpts{
Name: configLastReloadFailureName,
Help: "Last config reload failure",
}, []string{})
tlsCertsNotAfterTimestamp := newGaugeFrom(promState.collectors, stdprometheus.GaugeOpts{
tlsCertsNotAfterTimestamp := newGaugeFrom(stdprometheus.GaugeOpts{
Name: tlsCertsNotAfterTimestamp,
Help: "Certificate expiration timestamp",
}, []string{"cn", "serial", "sans"})
promState.describers = []func(chan<- *stdprometheus.Desc){
configReloads.cv.Describe,
configReloadsFailures.cv.Describe,
lastConfigReloadSuccess.gv.Describe,
lastConfigReloadFailure.gv.Describe,
tlsCertsNotAfterTimestamp.gv.Describe,
promState.vectors = []vector{
configReloads.cv,
configReloadsFailures.cv,
lastConfigReloadSuccess.gv,
lastConfigReloadFailure.gv,
tlsCertsNotAfterTimestamp.gv,
}
reg := &standardRegistry{
@ -156,30 +149,30 @@ func initStandardRegistry(config *types.Prometheus) Registry {
}
if config.AddEntryPointsLabels {
entryPointReqs := newCounterFrom(promState.collectors, stdprometheus.CounterOpts{
entryPointReqs := newCounterFrom(stdprometheus.CounterOpts{
Name: entryPointReqsTotalName,
Help: "How many HTTP requests processed on an entrypoint, partitioned by status code, protocol, and method.",
}, []string{"code", "method", "protocol", "entrypoint"})
entryPointReqsTLS := newCounterFrom(promState.collectors, stdprometheus.CounterOpts{
entryPointReqsTLS := newCounterFrom(stdprometheus.CounterOpts{
Name: entryPointReqsTLSTotalName,
Help: "How many HTTP requests with TLS processed on an entrypoint, partitioned by TLS Version and TLS cipher Used.",
}, []string{"tls_version", "tls_cipher", "entrypoint"})
entryPointReqDurations := newHistogramFrom(promState.collectors, stdprometheus.HistogramOpts{
entryPointReqDurations := newHistogramFrom(stdprometheus.HistogramOpts{
Name: entryPointReqDurationName,
Help: "How long it took to process the request on an entrypoint, partitioned by status code, protocol, and method.",
Buckets: buckets,
}, []string{"code", "method", "protocol", "entrypoint"})
entryPointOpenConns := newGaugeFrom(promState.collectors, stdprometheus.GaugeOpts{
entryPointOpenConns := newGaugeFrom(stdprometheus.GaugeOpts{
Name: entryPointOpenConnsName,
Help: "How many open connections exist on an entrypoint, partitioned by method and protocol.",
}, []string{"method", "protocol", "entrypoint"})
promState.describers = append(promState.describers, []func(chan<- *stdprometheus.Desc){
entryPointReqs.cv.Describe,
entryPointReqsTLS.cv.Describe,
entryPointReqDurations.hv.Describe,
entryPointOpenConns.gv.Describe,
}...)
promState.vectors = append(promState.vectors,
entryPointReqs.cv,
entryPointReqsTLS.cv,
entryPointReqDurations.hv,
entryPointOpenConns.gv,
)
reg.entryPointReqsCounter = entryPointReqs
reg.entryPointReqsTLSCounter = entryPointReqsTLS
@ -188,30 +181,30 @@ func initStandardRegistry(config *types.Prometheus) Registry {
}
if config.AddRoutersLabels {
routerReqs := newCounterFrom(promState.collectors, stdprometheus.CounterOpts{
routerReqs := newCounterFrom(stdprometheus.CounterOpts{
Name: routerReqsTotalName,
Help: "How many HTTP requests are processed on a router, partitioned by service, status code, protocol, and method.",
}, []string{"code", "method", "protocol", "router", "service"})
routerReqsTLS := newCounterFrom(promState.collectors, stdprometheus.CounterOpts{
routerReqsTLS := newCounterFrom(stdprometheus.CounterOpts{
Name: routerReqsTLSTotalName,
Help: "How many HTTP requests with TLS are processed on a router, partitioned by service, TLS Version, and TLS cipher Used.",
}, []string{"tls_version", "tls_cipher", "router", "service"})
routerReqDurations := newHistogramFrom(promState.collectors, stdprometheus.HistogramOpts{
routerReqDurations := newHistogramFrom(stdprometheus.HistogramOpts{
Name: routerReqDurationName,
Help: "How long it took to process the request on a router, partitioned by service, status code, protocol, and method.",
Buckets: buckets,
}, []string{"code", "method", "protocol", "router", "service"})
routerOpenConns := newGaugeFrom(promState.collectors, stdprometheus.GaugeOpts{
routerOpenConns := newGaugeFrom(stdprometheus.GaugeOpts{
Name: routerOpenConnsName,
Help: "How many open connections exist on a router, partitioned by service, method, and protocol.",
}, []string{"method", "protocol", "router", "service"})
promState.describers = append(promState.describers, []func(chan<- *stdprometheus.Desc){
routerReqs.cv.Describe,
routerReqsTLS.cv.Describe,
routerReqDurations.hv.Describe,
routerOpenConns.gv.Describe,
}...)
promState.vectors = append(promState.vectors,
routerReqs.cv,
routerReqsTLS.cv,
routerReqDurations.hv,
routerOpenConns.gv,
)
reg.routerReqsCounter = routerReqs
reg.routerReqsTLSCounter = routerReqsTLS
reg.routerReqDurationHistogram, _ = NewHistogramWithScale(routerReqDurations, time.Second)
@ -219,40 +212,40 @@ func initStandardRegistry(config *types.Prometheus) Registry {
}
if config.AddServicesLabels {
serviceReqs := newCounterFrom(promState.collectors, stdprometheus.CounterOpts{
serviceReqs := newCounterFrom(stdprometheus.CounterOpts{
Name: serviceReqsTotalName,
Help: "How many HTTP requests processed on a service, partitioned by status code, protocol, and method.",
}, []string{"code", "method", "protocol", "service"})
serviceReqsTLS := newCounterFrom(promState.collectors, stdprometheus.CounterOpts{
serviceReqsTLS := newCounterFrom(stdprometheus.CounterOpts{
Name: serviceReqsTLSTotalName,
Help: "How many HTTP requests with TLS processed on a service, partitioned by TLS version and TLS cipher.",
}, []string{"tls_version", "tls_cipher", "service"})
serviceReqDurations := newHistogramFrom(promState.collectors, stdprometheus.HistogramOpts{
serviceReqDurations := newHistogramFrom(stdprometheus.HistogramOpts{
Name: serviceReqDurationName,
Help: "How long it took to process the request on a service, partitioned by status code, protocol, and method.",
Buckets: buckets,
}, []string{"code", "method", "protocol", "service"})
serviceOpenConns := newGaugeFrom(promState.collectors, stdprometheus.GaugeOpts{
serviceOpenConns := newGaugeFrom(stdprometheus.GaugeOpts{
Name: serviceOpenConnsName,
Help: "How many open connections exist on a service, partitioned by method and protocol.",
}, []string{"method", "protocol", "service"})
serviceRetries := newCounterFrom(promState.collectors, stdprometheus.CounterOpts{
serviceRetries := newCounterFrom(stdprometheus.CounterOpts{
Name: serviceRetriesTotalName,
Help: "How many request retries happened on a service.",
}, []string{"service"})
serviceServerUp := newGaugeFrom(promState.collectors, stdprometheus.GaugeOpts{
serviceServerUp := newGaugeFrom(stdprometheus.GaugeOpts{
Name: serviceServerUpName,
Help: "service server is up, described by gauge value of 0 or 1.",
}, []string{"service", "url"})
promState.describers = append(promState.describers, []func(chan<- *stdprometheus.Desc){
serviceReqs.cv.Describe,
serviceReqsTLS.cv.Describe,
serviceReqDurations.hv.Describe,
serviceOpenConns.gv.Describe,
serviceRetries.cv.Describe,
serviceServerUp.gv.Describe,
}...)
promState.vectors = append(promState.vectors,
serviceReqs.cv,
serviceReqsTLS.cv,
serviceReqDurations.hv,
serviceOpenConns.gv,
serviceRetries.cv,
serviceServerUp.gv,
)
reg.serviceReqsCounter = serviceReqs
reg.serviceReqsTLSCounter = serviceReqsTLS
@ -287,64 +280,92 @@ func registerPromState(ctx context.Context) bool {
// It then converts the configuration to the optimized package internal format
// and sets it to the promState.
func OnConfigurationUpdate(conf dynamic.Configuration, entryPoints []string) {
dynamicConfig := newDynamicConfig()
dynCfg := newDynamicConfig()
for _, value := range entryPoints {
dynamicConfig.entryPoints[value] = true
dynCfg.entryPoints[value] = true
}
if conf.HTTP == nil {
promState.SetDynamicConfig(dynCfg)
return
}
for name := range conf.HTTP.Routers {
dynamicConfig.routers[name] = true
dynCfg.routers[name] = true
}
for serviceName, service := range conf.HTTP.Services {
dynamicConfig.services[serviceName] = make(map[string]bool)
dynCfg.services[serviceName] = make(map[string]bool)
if service.LoadBalancer != nil {
for _, server := range service.LoadBalancer.Servers {
dynamicConfig.services[serviceName][server.URL] = true
dynCfg.services[serviceName][server.URL] = true
}
}
}
promState.SetDynamicConfig(dynamicConfig)
promState.SetDynamicConfig(dynCfg)
}
func newPrometheusState() *prometheusState {
return &prometheusState{
collectors: make(chan *collector),
dynamicConfig: newDynamicConfig(),
state: make(map[string]*collector),
deletedURLs: make(map[string][]string),
}
}
type prometheusState struct {
collectors chan *collector
describers []func(ch chan<- *stdprometheus.Desc)
type vector interface {
stdprometheus.Collector
DeletePartialMatch(labels stdprometheus.Labels) int
}
mtx sync.Mutex
dynamicConfig *dynamicConfig
state map[string]*collector
type prometheusState struct {
vectors []vector
mtx sync.Mutex
dynamicConfig *dynamicConfig
deletedEP []string
deletedRouters []string
deletedServices []string
deletedURLs map[string][]string
}
func (ps *prometheusState) SetDynamicConfig(dynamicConfig *dynamicConfig) {
ps.mtx.Lock()
defer ps.mtx.Unlock()
ps.dynamicConfig = dynamicConfig
}
func (ps *prometheusState) ListenValueUpdates() {
for collector := range ps.collectors {
ps.mtx.Lock()
ps.state[collector.id] = collector
ps.mtx.Unlock()
for ep := range ps.dynamicConfig.entryPoints {
if _, ok := dynamicConfig.entryPoints[ep]; !ok {
ps.deletedEP = append(ps.deletedEP, ep)
}
}
for router := range ps.dynamicConfig.routers {
if _, ok := dynamicConfig.routers[router]; !ok {
ps.deletedRouters = append(ps.deletedRouters, router)
}
}
for service, serV := range ps.dynamicConfig.services {
actualService, ok := dynamicConfig.services[service]
if !ok {
ps.deletedServices = append(ps.deletedServices, service)
}
for url := range serV {
if _, ok := actualService[url]; !ok {
ps.deletedURLs[service] = append(ps.deletedURLs[service], url)
}
}
}
ps.dynamicConfig = dynamicConfig
}
// Describe implements prometheus.Collector and simply calls
// the registered describer functions.
func (ps *prometheusState) Describe(ch chan<- *stdprometheus.Desc) {
for _, desc := range ps.describers {
desc(ch)
for _, v := range ps.vectors {
v.Describe(ch)
}
}
@ -354,49 +375,54 @@ func (ps *prometheusState) Describe(ch chan<- *stdprometheus.Desc) {
// The removal happens only after their Collect method was called to ensure that
// also those metrics will be exported on the current scrape.
func (ps *prometheusState) Collect(ch chan<- stdprometheus.Metric) {
for _, v := range ps.vectors {
v.Collect(ch)
}
ps.mtx.Lock()
defer ps.mtx.Unlock()
var outdatedKeys []string
for key, cs := range ps.state {
cs.collector.Collect(ch)
if ps.isOutdated(cs) {
outdatedKeys = append(outdatedKeys, key)
for _, ep := range ps.deletedEP {
if !ps.dynamicConfig.hasEntryPoint(ep) {
ps.DeletePartialMatch(map[string]string{"entrypoint": ep})
}
}
for _, key := range outdatedKeys {
ps.state[key].delete()
delete(ps.state, key)
for _, router := range ps.deletedRouters {
if !ps.dynamicConfig.hasRouter(router) {
ps.DeletePartialMatch(map[string]string{"router": router})
}
}
for _, service := range ps.deletedServices {
if !ps.dynamicConfig.hasService(service) {
ps.DeletePartialMatch(map[string]string{"service": service})
}
}
for service, urls := range ps.deletedURLs {
for _, url := range urls {
if !ps.dynamicConfig.hasServerURL(service, url) {
ps.DeletePartialMatch(map[string]string{"service": service, "url": url})
}
}
}
ps.deletedEP = nil
ps.deletedRouters = nil
ps.deletedServices = nil
ps.deletedURLs = make(map[string][]string)
}
// isOutdated checks whether the passed collector has labels that mark
// it as belonging to an outdated configuration of Traefik.
func (ps *prometheusState) isOutdated(collector *collector) bool {
labels := collector.labels
if entrypointName, ok := labels["entrypoint"]; ok && !ps.dynamicConfig.hasEntryPoint(entrypointName) {
return true
// DeletePartialMatch deletes all metrics where the variable labels contain all of those passed in as labels.
// The order of the labels does not matter.
// It returns the number of metrics deleted.
func (ps *prometheusState) DeletePartialMatch(labels stdprometheus.Labels) int {
var count int
for _, elem := range ps.vectors {
count += elem.DeletePartialMatch(labels)
}
if routerName, ok := labels["router"]; ok {
if !ps.dynamicConfig.hasRouter(routerName) {
return true
}
}
if serviceName, ok := labels["service"]; ok {
if !ps.dynamicConfig.hasService(serviceName) {
return true
}
if url, ok := labels["url"]; ok && !ps.dynamicConfig.hasServerURL(serviceName, url) {
return true
}
}
return false
return count
}
func newDynamicConfig() *dynamicConfig {
@ -440,42 +466,15 @@ func (d *dynamicConfig) hasServerURL(serviceName, serverURL string) bool {
return false
}
func newCollector(metricName string, labels stdprometheus.Labels, c stdprometheus.Collector, deleteFn func()) *collector {
return &collector{
id: buildMetricID(metricName, labels),
labels: labels,
collector: c,
delete: deleteFn,
}
}
// collector wraps a Collector object from the Prometheus client library.
// It adds information on how many generations this metric should be present
// in the /metrics output, relative to the time it was last tracked.
type collector struct {
id string
labels stdprometheus.Labels
collector stdprometheus.Collector
delete func()
}
func buildMetricID(metricName string, labels stdprometheus.Labels) string {
var labelNamesValues []string
for name, value := range labels {
labelNamesValues = append(labelNamesValues, name, value)
}
sort.Strings(labelNamesValues)
return metricName + ":" + strings.Join(labelNamesValues, "|")
}
func newCounterFrom(collectors chan<- *collector, opts stdprometheus.CounterOpts, labelNames []string) *counter {
func newCounterFrom(opts stdprometheus.CounterOpts, labelNames []string) *counter {
cv := stdprometheus.NewCounterVec(opts, labelNames)
c := &counter{
name: opts.Name,
cv: cv,
collectors: collectors,
name: opts.Name,
cv: cv,
labelNamesValues: make([]string, 0, 16),
}
if len(labelNames) == 0 {
c.collector = cv.WithLabelValues()
c.Add(0)
}
return c
@ -485,39 +484,37 @@ type counter struct {
name string
cv *stdprometheus.CounterVec
labelNamesValues labelNamesValues
collectors chan<- *collector
collector stdprometheus.Counter
}
func (c *counter) With(labelValues ...string) metrics.Counter {
lnv := c.labelNamesValues.With(labelValues...)
return &counter{
name: c.name,
cv: c.cv,
labelNamesValues: c.labelNamesValues.With(labelValues...),
collectors: c.collectors,
labelNamesValues: lnv,
collector: c.cv.With(lnv.ToLabels()),
}
}
func (c *counter) Add(delta float64) {
labels := c.labelNamesValues.ToLabels()
collector := c.cv.With(labels)
collector.Add(delta)
c.collectors <- newCollector(c.name, labels, collector, func() {
c.cv.Delete(labels)
})
c.collector.Add(delta)
}
func (c *counter) Describe(ch chan<- *stdprometheus.Desc) {
c.cv.Describe(ch)
}
func newGaugeFrom(collectors chan<- *collector, opts stdprometheus.GaugeOpts, labelNames []string) *gauge {
func newGaugeFrom(opts stdprometheus.GaugeOpts, labelNames []string) *gauge {
gv := stdprometheus.NewGaugeVec(opts, labelNames)
g := &gauge{
name: opts.Name,
gv: gv,
collectors: collectors,
name: opts.Name,
gv: gv,
labelNamesValues: make([]string, 0, 16),
}
if len(labelNames) == 0 {
g.collector = gv.WithLabelValues()
g.Set(0)
}
return g
@ -527,46 +524,37 @@ type gauge struct {
name string
gv *stdprometheus.GaugeVec
labelNamesValues labelNamesValues
collectors chan<- *collector
collector stdprometheus.Gauge
}
func (g *gauge) With(labelValues ...string) metrics.Gauge {
lnv := g.labelNamesValues.With(labelValues...)
return &gauge{
name: g.name,
gv: g.gv,
labelNamesValues: g.labelNamesValues.With(labelValues...),
collectors: g.collectors,
labelNamesValues: lnv,
collector: g.gv.With(lnv.ToLabels()),
}
}
func (g *gauge) Add(delta float64) {
labels := g.labelNamesValues.ToLabels()
collector := g.gv.With(labels)
collector.Add(delta)
g.collectors <- newCollector(g.name, labels, collector, func() {
g.gv.Delete(labels)
})
g.collector.Add(delta)
}
func (g *gauge) Set(value float64) {
labels := g.labelNamesValues.ToLabels()
collector := g.gv.With(labels)
collector.Set(value)
g.collectors <- newCollector(g.name, labels, collector, func() {
g.gv.Delete(labels)
})
g.collector.Set(value)
}
func (g *gauge) Describe(ch chan<- *stdprometheus.Desc) {
g.gv.Describe(ch)
}
func newHistogramFrom(collectors chan<- *collector, opts stdprometheus.HistogramOpts, labelNames []string) *histogram {
func newHistogramFrom(opts stdprometheus.HistogramOpts, labelNames []string) *histogram {
hv := stdprometheus.NewHistogramVec(opts, labelNames)
return &histogram{
name: opts.Name,
hv: hv,
collectors: collectors,
name: opts.Name,
hv: hv,
labelNamesValues: make([]string, 0, 16),
}
}
@ -574,28 +562,21 @@ type histogram struct {
name string
hv *stdprometheus.HistogramVec
labelNamesValues labelNamesValues
collectors chan<- *collector
collector stdprometheus.Observer
}
func (h *histogram) With(labelValues ...string) metrics.Histogram {
lnv := h.labelNamesValues.With(labelValues...)
return &histogram{
name: h.name,
hv: h.hv,
labelNamesValues: h.labelNamesValues.With(labelValues...),
collectors: h.collectors,
labelNamesValues: lnv,
collector: h.hv.With(lnv.ToLabels()),
}
}
func (h *histogram) Observe(value float64) {
labels := h.labelNamesValues.ToLabels()
observer := h.hv.With(labels)
observer.Observe(value)
// Do a type assertion to be sure that prometheus will be able to call the Collect method.
if collector, ok := observer.(stdprometheus.Histogram); ok {
h.collectors <- newCollector(h.name, labels, collector, func() {
h.hv.Delete(labels)
})
}
h.collector.Observe(value)
}
func (h *histogram) Describe(ch chan<- *stdprometheus.Desc) {
@ -618,7 +599,7 @@ func (lvs labelNamesValues) With(labelValues ...string) labelNamesValues {
// ToLabels is a convenience method to convert a labelNamesValues
// to the native prometheus.Labels.
func (lvs labelNamesValues) ToLabels() stdprometheus.Labels {
labels := stdprometheus.Labels{}
labels := make(map[string]string, len(lvs)/2)
for i := 0; i < len(lvs); i += 2 {
labels[lvs[i]] = lvs[i+1]
}

164
pkg/metrics/prometheus_test.go
View file

@ -17,8 +17,7 @@ import (
)
func TestRegisterPromState(t *testing.T) {
// Reset state of global promState.
defer promState.reset()
t.Cleanup(promState.reset)
testCases := []struct {
desc string
@ -88,21 +87,10 @@ func TestRegisterPromState(t *testing.T) {
}
}
// reset is a utility method for unit testing. It should be called after each
// test run that changes promState internally in order to avoid dependencies
// between unit tests.
func (ps *prometheusState) reset() {
ps.collectors = make(chan *collector)
ps.describers = []func(ch chan<- *prometheus.Desc){}
ps.dynamicConfig = newDynamicConfig()
ps.state = make(map[string]*collector)
}
func TestPrometheus(t *testing.T) {
promState = newPrometheusState()
promRegistry = prometheus.NewRegistry()
// Reset state of global promState.
defer promState.reset()
t.Cleanup(promState.reset)
prometheusRegistry := RegisterPrometheus(context.Background(), &types.Prometheus{AddEntryPointsLabels: true, AddRoutersLabels: true, AddServicesLabels: true})
defer promRegistry.Unregister(promState)
@ -361,30 +349,41 @@ func TestPrometheus(t *testing.T) {
func TestPrometheusMetricRemoval(t *testing.T) {
promState = newPrometheusState()
promRegistry = prometheus.NewRegistry()
// Reset state of global promState.
defer promState.reset()
t.Cleanup(promState.reset)
prometheusRegistry := RegisterPrometheus(context.Background(), &types.Prometheus{AddEntryPointsLabels: true, AddServicesLabels: true, AddRoutersLabels: true})
defer promRegistry.Unregister(promState)
conf := dynamic.Configuration{
conf1 := dynamic.Configuration{
HTTP: th.BuildConfiguration(
th.WithRouters(
th.WithRouter("foo@providerName",
th.WithServiceName("bar")),
th.WithRouter("foo@providerName", th.WithServiceName("bar")),
th.WithRouter("router2", th.WithServiceName("bar@providerName")),
),
th.WithLoadBalancerServices(th.WithService("bar@providerName",
th.WithServers(th.WithServer("http://localhost:9000"))),
th.WithLoadBalancerServices(
th.WithService("bar@providerName", th.WithServers(
th.WithServer("http://localhost:9000"),
th.WithServer("http://localhost:9999"),
th.WithServer("http://localhost:9998"),
)),
th.WithService("service1", th.WithServers(th.WithServer("http://localhost:9000"))),
),
func(cfg *dynamic.HTTPConfiguration) {
cfg.Services["fii"] = &dynamic.Service{
Weighted: &dynamic.WeightedRoundRobin{},
}
},
),
}
OnConfigurationUpdate(conf, []string{"entrypoint1"})
conf2 := dynamic.Configuration{
HTTP: th.BuildConfiguration(
th.WithRouters(
th.WithRouter("foo@providerName", th.WithServiceName("bar")),
),
th.WithLoadBalancerServices(
th.WithService("bar@providerName", th.WithServers(th.WithServer("http://localhost:9000"))),
),
),
}
OnConfigurationUpdate(conf1, []string{"entrypoint1", "entrypoint2"})
OnConfigurationUpdate(conf2, []string{"entrypoint1"})
// Register some metrics manually that are not part of the active configuration.
// Those metrics should be part of the /metrics output on the first scrape but
@ -393,22 +392,25 @@ func TestPrometheusMetricRemoval(t *testing.T) {
EntryPointReqsCounter().
With("entrypoint", "entrypoint2", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
prometheusRegistry.
RouterReqsCounter().
With("router", "router2", "service", "bar@providerName", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
prometheusRegistry.
ServiceReqsCounter().
With("service", "service2", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
With("service", "service1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
prometheusRegistry.
ServiceServerUpGauge().
With("service", "service1", "url", "http://localhost:9999").
With("service", "bar@providerName", "url", "http://localhost:9999").
Set(1)
prometheusRegistry.
RouterReqsCounter().
With("router", "router2", "service", "service2", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
ServiceServerUpGauge().
With("service", "bar@providerName", "url", "http://localhost:9998").
Set(1)
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName, serviceReqsTotalName, serviceServerUpName)
assertMetricsAbsent(t, mustScrape(), entryPointReqsTotalName, serviceReqsTotalName, serviceServerUpName)
assertMetricsAbsent(t, mustScrape(), routerReqsTotalName, routerReqDurationName, routerOpenConnsName)
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName, routerReqsTotalName, serviceReqsTotalName, serviceServerUpName)
assertMetricsAbsent(t, mustScrape(), entryPointReqsTotalName, routerReqsTotalName, serviceReqsTotalName, serviceServerUpName)
// To verify that metrics belonging to active configurations are not removed
// here the counter examples.
@ -418,24 +420,80 @@ func TestPrometheusMetricRemoval(t *testing.T) {
Add(1)
prometheusRegistry.
RouterReqsCounter().
With("router", "foo@providerName", "service", "bar@providerName", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
With("router", "foo@providerName", "service", "bar", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
prometheusRegistry.
ServiceReqsCounter().
With("service", "bar@providerName", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
prometheusRegistry.
ServiceServerUpGauge().
With("service", "bar@providerName", "url", "http://localhost:9000").
Set(1)
delayForTrackingCompletion()
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName)
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName)
assertMetricsExist(t, mustScrape(), routerReqsTotalName)
assertMetricsExist(t, mustScrape(), routerReqsTotalName)
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName, serviceReqsTotalName, serviceServerUpName, routerReqsTotalName)
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName, serviceReqsTotalName, serviceServerUpName, routerReqsTotalName)
}
func TestPrometheusMetricRemoveEndpointForRecoveredService(t *testing.T) {
promState = newPrometheusState()
promRegistry = prometheus.NewRegistry()
t.Cleanup(promState.reset)
prometheusRegistry := RegisterPrometheus(context.Background(), &types.Prometheus{AddServicesLabels: true})
defer promRegistry.Unregister(promState)
conf1 := dynamic.Configuration{
HTTP: th.BuildConfiguration(
th.WithLoadBalancerServices(
th.WithService("service1", th.WithServers(th.WithServer("http://localhost:9000"))),
),
),
}
conf2 := dynamic.Configuration{
HTTP: th.BuildConfiguration(),
}
conf3 := dynamic.Configuration{
HTTP: th.BuildConfiguration(
th.WithLoadBalancerServices(
th.WithService("service1", th.WithServers(th.WithServer("http://localhost:9001"))),
),
),
}
OnConfigurationUpdate(conf1, []string{})
OnConfigurationUpdate(conf2, []string{})
OnConfigurationUpdate(conf3, []string{})
prometheusRegistry.
ServiceServerUpGauge().
With("service", "service1", "url", "http://localhost:9000").
Add(1)
assertMetricsExist(t, mustScrape(), serviceServerUpName)
assertMetricsAbsent(t, mustScrape(), serviceServerUpName)
}
func TestPrometheusRemovedMetricsReset(t *testing.T) {
// Reset state of global promState.
defer promState.reset()
t.Cleanup(promState.reset)
prometheusRegistry := RegisterPrometheus(context.Background(), &types.Prometheus{AddEntryPointsLabels: true, AddServicesLabels: true})
defer promRegistry.Unregister(promState)
conf1 := dynamic.Configuration{
HTTP: th.BuildConfiguration(
th.WithLoadBalancerServices(th.WithService("service",
th.WithServers(th.WithServer("http://localhost:9000"))),
),
),
}
OnConfigurationUpdate(conf1, []string{"entrypoint1", "entrypoint2"})
OnConfigurationUpdate(dynamic.Configuration{}, nil)
labelNamesValues := []string{
"service", "service",
"code", strconv.Itoa(http.StatusOK),
@ -467,12 +525,24 @@ func TestPrometheusRemovedMetricsReset(t *testing.T) {
assertCounterValue(t, 1, findMetricFamily(serviceReqsTotalName, metricsFamilies), labelNamesValues...)
}
// reset is a utility method for unit testing.
// It should be called after each test run that changes promState internally
// in order to avoid dependencies between unit tests.
func (ps *prometheusState) reset() {
ps.dynamicConfig = newDynamicConfig()
ps.vectors = nil
ps.deletedEP = nil
ps.deletedRouters = nil
ps.deletedServices = nil
ps.deletedURLs = make(map[string][]string)
}
// Tracking and gathering the metrics happens concurrently.
// In practice this is no problem, because in case a tracked metric would miss
// the current scrape, it would just be there in the next one.
// That we can test reliably the tracking of all metrics here, we sleep
// for a short amount of time, to make sure the metric will be present
// in the next scrape.
// In practice this is no problem, because in case a tracked metric would miss the current scrape,
// it would just be there in the next one.
// That we can test reliably the tracking of all metrics here,
// we sleep for a short amount of time,
// to make sure the metric will be present in the next scrape.
func delayForTrackingCompletion() {
time.Sleep(250 * time.Millisecond)
}

2
pkg/metrics/statsd.go
View file

@ -50,7 +50,7 @@ func RegisterStatsd(ctx context.Context, config *types.Statsd) Registry {
}
statsdClient = statsd.New(config.Prefix+".", kitlog.LoggerFunc(func(keyvals ...interface{}) error {
log.WithoutContext().WithField(log.MetricsProviderName, "statsd").Info(keyvals)
log.WithoutContext().WithField(log.MetricsProviderName, "statsd").Info(keyvals...)
return nil
}))

3
pkg/middlewares/forwardedheaders/forwarded_header.go
View file

@ -142,6 +142,9 @@ func (x *XForwarded) rewrite(outreq *http.Request) {
xfProto := unsafeHeader(outreq.Header).Get(xForwardedProto)
if xfProto == "" {
// TODO: is this expected to set the X-Forwarded-Proto header value to
// ws(s) as the underlying request used to upgrade the connection is
// made over HTTP(S)?
if isWebsocketRequest(outreq) {
if outreq.TLS != nil {
unsafeHeader(outreq.Header).Set(xForwardedProto, "wss")

8
pkg/middlewares/metrics/metrics.go
View file

@ -103,8 +103,9 @@ func (m *metricsMiddleware) ServeHTTP(rw http.ResponseWriter, req *http.Request)
labels = append(labels, m.baseLabels...)
labels = append(labels, "method", getMethod(req), "protocol", getRequestProtocol(req))
m.openConnsGauge.With(labels...).Add(1)
defer m.openConnsGauge.With(labels...).Add(-1)
openConnsGauge := m.openConnsGauge.With(labels...)
openConnsGauge.Add(1)
defer openConnsGauge.Add(-1)
// TLS metrics
if req.TLS != nil {
@ -122,8 +123,7 @@ func (m *metricsMiddleware) ServeHTTP(rw http.ResponseWriter, req *http.Request)
labels = append(labels, "code", strconv.Itoa(recorder.getCode()))
histograms := m.reqDurationHistogram.With(labels...)
histograms.ObserveFromStart(start)
m.reqDurationHistogram.With(labels...).ObserveFromStart(start)
m.reqsCounter.With(labels...).Add(1)
}

20
pkg/middlewares/redirect/redirect_scheme.go
View file

@ -33,10 +33,10 @@ func NewRedirectScheme(ctx context.Context, next http.Handler, conf dynamic.Redi
port = ":" + conf.Port
}
return newRedirect(next, uriPattern, conf.Scheme+"://${2}"+port+"${4}", conf.Permanent, rawURLScheme, name)
return newRedirect(next, uriPattern, conf.Scheme+"://${2}"+port+"${4}", conf.Permanent, clientRequestURL, name)
}
func rawURLScheme(req *http.Request) string {
func clientRequestURL(req *http.Request) string {
scheme := schemeHTTP
host, port, err := net.SplitHostPort(req.Host)
if err != nil {
@ -64,8 +64,20 @@ func rawURLScheme(req *http.Request) string {
scheme = schemeHTTPS
}
if value := req.Header.Get(xForwardedProto); value != "" {
scheme = value
if xProto := req.Header.Get(xForwardedProto); xProto != "" {
// When the initial request is a connection upgrade request,
// X-Forwarded-Proto header might have been set by a previous hop to ws(s),
// even though the actual protocol used so far is HTTP(s).
// Given that we're in a middleware that is only used in the context of HTTP(s) requests,
// the only possible valid schemes are one of "http" or "https", so we convert back to them.
switch {
case strings.EqualFold(xProto, "ws"):
scheme = schemeHTTP
case strings.EqualFold(xProto, "wss"):
scheme = schemeHTTPS
default:
scheme = xProto
}
}
if scheme == schemeHTTP && port == ":80" || scheme == schemeHTTPS && port == ":443" {

35
pkg/middlewares/redirect/redirect_scheme_test.go
View file

@ -63,6 +63,41 @@ func TestRedirectSchemeHandler(t *testing.T) {
},
expectedStatus: http.StatusOK,
},
{
desc: "HTTP to HTTPS, with X-Forwarded-Proto to unknown value",
config: dynamic.RedirectScheme{
Scheme: "https",
},
url: "http://foo",
headers: map[string]string{
"X-Forwarded-Proto": "bar",
},
expectedURL: "https://bar://foo",
expectedStatus: http.StatusFound,
},
{
desc: "HTTP to HTTPS, with X-Forwarded-Proto to ws",
config: dynamic.RedirectScheme{
Scheme: "https",
},
url: "http://foo",
headers: map[string]string{
"X-Forwarded-Proto": "ws",
},
expectedURL: "https://foo",
expectedStatus: http.StatusFound,
},
{
desc: "HTTP to HTTPS, with X-Forwarded-Proto to wss",
config: dynamic.RedirectScheme{
Scheme: "https",
},
url: "http://foo",
headers: map[string]string{
"X-Forwarded-Proto": "wss",
},
expectedStatus: http.StatusOK,
},
{
desc: "HTTP with port to HTTPS without port",
config: dynamic.RedirectScheme{

2
pkg/provider/consulcatalog/consul_catalog.go
View file

@ -46,7 +46,7 @@ type itemData struct {
// ProviderBuilder is responsible for constructing namespaced instances of the Consul Catalog provider.
type ProviderBuilder struct {
Configuration `export:"true"`
Configuration `yaml:",inline" export:"true"`
// Deprecated: use Namespaces option instead.
Namespace string `description:"Sets the namespace used to discover services (Consul Enterprise only)." json:"namespace,omitempty" toml:"namespace,omitempty" yaml:"namespace,omitempty"`

4
pkg/provider/kubernetes/crd/traefik/v1alpha1/service.go
View file

@ -11,8 +11,8 @@ import (
// TraefikService is the CRD implementation of a Traefik Service.
// TraefikService object allows to:
// - Apply weight to Services on load-balancing
// - Mirror traffic on services
// - Apply weight to Services on load-balancing
// - Mirror traffic on services
// More info: https://doc.traefik.io/traefik/v2.8/routing/providers/kubernetes-crd/#kind-traefikservice
type TraefikService struct {
metav1.TypeMeta `json:",inline"`

2
pkg/provider/kubernetes/ingress/client.go
View file

@ -570,7 +570,7 @@ func filterIngressClassByName(ingressClassName string, ics []*networkingv1.Ingre
return ingressClasses
}
// Ingress in networking.k8s.io/v1 is supported starting 1.19.
// Ingress in networking.k8s.io/v1 is supported starting 1.19.
// thus, we query it in K8s starting 1.19.
func supportsNetworkingV1Ingress(serverVersion *version.Version) bool {
ingressNetworkingVersion := version.Must(version.NewVersion("1.19"))

2
pkg/provider/kv/consul/consul.go
View file

@ -16,7 +16,7 @@ var _ provider.Provider = (*Provider)(nil)
// ProviderBuilder is responsible for constructing namespaced instances of the Consul provider.
type ProviderBuilder struct {
kv.Provider `export:"true"`
kv.Provider `yaml:",inline" export:"true"`
// Deprecated: use Namespaces instead.
Namespace string `description:"Sets the namespace used to discover the configuration (Consul Enterprise only)." json:"namespace,omitempty" toml:"namespace,omitempty" yaml:"namespace,omitempty"`

1
pkg/server/router/tcp/router_test.go
View file

@ -94,7 +94,6 @@ func (h *httpForwarder) Accept() (net.Conn, error) {
//
// - TCP-TLS HostSNI(`foobar`) and HTTPS PathPrefix(`/`)
// - On v2.6 and v2.7, the TCP-TLS one takes precedence.
//
func Test_Routing(t *testing.T) {
// This listener simulates the backend service.
// It is capable of switching into server first communication mode,

14
pkg/server/service/roundtripper_test.go
View file

@ -55,13 +55,13 @@ QPZ6VGR7+w1jB5BQXqEZcpHQIPSzeQJBAIy9tZJ/AYNlNbcegxEnsSjy/6VdlLsY
rqPRSAtd/h6oZbs=
-----END PRIVATE KEY-----`)
// openssl req -newkey rsa:2048 \
// -new -nodes -x509 \
// -days 3650 \
// -out cert.pem \
// -keyout key.pem \
// -subj "/CN=example.com"
// -addext "subjectAltName = DNS:example.com"
// openssl req -newkey rsa:2048 \
// -new -nodes -x509 \
// -days 3650 \
// -out cert.pem \
// -keyout key.pem \
// -subj "/CN=example.com"
// -addext "subjectAltName = DNS:example.com"
var mTLSCert = []byte(`-----BEGIN CERTIFICATE-----
MIIDJTCCAg2gAwIBAgIUYKnGcLnmMosOSKqTn4ydAMURE4gwDQYJKoZIhvcNAQEL
BQAwFjEUMBIGA1UEAwwLZXhhbXBsZS5jb20wHhcNMjAwODEzMDkyNzIwWhcNMzAw

37
pkg/tcp/chain.go
View file

@ -48,10 +48,10 @@ func (c Chain) Then(h Handler) (Handler, error) {
//
// Append returns a new chain, leaving the original one untouched.
//
// stdChain := tcp.NewChain(m1, m2)
// extChain := stdChain.Append(m3, m4)
// // requests in stdChain go m1 -> m2
// // requests in extChain go m1 -> m2 -> m3 -> m4
// stdChain := tcp.NewChain(m1, m2)
// extChain := stdChain.Append(m3, m4)
// // requests in stdChain go m1 -> m2
// // requests in extChain go m1 -> m2 -> m3 -> m4
func (c Chain) Append(constructors ...Constructor) Chain {
newCons := make([]Constructor, 0, len(c.constructors)+len(constructors))
newCons = append(newCons, c.constructors...)
@ -65,22 +65,23 @@ func (c Chain) Append(constructors ...Constructor) Chain {
//
// Extend returns a new chain, leaving the original one untouched.
//
// stdChain := tcp.NewChain(m1, m2)
// ext1Chain := tcp.NewChain(m3, m4)
// ext2Chain := stdChain.Extend(ext1Chain)
// // requests in stdChain go m1 -> m2
// // requests in ext1Chain go m3 -> m4
// // requests in ext2Chain go m1 -> m2 -> m3 -> m4
// stdChain := tcp.NewChain(m1, m2)
// ext1Chain := tcp.NewChain(m3, m4)
// ext2Chain := stdChain.Extend(ext1Chain)
// // requests in stdChain go m1 -> m2
// // requests in ext1Chain go m3 -> m4
// // requests in ext2Chain go m1 -> m2 -> m3 -> m4
//
// Another example:
// aHtmlAfterNosurf := tcp.NewChain(m2)
// aHtml := tcp.NewChain(m1, func(h tcp.Handler) tcp.Handler {
// csrf := nosurf.New(h)
// csrf.SetFailureHandler(aHtmlAfterNosurf.ThenFunc(csrfFail))
// return csrf
// }).Extend(aHtmlAfterNosurf)
// // requests to aHtml hitting nosurfs success handler go m1 -> nosurf -> m2 -> target-handler
// // requests to aHtml hitting nosurfs failure handler go m1 -> nosurf -> m2 -> csrfFail
//
// aHtmlAfterNosurf := tcp.NewChain(m2)
// aHtml := tcp.NewChain(m1, func(h tcp.Handler) tcp.Handler {
// csrf := nosurf.New(h)
// csrf.SetFailureHandler(aHtmlAfterNosurf.ThenFunc(csrfFail))
// return csrf
// }).Extend(aHtmlAfterNosurf)
// // requests to aHtml hitting nosurfs success handler go m1 -> nosurf -> m2 -> target-handler
// // requests to aHtml hitting nosurfs failure handler go m1 -> nosurf -> m2 -> csrfFail
func (c Chain) Extend(chain Chain) Chain {
return c.Append(chain.constructors...)
}

25
pkg/tls/certificate_store.go
View file

@ -73,17 +73,17 @@ func (c *CertificateStore) GetBestCertificate(clientHello *tls.ClientHelloInfo)
if c == nil {
return nil
}
domainToCheck := strings.ToLower(strings.TrimSpace(clientHello.ServerName))
if len(domainToCheck) == 0 {
serverName := strings.ToLower(strings.TrimSpace(clientHello.ServerName))
if len(serverName) == 0 {
// If no ServerName is provided, Check for local IP address matches
host, _, err := net.SplitHostPort(clientHello.Conn.LocalAddr().String())
if err != nil {
log.Debugf("Could not split host/port: %v", err)
log.WithoutContext().Debugf("Could not split host/port: %v", err)
}
domainToCheck = strings.TrimSpace(host)
serverName = strings.TrimSpace(host)
}
if cert, ok := c.CertCache.Get(domainToCheck); ok {
if cert, ok := c.CertCache.Get(serverName); ok {
return cert.(*tls.Certificate)
}
@ -91,7 +91,7 @@ func (c *CertificateStore) GetBestCertificate(clientHello *tls.ClientHelloInfo)
if c.DynamicCerts != nil && c.DynamicCerts.Get() != nil {
for domains, cert := range c.DynamicCerts.Get().(map[string]*tls.Certificate) {
for _, certDomain := range strings.Split(domains, ",") {
if MatchDomain(domainToCheck, certDomain) {
if matchDomain(serverName, certDomain) {
matchedCerts[certDomain] = cert
}
}
@ -107,7 +107,7 @@ func (c *CertificateStore) GetBestCertificate(clientHello *tls.ClientHelloInfo)
sort.Strings(keys)
// cache best match
c.CertCache.SetDefault(domainToCheck, matchedCerts[keys[len(keys)-1]])
c.CertCache.SetDefault(serverName, matchedCerts[keys[len(keys)-1]])
return matchedCerts[keys[len(keys)-1]]
}
@ -121,9 +121,12 @@ func (c CertificateStore) ResetCache() {
}
}
// MatchDomain return true if a domain match the cert domain.
func MatchDomain(domain, certDomain string) bool {
if domain == certDomain {
// matchDomain returns whether the server name matches the cert domain.
// The server name, from TLS SNI, must not have trailing dots (https://datatracker.ietf.org/doc/html/rfc6066#section-3).
// This is enforced by https://github.com/golang/go/blob/d3d7998756c33f69706488cade1cd2b9b10a4c7f/src/crypto/tls/handshake_messages.go#L423-L427.
func matchDomain(serverName, certDomain string) bool {
// TODO: assert equality after removing the trailing dots?
if serverName == certDomain {
return true
}
@ -131,7 +134,7 @@ func MatchDomain(domain, certDomain string) bool {
certDomain = certDomain[:len(certDomain)-1]
}
labels := strings.Split(domain, ".")
labels := strings.Split(serverName, ".")
for i := range labels {
labels[i] = "*"
candidate := strings.Join(labels, ".")

6
pkg/tracing/haystack/logger.go
View file

@ -10,15 +10,15 @@ type haystackLogger struct {
// Error prints the error message.
func (l haystackLogger) Error(format string, v ...interface{}) {
l.logger.Errorf(format, v)
l.logger.Errorf(format, v...)
}
// Info prints the info message.
func (l haystackLogger) Info(format string, v ...interface{}) {
l.logger.Infof(format, v)
l.logger.Infof(format, v...)
}
// Debug prints the info message.
func (l haystackLogger) Debug(format string, v ...interface{}) {
l.logger.Debug(format, v)
l.logger.Debugf(format, v...)
}