homelab/traefik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve performances when Prometheus metrics are enabled

Browse source
This commit is contained in:
Julien Salleyron 2022-07-07 18:00:09 +02:00 • committed by GitHub
parent 8f6463ba7a
commit 0231db05b4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 270 additions and 251 deletions
Download patch file Download diff file Expand all files Collapse all files

120
pkg/metrics/prometheus_test.go
View file

@ -17,8 +17,7 @@ import (
)
func TestRegisterPromState(t *testing.T) {
// Reset state of global promState.
defer promState.reset()
t.Cleanup(promState.reset)
testCases := []struct {
desc string
@ -88,21 +87,10 @@ func TestRegisterPromState(t *testing.T) {
}
}
// reset is a utility method for unit testing. It should be called after each
// test run that changes promState internally in order to avoid dependencies
// between unit tests.
func (ps *prometheusState) reset() {
ps.collectors = make(chan *collector)
ps.describers = []func(ch chan<- *prometheus.Desc){}
ps.dynamicConfig = newDynamicConfig()
ps.state = make(map[string]*collector)
}
func TestPrometheus(t *testing.T) {
promState = newPrometheusState()
promRegistry = prometheus.NewRegistry()
// Reset state of global promState.
defer promState.reset()
t.Cleanup(promState.reset)
prometheusRegistry := RegisterPrometheus(context.Background(), &types.Prometheus{AddEntryPointsLabels: true, AddRoutersLabels: true, AddServicesLabels: true})
defer promRegistry.Unregister(promState)
@ -361,30 +349,40 @@ func TestPrometheus(t *testing.T) {
func TestPrometheusMetricRemoval(t *testing.T) {
promState = newPrometheusState()
promRegistry = prometheus.NewRegistry()
// Reset state of global promState.
defer promState.reset()
t.Cleanup(promState.reset)
prometheusRegistry := RegisterPrometheus(context.Background(), &types.Prometheus{AddEntryPointsLabels: true, AddServicesLabels: true, AddRoutersLabels: true})
defer promRegistry.Unregister(promState)
conf := dynamic.Configuration{
conf1 := dynamic.Configuration{
HTTP: th.BuildConfiguration(
th.WithRouters(
th.WithRouter("foo@providerName",
th.WithServiceName("bar")),
th.WithRouter("foo@providerName", th.WithServiceName("bar")),
th.WithRouter("router2", th.WithServiceName("bar@providerName")),
),
th.WithLoadBalancerServices(th.WithService("bar@providerName",
th.WithServers(th.WithServer("http://localhost:9000"))),
th.WithLoadBalancerServices(
th.WithService("bar@providerName", th.WithServers(
th.WithServer("http://localhost:9000"),
th.WithServer("http://localhost:9999"),
)),
th.WithService("service1", th.WithServers(th.WithServer("http://localhost:9000"))),
),
func(cfg *dynamic.HTTPConfiguration) {
cfg.Services["fii"] = &dynamic.Service{
Weighted: &dynamic.WeightedRoundRobin{},
}
},
),
}
OnConfigurationUpdate(conf, []string{"entrypoint1"})
conf2 := dynamic.Configuration{
HTTP: th.BuildConfiguration(
th.WithRouters(
th.WithRouter("foo@providerName", th.WithServiceName("bar")),
),
th.WithLoadBalancerServices(
th.WithService("bar@providerName", th.WithServers(th.WithServer("http://localhost:9000"))),
),
),
}
OnConfigurationUpdate(conf1, []string{"entrypoint1", "entrypoint2"})
OnConfigurationUpdate(conf2, []string{"entrypoint1"})
// Register some metrics manually that are not part of the active configuration.
// Those metrics should be part of the /metrics output on the first scrape but
@ -393,22 +391,21 @@ func TestPrometheusMetricRemoval(t *testing.T) {
EntryPointReqsCounter().
With("entrypoint", "entrypoint2", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
prometheusRegistry.
RouterReqsCounter().
With("router", "router2", "service", "bar@providerName", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
prometheusRegistry.
ServiceReqsCounter().
With("service", "service2", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
With("service", "service1", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
prometheusRegistry.
ServiceServerUpGauge().
With("service", "service1", "url", "http://localhost:9999").
With("service", "bar@providerName", "url", "http://localhost:9999").
Set(1)
prometheusRegistry.
RouterReqsCounter().
With("router", "router2", "service", "service2", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName, serviceReqsTotalName, serviceServerUpName)
assertMetricsAbsent(t, mustScrape(), entryPointReqsTotalName, serviceReqsTotalName, serviceServerUpName)
assertMetricsAbsent(t, mustScrape(), routerReqsTotalName, routerReqDurationName, routerOpenConnsName)
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName, routerReqsTotalName, serviceReqsTotalName, serviceServerUpName)
assertMetricsAbsent(t, mustScrape(), entryPointReqsTotalName, routerReqsTotalName, serviceReqsTotalName, serviceServerUpName)
// To verify that metrics belonging to active configurations are not removed
// here the counter examples.
@ -418,24 +415,39 @@ func TestPrometheusMetricRemoval(t *testing.T) {
Add(1)
prometheusRegistry.
RouterReqsCounter().
With("router", "foo@providerName", "service", "bar@providerName", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
With("router", "foo@providerName", "service", "bar", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
prometheusRegistry.
ServiceReqsCounter().
With("service", "bar@providerName", "code", strconv.Itoa(http.StatusOK), "method", http.MethodGet, "protocol", "http").
Add(1)
prometheusRegistry.
ServiceServerUpGauge().
With("service", "bar@providerName", "url", "http://localhost:9000").
Set(1)
delayForTrackingCompletion()
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName)
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName)
assertMetricsExist(t, mustScrape(), routerReqsTotalName)
assertMetricsExist(t, mustScrape(), routerReqsTotalName)
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName, serviceReqsTotalName, serviceServerUpName, routerReqsTotalName)
assertMetricsExist(t, mustScrape(), entryPointReqsTotalName, serviceReqsTotalName, serviceServerUpName, routerReqsTotalName)
}
func TestPrometheusRemovedMetricsReset(t *testing.T) {
// Reset state of global promState.
defer promState.reset()
t.Cleanup(promState.reset)
prometheusRegistry := RegisterPrometheus(context.Background(), &types.Prometheus{AddEntryPointsLabels: true, AddServicesLabels: true})
defer promRegistry.Unregister(promState)
conf1 := dynamic.Configuration{
HTTP: th.BuildConfiguration(
th.WithLoadBalancerServices(th.WithService("service",
th.WithServers(th.WithServer("http://localhost:9000"))),
),
),
}
OnConfigurationUpdate(conf1, []string{"entrypoint1", "entrypoint2"})
OnConfigurationUpdate(dynamic.Configuration{}, nil)
labelNamesValues := []string{
"service", "service",
"code", strconv.Itoa(http.StatusOK),
@ -467,12 +479,24 @@ func TestPrometheusRemovedMetricsReset(t *testing.T) {
assertCounterValue(t, 1, findMetricFamily(serviceReqsTotalName, metricsFamilies), labelNamesValues...)
}
// reset is a utility method for unit testing.
// It should be called after each test run that changes promState internally
// in order to avoid dependencies between unit tests.
func (ps *prometheusState) reset() {
ps.dynamicConfig = newDynamicConfig()
ps.vectors = nil
ps.deletedEP = nil
ps.deletedRouters = nil
ps.deletedServices = nil
ps.deletedURLs = make(map[string]string)
}
// Tracking and gathering the metrics happens concurrently.
// In practice this is no problem, because in case a tracked metric would miss
// the current scrape, it would just be there in the next one.
// That we can test reliably the tracking of all metrics here, we sleep
// for a short amount of time, to make sure the metric will be present
// in the next scrape.
// In practice this is no problem, because in case a tracked metric would miss the current scrape,
// it would just be there in the next one.
// That we can test reliably the tracking of all metrics here,
// we sleep for a short amount of time,
// to make sure the metric will be present in the next scrape.
func delayForTrackingCompletion() {
time.Sleep(250 * time.Millisecond)
}