homelab/traefik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Manage observability at entrypoint and router level

Browse source 
Co-authored-by: Kevin Pollet <pollet.kevin@gmail.com>
This commit is contained in:
Romain 2024-12-12 09:52:07 +01:00 • committed by GitHub
parent 9588e51146
commit b1934231ca
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
58 changed files with 1216 additions and 303 deletions
Download patch file Download diff file Expand all files Collapse all files

16
pkg/server/aggregator.go
View file

@ -178,6 +178,22 @@ func applyModel(cfg dynamic.Configuration) dynamic.Configuration {
cp.Middlewares = append(m.Middlewares, cp.Middlewares...)
if cp.Observability == nil {
cp.Observability = &dynamic.RouterObservabilityConfig{}
}
if cp.Observability.AccessLogs == nil {
cp.Observability.AccessLogs = m.Observability.AccessLogs
}
if cp.Observability.Tracing == nil {
cp.Observability.Tracing = m.Observability.Tracing
}
if cp.Observability.Metrics == nil {
cp.Observability.Metrics = m.Observability.Metrics
}
rtName := name
if len(eps) > 1 {
rtName = epName + "-" + name

75
pkg/server/aggregator_test.go
View file

@ -9,6 +9,8 @@ import (
"github.com/traefik/traefik/v3/pkg/tls"
)
func pointer[T any](v T) *T { return &v }
func Test_mergeConfiguration(t *testing.T) {
testCases := []struct {
desc string
@ -558,9 +560,10 @@ func Test_applyModel(t *testing.T) {
HTTP: &dynamic.HTTPConfiguration{
Routers: map[string]*dynamic.Router{
"test": {
EntryPoints: []string{"websecure"},
Middlewares: []string{"test"},
TLS: &dynamic.RouterTLSConfig{},
EntryPoints: []string{"websecure"},
Middlewares: []string{"test"},
TLS: &dynamic.RouterTLSConfig{},
Observability: &dynamic.RouterObservabilityConfig{},
},
},
Middlewares: make(map[string]*dynamic.Middleware),
@ -574,6 +577,60 @@ func Test_applyModel(t *testing.T) {
},
},
},
{
desc: "with model, one entry point with observability",
input: dynamic.Configuration{
HTTP: &dynamic.HTTPConfiguration{
Routers: map[string]*dynamic.Router{
"test": {
EntryPoints: []string{"websecure"},
},
},
Middlewares: make(map[string]*dynamic.Middleware),
Services: make(map[string]*dynamic.Service),
Models: map[string]*dynamic.Model{
"websecure@internal": {
Middlewares: []string{"test"},
TLS: &dynamic.RouterTLSConfig{},
Observability: dynamic.RouterObservabilityConfig{
AccessLogs: pointer(true),
Tracing: pointer(true),
Metrics: pointer(true),
},
},
},
},
},
expected: dynamic.Configuration{
HTTP: &dynamic.HTTPConfiguration{
Routers: map[string]*dynamic.Router{
"test": {
EntryPoints: []string{"websecure"},
Middlewares: []string{"test"},
TLS: &dynamic.RouterTLSConfig{},
Observability: &dynamic.RouterObservabilityConfig{
AccessLogs: pointer(true),
Tracing: pointer(true),
Metrics: pointer(true),
},
},
},
Middlewares: make(map[string]*dynamic.Middleware),
Services: make(map[string]*dynamic.Service),
Models: map[string]*dynamic.Model{
"websecure@internal": {
Middlewares: []string{"test"},
TLS: &dynamic.RouterTLSConfig{},
Observability: dynamic.RouterObservabilityConfig{
AccessLogs: pointer(true),
Tracing: pointer(true),
Metrics: pointer(true),
},
},
},
},
},
},
{
desc: "with model, one entry point, and router with tls",
input: dynamic.Configuration{
@ -601,6 +658,11 @@ func Test_applyModel(t *testing.T) {
EntryPoints: []string{"websecure"},
Middlewares: []string{"test"},
TLS: &dynamic.RouterTLSConfig{CertResolver: "router"},
Observability: &dynamic.RouterObservabilityConfig{
AccessLogs: nil,
Tracing: nil,
Metrics: nil,
},
},
},
Middlewares: make(map[string]*dynamic.Middleware),
@ -640,9 +702,10 @@ func Test_applyModel(t *testing.T) {
EntryPoints: []string{"web"},
},
"websecure-test": {
EntryPoints: []string{"websecure"},
Middlewares: []string{"test"},
TLS: &dynamic.RouterTLSConfig{},
EntryPoints: []string{"websecure"},
Middlewares: []string{"test"},
TLS: &dynamic.RouterTLSConfig{},
Observability: &dynamic.RouterObservabilityConfig{},
},
},
Middlewares: make(map[string]*dynamic.Middleware),

76
pkg/server/middleware/observability.go
View file

@ -8,12 +8,13 @@ import (
"github.com/containous/alice"
"github.com/rs/zerolog/log"
"github.com/traefik/traefik/v3/pkg/config/dynamic"
"github.com/traefik/traefik/v3/pkg/config/static"
"github.com/traefik/traefik/v3/pkg/logs"
"github.com/traefik/traefik/v3/pkg/metrics"
"github.com/traefik/traefik/v3/pkg/middlewares/accesslog"
"github.com/traefik/traefik/v3/pkg/middlewares/capture"
metricsMiddle "github.com/traefik/traefik/v3/pkg/middlewares/metrics"
mmetrics "github.com/traefik/traefik/v3/pkg/middlewares/metrics"
"github.com/traefik/traefik/v3/pkg/middlewares/observability"
"github.com/traefik/traefik/v3/pkg/tracing"
)
@ -41,7 +42,7 @@ func NewObservabilityMgr(config static.Configuration, metricsRegistry metrics.Re
}
// BuildEPChain an observability middleware chain by entry point.
func (o *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName string, resourceName string) alice.Chain {
func (o *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName string, resourceName string, observabilityConfig *dynamic.RouterObservabilityConfig) alice.Chain {
chain := alice.New()
if o == nil {
@ -49,62 +50,101 @@ func (o *ObservabilityMgr) BuildEPChain(ctx context.Context, entryPointName stri
}
if o.accessLoggerMiddleware != nil || o.metricsRegistry != nil && (o.metricsRegistry.IsEpEnabled() || o.metricsRegistry.IsRouterEnabled() || o.metricsRegistry.IsSvcEnabled()) {
if o.ShouldAddAccessLogs(resourceName) || o.ShouldAddMetrics(resourceName) {
if o.ShouldAddAccessLogs(resourceName, observabilityConfig) || o.ShouldAddMetrics(resourceName, observabilityConfig) {
chain = chain.Append(capture.Wrap)
}
}
// As the Entry point observability middleware ensures that the tracing is added to the request and logger context,
// it needs to be added before the access log middleware to ensure that the trace ID is logged.
if (o.tracer != nil && o.ShouldAddTracing(resourceName)) || (o.metricsRegistry != nil && o.metricsRegistry.IsEpEnabled() && o.ShouldAddMetrics(resourceName)) {
chain = chain.Append(observability.WrapEntryPointHandler(ctx, o.tracer, o.semConvMetricRegistry, entryPointName))
if o.tracer != nil && o.ShouldAddTracing(resourceName, observabilityConfig) {
chain = chain.Append(observability.EntryPointHandler(ctx, o.tracer, entryPointName))
}
if o.accessLoggerMiddleware != nil && o.ShouldAddAccessLogs(resourceName) {
if o.accessLoggerMiddleware != nil && o.ShouldAddAccessLogs(resourceName, observabilityConfig) {
chain = chain.Append(accesslog.WrapHandler(o.accessLoggerMiddleware))
chain = chain.Append(func(next http.Handler) (http.Handler, error) {
return accesslog.NewFieldHandler(next, logs.EntryPointName, entryPointName, accesslog.InitServiceFields), nil
})
}
if o.metricsRegistry != nil && o.metricsRegistry.IsEpEnabled() && o.ShouldAddMetrics(resourceName) {
metricsHandler := metricsMiddle.WrapEntryPointHandler(ctx, o.metricsRegistry, entryPointName)
// Semantic convention server metrics handler.
if o.semConvMetricRegistry != nil && o.ShouldAddMetrics(resourceName, observabilityConfig) {
chain = chain.Append(observability.SemConvServerMetricsHandler(ctx, o.semConvMetricRegistry))
}
if o.tracer != nil && o.ShouldAddTracing(resourceName) {
if o.metricsRegistry != nil && o.metricsRegistry.IsEpEnabled() && o.ShouldAddMetrics(resourceName, observabilityConfig) {
metricsHandler := mmetrics.WrapEntryPointHandler(ctx, o.metricsRegistry, entryPointName)
if o.tracer != nil && o.ShouldAddTracing(resourceName, observabilityConfig) {
chain = chain.Append(observability.WrapMiddleware(ctx, metricsHandler))
} else {
chain = chain.Append(metricsHandler)
}
}
// Inject context keys to control whether to produce metrics further downstream (services, round-tripper),
// because the router configuration cannot be evaluated during build time for services.
if observabilityConfig != nil && observabilityConfig.Metrics != nil && !*observabilityConfig.Metrics {
chain = chain.Append(func(next http.Handler) (http.Handler, error) {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
next.ServeHTTP(rw, req.WithContext(context.WithValue(req.Context(), observability.DisableMetricsKey, true)))
}), nil
})
}
return chain
}
// ShouldAddAccessLogs returns whether the access logs should be enabled for the given resource.
func (o *ObservabilityMgr) ShouldAddAccessLogs(resourceName string) bool {
// ShouldAddAccessLogs returns whether the access logs should be enabled for the given serviceName and the observability config.
func (o *ObservabilityMgr) ShouldAddAccessLogs(serviceName string, observabilityConfig *dynamic.RouterObservabilityConfig) bool {
if o == nil {
return false
}
return o.config.AccessLog != nil && (o.config.AccessLog.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
if o.config.AccessLog == nil {
return false
}
if strings.HasSuffix(serviceName, "@internal") && !o.config.AccessLog.AddInternals {
return false
}
return observabilityConfig == nil || observabilityConfig.AccessLogs != nil && *observabilityConfig.AccessLogs
}
// ShouldAddMetrics returns whether the metrics should be enabled for the given resource.
func (o *ObservabilityMgr) ShouldAddMetrics(resourceName string) bool {
// ShouldAddMetrics returns whether the metrics should be enabled for the given resource and the observability config.
func (o *ObservabilityMgr) ShouldAddMetrics(serviceName string, observabilityConfig *dynamic.RouterObservabilityConfig) bool {
if o == nil {
return false
}
return o.config.Metrics != nil && (o.config.Metrics.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
if o.config.Metrics == nil {
return false
}
if strings.HasSuffix(serviceName, "@internal") && !o.config.Metrics.AddInternals {
return false
}
return observabilityConfig == nil || observabilityConfig.Metrics != nil && *observabilityConfig.Metrics
}
// ShouldAddTracing returns whether the tracing should be enabled for the given resource.
func (o *ObservabilityMgr) ShouldAddTracing(resourceName string) bool {
// ShouldAddTracing returns whether the tracing should be enabled for the given serviceName and the observability config.
func (o *ObservabilityMgr) ShouldAddTracing(serviceName string, observabilityConfig *dynamic.RouterObservabilityConfig) bool {
if o == nil {
return false
}
return o.config.Tracing != nil && (o.config.Tracing.AddInternals || !strings.HasSuffix(resourceName, "@internal"))
if o.config.Tracing == nil {
return false
}
if strings.HasSuffix(serviceName, "@internal") && !o.config.Tracing.AddInternals {
return false
}
return observabilityConfig == nil || observabilityConfig.Tracing != nil && *observabilityConfig.Tracing
}
// MetricsRegistry is an accessor to the metrics registry.

14
pkg/server/router/router.go
View file

@ -91,12 +91,12 @@ func (m *Manager) BuildHandlers(rootCtx context.Context, entryPoints []string, t
continue
}
handler, err := m.observabilityMgr.BuildEPChain(ctx, entryPointName, "").Then(BuildDefaultHTTPRouter())
defaultHandler, err := m.observabilityMgr.BuildEPChain(ctx, entryPointName, "", nil).Then(BuildDefaultHTTPRouter())
if err != nil {
logger.Error().Err(err).Send()
continue
}
entryPointHandlers[entryPointName] = handler
entryPointHandlers[entryPointName] = defaultHandler
}
return entryPointHandlers
@ -108,7 +108,7 @@ func (m *Manager) buildEntryPointHandler(ctx context.Context, entryPointName str
return nil, err
}
defaultHandler, err := m.observabilityMgr.BuildEPChain(ctx, entryPointName, "defaultHandler").Then(http.NotFoundHandler())
defaultHandler, err := m.observabilityMgr.BuildEPChain(ctx, entryPointName, "", nil).Then(http.NotFoundHandler())
if err != nil {
return nil, err
}
@ -137,7 +137,7 @@ func (m *Manager) buildEntryPointHandler(ctx context.Context, entryPointName str
continue
}
observabilityChain := m.observabilityMgr.BuildEPChain(ctx, entryPointName, routerConfig.Service)
observabilityChain := m.observabilityMgr.BuildEPChain(ctx, entryPointName, routerConfig.Service, routerConfig.Observability)
handler, err = observabilityChain.Then(handler)
if err != nil {
routerConfig.AddError(err, true)
@ -182,7 +182,7 @@ func (m *Manager) buildRouterHandler(ctx context.Context, routerName string, rou
}
// Prevents from enabling observability for internal resources.
if !m.observabilityMgr.ShouldAddAccessLogs(provider.GetQualifiedName(ctx, routerConfig.Service)) {
if !m.observabilityMgr.ShouldAddAccessLogs(provider.GetQualifiedName(ctx, routerConfig.Service), routerConfig.Observability) {
m.routerHandlers[routerName] = handler
return m.routerHandlers[routerName], nil
}
@ -221,12 +221,12 @@ func (m *Manager) buildHTTPHandler(ctx context.Context, router *runtime.RouterIn
chain := alice.New()
if m.observabilityMgr.MetricsRegistry() != nil && m.observabilityMgr.MetricsRegistry().IsRouterEnabled() &&
m.observabilityMgr.ShouldAddMetrics(provider.GetQualifiedName(ctx, router.Service)) {
m.observabilityMgr.ShouldAddMetrics(provider.GetQualifiedName(ctx, router.Service), router.Observability) {
chain = chain.Append(metricsMiddle.WrapRouterHandler(ctx, m.observabilityMgr.MetricsRegistry(), routerName, provider.GetQualifiedName(ctx, router.Service)))
}
// Prevents from enabling tracing for internal resources.
if !m.observabilityMgr.ShouldAddTracing(provider.GetQualifiedName(ctx, router.Service)) {
if !m.observabilityMgr.ShouldAddTracing(provider.GetQualifiedName(ctx, router.Service), router.Observability) {
return chain.Extend(*mHandler).Then(sHandler)
}

10
pkg/server/service/service.go
View file

@ -356,7 +356,7 @@ func (m *Manager) getLoadBalancerServiceHandler(ctx context.Context, serviceName
qualifiedSvcName := provider.GetQualifiedName(ctx, serviceName)
shouldObserve := m.observabilityMgr.ShouldAddTracing(qualifiedSvcName) || m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName)
shouldObserve := m.observabilityMgr.ShouldAddTracing(qualifiedSvcName, nil) || m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName, nil)
proxy, err := m.proxyBuilder.Build(service.ServersTransport, target, shouldObserve, passHostHeader, server.PreservePath, flushInterval)
if err != nil {
return nil, fmt.Errorf("error building proxy for server URL %s: %w", server.URL, err)
@ -364,14 +364,14 @@ func (m *Manager) getLoadBalancerServiceHandler(ctx context.Context, serviceName
// Prevents from enabling observability for internal resources.
if m.observabilityMgr.ShouldAddAccessLogs(qualifiedSvcName) {
if m.observabilityMgr.ShouldAddAccessLogs(qualifiedSvcName, nil) {
proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceURL, target.String(), nil)
proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceAddr, target.Host, nil)
proxy = accesslog.NewFieldHandler(proxy, accesslog.ServiceName, serviceName, accesslog.AddServiceFields)
}
if m.observabilityMgr.MetricsRegistry() != nil && m.observabilityMgr.MetricsRegistry().IsSvcEnabled() &&
m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName) {
m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName, nil) {
metricsHandler := metricsMiddle.WrapServiceHandler(ctx, m.observabilityMgr.MetricsRegistry(), serviceName)
proxy, err = alice.New().
@ -382,11 +382,11 @@ func (m *Manager) getLoadBalancerServiceHandler(ctx context.Context, serviceName
}
}
if m.observabilityMgr.ShouldAddTracing(qualifiedSvcName) {
if m.observabilityMgr.ShouldAddTracing(qualifiedSvcName, nil) {
proxy = observability.NewService(ctx, serviceName, proxy)
}
if m.observabilityMgr.ShouldAddAccessLogs(qualifiedSvcName) || m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName) {
if m.observabilityMgr.ShouldAddAccessLogs(qualifiedSvcName, nil) || m.observabilityMgr.ShouldAddMetrics(qualifiedSvcName, nil) {
// Some piece of middleware, like the ErrorPage, are relying on this serviceBuilder to get the handler for a given service,
// to re-target the request to it.
// Those pieces of middleware can be configured on routes that expose a Traefik internal service.