homelab/traefik
1
0
Fork 0
Code Activity

OpenTelemetry Logs and Access Logs

Browse source 
Co-authored-by: Kevin Pollet <pollet.kevin@gmail.com>
This commit is contained in:
Romain 2024-12-06 14:50:04 +01:00 • committed by GitHub
parent 33c1d700c0
commit 826a2b74aa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 2297 additions and 475 deletions
Download patch file Download diff file Expand all files Collapse all files

147
pkg/types/logs.go
View file

@ -1,6 +1,22 @@
package types
import "github.com/traefik/paerser/types"
import (
"context"
"fmt"
"net"
"net/url"
"github.com/traefik/paerser/types"
"github.com/traefik/traefik/v3/pkg/version"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
otelsdk "go.opentelemetry.io/otel/sdk/log"
"go.opentelemetry.io/otel/sdk/resource"
semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/encoding/gzip"
)
const (
// AccessLogKeep is the keep string value.
@ -14,11 +30,10 @@ const (
const (
// CommonFormat is the common logging format (CLF).
CommonFormat string = "common"
// JSONFormat is the JSON logging format.
JSONFormat string = "json"
)
const OTelTraefikServiceName = "traefik"
// TraefikLog holds the configuration settings for the traefik logger.
type TraefikLog struct {
Level string `description:"Log level set to traefik logs." json:"level,omitempty" toml:"level,omitempty" yaml:"level,omitempty" export:"true"`
@ -30,6 +45,8 @@ type TraefikLog struct {
MaxAge int `description:"Maximum number of days to retain old log files based on the timestamp encoded in their filename." json:"maxAge,omitempty" toml:"maxAge,omitempty" yaml:"maxAge,omitempty" export:"true"`
MaxBackups int `description:"Maximum number of old log files to retain." json:"maxBackups,omitempty" toml:"maxBackups,omitempty" yaml:"maxBackups,omitempty" export:"true"`
Compress bool `description:"Determines if the rotated log files should be compressed using gzip." json:"compress,omitempty" toml:"compress,omitempty" yaml:"compress,omitempty" export:"true"`
OTLP *OTelLog `description:"Settings for OpenTelemetry." json:"otlp,omitempty" toml:"otlp,omitempty" yaml:"otlp,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
}
// SetDefaults sets the default values.
@ -46,6 +63,8 @@ type AccessLog struct {
Fields *AccessLogFields `description:"AccessLogFields." json:"fields,omitempty" toml:"fields,omitempty" yaml:"fields,omitempty" export:"true"`
BufferingSize int64 `description:"Number of access log lines to process in a buffered way." json:"bufferingSize,omitempty" toml:"bufferingSize,omitempty" yaml:"bufferingSize,omitempty" export:"true"`
AddInternals bool `description:"Enables access log for internal services (ping, dashboard, etc...)." json:"addInternals,omitempty" toml:"addInternals,omitempty" yaml:"addInternals,omitempty" export:"true"`
OTLP *OTelLog `description:"Settings for OpenTelemetry." json:"otlp,omitempty" toml:"otlp,omitempty" yaml:"otlp,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
}
// SetDefaults sets the default values.
@ -128,3 +147,123 @@ func checkFieldHeaderValue(value, defaultValue string) string {
}
return defaultValue
}
// OTelLog provides configuration settings for the open-telemetry logger.
type OTelLog struct {
ServiceName string `description:"Set the name for this service." json:"serviceName,omitempty" toml:"serviceName,omitempty" yaml:"serviceName,omitempty" export:"true"`
ResourceAttributes map[string]string `description:"Defines additional resource attributes (key:value)." json:"resourceAttributes,omitempty" toml:"resourceAttributes,omitempty" yaml:"resourceAttributes,omitempty"`
GRPC *OTelGRPC `description:"gRPC configuration for the OpenTelemetry collector." json:"grpc,omitempty" toml:"grpc,omitempty" yaml:"grpc,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
HTTP *OTelHTTP `description:"HTTP configuration for the OpenTelemetry collector." json:"http,omitempty" toml:"http,omitempty" yaml:"http,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
}
// SetDefaults sets the default values.
func (o *OTelLog) SetDefaults() {
o.ServiceName = OTelTraefikServiceName
o.HTTP = &OTelHTTP{}
o.HTTP.SetDefaults()
}
// NewLoggerProvider creates a new OpenTelemetry logger provider.
func (o *OTelLog) NewLoggerProvider() (*otelsdk.LoggerProvider, error) {
var (
err error
exporter otelsdk.Exporter
)
if o.GRPC != nil {
exporter, err = o.buildGRPCExporter()
} else {
exporter, err = o.buildHTTPExporter()
}
if err != nil {
return nil, fmt.Errorf("setting up exporter: %w", err)
}
attr := []attribute.KeyValue{
semconv.ServiceNameKey.String(o.ServiceName),
semconv.ServiceVersionKey.String(version.Version),
}
for k, v := range o.ResourceAttributes {
attr = append(attr, attribute.String(k, v))
}
res, err := resource.New(context.Background(),
resource.WithAttributes(attr...),
resource.WithFromEnv(),
resource.WithTelemetrySDK(),
resource.WithOSType(),
resource.WithProcessCommandArgs(),
)
if err != nil {
return nil, fmt.Errorf("building resource: %w", err)
}
// Register the trace provider to allow the global logger to access it.
bp := otelsdk.NewBatchProcessor(exporter)
loggerProvider := otelsdk.NewLoggerProvider(
otelsdk.WithResource(res),
otelsdk.WithProcessor(bp),
)
return loggerProvider, nil
}
func (o *OTelLog) buildHTTPExporter() (*otlploghttp.Exporter, error) {
endpoint, err := url.Parse(o.HTTP.Endpoint)
if err != nil {
return nil, fmt.Errorf("invalid collector endpoint %q: %w", o.HTTP.Endpoint, err)
}
opts := []otlploghttp.Option{
otlploghttp.WithEndpoint(endpoint.Host),
otlploghttp.WithHeaders(o.HTTP.Headers),
otlploghttp.WithCompression(otlploghttp.GzipCompression),
}
if endpoint.Scheme == "http" {
opts = append(opts, otlploghttp.WithInsecure())
}
if endpoint.Path != "" {
opts = append(opts, otlploghttp.WithURLPath(endpoint.Path))
}
if o.HTTP.TLS != nil {
tlsConfig, err := o.HTTP.TLS.CreateTLSConfig(context.Background())
if err != nil {
return nil, fmt.Errorf("creating TLS client config: %w", err)
}
opts = append(opts, otlploghttp.WithTLSClientConfig(tlsConfig))
}
return otlploghttp.New(context.Background(), opts...)
}
func (o *OTelLog) buildGRPCExporter() (*otlploggrpc.Exporter, error) {
host, port, err := net.SplitHostPort(o.GRPC.Endpoint)
if err != nil {
return nil, fmt.Errorf("invalid collector endpoint %q: %w", o.GRPC.Endpoint, err)
}
opts := []otlploggrpc.Option{
otlploggrpc.WithEndpoint(fmt.Sprintf("%s:%s", host, port)),
otlploggrpc.WithHeaders(o.GRPC.Headers),
otlploggrpc.WithCompressor(gzip.Name),
}
if o.GRPC.Insecure {
opts = append(opts, otlploggrpc.WithInsecure())
}
if o.GRPC.TLS != nil {
tlsConfig, err := o.GRPC.TLS.CreateTLSConfig(context.Background())
if err != nil {
return nil, fmt.Errorf("creating TLS client config: %w", err)
}
opts = append(opts, otlploggrpc.WithTLSCredentials(credentials.NewTLS(tlsConfig)))
}
return otlploggrpc.New(context.Background(), opts...)
}

33
pkg/types/metrics.go
View file

@ -108,8 +108,8 @@ func (i *InfluxDB2) SetDefaults() {
// OTLP contains specific configuration used by the OpenTelemetry Metrics exporter.
type OTLP struct {
GRPC *OtelGRPC `description:"gRPC configuration for the OpenTelemetry collector." json:"grpc,omitempty" toml:"grpc,omitempty" yaml:"grpc,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
HTTP *OtelHTTP `description:"HTTP configuration for the OpenTelemetry collector." json:"http,omitempty" toml:"http,omitempty" yaml:"http,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
GRPC *OTelGRPC `description:"gRPC configuration for the OpenTelemetry collector." json:"grpc,omitempty" toml:"grpc,omitempty" yaml:"grpc,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
HTTP *OTelHTTP `description:"HTTP configuration for the OpenTelemetry collector." json:"http,omitempty" toml:"http,omitempty" yaml:"http,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
AddEntryPointsLabels bool `description:"Enable metrics on entry points." json:"addEntryPointsLabels,omitempty" toml:"addEntryPointsLabels,omitempty" yaml:"addEntryPointsLabels,omitempty" export:"true"`
AddRoutersLabels bool `description:"Enable metrics on routers." json:"addRoutersLabels,omitempty" toml:"addRoutersLabels,omitempty" yaml:"addRoutersLabels,omitempty" export:"true"`
@ -121,14 +121,14 @@ type OTLP struct {
// SetDefaults sets the default values.
func (o *OTLP) SetDefaults() {
o.HTTP = &OtelHTTP{}
o.HTTP = &OTelHTTP{}
o.HTTP.SetDefaults()
o.AddEntryPointsLabels = true
o.AddServicesLabels = true
o.ExplicitBoundaries = []float64{.005, .01, .025, .05, .075, .1, .25, .5, .75, 1, 2.5, 5, 7.5, 10}
o.PushInterval = types.Duration(10 * time.Second)
o.ServiceName = "traefik"
o.ServiceName = OTelTraefikServiceName
}
// Statistics provides options for monitoring request and response stats.
@ -140,28 +140,3 @@ type Statistics struct {
func (s *Statistics) SetDefaults() {
s.RecentErrors = 10
}
// OtelGRPC provides configuration settings for the gRPC open-telemetry.
type OtelGRPC struct {
Endpoint string `description:"Sets the gRPC endpoint (host:port) of the collector." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"`
Insecure bool `description:"Disables client transport security for the exporter." json:"insecure,omitempty" toml:"insecure,omitempty" yaml:"insecure,omitempty" export:"true"`
TLS *ClientTLS `description:"Defines client transport security parameters." json:"tls,omitempty" toml:"tls,omitempty" yaml:"tls,omitempty" export:"true"`
Headers map[string]string `description:"Headers sent with payload." json:"headers,omitempty" toml:"headers,omitempty" yaml:"headers,omitempty"`
}
// SetDefaults sets the default values.
func (c *OtelGRPC) SetDefaults() {
c.Endpoint = "localhost:4317"
}
// OtelHTTP provides configuration settings for the HTTP open-telemetry.
type OtelHTTP struct {
Endpoint string `description:"Sets the HTTP endpoint (scheme://host:port/path) of the collector." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"`
TLS *ClientTLS `description:"Defines client transport security parameters." json:"tls,omitempty" toml:"tls,omitempty" yaml:"tls,omitempty" export:"true"`
Headers map[string]string `description:"Headers sent with payload." json:"headers,omitempty" toml:"headers,omitempty" yaml:"headers,omitempty"`
}
// SetDefaults sets the default values.
func (c *OtelHTTP) SetDefaults() {
c.Endpoint = "https://localhost:4318"
}

26
pkg/types/otel.go Normal file
View file

@ -0,0 +1,26 @@
package types
// OTelGRPC provides configuration settings for the gRPC open-telemetry.
type OTelGRPC struct {
Endpoint string `description:"Sets the gRPC endpoint (host:port) of the collector." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"`
Insecure bool `description:"Disables client transport security for the exporter." json:"insecure,omitempty" toml:"insecure,omitempty" yaml:"insecure,omitempty" export:"true"`
TLS *ClientTLS `description:"Defines client transport security parameters." json:"tls,omitempty" toml:"tls,omitempty" yaml:"tls,omitempty" export:"true"`
Headers map[string]string `description:"Headers sent with payload." json:"headers,omitempty" toml:"headers,omitempty" yaml:"headers,omitempty"`
}
// SetDefaults sets the default values.
func (o *OTelGRPC) SetDefaults() {
o.Endpoint = "localhost:4317"
}
// OTelHTTP provides configuration settings for the HTTP open-telemetry.
type OTelHTTP struct {
Endpoint string `description:"Sets the HTTP endpoint (scheme://host:port/path) of the collector." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"`
TLS *ClientTLS `description:"Defines client transport security parameters." json:"tls,omitempty" toml:"tls,omitempty" yaml:"tls,omitempty" export:"true"`
Headers map[string]string `description:"Headers sent with payload." json:"headers,omitempty" toml:"headers,omitempty" yaml:"headers,omitempty"`
}
// SetDefaults sets the default values.
func (o *OTelHTTP) SetDefaults() {
o.Endpoint = "https://localhost:4318"
}

163
pkg/types/tracing.go Normal file
View file

@ -0,0 +1,163 @@
package types
import (
"context"
"fmt"
"io"
"net"
"net/url"
"time"
"github.com/rs/zerolog/log"
"github.com/traefik/traefik/v3/pkg/version"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
"go.opentelemetry.io/otel/trace"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/encoding/gzip"
)
// OTelTracing provides configuration settings for the open-telemetry tracer.
type OTelTracing struct {
GRPC *OTelGRPC `description:"gRPC configuration for the OpenTelemetry collector." json:"grpc,omitempty" toml:"grpc,omitempty" yaml:"grpc,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
HTTP *OTelHTTP `description:"HTTP configuration for the OpenTelemetry collector." json:"http,omitempty" toml:"http,omitempty" yaml:"http,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
}
// SetDefaults sets the default values.
func (c *OTelTracing) SetDefaults() {
c.HTTP = &OTelHTTP{}
c.HTTP.SetDefaults()
}
// Setup sets up the tracer.
func (c *OTelTracing) Setup(serviceName string, sampleRate float64, globalAttributes map[string]string) (trace.Tracer, io.Closer, error) {
var (
err error
exporter *otlptrace.Exporter
)
if c.GRPC != nil {
exporter, err = c.setupGRPCExporter()
} else {
exporter, err = c.setupHTTPExporter()
}
if err != nil {
return nil, nil, fmt.Errorf("setting up exporter: %w", err)
}
attr := []attribute.KeyValue{
semconv.ServiceNameKey.String(serviceName),
semconv.ServiceVersionKey.String(version.Version),
}
for k, v := range globalAttributes {
attr = append(attr, attribute.String(k, v))
}
res, err := resource.New(context.Background(),
resource.WithAttributes(attr...),
resource.WithFromEnv(),
resource.WithTelemetrySDK(),
resource.WithOSType(),
resource.WithProcessCommandArgs(),
)
if err != nil {
return nil, nil, fmt.Errorf("building resource: %w", err)
}
// Register the trace exporter with a TracerProvider, using a batch
// span processor to aggregate spans before export.
bsp := sdktrace.NewBatchSpanProcessor(exporter)
tracerProvider := sdktrace.NewTracerProvider(
sdktrace.WithSampler(sdktrace.TraceIDRatioBased(sampleRate)),
sdktrace.WithResource(res),
sdktrace.WithSpanProcessor(bsp),
)
otel.SetTracerProvider(tracerProvider)
log.Debug().Msg("OpenTelemetry tracer configured")
return tracerProvider.Tracer("github.com/traefik/traefik"), &tpCloser{provider: tracerProvider}, err
}
func (c *OTelTracing) setupHTTPExporter() (*otlptrace.Exporter, error) {
endpoint, err := url.Parse(c.HTTP.Endpoint)
if err != nil {
return nil, fmt.Errorf("invalid collector endpoint %q: %w", c.HTTP.Endpoint, err)
}
opts := []otlptracehttp.Option{
otlptracehttp.WithEndpoint(endpoint.Host),
otlptracehttp.WithHeaders(c.HTTP.Headers),
otlptracehttp.WithCompression(otlptracehttp.GzipCompression),
}
if endpoint.Scheme == "http" {
opts = append(opts, otlptracehttp.WithInsecure())
}
if endpoint.Path != "" {
opts = append(opts, otlptracehttp.WithURLPath(endpoint.Path))
}
if c.HTTP.TLS != nil {
tlsConfig, err := c.HTTP.TLS.CreateTLSConfig(context.Background())
if err != nil {
return nil, fmt.Errorf("creating TLS client config: %w", err)
}
opts = append(opts, otlptracehttp.WithTLSClientConfig(tlsConfig))
}
return otlptrace.New(context.Background(), otlptracehttp.NewClient(opts...))
}
func (c *OTelTracing) setupGRPCExporter() (*otlptrace.Exporter, error) {
host, port, err := net.SplitHostPort(c.GRPC.Endpoint)
if err != nil {
return nil, fmt.Errorf("invalid collector endpoint %q: %w", c.GRPC.Endpoint, err)
}
opts := []otlptracegrpc.Option{
otlptracegrpc.WithEndpoint(fmt.Sprintf("%s:%s", host, port)),
otlptracegrpc.WithHeaders(c.GRPC.Headers),
otlptracegrpc.WithCompressor(gzip.Name),
}
if c.GRPC.Insecure {
opts = append(opts, otlptracegrpc.WithInsecure())
}
if c.GRPC.TLS != nil {
tlsConfig, err := c.GRPC.TLS.CreateTLSConfig(context.Background())
if err != nil {
return nil, fmt.Errorf("creating TLS client config: %w", err)
}
opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(tlsConfig)))
}
return otlptrace.New(context.Background(), otlptracegrpc.NewClient(opts...))
}
// tpCloser converts a TraceProvider into an io.Closer.
type tpCloser struct {
provider *sdktrace.TracerProvider
}
func (t *tpCloser) Close() error {
if t == nil {
return nil
}
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second))
defer cancel()
return t.provider.Shutdown(ctx)
}