1
0
Fork 0

Semconv OTLP stable HTTP metrics

This commit is contained in:
Michael 2024-03-12 09:48:04 +01:00 committed by GitHub
parent 709ff6fb09
commit 6c9687f410
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
44 changed files with 803 additions and 432 deletions

View file

@ -0,0 +1,98 @@
package observability
import (
"context"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/containous/alice"
"github.com/traefik/traefik/v3/pkg/metrics"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/noop"
)
const (
entryPointTypeName = "TracingEntryPoint"
)
type entryPointTracing struct {
tracer *tracing.Tracer
entryPoint string
next http.Handler
semConvMetricRegistry *metrics.SemConvMetricsRegistry
}
// WrapEntryPointHandler Wraps tracing to alice.Constructor.
func WrapEntryPointHandler(ctx context.Context, tracer *tracing.Tracer, semConvMetricRegistry *metrics.SemConvMetricsRegistry, entryPointName string) alice.Constructor {
return func(next http.Handler) (http.Handler, error) {
if tracer == nil {
tracer = tracing.NewTracer(noop.Tracer{}, nil, nil)
}
return newEntryPoint(ctx, tracer, semConvMetricRegistry, entryPointName, next), nil
}
}
// newEntryPoint creates a new tracing middleware for incoming requests.
func newEntryPoint(ctx context.Context, tracer *tracing.Tracer, semConvMetricRegistry *metrics.SemConvMetricsRegistry, entryPointName string, next http.Handler) http.Handler {
middlewares.GetLogger(ctx, "tracing", entryPointTypeName).Debug().Msg("Creating middleware")
if tracer == nil {
tracer = tracing.NewTracer(noop.Tracer{}, nil, nil)
}
return &entryPointTracing{
entryPoint: entryPointName,
tracer: tracer,
semConvMetricRegistry: semConvMetricRegistry,
next: next,
}
}
func (e *entryPointTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
tracingCtx := tracing.ExtractCarrierIntoContext(req.Context(), req.Header)
start := time.Now()
tracingCtx, span := e.tracer.Start(tracingCtx, "EntryPoint", trace.WithSpanKind(trace.SpanKindServer), trace.WithTimestamp(start))
req = req.WithContext(tracingCtx)
span.SetAttributes(attribute.String("entry_point", e.entryPoint))
e.tracer.CaptureServerRequest(span, req)
recorder := newStatusCodeRecorder(rw, http.StatusOK)
e.next.ServeHTTP(recorder, req)
e.tracer.CaptureResponse(span, recorder.Header(), recorder.Status(), trace.SpanKindServer)
end := time.Now()
span.End(trace.WithTimestamp(end))
if e.semConvMetricRegistry != nil && e.semConvMetricRegistry.HTTPServerRequestDuration() != nil {
var attrs []attribute.KeyValue
if recorder.Status() < 100 || recorder.Status() >= 600 {
attrs = append(attrs, attribute.Key("error.type").String(fmt.Sprintf("Invalid HTTP status code ; %d", recorder.Status())))
} else if recorder.Status() >= 400 {
attrs = append(attrs, attribute.Key("error.type").String(strconv.Itoa(recorder.Status())))
}
attrs = append(attrs, semconv.HTTPRequestMethodKey.String(req.Method))
attrs = append(attrs, semconv.HTTPResponseStatusCode(recorder.Status()))
attrs = append(attrs, semconv.NetworkProtocolName(strings.ToLower(req.Proto)))
attrs = append(attrs, semconv.NetworkProtocolVersion(Proto(req.Proto)))
attrs = append(attrs, semconv.ServerAddress(req.Host))
attrs = append(attrs, semconv.URLScheme(req.Header.Get("X-Forwarded-Proto")))
e.semConvMetricRegistry.HTTPServerRequestDuration().Record(req.Context(), end.Sub(start).Seconds(), metric.WithAttributes(attrs...))
}
}

View file

@ -0,0 +1,185 @@
package observability
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
ptypes "github.com/traefik/paerser/types"
"github.com/traefik/traefik/v3/pkg/metrics"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/types"
"go.opentelemetry.io/otel/attribute"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
)
func TestEntryPointMiddleware_tracing(t *testing.T) {
type expected struct {
name string
attributes []attribute.KeyValue
}
testCases := []struct {
desc string
entryPoint string
expected expected
}{
{
desc: "basic test",
entryPoint: "test",
expected: expected{
name: "EntryPoint",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "server"),
attribute.String("entry_point", "test"),
attribute.String("http.request.method", "GET"),
attribute.String("network.protocol.version", "1.1"),
attribute.Int64("http.request.body.size", int64(0)),
attribute.String("url.path", "/search"),
attribute.String("url.query", "q=Opentelemetry"),
attribute.String("url.scheme", "http"),
attribute.String("user_agent.original", "entrypoint-test"),
attribute.String("server.address", "www.test.com"),
attribute.String("network.peer.address", "10.0.0.1"),
attribute.String("network.peer.port", "1234"),
attribute.String("client.address", "10.0.0.1"),
attribute.Int64("client.port", int64(1234)),
attribute.String("client.socket.address", ""),
attribute.StringSlice("http.request.header.x-foo", []string{"foo", "bar"}),
attribute.Int64("http.response.status_code", int64(404)),
attribute.StringSlice("http.response.header.x-bar", []string{"foo", "bar"}),
},
},
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
rw := httptest.NewRecorder()
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "entrypoint-test")
req.Header.Set("X-Forwarded-Proto", "http")
req.Header.Set("X-Foo", "foo")
req.Header.Add("X-Foo", "bar")
next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
rw.Header().Set("X-Bar", "foo")
rw.Header().Add("X-Bar", "bar")
rw.WriteHeader(http.StatusNotFound)
})
tracer := &mockTracer{}
handler := newEntryPoint(context.Background(), tracing.NewTracer(tracer, []string{"X-Foo"}, []string{"X-Bar"}), nil, test.entryPoint, next)
handler.ServeHTTP(rw, req)
for _, span := range tracer.spans {
assert.Equal(t, test.expected.name, span.name)
assert.Equal(t, test.expected.attributes, span.attributes)
}
})
}
}
func TestEntryPointMiddleware_metrics(t *testing.T) {
tests := []struct {
desc string
statusCode int
wantAttributes attribute.Set
}{
{
desc: "not found status",
statusCode: http.StatusNotFound,
wantAttributes: attribute.NewSet(
attribute.Key("error.type").String("404"),
attribute.Key("http.request.method").String("GET"),
attribute.Key("http.response.status_code").Int(404),
attribute.Key("network.protocol.name").String("http/1.1"),
attribute.Key("network.protocol.version").String("1.1"),
attribute.Key("server.address").String("www.test.com"),
attribute.Key("url.scheme").String("http"),
),
},
{
desc: "created status",
statusCode: http.StatusCreated,
wantAttributes: attribute.NewSet(
attribute.Key("http.request.method").String("GET"),
attribute.Key("http.response.status_code").Int(201),
attribute.Key("network.protocol.name").String("http/1.1"),
attribute.Key("network.protocol.version").String("1.1"),
attribute.Key("server.address").String("www.test.com"),
attribute.Key("url.scheme").String("http"),
),
},
}
for _, test := range tests {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
var cfg types.OTLP
(&cfg).SetDefaults()
cfg.AddRoutersLabels = true
cfg.PushInterval = ptypes.Duration(10 * time.Millisecond)
rdr := sdkmetric.NewManualReader()
meterProvider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(rdr))
// force the meter provider with manual reader to collect metrics for the test.
metrics.SetMeterProvider(meterProvider)
semConvMetricRegistry, err := metrics.NewSemConvMetricRegistry(context.Background(), &cfg)
require.NoError(t, err)
require.NotNil(t, semConvMetricRegistry)
req := httptest.NewRequest(http.MethodGet, "http://www.test.com/search?q=Opentelemetry", nil)
rw := httptest.NewRecorder()
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "entrypoint-test")
req.Header.Set("X-Forwarded-Proto", "http")
next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
rw.WriteHeader(test.statusCode)
})
handler := newEntryPoint(context.Background(), nil, semConvMetricRegistry, "test", next)
handler.ServeHTTP(rw, req)
got := metricdata.ResourceMetrics{}
err = rdr.Collect(context.Background(), &got)
require.NoError(t, err)
require.Len(t, got.ScopeMetrics, 1)
expected := metricdata.Metrics{
Name: "http.server.request.duration",
Description: "Duration of HTTP server requests.",
Unit: "s",
Data: metricdata.Histogram[float64]{
DataPoints: []metricdata.HistogramDataPoint[float64]{
{
Attributes: test.wantAttributes,
Count: 1,
Bounds: []float64{0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10},
BucketCounts: []uint64{0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
Min: metricdata.NewExtrema[float64](1),
Max: metricdata.NewExtrema[float64](1),
Sum: 1,
},
},
Temporality: metricdata.CumulativeTemporality,
},
}
metricdatatest.AssertEqual[metricdata.Metrics](t, expected, got.ScopeMetrics[0].Metrics[0], metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreValue())
})
}
}

View file

@ -0,0 +1,71 @@
package observability
import (
"context"
"net/http"
"github.com/containous/alice"
"github.com/rs/zerolog/log"
"github.com/traefik/traefik/v3/pkg/logs"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
// Traceable embeds tracing information.
type Traceable interface {
GetTracingInformation() (name string, typeName string, spanKind trace.SpanKind)
}
// WrapMiddleware adds traceability to an alice.Constructor.
func WrapMiddleware(ctx context.Context, constructor alice.Constructor) alice.Constructor {
return func(next http.Handler) (http.Handler, error) {
if constructor == nil {
return nil, nil
}
handler, err := constructor(next)
if err != nil {
return nil, err
}
if traceableHandler, ok := handler.(Traceable); ok {
name, typeName, spanKind := traceableHandler.GetTracingInformation()
log.Ctx(ctx).Debug().Str(logs.MiddlewareName, name).Msg("Adding tracing to middleware")
return NewMiddleware(handler, name, typeName, spanKind), nil
}
return handler, nil
}
}
// NewMiddleware returns a http.Handler struct.
func NewMiddleware(next http.Handler, name string, typeName string, spanKind trace.SpanKind) http.Handler {
return &middlewareTracing{
next: next,
name: name,
typeName: typeName,
spanKind: spanKind,
}
}
// middlewareTracing is used to wrap http handler middleware.
type middlewareTracing struct {
next http.Handler
name string
typeName string
spanKind trace.SpanKind
}
func (w *middlewareTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if tracer := tracing.TracerFromContext(req.Context()); tracer != nil {
tracingCtx, span := tracer.Start(req.Context(), w.typeName, trace.WithSpanKind(w.spanKind))
defer span.End()
req = req.WithContext(tracingCtx)
span.SetAttributes(attribute.String("traefik.middleware.name", w.name))
}
if w.next != nil {
w.next.ServeHTTP(rw, req)
}
}

View file

@ -0,0 +1,66 @@
package observability
import (
"context"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/embedded"
)
type mockTracerProvider struct {
embedded.TracerProvider
}
var _ trace.TracerProvider = mockTracerProvider{}
func (p mockTracerProvider) Tracer(string, ...trace.TracerOption) trace.Tracer {
return &mockTracer{}
}
type mockTracer struct {
embedded.Tracer
spans []*mockSpan
}
var _ trace.Tracer = &mockTracer{}
func (t *mockTracer) Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
config := trace.NewSpanStartConfig(opts...)
span := &mockSpan{}
span.SetName(name)
span.SetAttributes(attribute.String("span.kind", config.SpanKind().String()))
span.SetAttributes(config.Attributes()...)
t.spans = append(t.spans, span)
return trace.ContextWithSpan(ctx, span), span
}
// mockSpan is an implementation of Span that preforms no operations.
type mockSpan struct {
embedded.Span
name string
attributes []attribute.KeyValue
}
var _ trace.Span = &mockSpan{}
func (*mockSpan) SpanContext() trace.SpanContext {
return trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{1}})
}
func (*mockSpan) IsRecording() bool { return false }
func (s *mockSpan) SetStatus(_ codes.Code, _ string) {}
func (s *mockSpan) SetAttributes(kv ...attribute.KeyValue) {
s.attributes = append(s.attributes, kv...)
}
func (s *mockSpan) End(...trace.SpanEndOption) {}
func (s *mockSpan) RecordError(_ error, _ ...trace.EventOption) {}
func (s *mockSpan) AddEvent(_ string, _ ...trace.EventOption) {}
func (s *mockSpan) SetName(name string) { s.name = name }
func (s *mockSpan) TracerProvider() trace.TracerProvider {
return nil
}

View file

@ -0,0 +1,31 @@
package observability
import (
"context"
"fmt"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
)
// SetStatusErrorf flags the span as in error and log an event.
func SetStatusErrorf(ctx context.Context, format string, args ...interface{}) {
if span := trace.SpanFromContext(ctx); span != nil {
span.SetStatus(codes.Error, fmt.Sprintf(format, args...))
}
}
func Proto(proto string) string {
switch proto {
case "HTTP/1.0":
return "1.0"
case "HTTP/1.1":
return "1.1"
case "HTTP/2":
return "2"
case "HTTP/3":
return "3"
default:
return proto
}
}

View file

@ -0,0 +1,60 @@
package observability
import (
"context"
"net/http"
"github.com/containous/alice"
"github.com/traefik/traefik/v3/pkg/logs"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/attribute"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
"go.opentelemetry.io/otel/trace"
)
const (
routerTypeName = "TracingRouter"
)
type routerTracing struct {
router string
routerRule string
service string
next http.Handler
}
// WrapRouterHandler Wraps tracing to alice.Constructor.
func WrapRouterHandler(ctx context.Context, router, routerRule, service string) alice.Constructor {
return func(next http.Handler) (http.Handler, error) {
return newRouter(ctx, router, routerRule, service, next), nil
}
}
// newRouter creates a new tracing middleware that traces the internal requests.
func newRouter(ctx context.Context, router, routerRule, service string, next http.Handler) http.Handler {
middlewares.GetLogger(ctx, "tracing", routerTypeName).
Debug().Str(logs.RouterName, router).Str(logs.ServiceName, service).Msg("Added outgoing tracing middleware")
return &routerTracing{
router: router,
routerRule: routerRule,
service: service,
next: next,
}
}
func (f *routerTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if tracer := tracing.TracerFromContext(req.Context()); tracer != nil {
tracingCtx, span := tracer.Start(req.Context(), "Router", trace.WithSpanKind(trace.SpanKindInternal))
defer span.End()
req = req.WithContext(tracingCtx)
span.SetAttributes(attribute.String("traefik.service.name", f.service))
span.SetAttributes(attribute.String("traefik.router.name", f.router))
span.SetAttributes(semconv.HTTPRoute(f.routerRule))
}
f.next.ServeHTTP(rw, req)
}

View file

@ -0,0 +1,86 @@
package observability
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
func TestNewRouter(t *testing.T) {
type expected struct {
attributes []attribute.KeyValue
name string
}
testCases := []struct {
desc string
service string
router string
routerRule string
expected []expected
}{
{
desc: "base",
service: "myService",
router: "myRouter",
routerRule: "Path(`/`)",
expected: []expected{
{
name: "EntryPoint",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "server"),
},
},
{
name: "Router",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "internal"),
attribute.String("http.request.method", "GET"),
attribute.Int64("http.response.status_code", int64(404)),
attribute.String("network.protocol.version", "1.1"),
attribute.String("server.address", "www.test.com"),
attribute.Int64("server.port", int64(80)),
attribute.String("url.full", "http://www.test.com/traces?p=OpenTelemetry"),
attribute.String("url.scheme", "http"),
attribute.String("traefik.service.name", "myService"),
attribute.String("traefik.router.name", "myRouter"),
attribute.String("http.route", "Path(`/`)"),
attribute.String("user_agent.original", "router-test"),
},
},
},
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "http://www.test.com/traces?p=OpenTelemetry", nil)
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "router-test")
tracer := &mockTracer{}
tracingCtx, entryPointSpan := tracer.Start(req.Context(), "EntryPoint", trace.WithSpanKind(trace.SpanKindServer))
defer entryPointSpan.End()
req = req.WithContext(tracingCtx)
rw := httptest.NewRecorder()
next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
rw.WriteHeader(http.StatusNotFound)
})
handler := newRouter(context.Background(), test.router, test.routerRule, test.service, next)
handler.ServeHTTP(rw, req)
for i, span := range tracer.spans {
assert.Equal(t, test.expected[i].name, span.name)
assert.Equal(t, test.expected[i].attributes, span.attributes)
}
})
}
}

View file

@ -0,0 +1,45 @@
package observability
import (
"context"
"net/http"
"github.com/traefik/traefik/v3/pkg/logs"
"github.com/traefik/traefik/v3/pkg/middlewares"
"github.com/traefik/traefik/v3/pkg/tracing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
const (
serviceTypeName = "TracingService"
)
type serviceTracing struct {
service string
next http.Handler
}
// NewService creates a new tracing middleware that traces the outgoing requests.
func NewService(ctx context.Context, service string, next http.Handler) http.Handler {
middlewares.GetLogger(ctx, "tracing", serviceTypeName).
Debug().Str(logs.ServiceName, service).Msg("Added outgoing tracing middleware")
return &serviceTracing{
service: service,
next: next,
}
}
func (t *serviceTracing) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if tracer := tracing.TracerFromContext(req.Context()); tracer != nil {
tracingCtx, span := tracer.Start(req.Context(), "Service", trace.WithSpanKind(trace.SpanKindInternal))
defer span.End()
req = req.WithContext(tracingCtx)
span.SetAttributes(attribute.String("traefik.service.name", t.service))
}
t.next.ServeHTTP(rw, req)
}

View file

@ -0,0 +1,80 @@
package observability
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
)
func TestNewService(t *testing.T) {
type expected struct {
attributes []attribute.KeyValue
name string
}
testCases := []struct {
desc string
service string
expected []expected
}{
{
desc: "base",
service: "myService",
expected: []expected{
{
name: "EntryPoint",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "server"),
},
},
{
name: "Service",
attributes: []attribute.KeyValue{
attribute.String("span.kind", "internal"),
attribute.String("http.request.method", "GET"),
attribute.Int64("http.response.status_code", int64(404)),
attribute.String("network.protocol.version", "1.1"),
attribute.String("server.address", "www.test.com"),
attribute.Int64("server.port", int64(80)),
attribute.String("url.full", "http://www.test.com/traces?p=OpenTelemetry"),
attribute.String("url.scheme", "http"),
attribute.String("traefik.service.name", "myService"),
attribute.String("user_agent.original", "service-test"),
},
},
},
},
}
for _, test := range testCases {
t.Run(test.desc, func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "http://www.test.com/traces?p=OpenTelemetry", nil)
req.RemoteAddr = "10.0.0.1:1234"
req.Header.Set("User-Agent", "service-test")
tracer := &mockTracer{}
tracingCtx, entryPointSpan := tracer.Start(req.Context(), "EntryPoint", trace.WithSpanKind(trace.SpanKindServer))
defer entryPointSpan.End()
req = req.WithContext(tracingCtx)
rw := httptest.NewRecorder()
next := http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
rw.WriteHeader(http.StatusNotFound)
})
handler := NewService(context.Background(), test.service, next)
handler.ServeHTTP(rw, req)
for i, span := range tracer.spans {
assert.Equal(t, test.expected[i].name, span.name)
assert.Equal(t, test.expected[i].attributes, span.attributes)
}
})
}
}

View file

@ -0,0 +1,40 @@
package observability
import (
"bufio"
"net"
"net/http"
)
// newStatusCodeRecorder returns an initialized statusCodeRecoder.
func newStatusCodeRecorder(rw http.ResponseWriter, status int) *statusCodeRecorder {
return &statusCodeRecorder{rw, status}
}
type statusCodeRecorder struct {
http.ResponseWriter
status int
}
// WriteHeader captures the status code for later retrieval.
func (s *statusCodeRecorder) WriteHeader(status int) {
s.status = status
s.ResponseWriter.WriteHeader(status)
}
// Status get response status.
func (s *statusCodeRecorder) Status() int {
return s.status
}
// Hijack hijacks the connection.
func (s *statusCodeRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) {
return s.ResponseWriter.(http.Hijacker).Hijack()
}
// Flush sends any buffered data to the client.
func (s *statusCodeRecorder) Flush() {
if flusher, ok := s.ResponseWriter.(http.Flusher); ok {
flusher.Flush()
}
}