DataDog and StatsD Metrics Support
* Added support for DataDog and StatsD monitoring * Added documentation
This commit is contained in:
Alex Antonov
Ludovic Fernandez
parent
cd28e7b24f
commit
69c628b626
39 changed files with 3921 additions and 13 deletions
92
middlewares/datadog.go
Normal file
92
middlewares/datadog.go
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
package middlewares
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/containous/traefik/log"
|
||||
"github.com/containous/traefik/safe"
|
||||
"github.com/containous/traefik/types"
|
||||
kitlog "github.com/go-kit/kit/log"
|
||||
"github.com/go-kit/kit/metrics"
|
||||
"github.com/go-kit/kit/metrics/dogstatsd"
|
||||
)
|
||||
|
||||
var _ Metrics = (Metrics)(nil)
|
||||
|
||||
var datadogClient = dogstatsd.New("traefik.", kitlog.LoggerFunc(func(keyvals ...interface{}) error {
|
||||
log.Info(keyvals)
|
||||
return nil
|
||||
}))
|
||||
|
||||
var datadogTicker *time.Ticker
|
||||
|
||||
// Metric names consistent with https://github.com/DataDog/integrations-extras/pull/64
|
||||
const (
|
||||
ddMetricsReqsName = "requests.total"
|
||||
ddMetricsLatencyName = "request.duration"
|
||||
)
|
||||
|
||||
// Datadog is an Implementation for Metrics that exposes datadog metrics for the latency
|
||||
// and the number of requests partitioned by status code and method.
|
||||
// - number of requests partitioned by status code and method
|
||||
// - request durations
|
||||
// - amount of retries happened
|
||||
type Datadog struct {
|
||||
reqsCounter metrics.Counter
|
||||
reqDurationHistogram metrics.Histogram
|
||||
retryCounter metrics.Counter
|
||||
}
|
||||
|
||||
func (dd *Datadog) getReqsCounter() metrics.Counter {
|
||||
return dd.reqsCounter
|
||||
}
|
||||
|
||||
func (dd *Datadog) getReqDurationHistogram() metrics.Histogram {
|
||||
return dd.reqDurationHistogram
|
||||
}
|
||||
|
||||
func (dd *Datadog) getRetryCounter() metrics.Counter {
|
||||
return dd.retryCounter
|
||||
}
|
||||
|
||||
// NewDataDog creates new instance of Datadog
|
||||
func NewDataDog(name string) *Datadog {
|
||||
var m Datadog
|
||||
|
||||
m.reqsCounter = datadogClient.NewCounter(ddMetricsReqsName, 1.0).With("service", name)
|
||||
m.reqDurationHistogram = datadogClient.NewHistogram(ddMetricsLatencyName, 1.0).With("service", name)
|
||||
|
||||
return &m
|
||||
}
|
||||
|
||||
// InitDatadogClient initializes metrics pusher and creates a datadogClient if not created already
|
||||
func InitDatadogClient(config *types.Datadog) *time.Ticker {
|
||||
if datadogTicker == nil {
|
||||
address := config.Address
|
||||
if len(address) == 0 {
|
||||
address = "localhost:8125"
|
||||
}
|
||||
pushInterval, err := time.ParseDuration(config.PushInterval)
|
||||
if err != nil {
|
||||
log.Warnf("Unable to parse %s into pushInterval, using 10s as default value", config.PushInterval)
|
||||
pushInterval = 10 * time.Second
|
||||
}
|
||||
|
||||
report := time.NewTicker(pushInterval)
|
||||
|
||||
safe.Go(func() {
|
||||
datadogClient.SendLoop(report.C, "udp", address)
|
||||
})
|
||||
|
||||
datadogTicker = report
|
||||
}
|
||||
return datadogTicker
|
||||
}
|
||||
|
||||
// StopDatadogClient stops internal datadogTicker which controls the pushing of metrics to DD Agent and resets it to `nil`
|
||||
func StopDatadogClient() {
|
||||
if datadogTicker != nil {
|
||||
datadogTicker.Stop()
|
||||
}
|
||||
datadogTicker = nil
|
||||
}
|
||||
53
middlewares/datadog_test.go
Normal file
53
middlewares/datadog_test.go
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
package middlewares
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/containous/traefik/testhelpers"
|
||||
"github.com/containous/traefik/types"
|
||||
"github.com/stvp/go-udp-testing"
|
||||
"github.com/urfave/negroni"
|
||||
)
|
||||
|
||||
func TestDatadog(t *testing.T) {
|
||||
udp.SetAddr(":18125")
|
||||
// This is needed to make sure that UDP Listener listens for data a bit longer, otherwise it will quit after a millisecond
|
||||
udp.Timeout = 5 * time.Second
|
||||
recorder := httptest.NewRecorder()
|
||||
InitDatadogClient(&types.Datadog{":18125", "1s"})
|
||||
|
||||
n := negroni.New()
|
||||
dd := NewDataDog("test")
|
||||
defer StopDatadogClient()
|
||||
metricsMiddlewareBackend := NewMetricsWrapper(dd)
|
||||
|
||||
n.Use(metricsMiddlewareBackend)
|
||||
r := http.NewServeMux()
|
||||
r.HandleFunc(`/ok`, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintln(w, "ok")
|
||||
})
|
||||
r.HandleFunc(`/not-found`, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
fmt.Fprintln(w, "not-found")
|
||||
})
|
||||
n.UseHandler(r)
|
||||
|
||||
req1 := testhelpers.MustNewRequest(http.MethodGet, "http://localhost:3000/ok", nil)
|
||||
req2 := testhelpers.MustNewRequest(http.MethodGet, "http://localhost:3000/not-found", nil)
|
||||
|
||||
expected := []string{
|
||||
// We are only validating counts, as it is nearly impossible to validate latency, since it varies every run
|
||||
"traefik.requests.total:1.000000|c|#service:test,code:404,method:GET\n",
|
||||
"traefik.requests.total:1.000000|c|#service:test,code:200,method:GET\n",
|
||||
}
|
||||
|
||||
udp.ShouldReceiveAll(t, expected, func() {
|
||||
n.ServeHTTP(recorder, req1)
|
||||
n.ServeHTTP(recorder, req2)
|
||||
})
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/go-kit/kit/metrics"
|
||||
"github.com/go-kit/kit/metrics/multi"
|
||||
)
|
||||
|
||||
// Metrics is an Interface that must be satisfied by any system that
|
||||
|
|
@ -22,6 +23,52 @@ type RetryMetrics interface {
|
|||
getRetryCounter() metrics.Counter
|
||||
}
|
||||
|
||||
// MultiMetrics is a struct that provides a wrapper container for multiple Metrics, if they are configured
|
||||
type MultiMetrics struct {
|
||||
wrappedMetrics *[]Metrics
|
||||
reqsCounter metrics.Counter
|
||||
reqDurationHistogram metrics.Histogram
|
||||
retryCounter metrics.Counter
|
||||
}
|
||||
|
||||
// NewMultiMetrics creates a new instance of MultiMetrics
|
||||
func NewMultiMetrics(manyMetrics []Metrics) *MultiMetrics {
|
||||
counters := []metrics.Counter{}
|
||||
histograms := []metrics.Histogram{}
|
||||
retryCounters := []metrics.Counter{}
|
||||
|
||||
for _, m := range manyMetrics {
|
||||
counters = append(counters, m.getReqsCounter())
|
||||
histograms = append(histograms, m.getReqDurationHistogram())
|
||||
retryCounters = append(retryCounters, m.getRetryCounter())
|
||||
}
|
||||
|
||||
var mm MultiMetrics
|
||||
|
||||
mm.wrappedMetrics = &manyMetrics
|
||||
mm.reqsCounter = multi.NewCounter(counters...)
|
||||
mm.reqDurationHistogram = multi.NewHistogram(histograms...)
|
||||
mm.retryCounter = multi.NewCounter(retryCounters...)
|
||||
|
||||
return &mm
|
||||
}
|
||||
|
||||
func (mm *MultiMetrics) getReqsCounter() metrics.Counter {
|
||||
return mm.reqsCounter
|
||||
}
|
||||
|
||||
func (mm *MultiMetrics) getReqDurationHistogram() metrics.Histogram {
|
||||
return mm.reqDurationHistogram
|
||||
}
|
||||
|
||||
func (mm *MultiMetrics) getRetryCounter() metrics.Counter {
|
||||
return mm.retryCounter
|
||||
}
|
||||
|
||||
func (mm *MultiMetrics) getWrappedMetrics() *[]Metrics {
|
||||
return mm.wrappedMetrics
|
||||
}
|
||||
|
||||
// MetricsWrapper is a Negroni compatible Handler which relies on a
|
||||
// given Metrics implementation to expose and monitor Traefik Metrics.
|
||||
type MetricsWrapper struct {
|
||||
|
|
|
|||
85
middlewares/statsd.go
Normal file
85
middlewares/statsd.go
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
package middlewares
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/containous/traefik/log"
|
||||
"github.com/containous/traefik/safe"
|
||||
"github.com/containous/traefik/types"
|
||||
kitlog "github.com/go-kit/kit/log"
|
||||
"github.com/go-kit/kit/metrics"
|
||||
"github.com/go-kit/kit/metrics/statsd"
|
||||
)
|
||||
|
||||
var _ Metrics = (Metrics)(nil)
|
||||
|
||||
var statsdClient = statsd.New("traefik.", kitlog.LoggerFunc(func(keyvals ...interface{}) error {
|
||||
log.Info(keyvals)
|
||||
return nil
|
||||
}))
|
||||
var statsdTicker *time.Ticker
|
||||
|
||||
// Statsd is an Implementation for Metrics that exposes statsd metrics for the latency
|
||||
// and the number of requests partitioned by status code and method.
|
||||
// - number of requests partitioned by status code and method
|
||||
// - request durations
|
||||
// - amount of retries happened
|
||||
type Statsd struct {
|
||||
reqsCounter metrics.Counter
|
||||
reqDurationHistogram metrics.Histogram
|
||||
retryCounter metrics.Counter
|
||||
}
|
||||
|
||||
func (s *Statsd) getReqsCounter() metrics.Counter {
|
||||
return s.reqsCounter
|
||||
}
|
||||
|
||||
func (s *Statsd) getReqDurationHistogram() metrics.Histogram {
|
||||
return s.reqDurationHistogram
|
||||
}
|
||||
|
||||
func (s *Statsd) getRetryCounter() metrics.Counter {
|
||||
return s.retryCounter
|
||||
}
|
||||
|
||||
// NewStatsD creates new instance of StatsD
|
||||
func NewStatsD(name string) *Statsd {
|
||||
var m Statsd
|
||||
|
||||
m.reqsCounter = statsdClient.NewCounter(ddMetricsReqsName, 1.0).With("service", name)
|
||||
m.reqDurationHistogram = statsdClient.NewTiming(ddMetricsLatencyName, 1.0).With("service", name)
|
||||
|
||||
return &m
|
||||
}
|
||||
|
||||
// InitStatsdClient initializes metrics pusher and creates a statsdClient if not created already
|
||||
func InitStatsdClient(config *types.Statsd) *time.Ticker {
|
||||
if statsdTicker == nil {
|
||||
address := config.Address
|
||||
if len(address) == 0 {
|
||||
address = "localhost:8125"
|
||||
}
|
||||
pushInterval, err := time.ParseDuration(config.PushInterval)
|
||||
if err != nil {
|
||||
log.Warnf("Unable to parse %s into pushInterval, using 10s as default value", config.PushInterval)
|
||||
pushInterval = 10 * time.Second
|
||||
}
|
||||
|
||||
report := time.NewTicker(pushInterval)
|
||||
|
||||
safe.Go(func() {
|
||||
statsdClient.SendLoop(report.C, "udp", address)
|
||||
})
|
||||
|
||||
statsdTicker = report
|
||||
}
|
||||
return statsdTicker
|
||||
}
|
||||
|
||||
// StopStatsdClient stops internal statsdTicker which controls the pushing of metrics to StatsD Agent and resets it to `nil`
|
||||
func StopStatsdClient() {
|
||||
if statsdTicker != nil {
|
||||
statsdTicker.Stop()
|
||||
}
|
||||
statsdTicker = nil
|
||||
}
|
||||
52
middlewares/statsd_test.go
Normal file
52
middlewares/statsd_test.go
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
package middlewares
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/containous/traefik/testhelpers"
|
||||
"github.com/containous/traefik/types"
|
||||
"github.com/stvp/go-udp-testing"
|
||||
"github.com/urfave/negroni"
|
||||
)
|
||||
|
||||
func TestStatsD(t *testing.T) {
|
||||
udp.SetAddr(":18125")
|
||||
// This is needed to make sure that UDP Listener listens for data a bit longer, otherwise it will quit after a millisecond
|
||||
udp.Timeout = 5 * time.Second
|
||||
recorder := httptest.NewRecorder()
|
||||
InitStatsdClient(&types.Statsd{":18125", "1s"})
|
||||
|
||||
n := negroni.New()
|
||||
c := NewStatsD("test")
|
||||
defer StopStatsdClient()
|
||||
metricsMiddlewareBackend := NewMetricsWrapper(c)
|
||||
|
||||
n.Use(metricsMiddlewareBackend)
|
||||
r := http.NewServeMux()
|
||||
r.HandleFunc(`/ok`, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintln(w, "ok")
|
||||
})
|
||||
r.HandleFunc(`/not-found`, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
fmt.Fprintln(w, "not-found")
|
||||
})
|
||||
n.UseHandler(r)
|
||||
|
||||
req1 := testhelpers.MustNewRequest(http.MethodGet, "http://localhost:3000/ok", nil)
|
||||
req2 := testhelpers.MustNewRequest(http.MethodGet, "http://localhost:3000/not-found", nil)
|
||||
|
||||
expected := []string{
|
||||
// We are only validating counts, as it is nearly impossible to validate latency, since it varies every run
|
||||
"traefik.requests.total:2.000000|c\n",
|
||||
}
|
||||
|
||||
udp.ShouldReceiveAll(t, expected, func() {
|
||||
n.ServeHTTP(recorder, req1)
|
||||
n.ServeHTTP(recorder, req2)
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue