1
0
Fork 0

Fix duplicated tags in InfluxDB

This commit is contained in:
Michael 2018-04-16 10:28:04 +02:00 committed by Traefiker Bot
parent 749d833f65
commit ebd77f314d
20 changed files with 530 additions and 209 deletions

View file

@ -1,23 +1,54 @@
// Package metrics provides a framework for application instrumentation. All
// metrics are safe for concurrent use. Considerable design influence has been
// taken from https://github.com/codahale/metrics and https://prometheus.io.
// Package metrics provides a framework for application instrumentation. It's
// primarily designed to help you get started with good and robust
// instrumentation, and to help you migrate from a less-capable system like
// Graphite to a more-capable system like Prometheus. If your organization has
// already standardized on an instrumentation system like Prometheus, and has no
// plans to change, it may make sense to use that system's instrumentation
// library directly.
//
// This package contains the common interfaces. Your code should take these
// interfaces as parameters. Implementations are provided for different
// instrumentation systems in the various subdirectories.
// This package provides three core metric abstractions (Counter, Gauge, and
// Histogram) and implementations for almost all common instrumentation
// backends. Each metric has an observation method (Add, Set, or Observe,
// respectively) used to record values, and a With method to "scope" the
// observation by various parameters. For example, you might have a Histogram to
// record request durations, parameterized by the method that's being called.
//
// var requestDuration metrics.Histogram
// // ...
// requestDuration.With("method", "MyMethod").Observe(time.Since(begin))
//
// This allows a single high-level metrics object (requestDuration) to work with
// many code paths somewhat dynamically. The concept of With is fully supported
// in some backends like Prometheus, and not supported in other backends like
// Graphite. So, With may be a no-op, depending on the concrete implementation
// you choose. Please check the implementation to know for sure. For
// implementations that don't provide With, it's necessary to fully parameterize
// each metric in the metric name, e.g.
//
// // Statsd
// c := statsd.NewCounter("request_duration_MyMethod_200")
// c.Add(1)
//
// // Prometheus
// c := prometheus.NewCounter(stdprometheus.CounterOpts{
// Name: "request_duration",
// ...
// }, []string{"method", "status_code"})
// c.With("method", "MyMethod", "status_code", strconv.Itoa(code)).Add(1)
//
// Usage
//
// Metrics are dependencies and should be passed to the components that need
// Metrics are dependencies, and should be passed to the components that need
// them in the same way you'd construct and pass a database handle, or reference
// to another component. So, create metrics in your func main, using whichever
// concrete implementation is appropriate for your organization.
// to another component. Metrics should *not* be created in the global scope.
// Instead, instantiate metrics in your func main, using whichever concrete
// implementation is appropriate for your organization.
//
// latency := prometheus.NewSummaryFrom(stdprometheus.SummaryOpts{
// Namespace: "myteam",
// Subsystem: "foosvc",
// Name: "request_latency_seconds",
// Help: "Incoming request latency in seconds."
// Help: "Incoming request latency in seconds.",
// }, []string{"method", "status_code"})
//
// Write your components to take the metrics they will use as parameters to
@ -40,8 +71,14 @@
// api := NewAPI(store, logger, latency)
// http.ListenAndServe("/", api)
//
// Note that metrics are "write-only" interfaces.
//
// Implementation details
//
// All metrics are safe for concurrent use. Considerable design influence has
// been taken from https://github.com/codahale/metrics and
// https://prometheus.io.
//
// Each telemetry system has different semantics for label values, push vs.
// pull, support for histograms, etc. These properties influence the design of
// their respective packages. This table attempts to summarize the key points of
@ -54,7 +91,7 @@
// expvar 1 atomic atomic synthetic, batch, in-place expose
// influx n custom custom custom
// prometheus n native native native
// circonus 1 native native native
// pcp 1 native native native
// cloudwatch n batch push-aggregate batch push-aggregate synthetic, batch, push-aggregate
//
package metrics

View file

@ -14,10 +14,13 @@ import (
"fmt"
"io"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/go-kit/kit/log"
"github.com/go-kit/kit/metrics"
"github.com/go-kit/kit/metrics/generic"
"github.com/go-kit/kit/metrics/internal/lv"
"github.com/go-kit/kit/metrics/internal/ratemap"
"github.com/go-kit/kit/util/conn"
@ -34,53 +37,63 @@ import (
// To regularly report metrics to an io.Writer, use the WriteLoop helper method.
// To send to a DogStatsD server, use the SendLoop helper method.
type Dogstatsd struct {
mtx sync.RWMutex
prefix string
rates *ratemap.RateMap
counters *lv.Space
gauges *lv.Space
gauges map[string]*gaugeNode
timings *lv.Space
histograms *lv.Space
logger log.Logger
lvs lv.LabelValues
}
// New returns a Dogstatsd object that may be used to create metrics. Prefix is
// applied to all created metrics. Callers must ensure that regular calls to
// WriteTo are performed, either manually or with one of the helper methods.
func New(prefix string, logger log.Logger) *Dogstatsd {
func New(prefix string, logger log.Logger, lvs ...string) *Dogstatsd {
if len(lvs)%2 != 0 {
panic("odd number of LabelValues; programmer error!")
}
return &Dogstatsd{
prefix: prefix,
rates: ratemap.New(),
counters: lv.NewSpace(),
gauges: lv.NewSpace(),
gauges: map[string]*gaugeNode{},
timings: lv.NewSpace(),
histograms: lv.NewSpace(),
logger: logger,
lvs: lvs,
}
}
// NewCounter returns a counter, sending observations to this Dogstatsd object.
func (d *Dogstatsd) NewCounter(name string, sampleRate float64) *Counter {
d.rates.Set(d.prefix+name, sampleRate)
d.rates.Set(name, sampleRate)
return &Counter{
name: d.prefix + name,
name: name,
obs: d.counters.Observe,
}
}
// NewGauge returns a gauge, sending observations to this Dogstatsd object.
func (d *Dogstatsd) NewGauge(name string) *Gauge {
return &Gauge{
name: d.prefix + name,
obs: d.gauges.Observe,
d.mtx.Lock()
n, ok := d.gauges[name]
if !ok {
n = &gaugeNode{gauge: &Gauge{g: generic.NewGauge(name), ddog: d}}
d.gauges[name] = n
}
d.mtx.Unlock()
return n.gauge
}
// NewTiming returns a histogram whose observations are interpreted as
// millisecond durations, and are forwarded to this Dogstatsd object.
func (d *Dogstatsd) NewTiming(name string, sampleRate float64) *Timing {
d.rates.Set(d.prefix+name, sampleRate)
d.rates.Set(name, sampleRate)
return &Timing{
name: d.prefix + name,
name: name,
obs: d.timings.Observe,
}
}
@ -88,9 +101,9 @@ func (d *Dogstatsd) NewTiming(name string, sampleRate float64) *Timing {
// NewHistogram returns a histogram whose observations are of an unspecified
// unit, and are forwarded to this Dogstatsd object.
func (d *Dogstatsd) NewHistogram(name string, sampleRate float64) *Histogram {
d.rates.Set(d.prefix+name, sampleRate)
d.rates.Set(name, sampleRate)
return &Histogram{
name: d.prefix + name,
name: name,
obs: d.histograms.Observe,
}
}
@ -124,7 +137,7 @@ func (d *Dogstatsd) WriteTo(w io.Writer) (count int64, err error) {
var n int
d.counters.Reset().Walk(func(name string, lvs lv.LabelValues, values []float64) bool {
n, err = fmt.Fprintf(w, "%s:%f|c%s%s\n", name, sum(values), sampling(d.rates.Get(name)), tagValues(lvs))
n, err = fmt.Fprintf(w, "%s%s:%f|c%s%s\n", d.prefix, name, sum(values), sampling(d.rates.Get(name)), d.tagValues(lvs))
if err != nil {
return false
}
@ -135,22 +148,23 @@ func (d *Dogstatsd) WriteTo(w io.Writer) (count int64, err error) {
return count, err
}
d.gauges.Reset().Walk(func(name string, lvs lv.LabelValues, values []float64) bool {
n, err = fmt.Fprintf(w, "%s:%f|g%s\n", name, last(values), tagValues(lvs))
if err != nil {
return false
}
count += int64(n)
return true
})
if err != nil {
return count, err
d.mtx.RLock()
for _, root := range d.gauges {
root.walk(func(name string, lvs lv.LabelValues, value float64) bool {
n, err = fmt.Fprintf(w, "%s%s:%f|g%s\n", d.prefix, name, value, d.tagValues(lvs))
if err != nil {
return false
}
count += int64(n)
return true
})
}
d.mtx.RUnlock()
d.timings.Reset().Walk(func(name string, lvs lv.LabelValues, values []float64) bool {
sampleRate := d.rates.Get(name)
for _, value := range values {
n, err = fmt.Fprintf(w, "%s:%f|ms%s%s\n", name, value, sampling(sampleRate), tagValues(lvs))
n, err = fmt.Fprintf(w, "%s%s:%f|ms%s%s\n", d.prefix, name, value, sampling(sampleRate), d.tagValues(lvs))
if err != nil {
return false
}
@ -165,7 +179,7 @@ func (d *Dogstatsd) WriteTo(w io.Writer) (count int64, err error) {
d.histograms.Reset().Walk(func(name string, lvs lv.LabelValues, values []float64) bool {
sampleRate := d.rates.Get(name)
for _, value := range values {
n, err = fmt.Fprintf(w, "%s:%f|h%s%s\n", name, value, sampling(sampleRate), tagValues(lvs))
n, err = fmt.Fprintf(w, "%s%s:%f|h%s%s\n", d.prefix, name, value, sampling(sampleRate), d.tagValues(lvs))
if err != nil {
return false
}
@ -200,14 +214,17 @@ func sampling(r float64) string {
return sv
}
func tagValues(labelValues []string) string {
if len(labelValues) == 0 {
func (d *Dogstatsd) tagValues(labelValues []string) string {
if len(labelValues) == 0 && len(d.lvs) == 0 {
return ""
}
if len(labelValues)%2 != 0 {
panic("tagValues received a labelValues with an odd number of strings")
}
pairs := make([]string, 0, len(labelValues)/2)
pairs := make([]string, 0, (len(d.lvs)+len(labelValues))/2)
for i := 0; i < len(d.lvs); i += 2 {
pairs = append(pairs, d.lvs[i]+":"+d.lvs[i+1])
}
for i := 0; i < len(labelValues); i += 2 {
pairs = append(pairs, labelValues[i]+":"+labelValues[i+1])
}
@ -241,23 +258,31 @@ func (c *Counter) Add(delta float64) {
// Gauge is a DogStatsD gauge. Observations are forwarded to a Dogstatsd
// object, and aggregated (the last observation selected) per timeseries.
type Gauge struct {
name string
lvs lv.LabelValues
obs observeFunc
g *generic.Gauge
ddog *Dogstatsd
set int32
}
// With implements metrics.Gauge.
func (g *Gauge) With(labelValues ...string) metrics.Gauge {
return &Gauge{
name: g.name,
lvs: g.lvs.With(labelValues...),
obs: g.obs,
}
g.ddog.mtx.RLock()
node := g.ddog.gauges[g.g.Name]
g.ddog.mtx.RUnlock()
ga := &Gauge{g: g.g.With(labelValues...).(*generic.Gauge), ddog: g.ddog}
return node.addGauge(ga, ga.g.LabelValues())
}
// Set implements metrics.Gauge.
func (g *Gauge) Set(value float64) {
g.obs(g.name, g.lvs, value)
g.g.Set(value)
g.touch()
}
// Add implements metrics.Gauge.
func (g *Gauge) Add(delta float64) {
g.g.Add(delta)
g.touch()
}
// Timing is a DogStatsD timing, or metrics.Histogram. Observations are
@ -304,3 +329,61 @@ func (h *Histogram) With(labelValues ...string) metrics.Histogram {
func (h *Histogram) Observe(value float64) {
h.obs(h.name, h.lvs, value)
}
type pair struct{ label, value string }
type gaugeNode struct {
mtx sync.RWMutex
gauge *Gauge
children map[pair]*gaugeNode
}
func (n *gaugeNode) addGauge(g *Gauge, lvs lv.LabelValues) *Gauge {
n.mtx.Lock()
defer n.mtx.Unlock()
if len(lvs) == 0 {
if n.gauge == nil {
n.gauge = g
}
return n.gauge
}
if len(lvs) < 2 {
panic("too few LabelValues; programmer error!")
}
head, tail := pair{lvs[0], lvs[1]}, lvs[2:]
if n.children == nil {
n.children = map[pair]*gaugeNode{}
}
child, ok := n.children[head]
if !ok {
child = &gaugeNode{}
n.children[head] = child
}
return child.addGauge(g, tail)
}
func (n *gaugeNode) walk(fn func(string, lv.LabelValues, float64) bool) bool {
n.mtx.RLock()
defer n.mtx.RUnlock()
if n.gauge != nil {
value, ok := n.gauge.read()
if ok && !fn(n.gauge.g.Name, n.gauge.g.LabelValues(), value) {
return false
}
}
for _, child := range n.children {
if !child.walk(fn) {
return false
}
}
return true
}
func (g *Gauge) touch() {
atomic.StoreInt32(&(g.set), 1)
}
func (g *Gauge) read() (float64, bool) {
set := atomic.SwapInt32(&(g.set), 0)
return g.g.Value(), set != 0
}

View file

@ -33,6 +33,7 @@ func NewCounter(name string) *Counter {
// With implements Counter.
func (c *Counter) With(labelValues ...string) metrics.Counter {
return &Counter{
Name: c.Name,
bits: atomic.LoadUint64(&c.bits),
lvs: c.lvs.With(labelValues...),
}
@ -95,6 +96,7 @@ func NewGauge(name string) *Gauge {
// With implements Gauge.
func (g *Gauge) With(labelValues ...string) metrics.Gauge {
return &Gauge{
Name: g.Name,
bits: atomic.LoadUint64(&g.bits),
lvs: g.lvs.With(labelValues...),
}
@ -105,6 +107,20 @@ func (g *Gauge) Set(value float64) {
atomic.StoreUint64(&g.bits, math.Float64bits(value))
}
// Add implements metrics.Gauge.
func (g *Gauge) Add(delta float64) {
for {
var (
old = atomic.LoadUint64(&g.bits)
newf = math.Float64frombits(old) + delta
new = math.Float64bits(newf)
)
if atomic.CompareAndSwapUint64(&g.bits, old, new) {
break
}
}
}
// Value returns the current value of the gauge.
func (g *Gauge) Value() float64 {
return math.Float64frombits(atomic.LoadUint64(&g.bits))
@ -121,7 +137,7 @@ func (g *Gauge) LabelValues() []string {
type Histogram struct {
Name string
lvs lv.LabelValues
h gohistogram.Histogram
h *safeHistogram
}
// NewHistogram returns a numeric histogram based on VividCortex/gohistogram. A
@ -129,25 +145,30 @@ type Histogram struct {
func NewHistogram(name string, buckets int) *Histogram {
return &Histogram{
Name: name,
h: gohistogram.NewHistogram(buckets),
h: &safeHistogram{Histogram: gohistogram.NewHistogram(buckets)},
}
}
// With implements Histogram.
func (h *Histogram) With(labelValues ...string) metrics.Histogram {
return &Histogram{
lvs: h.lvs.With(labelValues...),
h: h.h,
Name: h.Name,
lvs: h.lvs.With(labelValues...),
h: h.h,
}
}
// Observe implements Histogram.
func (h *Histogram) Observe(value float64) {
h.h.Lock()
defer h.h.Unlock()
h.h.Add(value)
}
// Quantile returns the value of the quantile q, 0.0 < q < 1.0.
func (h *Histogram) Quantile(q float64) float64 {
h.h.RLock()
defer h.h.RUnlock()
return h.h.Quantile(q)
}
@ -159,9 +180,17 @@ func (h *Histogram) LabelValues() []string {
// Print writes a string representation of the histogram to the passed writer.
// Useful for printing to a terminal.
func (h *Histogram) Print(w io.Writer) {
h.h.RLock()
defer h.h.RUnlock()
fmt.Fprintf(w, h.h.String())
}
// safeHistogram exists as gohistogram.Histogram is not goroutine-safe.
type safeHistogram struct {
sync.RWMutex
gohistogram.Histogram
}
// Bucket is a range in a histogram which aggregates observations.
type Bucket struct {
From, To, Count int64

View file

@ -66,6 +66,7 @@ func (in *Influx) NewGauge(name string) *Gauge {
return &Gauge{
name: name,
obs: in.gauges.Observe,
add: in.gauges.Add,
}
}
@ -168,10 +169,14 @@ func mergeTags(tags map[string]string, labelValues []string) map[string]string {
if len(labelValues)%2 != 0 {
panic("mergeTags received a labelValues with an odd number of strings")
}
for i := 0; i < len(labelValues); i += 2 {
tags[labelValues[i]] = labelValues[i+1]
ret := make(map[string]string, len(tags)+len(labelValues)/2)
for k, v := range tags {
ret[k] = v
}
return tags
for i := 0; i < len(labelValues); i += 2 {
ret[labelValues[i]] = labelValues[i+1]
}
return ret
}
func sum(a []float64) float64 {
@ -216,6 +221,7 @@ type Gauge struct {
name string
lvs lv.LabelValues
obs observeFunc
add observeFunc
}
// With implements metrics.Gauge.
@ -224,6 +230,7 @@ func (g *Gauge) With(labelValues ...string) metrics.Gauge {
name: g.name,
lvs: g.lvs.With(labelValues...),
obs: g.obs,
add: g.add,
}
}
@ -232,6 +239,11 @@ func (g *Gauge) Set(value float64) {
g.obs(g.name, g.lvs, value)
}
// Add implements metrics.Gauge.
func (g *Gauge) Add(delta float64) {
g.add(g.name, g.lvs, delta)
}
// Histogram is an Influx histrogram. Observations are aggregated into a
// generic.Histogram and emitted as per-quantile gauges to the Influx server.
type Histogram struct {

View file

@ -21,6 +21,13 @@ func (s *Space) Observe(name string, lvs LabelValues, value float64) {
s.nodeFor(name).observe(lvs, value)
}
// Add locates the time series identified by the name and label values in
// the vector space, and appends the delta to the last value in the list of
// observations.
func (s *Space) Add(name string, lvs LabelValues, delta float64) {
s.nodeFor(name).add(lvs, delta)
}
// Walk traverses the vector space and invokes fn for each non-empty time series
// which is encountered. Return false to abort the traversal.
func (s *Space) Walk(fn func(name string, lvs LabelValues, observations []float64) bool) {
@ -91,6 +98,34 @@ func (n *node) observe(lvs LabelValues, value float64) {
child.observe(tail, value)
}
func (n *node) add(lvs LabelValues, delta float64) {
n.mtx.Lock()
defer n.mtx.Unlock()
if len(lvs) == 0 {
var value float64
if len(n.observations) > 0 {
value = last(n.observations) + delta
} else {
value = delta
}
n.observations = append(n.observations, value)
return
}
if len(lvs) < 2 {
panic("too few LabelValues; programmer error!")
}
head, tail := pair{lvs[0], lvs[1]}, lvs[2:]
if n.children == nil {
n.children = map[pair]*node{}
}
child, ok := n.children[head]
if !ok {
child = &node{}
n.children[head] = child
}
child.add(tail, delta)
}
func (n *node) walk(lvs LabelValues, fn func(LabelValues, []float64) bool) bool {
n.mtx.RLock()
defer n.mtx.RUnlock()
@ -104,3 +139,7 @@ func (n *node) walk(lvs LabelValues, fn func(LabelValues, []float64) bool) bool
}
return true
}
func last(a []float64) float64 {
return a[len(a)-1]
}

View file

@ -12,6 +12,7 @@ type Counter interface {
type Gauge interface {
With(labelValues ...string) Gauge
Set(value float64)
Add(delta float64)
}
// Histogram describes a metric that takes repeated observations of the same

View file

@ -54,6 +54,13 @@ func (g Gauge) With(labelValues ...string) metrics.Gauge {
return next
}
// Add implements metrics.Gauge.
func (g Gauge) Add(delta float64) {
for _, gauge := range g {
gauge.Add(delta)
}
}
// Histogram collects multiple individual histograms and treats them as a unit.
type Histogram []metrics.Histogram

View file

@ -74,6 +74,7 @@ func (s *Statsd) NewGauge(name string) *Gauge {
return &Gauge{
name: s.prefix + name,
obs: s.gauges.Observe,
add: s.gauges.Add,
}
}
@ -201,6 +202,7 @@ func (c *Counter) Add(delta float64) {
type Gauge struct {
name string
obs observeFunc
add observeFunc
}
// With is a no-op.
@ -213,6 +215,11 @@ func (g *Gauge) Set(value float64) {
g.obs(g.name, lv.LabelValues{}, value)
}
// Add implements metrics.Gauge.
func (g *Gauge) Add(delta float64) {
g.add(g.name, lv.LabelValues{}, delta)
}
// Timing is a StatsD timing, or metrics.Histogram. Observations are
// forwarded to a Statsd object, and collected (but not aggregated) per
// timeseries.

View file

@ -7,6 +7,7 @@ import "time"
type Timer struct {
h Histogram
t time.Time
u time.Duration
}
// NewTimer wraps the given histogram and records the current time.
@ -14,15 +15,22 @@ func NewTimer(h Histogram) *Timer {
return &Timer{
h: h,
t: time.Now(),
u: time.Second,
}
}
// ObserveDuration captures the number of seconds since the timer was
// constructed, and forwards that observation to the histogram.
func (t *Timer) ObserveDuration() {
d := time.Since(t.t).Seconds()
d := float64(time.Since(t.t).Nanoseconds()) / float64(t.u)
if d < 0 {
d = 0
}
t.h.Observe(d)
}
// Unit sets the unit of the float64 emitted by the timer.
// By default, the timer emits seconds.
func (t *Timer) Unit(u time.Duration) {
t.u = u
}