DataDog and StatsD Metrics Support

* Added support for DataDog and StatsD monitoring
* Added documentation

vendor/github.com/go-kit/kit/log/doc.go

@@ -0,0 +1,93 @@
+// Package log provides a structured logger.
+//
+// Structured logging produces logs easily consumed later by humans or
+// machines. Humans might be interested in debugging errors, or tracing
+// specific requests. Machines might be interested in counting interesting
+// events, or aggregating information for off-line processing. In both cases,
+// it is important that the log messages are structured and actionable.
+// Package log is designed to encourage both of these best practices.
+//
+// Basic Usage
+//
+// The fundamental interface is Logger. Loggers create log events from
+// key/value data. The Logger interface has a single method, Log, which
+// accepts a sequence of alternating key/value pairs, which this package names
+// keyvals.
+//
+//    type Logger interface {
+//        Log(keyvals ...interface{}) error
+//    }
+//
+// Here is an example of a function using a Logger to create log events.
+//
+//    func RunTask(task Task, logger log.Logger) string {
+//        logger.Log("taskID", task.ID, "event", "starting task")
+//        ...
+//        logger.Log("taskID", task.ID, "event", "task complete")
+//    }
+//
+// The keys in the above example are "taskID" and "event". The values are
+// task.ID, "starting task", and "task complete". Every key is followed
+// immediately by its value.
+//
+// Keys are usually plain strings. Values may be any type that has a sensible
+// encoding in the chosen log format. With structured logging it is a good
+// idea to log simple values without formatting them. This practice allows
+// the chosen logger to encode values in the most appropriate way.
+//
+// Log Context
+//
+// A log context stores keyvals that it includes in all log events. Building
+// appropriate log contexts reduces repetition and aids consistency in the
+// resulting log output. We can use a context to improve the RunTask example.
+//
+//    func RunTask(task Task, logger log.Logger) string {
+//        logger = log.NewContext(logger).With("taskID", task.ID)
+//        logger.Log("event", "starting task")
+//        ...
+//        taskHelper(task.Cmd, logger)
+//        ...
+//        logger.Log("event", "task complete")
+//    }
+//
+// The improved version emits the same log events as the original for the
+// first and last calls to Log. The call to taskHelper highlights that a
+// context may be passed as a logger to other functions. Each log event
+// created by the called function will include the task.ID even though the
+// function does not have access to that value. Using log contexts this way
+// simplifies producing log output that enables tracing the life cycle of
+// individual tasks. (See the Context example for the full code of the
+// above snippet.)
+//
+// Dynamic Context Values
+//
+// A Valuer function stored in a log context generates a new value each time
+// the context logs an event. The Valuer example demonstrates how this
+// feature works.
+//
+// Valuers provide the basis for consistently logging timestamps and source
+// code location. The log package defines several valuers for that purpose.
+// See Timestamp, DefaultTimestamp, DefaultTimestampUTC, Caller, and
+// DefaultCaller. A common logger initialization sequence that ensures all log
+// entries contain a timestamp and source location looks like this:
+//
+//    logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout))
+//    logger = log.NewContext(logger).With("ts", log.DefaultTimestampUTC, "caller", log.DefaultCaller)
+//
+// Concurrent Safety
+//
+// Applications with multiple goroutines want each log event written to the
+// same logger to remain separate from other log events. Package log provides
+// two simple solutions for concurrent safe logging.
+//
+// NewSyncWriter wraps an io.Writer and serializes each call to its Write
+// method. Using a SyncWriter has the benefit that the smallest practical
+// portion of the logging logic is performed within a mutex, but it requires
+// the formatting Logger to make only one call to Write per log event.
+//
+// NewSyncLogger wraps any Logger and serializes each call to its Log method.
+// Using a SyncLogger has the benefit that it guarantees each log event is
+// handled atomically within the wrapped logger, but it typically serializes
+// both the formatting and output logic. Use a SyncLogger if the formatting
+// logger may perform multiple writes per log event.
+package log

vendor/github.com/go-kit/kit/log/json_logger.go

@@ -0,0 +1,92 @@
+package log
+
+import (
+	"encoding"
+	"encoding/json"
+	"fmt"
+	"io"
+	"reflect"
+)
+
+type jsonLogger struct {
+	io.Writer
+}
+
+// NewJSONLogger returns a Logger that encodes keyvals to the Writer as a
+// single JSON object. Each log event produces no more than one call to
+// w.Write. The passed Writer must be safe for concurrent use by multiple
+// goroutines if the returned Logger will be used concurrently.
+func NewJSONLogger(w io.Writer) Logger {
+	return &jsonLogger{w}
+}
+
+func (l *jsonLogger) Log(keyvals ...interface{}) error {
+	n := (len(keyvals) + 1) / 2 // +1 to handle case when len is odd
+	m := make(map[string]interface{}, n)
+	for i := 0; i < len(keyvals); i += 2 {
+		k := keyvals[i]
+		var v interface{} = ErrMissingValue
+		if i+1 < len(keyvals) {
+			v = keyvals[i+1]
+		}
+		merge(m, k, v)
+	}
+	return json.NewEncoder(l.Writer).Encode(m)
+}
+
+func merge(dst map[string]interface{}, k, v interface{}) {
+	var key string
+	switch x := k.(type) {
+	case string:
+		key = x
+	case fmt.Stringer:
+		key = safeString(x)
+	default:
+		key = fmt.Sprint(x)
+	}
+	if x, ok := v.(error); ok {
+		v = safeError(x)
+	}
+
+	// We want json.Marshaler and encoding.TextMarshaller to take priority over
+	// err.Error() and v.String(). But json.Marshall (called later) does that by
+	// default so we force a no-op if it's one of those 2 case.
+	switch x := v.(type) {
+	case json.Marshaler:
+	case encoding.TextMarshaler:
+	case error:
+		v = safeError(x)
+	case fmt.Stringer:
+		v = safeString(x)
+	}
+
+	dst[key] = v
+}
+
+func safeString(str fmt.Stringer) (s string) {
+	defer func() {
+		if panicVal := recover(); panicVal != nil {
+			if v := reflect.ValueOf(str); v.Kind() == reflect.Ptr && v.IsNil() {
+				s = "NULL"
+			} else {
+				panic(panicVal)
+			}
+		}
+	}()
+	s = str.String()
+	return
+}
+
+func safeError(err error) (s interface{}) {
+	defer func() {
+		if panicVal := recover(); panicVal != nil {
+			if v := reflect.ValueOf(err); v.Kind() == reflect.Ptr && v.IsNil() {
+				s = nil
+			} else {
+				panic(panicVal)
+			}
+		}
+	}()
+	s = err.Error()
+	return
+}

vendor/github.com/go-kit/kit/log/log.go

@@ -0,0 +1,144 @@
+package log
+
+import "errors"
+
+// Logger is the fundamental interface for all log operations. Log creates a
+// log event from keyvals, a variadic sequence of alternating keys and values.
+// Implementations must be safe for concurrent use by multiple goroutines. In
+// particular, any implementation of Logger that appends to keyvals or
+// modifies any of its elements must make a copy first.
+type Logger interface {
+	Log(keyvals ...interface{}) error
+}
+
+// ErrMissingValue is appended to keyvals slices with odd length to substitute
+// the missing value.
+var ErrMissingValue = errors.New("(MISSING)")
+
+// NewContext returns a new Context that logs to logger.
+func NewContext(logger Logger) *Context {
+	if c, ok := logger.(*Context); ok {
+		return c
+	}
+	return &Context{logger: logger}
+}
+
+// Context must always have the same number of stack frames between calls to
+// its Log method and the eventual binding of Valuers to their value. This
+// requirement comes from the functional requirement to allow a context to
+// resolve application call site information for a log.Caller stored in the
+// context. To do this we must be able to predict the number of logging
+// functions on the stack when bindValues is called.
+//
+// Three implementation details provide the needed stack depth consistency.
+// The first two of these details also result in better amortized performance,
+// and thus make sense even without the requirements regarding stack depth.
+// The third detail, however, is subtle and tied to the implementation of the
+// Go compiler.
+//
+//    1. NewContext avoids introducing an additional layer when asked to
+//       wrap another Context.
+//    2. With avoids introducing an additional layer by returning a newly
+//       constructed Context with a merged keyvals rather than simply
+//       wrapping the existing Context.
+//    3. All of Context's methods take pointer receivers even though they
+//       do not mutate the Context.
+//
+// Before explaining the last detail, first some background. The Go compiler
+// generates wrapper methods to implement the auto dereferencing behavior when
+// calling a value method through a pointer variable. These wrapper methods
+// are also used when calling a value method through an interface variable
+// because interfaces store a pointer to the underlying concrete value.
+// Calling a pointer receiver through an interface does not require generating
+// an additional function.
+//
+// If Context had value methods then calling Context.Log through a variable
+// with type Logger would have an extra stack frame compared to calling
+// Context.Log through a variable with type Context. Using pointer receivers
+// avoids this problem.
+
+// A Context wraps a Logger and holds keyvals that it includes in all log
+// events. When logging, a Context replaces all value elements (odd indexes)
+// containing a Valuer with their generated value for each call to its Log
+// method.
+type Context struct {
+	logger    Logger
+	keyvals   []interface{}
+	hasValuer bool
+}
+
+// Log replaces all value elements (odd indexes) containing a Valuer in the
+// stored context with their generated value, appends keyvals, and passes the
+// result to the wrapped Logger.
+func (l *Context) Log(keyvals ...interface{}) error {
+	kvs := append(l.keyvals, keyvals...)
+	if len(kvs)%2 != 0 {
+		kvs = append(kvs, ErrMissingValue)
+	}
+	if l.hasValuer {
+		// If no keyvals were appended above then we must copy l.keyvals so
+		// that future log events will reevaluate the stored Valuers.
+		if len(keyvals) == 0 {
+			kvs = append([]interface{}{}, l.keyvals...)
+		}
+		bindValues(kvs[:len(l.keyvals)])
+	}
+	return l.logger.Log(kvs...)
+}
+
+// With returns a new Context with keyvals appended to those of the receiver.
+func (l *Context) With(keyvals ...interface{}) *Context {
+	if len(keyvals) == 0 {
+		return l
+	}
+	kvs := append(l.keyvals, keyvals...)
+	if len(kvs)%2 != 0 {
+		kvs = append(kvs, ErrMissingValue)
+	}
+	return &Context{
+		logger: l.logger,
+		// Limiting the capacity of the stored keyvals ensures that a new
+		// backing array is created if the slice must grow in Log or With.
+		// Using the extra capacity without copying risks a data race that
+		// would violate the Logger interface contract.
+		keyvals:   kvs[:len(kvs):len(kvs)],
+		hasValuer: l.hasValuer || containsValuer(keyvals),
+	}
+}
+
+// WithPrefix returns a new Context with keyvals prepended to those of the
+// receiver.
+func (l *Context) WithPrefix(keyvals ...interface{}) *Context {
+	if len(keyvals) == 0 {
+		return l
+	}
+	// Limiting the capacity of the stored keyvals ensures that a new
+	// backing array is created if the slice must grow in Log or With.
+	// Using the extra capacity without copying risks a data race that
+	// would violate the Logger interface contract.
+	n := len(l.keyvals) + len(keyvals)
+	if len(keyvals)%2 != 0 {
+		n++
+	}
+	kvs := make([]interface{}, 0, n)
+	kvs = append(kvs, keyvals...)
+	if len(kvs)%2 != 0 {
+		kvs = append(kvs, ErrMissingValue)
+	}
+	kvs = append(kvs, l.keyvals...)
+	return &Context{
+		logger:    l.logger,
+		keyvals:   kvs,
+		hasValuer: l.hasValuer || containsValuer(keyvals),
+	}
+}
+
+// LoggerFunc is an adapter to allow use of ordinary functions as Loggers. If
+// f is a function with the appropriate signature, LoggerFunc(f) is a Logger
+// object that calls f.
+type LoggerFunc func(...interface{}) error
+
+// Log implements Logger by calling f(keyvals...).
+func (f LoggerFunc) Log(keyvals ...interface{}) error {
+	return f(keyvals...)
+}

vendor/github.com/go-kit/kit/log/logfmt_logger.go

@@ -0,0 +1,62 @@
+package log
+
+import (
+	"bytes"
+	"io"
+	"sync"
+
+	"github.com/go-logfmt/logfmt"
+)
+
+type logfmtEncoder struct {
+	*logfmt.Encoder
+	buf bytes.Buffer
+}
+
+func (l *logfmtEncoder) Reset() {
+	l.Encoder.Reset()
+	l.buf.Reset()
+}
+
+var logfmtEncoderPool = sync.Pool{
+	New: func() interface{} {
+		var enc logfmtEncoder
+		enc.Encoder = logfmt.NewEncoder(&enc.buf)
+		return &enc
+	},
+}
+
+type logfmtLogger struct {
+	w io.Writer
+}
+
+// NewLogfmtLogger returns a logger that encodes keyvals to the Writer in
+// logfmt format. Each log event produces no more than one call to w.Write.
+// The passed Writer must be safe for concurrent use by multiple goroutines if
+// the returned Logger will be used concurrently.
+func NewLogfmtLogger(w io.Writer) Logger {
+	return &logfmtLogger{w}
+}
+
+func (l logfmtLogger) Log(keyvals ...interface{}) error {
+	enc := logfmtEncoderPool.Get().(*logfmtEncoder)
+	enc.Reset()
+	defer logfmtEncoderPool.Put(enc)
+
+	if err := enc.EncodeKeyvals(keyvals...); err != nil {
+		return err
+	}
+
+	// Add newline to the end of the buffer
+	if err := enc.EndRecord(); err != nil {
+		return err
+	}
+
+	// The Logger interface requires implementations to be safe for concurrent
+	// use by multiple goroutines. For this implementation that means making
+	// only one call to l.w.Write() for each call to Log.
+	if _, err := l.w.Write(enc.buf.Bytes()); err != nil {
+		return err
+	}
+	return nil
+}

vendor/github.com/go-kit/kit/log/nop_logger.go

@@ -0,0 +1,8 @@
+package log
+
+type nopLogger struct{}
+
+// NewNopLogger returns a logger that doesn't do anything.
+func NewNopLogger() Logger { return nopLogger{} }
+
+func (nopLogger) Log(...interface{}) error { return nil }

vendor/github.com/go-kit/kit/log/stdlib.go

@@ -0,0 +1,116 @@
+package log
+
+import (
+	"io"
+	"log"
+	"regexp"
+	"strings"
+)
+
+// StdlibWriter implements io.Writer by invoking the stdlib log.Print. It's
+// designed to be passed to a Go kit logger as the writer, for cases where
+// it's necessary to redirect all Go kit log output to the stdlib logger.
+//
+// If you have any choice in the matter, you shouldn't use this. Prefer to
+// redirect the stdlib log to the Go kit logger via NewStdlibAdapter.
+type StdlibWriter struct{}
+
+// Write implements io.Writer.
+func (w StdlibWriter) Write(p []byte) (int, error) {
+	log.Print(strings.TrimSpace(string(p)))
+	return len(p), nil
+}
+
+// StdlibAdapter wraps a Logger and allows it to be passed to the stdlib
+// logger's SetOutput. It will extract date/timestamps, filenames, and
+// messages, and place them under relevant keys.
+type StdlibAdapter struct {
+	Logger
+	timestampKey string
+	fileKey      string
+	messageKey   string
+}
+
+// StdlibAdapterOption sets a parameter for the StdlibAdapter.
+type StdlibAdapterOption func(*StdlibAdapter)
+
+// TimestampKey sets the key for the timestamp field. By default, it's "ts".
+func TimestampKey(key string) StdlibAdapterOption {
+	return func(a *StdlibAdapter) { a.timestampKey = key }
+}
+
+// FileKey sets the key for the file and line field. By default, it's "file".
+func FileKey(key string) StdlibAdapterOption {
+	return func(a *StdlibAdapter) { a.fileKey = key }
+}
+
+// MessageKey sets the key for the actual log message. By default, it's "msg".
+func MessageKey(key string) StdlibAdapterOption {
+	return func(a *StdlibAdapter) { a.messageKey = key }
+}
+
+// NewStdlibAdapter returns a new StdlibAdapter wrapper around the passed
+// logger. It's designed to be passed to log.SetOutput.
+func NewStdlibAdapter(logger Logger, options ...StdlibAdapterOption) io.Writer {
+	a := StdlibAdapter{
+		Logger:       logger,
+		timestampKey: "ts",
+		fileKey:      "file",
+		messageKey:   "msg",
+	}
+	for _, option := range options {
+		option(&a)
+	}
+	return a
+}
+
+func (a StdlibAdapter) Write(p []byte) (int, error) {
+	result := subexps(p)
+	keyvals := []interface{}{}
+	var timestamp string
+	if date, ok := result["date"]; ok && date != "" {
+		timestamp = date
+	}
+	if time, ok := result["time"]; ok && time != "" {
+		if timestamp != "" {
+			timestamp += " "
+		}
+		timestamp += time
+	}
+	if timestamp != "" {
+		keyvals = append(keyvals, a.timestampKey, timestamp)
+	}
+	if file, ok := result["file"]; ok && file != "" {
+		keyvals = append(keyvals, a.fileKey, file)
+	}
+	if msg, ok := result["msg"]; ok {
+		keyvals = append(keyvals, a.messageKey, msg)
+	}
+	if err := a.Logger.Log(keyvals...); err != nil {
+		return 0, err
+	}
+	return len(p), nil
+}
+
+const (
+	logRegexpDate = `(?P<date>[0-9]{4}/[0-9]{2}/[0-9]{2})?[ ]?`
+	logRegexpTime = `(?P<time>[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?)?[ ]?`
+	logRegexpFile = `(?P<file>.+?:[0-9]+)?`
+	logRegexpMsg  = `(: )?(?P<msg>.*)`
+)
+
+var (
+	logRegexp = regexp.MustCompile(logRegexpDate + logRegexpTime + logRegexpFile + logRegexpMsg)
+)
+
+func subexps(line []byte) map[string]string {
+	m := logRegexp.FindSubmatch(line)
+	if len(m) < len(logRegexp.SubexpNames()) {
+		return map[string]string{}
+	}
+	result := map[string]string{}
+	for i, name := range logRegexp.SubexpNames() {
+		result[name] = string(m[i])
+	}
+	return result
+}

vendor/github.com/go-kit/kit/log/sync.go

@@ -0,0 +1,81 @@
+package log
+
+import (
+	"io"
+	"sync"
+	"sync/atomic"
+)
+
+// SwapLogger wraps another logger that may be safely replaced while other
+// goroutines use the SwapLogger concurrently. The zero value for a SwapLogger
+// will discard all log events without error.
+//
+// SwapLogger serves well as a package global logger that can be changed by
+// importers.
+type SwapLogger struct {
+	logger atomic.Value
+}
+
+type loggerStruct struct {
+	Logger
+}
+
+// Log implements the Logger interface by forwarding keyvals to the currently
+// wrapped logger. It does not log anything if the wrapped logger is nil.
+func (l *SwapLogger) Log(keyvals ...interface{}) error {
+	s, ok := l.logger.Load().(loggerStruct)
+	if !ok || s.Logger == nil {
+		return nil
+	}
+	return s.Log(keyvals...)
+}
+
+// Swap replaces the currently wrapped logger with logger. Swap may be called
+// concurrently with calls to Log from other goroutines.
+func (l *SwapLogger) Swap(logger Logger) {
+	l.logger.Store(loggerStruct{logger})
+}
+
+// SyncWriter synchronizes concurrent writes to an io.Writer.
+type SyncWriter struct {
+	mu sync.Mutex
+	w  io.Writer
+}
+
+// NewSyncWriter returns a new SyncWriter. The returned writer is safe for
+// concurrent use by multiple goroutines.
+func NewSyncWriter(w io.Writer) *SyncWriter {
+	return &SyncWriter{w: w}
+}
+
+// Write writes p to the underlying io.Writer. If another write is already in
+// progress, the calling goroutine blocks until the SyncWriter is available.
+func (w *SyncWriter) Write(p []byte) (n int, err error) {
+	w.mu.Lock()
+	n, err = w.w.Write(p)
+	w.mu.Unlock()
+	return n, err
+}
+
+// syncLogger provides concurrent safe logging for another Logger.
+type syncLogger struct {
+	mu     sync.Mutex
+	logger Logger
+}
+
+// NewSyncLogger returns a logger that synchronizes concurrent use of the
+// wrapped logger. When multiple goroutines use the SyncLogger concurrently
+// only one goroutine will be allowed to log to the wrapped logger at a time.
+// The other goroutines will block until the logger is available.
+func NewSyncLogger(logger Logger) Logger {
+	return &syncLogger{logger: logger}
+}
+
+// Log logs keyvals to the underlying Logger. If another log is already in
+// progress, the calling goroutine blocks until the syncLogger is available.
+func (l *syncLogger) Log(keyvals ...interface{}) error {
+	l.mu.Lock()
+	err := l.logger.Log(keyvals...)
+	l.mu.Unlock()
+	return err
+}

vendor/github.com/go-kit/kit/log/value.go

@@ -0,0 +1,62 @@
+package log
+
+import (
+	"time"
+
+	"github.com/go-stack/stack"
+)
+
+// A Valuer generates a log value. When passed to Context.With in a value
+// element (odd indexes), it represents a dynamic value which is re-evaluated
+// with each log event.
+type Valuer func() interface{}
+
+// bindValues replaces all value elements (odd indexes) containing a Valuer
+// with their generated value.
+func bindValues(keyvals []interface{}) {
+	for i := 1; i < len(keyvals); i += 2 {
+		if v, ok := keyvals[i].(Valuer); ok {
+			keyvals[i] = v()
+		}
+	}
+}
+
+// containsValuer returns true if any of the value elements (odd indexes)
+// contain a Valuer.
+func containsValuer(keyvals []interface{}) bool {
+	for i := 1; i < len(keyvals); i += 2 {
+		if _, ok := keyvals[i].(Valuer); ok {
+			return true
+		}
+	}
+	return false
+}
+
+// Timestamp returns a Valuer that invokes the underlying function when bound,
+// returning a time.Time. Users will probably want to use DefaultTimestamp or
+// DefaultTimestampUTC.
+func Timestamp(t func() time.Time) Valuer {
+	return func() interface{} { return t() }
+}
+
+var (
+	// DefaultTimestamp is a Valuer that returns the current wallclock time,
+	// respecting time zones, when bound.
+	DefaultTimestamp Valuer = func() interface{} { return time.Now().Format(time.RFC3339) }
+
+	// DefaultTimestampUTC is a Valuer that returns the current time in UTC
+	// when bound.
+	DefaultTimestampUTC Valuer = func() interface{} { return time.Now().UTC().Format(time.RFC3339) }
+)
+
+// Caller returns a Valuer that returns a file and line from a specified depth
+// in the callstack. Users will probably want to use DefaultCaller.
+func Caller(depth int) Valuer {
+	return func() interface{} { return stack.Caller(depth) }
+}
+
+var (
+	// DefaultCaller is a Valuer that returns the file and line where the Log
+	// method was invoked. It can only be used with log.With.
+	DefaultCaller = Caller(3)
+)