Vendor main dependencies.

vendor/github.com/vulcand/oxy/cbreaker/cbreaker.go

@@ -0,0 +1,358 @@
+// package cbreaker implements circuit breaker similar to  https://github.com/Netflix/Hystrix/wiki/How-it-Works
+//
+// Vulcan circuit breaker watches the error condtion to match
+// after which it activates the fallback scenario, e.g. returns the response code
+// or redirects the request to another location
+
+// Circuit breakers start in the Standby state first, observing responses and watching location metrics.
+//
+// Once the Circuit breaker condition is met, it enters the "Tripped" state, where it activates fallback scenario
+// for all requests during the FallbackDuration time period and reset the stats for the location.
+//
+// After FallbackDuration time period passes, Circuit breaker enters "Recovering" state, during that state it will
+// start passing some traffic back to the endpoints, increasing the amount of passed requests using linear function:
+//
+//    allowedRequestsRatio = 0.5 * (Now() - StartRecovery())/RecoveryDuration
+//
+// Two scenarios are possible in the "Recovering" state:
+// 1. Condition matches again, this will reset the state to "Tripped" and reset the timer.
+// 2. Condition does not match, circuit breaker enters "Standby" state
+//
+// It is possible to define actions (e.g. webhooks) of transitions between states:
+//
+// * OnTripped action is called on transition (Standby -> Tripped)
+// * OnStandby action is called on transition (Recovering -> Standby)
+//
+package cbreaker
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/mailgun/timetools"
+	"github.com/vulcand/oxy/memmetrics"
+	"github.com/vulcand/oxy/utils"
+)
+
+// CircuitBreaker is http.Handler that implements circuit breaker pattern
+type CircuitBreaker struct {
+	m       *sync.RWMutex
+	metrics *memmetrics.RTMetrics
+
+	condition hpredicate
+
+	fallbackDuration time.Duration
+	recoveryDuration time.Duration
+
+	onTripped SideEffect
+	onStandby SideEffect
+
+	state cbState
+	until time.Time
+
+	rc *ratioController
+
+	checkPeriod time.Duration
+	lastCheck   time.Time
+
+	fallback http.Handler
+	next     http.Handler
+
+	log   utils.Logger
+	clock timetools.TimeProvider
+}
+
+// New creates a new CircuitBreaker middleware
+func New(next http.Handler, expression string, options ...CircuitBreakerOption) (*CircuitBreaker, error) {
+	cb := &CircuitBreaker{
+		m:    &sync.RWMutex{},
+		next: next,
+		// Default values. Might be overwritten by options below.
+		clock:            &timetools.RealTime{},
+		checkPeriod:      defaultCheckPeriod,
+		fallbackDuration: defaultFallbackDuration,
+		recoveryDuration: defaultRecoveryDuration,
+		fallback:         defaultFallback,
+		log:              utils.NullLogger,
+	}
+
+	for _, s := range options {
+		if err := s(cb); err != nil {
+			return nil, err
+		}
+	}
+
+	condition, err := parseExpression(expression)
+	if err != nil {
+		return nil, err
+	}
+	cb.condition = condition
+
+	mt, err := memmetrics.NewRTMetrics()
+	if err != nil {
+		return nil, err
+	}
+	cb.metrics = mt
+
+	return cb, nil
+}
+
+func (c *CircuitBreaker) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	if c.activateFallback(w, req) {
+		c.fallback.ServeHTTP(w, req)
+		return
+	}
+	c.serve(w, req)
+}
+
+func (c *CircuitBreaker) Wrap(next http.Handler) {
+	c.next = next
+}
+
+// updateState updates internal state and returns true if fallback should be used and false otherwise
+func (c *CircuitBreaker) activateFallback(w http.ResponseWriter, req *http.Request) bool {
+	// Quick check with read locks optimized for normal operation use-case
+	if c.isStandby() {
+		return false
+	}
+	// Circuit breaker is in tripped or recovering state
+	c.m.Lock()
+	defer c.m.Unlock()
+
+	c.log.Infof("%v is in error state", c)
+
+	switch c.state {
+	case stateStandby:
+		// someone else has set it to standby just now
+		return false
+	case stateTripped:
+		if c.clock.UtcNow().Before(c.until) {
+			return true
+		}
+		// We have been in active state enough, enter recovering state
+		c.setRecovering()
+		fallthrough
+	case stateRecovering:
+		// We have been in recovering state enough, enter standby and allow request
+		if c.clock.UtcNow().After(c.until) {
+			c.setState(stateStandby, c.clock.UtcNow())
+			return false
+		}
+		// ratio controller allows this request
+		if c.rc.allowRequest() {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+func (c *CircuitBreaker) serve(w http.ResponseWriter, req *http.Request) {
+	start := c.clock.UtcNow()
+	p := &utils.ProxyWriter{W: w}
+
+	c.next.ServeHTTP(p, req)
+
+	latency := c.clock.UtcNow().Sub(start)
+	c.metrics.Record(p.Code, latency)
+
+	// Note that this call is less expensive than it looks -- checkCondition only performs the real check
+	// periodically. Because of that we can afford to call it here on every single response.
+	c.checkAndSet()
+}
+
+func (c *CircuitBreaker) isStandby() bool {
+	c.m.RLock()
+	defer c.m.RUnlock()
+	return c.state == stateStandby
+}
+
+// String returns log-friendly representation of the circuit breaker state
+func (c *CircuitBreaker) String() string {
+	switch c.state {
+	case stateTripped, stateRecovering:
+		return fmt.Sprintf("CircuitBreaker(state=%v, until=%v)", c.state, c.until)
+	default:
+		return fmt.Sprintf("CircuitBreaker(state=%v)", c.state)
+	}
+}
+
+// exec executes side effect
+func (c *CircuitBreaker) exec(s SideEffect) {
+	if s == nil {
+		return
+	}
+	go func() {
+		if err := s.Exec(); err != nil {
+			c.log.Errorf("%v side effect failure: %v", c, err)
+		}
+	}()
+}
+
+func (c *CircuitBreaker) setState(new cbState, until time.Time) {
+	c.log.Infof("%v setting state to %v, until %v", c, new, until)
+	c.state = new
+	c.until = until
+	switch new {
+	case stateTripped:
+		c.exec(c.onTripped)
+	case stateStandby:
+		c.exec(c.onStandby)
+	}
+}
+
+func (c *CircuitBreaker) timeToCheck() bool {
+	c.m.RLock()
+	defer c.m.RUnlock()
+	return c.clock.UtcNow().After(c.lastCheck)
+}
+
+// Checks if tripping condition matches and sets circuit breaker to the tripped state
+func (c *CircuitBreaker) checkAndSet() {
+	if !c.timeToCheck() {
+		return
+	}
+
+	c.m.Lock()
+	defer c.m.Unlock()
+
+	// Other goroutine could have updated the lastCheck variable before we grabbed mutex
+	if !c.clock.UtcNow().After(c.lastCheck) {
+		return
+	}
+	c.lastCheck = c.clock.UtcNow().Add(c.checkPeriod)
+
+	if c.state == stateTripped {
+		c.log.Infof("%v skip set tripped", c)
+		return
+	}
+
+	if !c.condition(c) {
+		return
+	}
+
+	c.setState(stateTripped, c.clock.UtcNow().Add(c.fallbackDuration))
+	c.metrics.Reset()
+}
+
+func (c *CircuitBreaker) setRecovering() {
+	c.setState(stateRecovering, c.clock.UtcNow().Add(c.recoveryDuration))
+	c.rc = newRatioController(c.clock, c.recoveryDuration)
+}
+
+// CircuitBreakerOption represents an option you can pass to New.
+// See the documentation for the individual options below.
+type CircuitBreakerOption func(*CircuitBreaker) error
+
+// Clock allows you to fake che CircuitBreaker's view of the current time.
+// Intended for unit tests.
+func Clock(clock timetools.TimeProvider) CircuitBreakerOption {
+	return func(c *CircuitBreaker) error {
+		c.clock = clock
+		return nil
+	}
+}
+
+// FallbackDuration is how long the CircuitBreaker will remain in the Tripped
+// state before trying to recover.
+func FallbackDuration(d time.Duration) CircuitBreakerOption {
+	return func(c *CircuitBreaker) error {
+		c.fallbackDuration = d
+		return nil
+	}
+}
+
+// RecoveryDuration is how long the CircuitBreaker will take to ramp up
+// requests during the Recovering state.
+func RecoveryDuration(d time.Duration) CircuitBreakerOption {
+	return func(c *CircuitBreaker) error {
+		c.recoveryDuration = d
+		return nil
+	}
+}
+
+// CheckPeriod is how long the CircuitBreaker will wait between successive
+// checks of the breaker condition.
+func CheckPeriod(d time.Duration) CircuitBreakerOption {
+	return func(c *CircuitBreaker) error {
+		c.checkPeriod = d
+		return nil
+	}
+}
+
+// OnTripped sets a SideEffect to run when entering the Tripped state.
+// Only one SideEffect can be set for this hook.
+func OnTripped(s SideEffect) CircuitBreakerOption {
+	return func(c *CircuitBreaker) error {
+		c.onTripped = s
+		return nil
+	}
+}
+
+// OnTripped sets a SideEffect to run when entering the Standby state.
+// Only one SideEffect can be set for this hook.
+func OnStandby(s SideEffect) CircuitBreakerOption {
+	return func(c *CircuitBreaker) error {
+		c.onStandby = s
+		return nil
+	}
+}
+
+// Fallback defines the http.Handler that the CircuitBreaker should route
+// requests to when it prevents a request from taking its normal path.
+func Fallback(h http.Handler) CircuitBreakerOption {
+	return func(c *CircuitBreaker) error {
+		c.fallback = h
+		return nil
+	}
+}
+
+// Logger adds logging for the CircuitBreaker.
+func Logger(l utils.Logger) CircuitBreakerOption {
+	return func(c *CircuitBreaker) error {
+		c.log = l
+		return nil
+	}
+}
+
+// cbState is the state of the circuit breaker
+type cbState int
+
+func (s cbState) String() string {
+	switch s {
+	case stateStandby:
+		return "standby"
+	case stateTripped:
+		return "tripped"
+	case stateRecovering:
+		return "recovering"
+	}
+	return "undefined"
+}
+
+const (
+	// CircuitBreaker is passing all requests and watching stats
+	stateStandby = iota
+	// CircuitBreaker activates fallback scenario for all requests
+	stateTripped
+	// CircuitBreaker passes some requests to go through, rejecting others
+	stateRecovering
+)
+
+const (
+	defaultFallbackDuration = 10 * time.Second
+	defaultRecoveryDuration = 10 * time.Second
+	defaultCheckPeriod      = 100 * time.Millisecond
+)
+
+var defaultFallback = &fallback{}
+
+type fallback struct {
+}
+
+func (f *fallback) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	w.WriteHeader(http.StatusServiceUnavailable)
+	w.Write([]byte(http.StatusText(http.StatusServiceUnavailable)))
+}

vendor/github.com/vulcand/oxy/cbreaker/effect.go

@@ -0,0 +1,76 @@
+package cbreaker
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/vulcand/oxy/utils"
+)
+
+type SideEffect interface {
+	Exec() error
+}
+
+type Webhook struct {
+	URL     string
+	Method  string
+	Headers http.Header
+	Form    url.Values
+	Body    []byte
+}
+
+type WebhookSideEffect struct {
+	w Webhook
+}
+
+func NewWebhookSideEffect(w Webhook) (*WebhookSideEffect, error) {
+	if w.Method == "" {
+		return nil, fmt.Errorf("Supply method")
+	}
+	_, err := url.Parse(w.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	return &WebhookSideEffect{w: w}, nil
+}
+
+func (w *WebhookSideEffect) getBody() io.Reader {
+	if len(w.w.Form) != 0 {
+		return strings.NewReader(w.w.Form.Encode())
+	}
+	if len(w.w.Body) != 0 {
+		return bytes.NewBuffer(w.w.Body)
+	}
+	return nil
+}
+
+func (w *WebhookSideEffect) Exec() error {
+	r, err := http.NewRequest(w.w.Method, w.w.URL, w.getBody())
+	if err != nil {
+		return err
+	}
+	if len(w.w.Headers) != 0 {
+		utils.CopyHeaders(r.Header, w.w.Headers)
+	}
+	if len(w.w.Form) != 0 {
+		r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	}
+	re, err := http.DefaultClient.Do(r)
+	if err != nil {
+		return err
+	}
+	if re.Body != nil {
+		defer re.Body.Close()
+	}
+	_, err = ioutil.ReadAll(re.Body)
+	if err != nil {
+		return err
+	}
+	return nil
+}

vendor/github.com/vulcand/oxy/cbreaker/fallback.go

@@ -0,0 +1,56 @@
+package cbreaker
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+)
+
+type Response struct {
+	StatusCode  int
+	ContentType string
+	Body        []byte
+}
+
+type ResponseFallback struct {
+	r Response
+}
+
+func NewResponseFallback(r Response) (*ResponseFallback, error) {
+	if r.StatusCode == 0 {
+		return nil, fmt.Errorf("response code should not be 0")
+	}
+	return &ResponseFallback{r: r}, nil
+}
+
+func (f *ResponseFallback) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	if f.r.ContentType != "" {
+		w.Header().Set("Content-Type", f.r.ContentType)
+	}
+	w.Header().Set("Content-Length", strconv.Itoa(len(f.r.Body)))
+	w.WriteHeader(f.r.StatusCode)
+	w.Write(f.r.Body)
+}
+
+type Redirect struct {
+	URL string
+}
+
+type RedirectFallback struct {
+	u *url.URL
+}
+
+func NewRedirectFallback(r Redirect) (*RedirectFallback, error) {
+	u, err := url.ParseRequestURI(r.URL)
+	if err != nil {
+		return nil, err
+	}
+	return &RedirectFallback{u: u}, nil
+}
+
+func (f *RedirectFallback) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	w.Header().Set("Location", f.u.String())
+	w.WriteHeader(http.StatusFound)
+	w.Write([]byte(http.StatusText(http.StatusFound)))
+}

vendor/github.com/vulcand/oxy/cbreaker/predicates.go

@@ -0,0 +1,232 @@
+package cbreaker
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/vulcand/predicate"
+)
+
+type hpredicate func(*CircuitBreaker) bool
+
+// parseExpression parses expression in the go language into predicates.
+func parseExpression(in string) (hpredicate, error) {
+	p, err := predicate.NewParser(predicate.Def{
+		Operators: predicate.Operators{
+			AND: and,
+			OR:  or,
+			EQ:  eq,
+			NEQ: neq,
+			LT:  lt,
+			LE:  le,
+			GT:  gt,
+			GE:  ge,
+		},
+		Functions: map[string]interface{}{
+			"LatencyAtQuantileMS": latencyAtQuantile,
+			"NetworkErrorRatio":   networkErrorRatio,
+			"ResponseCodeRatio":   responseCodeRatio,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	out, err := p.Parse(in)
+	if err != nil {
+		return nil, err
+	}
+	pr, ok := out.(hpredicate)
+	if !ok {
+		return nil, fmt.Errorf("expected predicate, got %T", out)
+	}
+	return pr, nil
+}
+
+type toInt func(c *CircuitBreaker) int
+type toFloat64 func(c *CircuitBreaker) float64
+
+func latencyAtQuantile(quantile float64) toInt {
+	return func(c *CircuitBreaker) int {
+		h, err := c.metrics.LatencyHistogram()
+		if err != nil {
+			c.log.Errorf("Failed to get latency histogram, for %v error: %v", c, err)
+			return 0
+		}
+		return int(h.LatencyAtQuantile(quantile) / time.Millisecond)
+	}
+}
+
+func networkErrorRatio() toFloat64 {
+	return func(c *CircuitBreaker) float64 {
+		return c.metrics.NetworkErrorRatio()
+	}
+}
+
+func responseCodeRatio(startA, endA, startB, endB int) toFloat64 {
+	return func(c *CircuitBreaker) float64 {
+		return c.metrics.ResponseCodeRatio(startA, endA, startB, endB)
+	}
+}
+
+// or returns predicate by joining the passed predicates with logical 'or'
+func or(fns ...hpredicate) hpredicate {
+	return func(c *CircuitBreaker) bool {
+		for _, fn := range fns {
+			if fn(c) {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+// and returns predicate by joining the passed predicates with logical 'and'
+func and(fns ...hpredicate) hpredicate {
+	return func(c *CircuitBreaker) bool {
+		for _, fn := range fns {
+			if !fn(c) {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+// not creates negation of the passed predicate
+func not(p hpredicate) hpredicate {
+	return func(c *CircuitBreaker) bool {
+		return !p(c)
+	}
+}
+
+// eq returns predicate that tests for equality of the value of the mapper and the constant
+func eq(m interface{}, value interface{}) (hpredicate, error) {
+	switch mapper := m.(type) {
+	case toInt:
+		return intEQ(mapper, value)
+	case toFloat64:
+		return float64EQ(mapper, value)
+	}
+	return nil, fmt.Errorf("eq: unsupported argument: %T", m)
+}
+
+// neq returns predicate that tests for inequality of the value of the mapper and the constant
+func neq(m interface{}, value interface{}) (hpredicate, error) {
+	p, err := eq(m, value)
+	if err != nil {
+		return nil, err
+	}
+	return not(p), nil
+}
+
+// lt returns predicate that tests that value of the mapper function is less than the constant
+func lt(m interface{}, value interface{}) (hpredicate, error) {
+	switch mapper := m.(type) {
+	case toInt:
+		return intLT(mapper, value)
+	case toFloat64:
+		return float64LT(mapper, value)
+	}
+	return nil, fmt.Errorf("lt: unsupported argument: %T", m)
+}
+
+// le returns predicate that tests that value of the mapper function is less or equal than the constant
+func le(m interface{}, value interface{}) (hpredicate, error) {
+	l, err := lt(m, value)
+	if err != nil {
+		return nil, err
+	}
+	e, err := eq(m, value)
+	if err != nil {
+		return nil, err
+	}
+	return func(c *CircuitBreaker) bool {
+		return l(c) || e(c)
+	}, nil
+}
+
+// gt returns predicate that tests that value of the mapper function is greater than the constant
+func gt(m interface{}, value interface{}) (hpredicate, error) {
+	switch mapper := m.(type) {
+	case toInt:
+		return intGT(mapper, value)
+	case toFloat64:
+		return float64GT(mapper, value)
+	}
+	return nil, fmt.Errorf("gt: unsupported argument: %T", m)
+}
+
+// ge returns predicate that tests that value of the mapper function is less or equal than the constant
+func ge(m interface{}, value interface{}) (hpredicate, error) {
+	g, err := gt(m, value)
+	if err != nil {
+		return nil, err
+	}
+	e, err := eq(m, value)
+	if err != nil {
+		return nil, err
+	}
+	return func(c *CircuitBreaker) bool {
+		return g(c) || e(c)
+	}, nil
+}
+
+func intEQ(m toInt, val interface{}) (hpredicate, error) {
+	value, ok := val.(int)
+	if !ok {
+		return nil, fmt.Errorf("expected int, got %T", val)
+	}
+	return func(c *CircuitBreaker) bool {
+		return m(c) == value
+	}, nil
+}
+
+func float64EQ(m toFloat64, val interface{}) (hpredicate, error) {
+	value, ok := val.(float64)
+	if !ok {
+		return nil, fmt.Errorf("expected float64, got %T", val)
+	}
+	return func(c *CircuitBreaker) bool {
+		return m(c) == value
+	}, nil
+}
+
+func intLT(m toInt, val interface{}) (hpredicate, error) {
+	value, ok := val.(int)
+	if !ok {
+		return nil, fmt.Errorf("expected int, got %T", val)
+	}
+	return func(c *CircuitBreaker) bool {
+		return m(c) < value
+	}, nil
+}
+
+func intGT(m toInt, val interface{}) (hpredicate, error) {
+	value, ok := val.(int)
+	if !ok {
+		return nil, fmt.Errorf("expected int, got %T", val)
+	}
+	return func(c *CircuitBreaker) bool {
+		return m(c) > value
+	}, nil
+}
+
+func float64LT(m toFloat64, val interface{}) (hpredicate, error) {
+	value, ok := val.(float64)
+	if !ok {
+		return nil, fmt.Errorf("expected int, got %T", val)
+	}
+	return func(c *CircuitBreaker) bool {
+		return m(c) < value
+	}, nil
+}
+
+func float64GT(m toFloat64, val interface{}) (hpredicate, error) {
+	value, ok := val.(float64)
+	if !ok {
+		return nil, fmt.Errorf("expected int, got %T", val)
+	}
+	return func(c *CircuitBreaker) bool {
+		return m(c) > value
+	}, nil
+}

vendor/github.com/vulcand/oxy/cbreaker/ratio.go

@@ -0,0 +1,66 @@
+package cbreaker
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/mailgun/timetools"
+)
+
+// ratioController allows passing portions traffic back to the endpoints,
+// increasing the amount of passed requests using linear function:
+//
+//   allowedRequestsRatio = 0.5 * (Now() - Start())/Duration
+//
+type ratioController struct {
+	duration time.Duration
+	start    time.Time
+	tm       timetools.TimeProvider
+	allowed  int
+	denied   int
+}
+
+func newRatioController(tm timetools.TimeProvider, rampUp time.Duration) *ratioController {
+	return &ratioController{
+		duration: rampUp,
+		tm:       tm,
+		start:    tm.UtcNow(),
+	}
+}
+
+func (r *ratioController) String() string {
+	return fmt.Sprintf("RatioController(target=%f, current=%f, allowed=%d, denied=%d)", r.targetRatio(), r.computeRatio(r.allowed, r.denied), r.allowed, r.denied)
+}
+
+func (r *ratioController) allowRequest() bool {
+	t := r.targetRatio()
+	// This condition answers the question - would we satisfy the target ratio if we allow this request?
+	e := r.computeRatio(r.allowed+1, r.denied)
+	if e < t {
+		r.allowed++
+		return true
+	}
+	r.denied++
+	return false
+}
+
+func (r *ratioController) computeRatio(allowed, denied int) float64 {
+	if denied+allowed == 0 {
+		return 0
+	}
+	return float64(allowed) / float64(denied+allowed)
+}
+
+func (r *ratioController) targetRatio() float64 {
+	// Here's why it's 0.5:
+	// We are watching the following ratio
+	// ratio = a / (a + d)
+	// We can notice, that once we get to 0.5
+	// 0.5 = a / (a + d)
+	// we can evaluate that a = d
+	// that means equilibrium, where we would allow all the requests
+	// after this point to achieve ratio of 1 (that can never be reached unless d is 0)
+	// so we stop from there
+	multiplier := 0.5 / float64(r.duration)
+	return multiplier * float64(r.tm.UtcNow().Sub(r.start))
+}