Refactor into dual Rancher API/Metadata providers

Introduces Rancher's metadata service as an optional provider source for
Traefik, enabled by setting `rancher.MetadataService`.

The provider uses a long polling technique to watch the metadata service and
obtain near instantaneous updates. Alternatively it can be configured to poll
the metadata service every `rancher.RefreshSeconds` by setting
`rancher.MetadataPoll`.

The refactor splits API and metadata service code into separate source
files respectively, and specific configuration is deferred to
sub-structs.

Incorporates bugfix #1414

vendor/github.com/rancher/go-rancher-metadata/main.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/rancher/go-rancher-metadata/metadata"
+)
+
+const (
+	metadataUrl = "http://rancher-metadata/2015-12-19"
+)
+
+func main() {
+	m := metadata.NewClient(metadataUrl)
+
+	version := "init"
+
+	for {
+		newVersion, err := m.GetVersion()
+		if err != nil {
+			logrus.Errorf("Error reading metadata version: %v", err)
+		} else if version == newVersion {
+			logrus.Debug("No changes in metadata version")
+		} else {
+			logrus.Debugf("Metadata version has changed, oldVersion=[%s], newVersion=[%s]", version, newVersion)
+			version = newVersion
+		}
+		time.Sleep(5 * time.Second)
+	}
+}

vendor/github.com/rancher/go-rancher-metadata/metadata/change.go

@@ -0,0 +1,64 @@
+package metadata
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+)
+
+func (m *client) OnChangeWithError(intervalSeconds int, do func(string)) error {
+	return m.onChangeFromVersionWithError("init", intervalSeconds, do)
+}
+
+func (m *client) onChangeFromVersionWithError(version string, intervalSeconds int, do func(string)) error {
+	for {
+		newVersion, err := m.waitVersion(intervalSeconds, version)
+		if err != nil {
+			return err
+		} else if version == newVersion {
+			logrus.Debug("No changes in metadata version")
+		} else {
+			logrus.Debugf("Metadata Version has been changed. Old version: %s. New version: %s.", version, newVersion)
+			version = newVersion
+			do(newVersion)
+		}
+	}
+
+	return nil
+}
+
+func (m *client) OnChange(intervalSeconds int, do func(string)) {
+	version := "init"
+	updateVersionAndDo := func(v string) {
+		version = v
+		do(version)
+	}
+	interval := time.Duration(intervalSeconds)
+	for {
+		if err := m.onChangeFromVersionWithError(version, intervalSeconds, updateVersionAndDo); err != nil {
+			logrus.Errorf("Error reading metadata version: %v", err)
+		}
+		time.Sleep(interval * time.Second)
+	}
+}
+
+type timeout interface {
+	Timeout() bool
+}
+
+func (m *client) waitVersion(maxWait int, version string) (string, error) {
+	for {
+		resp, err := m.SendRequest(fmt.Sprintf("/version?wait=true&value=%s&maxWait=%d", version, maxWait))
+		if err != nil {
+			t, ok := err.(timeout)
+			if ok && t.Timeout() {
+				continue
+			}
+			return "", err
+		}
+		err = json.Unmarshal(resp, &version)
+		return version, err
+	}
+}

vendor/github.com/rancher/go-rancher-metadata/metadata/metadata.go

@@ -0,0 +1,262 @@
+package metadata
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+type Client interface {
+	OnChangeWithError(int, func(string)) error
+	OnChange(int, func(string))
+	SendRequest(string) ([]byte, error)
+	GetVersion() (string, error)
+	GetSelfHost() (Host, error)
+	GetSelfContainer() (Container, error)
+	GetSelfServiceByName(string) (Service, error)
+	GetSelfService() (Service, error)
+	GetSelfStack() (Stack, error)
+	GetServices() ([]Service, error)
+	GetStacks() ([]Stack, error)
+	GetContainers() ([]Container, error)
+	GetServiceContainers(string, string) ([]Container, error)
+	GetHosts() ([]Host, error)
+	GetHost(string) (Host, error)
+	GetNetworks() ([]Network, error)
+}
+
+type client struct {
+	url    string
+	ip     string
+	client *http.Client
+}
+
+func newClient(url, ip string) *client {
+	return &client{url, ip, &http.Client{Timeout: 10 * time.Second}}
+}
+
+func NewClient(url string) Client {
+	ip := ""
+	return newClient(url, ip)
+}
+
+func NewClientWithIPAndWait(url, ip string) (Client, error) {
+	client := newClient(url, ip)
+
+	if err := testConnection(client); err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}
+
+func NewClientAndWait(url string) (Client, error) {
+	ip := ""
+	client := newClient(url, ip)
+
+	if err := testConnection(client); err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}
+
+func (m *client) SendRequest(path string) ([]byte, error) {
+	req, err := http.NewRequest("GET", m.url+path, nil)
+	req.Header.Add("Accept", "application/json")
+	if m.ip != "" {
+		req.Header.Add("X-Forwarded-For", m.ip)
+	}
+	resp, err := m.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("Error %v accessing %v path", resp.StatusCode, path)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	return body, nil
+}
+
+func (m *client) GetVersion() (string, error) {
+	resp, err := m.SendRequest("/version")
+	if err != nil {
+		return "", err
+	}
+	return string(resp[:]), nil
+}
+
+func (m *client) GetSelfHost() (Host, error) {
+	resp, err := m.SendRequest("/self/host")
+	var host Host
+	if err != nil {
+		return host, err
+	}
+
+	if err = json.Unmarshal(resp, &host); err != nil {
+		return host, err
+	}
+
+	return host, nil
+}
+
+func (m *client) GetSelfContainer() (Container, error) {
+	resp, err := m.SendRequest("/self/container")
+	var container Container
+	if err != nil {
+		return container, err
+	}
+
+	if err = json.Unmarshal(resp, &container); err != nil {
+		return container, err
+	}
+
+	return container, nil
+}
+
+func (m *client) GetSelfServiceByName(name string) (Service, error) {
+	resp, err := m.SendRequest("/self/stack/services/" + name)
+	var service Service
+	if err != nil {
+		return service, err
+	}
+
+	if err = json.Unmarshal(resp, &service); err != nil {
+		return service, err
+	}
+
+	return service, nil
+}
+
+func (m *client) GetSelfService() (Service, error) {
+	resp, err := m.SendRequest("/self/service")
+	var service Service
+	if err != nil {
+		return service, err
+	}
+
+	if err = json.Unmarshal(resp, &service); err != nil {
+		return service, err
+	}
+
+	return service, nil
+}
+
+func (m *client) GetSelfStack() (Stack, error) {
+	resp, err := m.SendRequest("/self/stack")
+	var stack Stack
+	if err != nil {
+		return stack, err
+	}
+
+	if err = json.Unmarshal(resp, &stack); err != nil {
+		return stack, err
+	}
+
+	return stack, nil
+}
+
+func (m *client) GetServices() ([]Service, error) {
+	resp, err := m.SendRequest("/services")
+	var services []Service
+	if err != nil {
+		return services, err
+	}
+
+	if err = json.Unmarshal(resp, &services); err != nil {
+		return services, err
+	}
+	return services, nil
+}
+
+func (m *client) GetStacks() ([]Stack, error) {
+	resp, err := m.SendRequest("/stacks")
+	var stacks []Stack
+	if err != nil {
+		return stacks, err
+	}
+
+	if err = json.Unmarshal(resp, &stacks); err != nil {
+		return stacks, err
+	}
+	return stacks, nil
+}
+
+func (m *client) GetContainers() ([]Container, error) {
+	resp, err := m.SendRequest("/containers")
+	var containers []Container
+	if err != nil {
+		return containers, err
+	}
+
+	if err = json.Unmarshal(resp, &containers); err != nil {
+		return containers, err
+	}
+	return containers, nil
+}
+
+func (m *client) GetServiceContainers(serviceName string, stackName string) ([]Container, error) {
+	var serviceContainers = []Container{}
+	containers, err := m.GetContainers()
+	if err != nil {
+		return serviceContainers, err
+	}
+
+	for _, container := range containers {
+		if container.StackName == stackName && container.ServiceName == serviceName {
+			serviceContainers = append(serviceContainers, container)
+		}
+	}
+
+	return serviceContainers, nil
+}
+
+func (m *client) GetHosts() ([]Host, error) {
+	resp, err := m.SendRequest("/hosts")
+	var hosts []Host
+	if err != nil {
+		return hosts, err
+	}
+
+	if err = json.Unmarshal(resp, &hosts); err != nil {
+		return hosts, err
+	}
+	return hosts, nil
+}
+
+func (m *client) GetHost(UUID string) (Host, error) {
+	var host Host
+	hosts, err := m.GetHosts()
+	if err != nil {
+		return host, err
+	}
+	for _, host := range hosts {
+		if host.UUID == UUID {
+			return host, nil
+		}
+	}
+
+	return host, fmt.Errorf("could not find host by UUID %v", UUID)
+}
+
+func (m *client) GetNetworks() ([]Network, error) {
+	resp, err := m.SendRequest("/networks")
+	var networks []Network
+	if err != nil {
+		return networks, err
+	}
+
+	if err = json.Unmarshal(resp, &networks); err != nil {
+		return networks, err
+	}
+
+	return networks, nil
+}

vendor/github.com/rancher/go-rancher-metadata/metadata/types.go

@@ -0,0 +1,149 @@
+package metadata
+
+type Stack struct {
+	EnvironmentName string    `json:"environment_name"`
+	EnvironmentUUID string    `json:"environment_uuid"`
+	Name            string    `json:"name"`
+	UUID            string    `json:"uuid"`
+	Services        []Service `json:"services"`
+	System          bool      `json:"system"`
+}
+
+type HealthCheck struct {
+	HealthyThreshold   int    `json:"healthy_threshold"`
+	Interval           int    `json:"interval"`
+	Port               int    `json:"port"`
+	RequestLine        string `json:"request_line"`
+	ResponseTimeout    int    `json:"response_timeout"`
+	UnhealthyThreshold int    `json:"unhealthy_threshold"`
+}
+
+type Service struct {
+	Scale              int                    `json:"scale"`
+	Name               string                 `json:"name"`
+	StackName          string                 `json:"stack_name"`
+	StackUUID          string                 `json:"stack_uuid"`
+	Kind               string                 `json:"kind"`
+	Hostname           string                 `json:"hostname"`
+	Vip                string                 `json:"vip"`
+	CreateIndex        int                    `json:"create_index"`
+	UUID               string                 `json:"uuid"`
+	ExternalIps        []string               `json:"external_ips"`
+	Sidekicks          []string               `json:"sidekicks"`
+	Containers         []Container            `json:"containers"`
+	Ports              []string               `json:"ports"`
+	Labels             map[string]string      `json:"labels"`
+	Links              map[string]string      `json:"links"`
+	Metadata           map[string]interface{} `json:"metadata"`
+	Token              string                 `json:"token"`
+	Fqdn               string                 `json:"fqdn"`
+	HealthCheck        HealthCheck            `json:"health_check"`
+	PrimaryServiceName string                 `json:"primary_service_name"`
+	LBConfig           LBConfig               `json:"lb_config"`
+	EnvironmentUUID    string                 `json:"environment_uuid"`
+	State              string                 `json:"state"`
+	System             bool                   `json:"system"`
+}
+
+type Container struct {
+	Name                     string            `json:"name"`
+	PrimaryIp                string            `json:"primary_ip"`
+	PrimaryMacAddress        string            `json:"primary_mac_address"`
+	Ips                      []string          `json:"ips"`
+	Ports                    []string          `json:"ports"`
+	ServiceName              string            `json:"service_name"`
+	ServiceIndex             string            `json:"service_index"`
+	StackName                string            `json:"stack_name"`
+	Labels                   map[string]string `json:"labels"`
+	CreateIndex              int               `json:"create_index"`
+	HostUUID                 string            `json:"host_uuid"`
+	UUID                     string            `json:"uuid"`
+	State                    string            `json:"state"`
+	HealthState              string            `json:"health_state"`
+	ExternalId               string            `json:"external_id"`
+	StartCount               int               `json:"start_count"`
+	MemoryReservation        int64             `json:"memory_reservation"`
+	MilliCPUReservation      int64             `json:"milli_cpu_reservation"`
+	Dns                      []string          `json:"dns"`
+	DnsSearch                []string          `json:"dns_search"`
+	HealthCheckHosts         []string          `json:"health_check_hosts"`
+	NetworkFromContainerUUID string            `json:"network_from_container_uuid"`
+	NetworkUUID              string            `json:"network_uuid"`
+	Links                    map[string]string `json:"links"`
+	System                   bool              `json:"system"`
+	EnvironmentUUID          string            `json:"environment_uuid"`
+	HealthCheck              HealthCheck       `json:"health_check"`
+}
+
+type Network struct {
+	Name                string                 `json:"name"`
+	UUID                string                 `json:"uuid"`
+	EnvironmentUUID     string                 `json:"environment_uuid"`
+	Metadata            map[string]interface{} `json:"metadata"`
+	HostPorts           bool                   `json:"host_ports"`
+	Default             bool                   `json:"is_default"`
+	Policy              []NetworkPolicyRule    `json:"policy,omitempty"`
+	DefaultPolicyAction string                 `json:"default_policy_action"`
+}
+
+type Host struct {
+	Name            string            `json:"name"`
+	AgentIP         string            `json:"agent_ip"`
+	HostId          int               `json:"host_id"`
+	Labels          map[string]string `json:"labels"`
+	UUID            string            `json:"uuid"`
+	Hostname        string            `json:"hostname"`
+	Memory          int64             `json:"memory"`
+	MilliCPU        int64             `json:"milli_cpu"`
+	LocalStorageMb  int64             `json:"local_storage_mb"`
+	EnvironmentUUID string            `json:"environment_uuid"`
+}
+
+type PortRule struct {
+	SourcePort  int    `json:"source_port"`
+	Protocol    string `json:"protocol"`
+	Path        string `json:"path"`
+	Hostname    string `json:"hostname"`
+	Service     string `json:"service"`
+	TargetPort  int    `json:"target_port"`
+	Priority    int    `json:"priority"`
+	BackendName string `json:"backend_name"`
+	Selector    string `json:"selector"`
+	Container   string `json:"container"`
+}
+
+type LBConfig struct {
+	Certs            []string           `json:"certs"`
+	DefaultCert      string             `json:"default_cert"`
+	PortRules        []PortRule         `json:"port_rules"`
+	Config           string             `json:"config"`
+	StickinessPolicy LBStickinessPolicy `json:"stickiness_policy"`
+}
+
+type LBStickinessPolicy struct {
+	Name     string `json:"name"`
+	Cookie   string `json:"cookie"`
+	Domain   string `json:"domain"`
+	Indirect bool   `json:"indirect"`
+	Nocache  bool   `json:"nocache"`
+	Postonly bool   `json:"postonly"`
+	Mode     string `json:"mode"`
+}
+
+type NetworkPolicyRuleBetween struct {
+	Selector string `yaml:"selector,omitempty"`
+	GroupBy  string `yaml:"groupBy,omitempty"`
+}
+
+type NetworkPolicyRuleMember struct {
+	Selector string `yaml:"selector,omitempty"`
+}
+
+type NetworkPolicyRule struct {
+	From    *NetworkPolicyRuleMember  `yaml:"from"`
+	To      *NetworkPolicyRuleMember  `yaml:"to"`
+	Ports   []string                  `yaml:"ports"`
+	Within  string                    `yaml:"within"`
+	Between *NetworkPolicyRuleBetween `yaml:"between"`
+	Action  string                    `yaml:"action"`
+}

vendor/github.com/rancher/go-rancher-metadata/metadata/utils.go

@@ -0,0 +1,19 @@
+package metadata
+
+import (
+	"time"
+)
+
+func testConnection(mdClient Client) error {
+	var err error
+	maxTime := 20 * time.Second
+
+	for i := 1 * time.Second; i < maxTime; i *= time.Duration(2) {
+		if _, err = mdClient.GetVersion(); err != nil {
+			time.Sleep(i)
+		} else {
+			return nil
+		}
+	}
+	return err
+}