1
0
Fork 0

Vendor main dependencies.

This commit is contained in:
Timo Reimann 2017-02-07 22:33:23 +01:00
parent 49a09ab7dd
commit dd5e3fba01
2738 changed files with 1045689 additions and 0 deletions

View file

@ -0,0 +1,80 @@
package records
import (
"github.com/mesosphere/mesos-dns/records/labels"
)
// chain is a generation func that consumes record-like strings and does
// something with them
type chain func(...string)
const (
protocolNone = "" // for readability
domainNone = "" // for readability
)
// withProtocol appends `._{protocol}.{framework}` to records. if protocol is "" then
// the protocols "tcp" and "udp" are assumed.
func withProtocol(protocol, framework string, spec labels.Func, gen chain) chain {
return func(records ...string) {
protocol = spec(protocol)
if protocol != protocolNone {
for i := range records {
records[i] += "._" + protocol + "." + framework
}
} else {
records = append(records, records...)
for i, j := 0, len(records)/2; j < len(records); {
records[i] += "._tcp." + framework
records[j] += "._udp." + framework
i++
j++
}
}
gen(records...)
}
}
// withSubdomains appends `.{subdomain}` (for each subdomain spec'd) to records.
// the empty subdomain "" indicates to generate records w/o a subdomain fragment.
func withSubdomains(subdomains []string, gen chain) chain {
if len(subdomains) == 0 {
return gen
}
return func(records ...string) {
var (
recordLen = len(records)
tmp = make([]string, recordLen*len(subdomains))
offset = 0
)
for s := range subdomains {
if subdomains[s] == domainNone {
copy(tmp[offset:], records)
} else {
for i := range records {
tmp[offset+i] = records[i] + "." + subdomains[s]
}
}
offset += recordLen
}
gen(tmp...)
}
}
// withNamedPort prepends a `_{discoveryInfo port name}.` to records
func withNamedPort(portName string, spec labels.Func, gen chain) chain {
portName = spec(portName)
if portName == "" {
return gen
}
return func(records ...string) {
// generate without port-name prefix
gen(records...)
// generate with port-name prefix
for i := range records {
records[i] = "_" + portName + "." + records[i]
}
gen(records...)
}
}

View file

@ -0,0 +1,261 @@
package records
import (
"encoding/json"
"fmt"
"io/ioutil"
"net"
"os"
"path/filepath"
"strings"
"time"
"github.com/mesosphere/mesos-dns/logging"
"github.com/miekg/dns"
)
// Config holds mesos dns configuration
type Config struct {
// Refresh frequency: the frequency in seconds of regenerating records (default 60)
RefreshSeconds int
// Resolver port: port used to listen for slave requests (default 53)
Port int
// Timeout is the default connect/read/write timeout for outbound
// queries
Timeout int
// Timeout in seconds waiting for the master to return data from StateJson
StateTimeoutSeconds int
// Zookeeper Detection Timeout: how long in seconds to wait for Zookeeper to
// be initially responsive. Default is 30 and 0 means no timeout.
ZkDetectionTimeout int
// NOTE(tsenart): HTTPPort, DNSOn and HTTPOn have defined JSON keys for
// backwards compatibility with external API clients.
HTTPPort int `json:"HttpPort"`
// TTL: the TTL value used for SRV and A records (default 60)
TTL int32
// SOA record fields (see http://tools.ietf.org/html/rfc1035#page-18)
SOASerial uint32 // initial version number (incremented on refresh)
SOARefresh uint32 // refresh interval
SOARetry uint32 // retry interval
SOAExpire uint32 // expiration time
SOAMinttl uint32 // minimum TTL
SOAMname string // primary name server
SOARname string // email of admin esponsible
// Mesos master(s): a list of IP:port pairs for one or more Mesos masters
Masters []string
// DNS server: IP address of the DNS server for forwarded accesses
Resolvers []string
// IPSources is the prioritized list of task IP sources
IPSources []string // e.g. ["host", "docker", "mesos", "rkt"]
// Zookeeper: a single Zk url
Zk string
// Domain: name of the domain used (default "mesos", ie .mesos domain)
Domain string
// File is the location of the config.json file
File string
// Listen is the server DNS listener IP address
Listener string
// HTTPListen is the server HTTP listener IP address
HTTPListener string
// Value of RecursionAvailable for responses in Mesos domain
RecurseOn bool
// Enable serving DSN and HTTP requests
DNSOn bool `json:"DnsOn"`
HTTPOn bool `json:"HttpOn"`
// Enable replies for external requests
ExternalOn bool
// EnforceRFC952 will enforce an older, more strict set of rules for DNS labels
EnforceRFC952 bool
// Enumeration enabled via the API enumeration endpoint
EnumerationOn bool
}
// NewConfig return the default config of the resolver
func NewConfig() Config {
return Config{
ZkDetectionTimeout: 30,
RefreshSeconds: 60,
TTL: 60,
Domain: "mesos",
Port: 53,
Timeout: 5,
StateTimeoutSeconds: 300,
SOARname: "root.ns1.mesos",
SOAMname: "ns1.mesos",
SOARefresh: 60,
SOARetry: 600,
SOAExpire: 86400,
SOAMinttl: 60,
Resolvers: []string{"8.8.8.8"},
Listener: "0.0.0.0",
HTTPListener: "0.0.0.0",
HTTPPort: 8123,
DNSOn: true,
HTTPOn: true,
ExternalOn: true,
RecurseOn: true,
IPSources: []string{"netinfo", "mesos", "host"},
EnumerationOn: true,
}
}
// SetConfig instantiates a Config struct read in from config.json
func SetConfig(cjson string) Config {
c, err := readConfig(cjson)
if err != nil {
logging.Error.Fatal(err)
}
logging.Verbose.Printf("config loaded from %q", c.File)
// validate and complete configuration file
err = validateEnabledServices(c)
if err != nil {
logging.Error.Fatalf("service validation failed: %v", err)
}
if err = validateMasters(c.Masters); err != nil {
logging.Error.Fatalf("Masters validation failed: %v", err)
}
if c.ExternalOn {
if len(c.Resolvers) == 0 {
c.Resolvers = GetLocalDNS()
}
if err = validateResolvers(c.Resolvers); err != nil {
logging.Error.Fatalf("Resolvers validation failed: %v", err)
}
}
if err = validateIPSources(c.IPSources); err != nil {
logging.Error.Fatalf("IPSources validation failed: %v", err)
}
c.Domain = strings.ToLower(c.Domain)
// SOA record fields
c.SOARname = strings.TrimRight(strings.Replace(c.SOARname, "@", ".", -1), ".") + "."
c.SOAMname = strings.TrimRight(c.SOAMname, ".") + "."
c.SOASerial = uint32(time.Now().Unix())
// print configuration file
logging.Verbose.Println("Mesos-DNS configuration:")
logging.Verbose.Println(" - Masters: " + strings.Join(c.Masters, ", "))
logging.Verbose.Println(" - Zookeeper: ", c.Zk)
logging.Verbose.Println(" - ZookeeperDetectionTimeout: ", c.ZkDetectionTimeout)
logging.Verbose.Println(" - RefreshSeconds: ", c.RefreshSeconds)
logging.Verbose.Println(" - Domain: " + c.Domain)
logging.Verbose.Println(" - Listener: " + c.Listener)
logging.Verbose.Println(" - HTTPListener: " + c.HTTPListener)
logging.Verbose.Println(" - Port: ", c.Port)
logging.Verbose.Println(" - DnsOn: ", c.DNSOn)
logging.Verbose.Println(" - TTL: ", c.TTL)
logging.Verbose.Println(" - Timeout: ", c.Timeout)
logging.Verbose.Println(" - StateTimeoutSeconds: ", c.StateTimeoutSeconds)
logging.Verbose.Println(" - Resolvers: " + strings.Join(c.Resolvers, ", "))
logging.Verbose.Println(" - ExternalOn: ", c.ExternalOn)
logging.Verbose.Println(" - SOAMname: " + c.SOAMname)
logging.Verbose.Println(" - SOARname: " + c.SOARname)
logging.Verbose.Println(" - SOASerial: ", c.SOASerial)
logging.Verbose.Println(" - SOARefresh: ", c.SOARefresh)
logging.Verbose.Println(" - SOARetry: ", c.SOARetry)
logging.Verbose.Println(" - SOAExpire: ", c.SOAExpire)
logging.Verbose.Println(" - SOAExpire: ", c.SOAMinttl)
logging.Verbose.Println(" - RecurseOn: ", c.RecurseOn)
logging.Verbose.Println(" - HttpPort: ", c.HTTPPort)
logging.Verbose.Println(" - HttpOn: ", c.HTTPOn)
logging.Verbose.Println(" - ConfigFile: ", c.File)
logging.Verbose.Println(" - EnforceRFC952: ", c.EnforceRFC952)
logging.Verbose.Println(" - IPSources: ", c.IPSources)
logging.Verbose.Println(" - EnumerationOn", c.EnumerationOn)
return *c
}
func readConfig(file string) (*Config, error) {
c := NewConfig()
workingDir := "."
for _, name := range []string{"HOME", "USERPROFILE"} { // *nix, windows
if dir := os.Getenv(name); dir != "" {
workingDir = dir
}
}
var err error
c.File, err = filepath.Abs(strings.Replace(file, "~/", workingDir+"/", 1))
if err != nil {
return nil, fmt.Errorf("cannot find configuration file")
} else if bs, err := ioutil.ReadFile(c.File); err != nil {
return nil, fmt.Errorf("missing configuration file: %q", c.File)
} else if err = json.Unmarshal(bs, &c); err != nil {
return nil, fmt.Errorf("failed to unmarshal config file %q: %v", c.File, err)
}
return &c, nil
}
func unique(ss []string) []string {
set := make(map[string]struct{}, len(ss))
out := make([]string, 0, len(ss))
for _, s := range ss {
if _, ok := set[s]; !ok {
set[s] = struct{}{}
out = append(out, s)
}
}
return out
}
// GetLocalDNS returns the first nameserver in /etc/resolv.conf
// Used for non-Mesos queries.
func GetLocalDNS() []string {
conf, err := dns.ClientConfigFromFile("/etc/resolv.conf")
if err != nil {
logging.Error.Fatalf("%v", err)
}
return nonLocalAddies(conf.Servers)
}
// Returns non-local nameserver entries
func nonLocalAddies(cservers []string) []string {
bad := localAddies()
good := []string{}
for i := 0; i < len(cservers); i++ {
local := false
for x := 0; x < len(bad); x++ {
if cservers[i] == bad[x] {
local = true
}
}
if !local {
good = append(good, cservers[i])
}
}
return good
}
// Returns an array of local ipv4 addresses
func localAddies() []string {
addies, err := net.InterfaceAddrs()
if err != nil {
logging.Error.Println(err)
}
bad := []string{}
for i := 0; i < len(addies); i++ {
ip, _, err := net.ParseCIDR(addies[i].String())
if err != nil {
logging.Error.Println(err)
}
t4 := ip.To4()
if t4 != nil {
bad = append(bad, t4.String())
}
}
return bad
}

View file

@ -0,0 +1,655 @@
// Package records contains functions to generate resource records from
// mesos master states to serve through a dns server
package records
import (
"crypto/sha1"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/mesosphere/mesos-dns/errorutil"
"github.com/mesosphere/mesos-dns/logging"
"github.com/mesosphere/mesos-dns/models"
"github.com/mesosphere/mesos-dns/records/labels"
"github.com/mesosphere/mesos-dns/records/state"
"github.com/tv42/zbase32"
)
// Map host/service name to DNS answer
// REFACTOR - when discoveryinfo is integrated
// Will likely become map[string][]discoveryinfo
// Effectively we're (ab)using the map type as a set
// It used to have the type: rrs map[string][]string
type rrs map[string]map[string]struct{}
func (r rrs) add(name, host string) bool {
if host == "" {
return false
}
v, ok := r[name]
if !ok {
v = make(map[string]struct{})
r[name] = v
} else {
// don't overwrite existing values
_, ok = v[host]
if ok {
return false
}
}
v[host] = struct{}{}
return true
}
func (r rrs) First(name string) (string, bool) {
for host := range r[name] {
return host, true
}
return "", false
}
// Transform the record set into something exportable via the REST API
func (r rrs) ToAXFRResourceRecordSet() models.AXFRResourceRecordSet {
ret := make(models.AXFRResourceRecordSet, len(r))
for host, values := range r {
ret[host] = make([]string, 0, len(values))
for record := range values {
ret[host] = append(ret[host], record)
}
}
return ret
}
type rrsKind string
const (
// A record types
A rrsKind = "A"
// SRV record types
SRV = "SRV"
)
func (kind rrsKind) rrs(rg *RecordGenerator) rrs {
switch kind {
case A:
return rg.As
case SRV:
return rg.SRVs
default:
return nil
}
}
// RecordGenerator contains DNS records and methods to access and manipulate
// them. TODO(kozyraki): Refactor when discovery id is available.
type RecordGenerator struct {
As rrs
SRVs rrs
SlaveIPs map[string]string
EnumData EnumerationData
httpClient http.Client
}
// EnumerableRecord is the lowest level object, and should map 1:1 with DNS records
type EnumerableRecord struct {
Name string `json:"name"`
Host string `json:"host"`
Rtype string `json:"rtype"`
}
// EnumerableTask consists of the records derived from a task
type EnumerableTask struct {
Name string `json:"name"`
ID string `json:"id"`
Records []EnumerableRecord `json:"records"`
}
// EnumerableFramework is consistent of enumerable tasks, and include the name of the framework
type EnumerableFramework struct {
Tasks []*EnumerableTask `json:"tasks"`
Name string `json:"name"`
}
// EnumerationData is the top level container pointing to the
// enumerable frameworks containing enumerable tasks
type EnumerationData struct {
Frameworks []*EnumerableFramework `json:"frameworks"`
}
// NewRecordGenerator returns a RecordGenerator that's been configured with a timeout.
func NewRecordGenerator(httpTimeout time.Duration) *RecordGenerator {
enumData := EnumerationData{
Frameworks: []*EnumerableFramework{},
}
rg := &RecordGenerator{
httpClient: http.Client{Timeout: httpTimeout},
EnumData: enumData,
}
return rg
}
// ParseState retrieves and parses the Mesos master /state.json and converts it
// into DNS records.
func (rg *RecordGenerator) ParseState(c Config, masters ...string) error {
// find master -- return if error
sj, err := rg.FindMaster(masters...)
if err != nil {
logging.Error.Println("no master")
return err
}
if sj.Leader == "" {
logging.Error.Println("Unexpected error")
err = errors.New("empty master")
return err
}
hostSpec := labels.RFC1123
if c.EnforceRFC952 {
hostSpec = labels.RFC952
}
return rg.InsertState(sj, c.Domain, c.SOARname, c.Listener, masters, c.IPSources, hostSpec)
}
// Tries each master and looks for the leader
// if no leader responds it errors
func (rg *RecordGenerator) FindMaster(masters ...string) (state.State, error) {
var sj state.State
var leader string
if len(masters) > 0 {
leader, masters = masters[0], masters[1:]
}
// Check if ZK leader is correct
if leader != "" {
logging.VeryVerbose.Println("Zookeeper says the leader is: ", leader)
ip, port, err := getProto(leader)
if err != nil {
logging.Error.Println(err)
}
if sj, err = rg.loadWrap(ip, port); err == nil && sj.Leader != "" {
return sj, nil
}
logging.Verbose.Println("Warning: Zookeeper is wrong about leader")
if len(masters) == 0 {
return sj, errors.New("no master")
}
logging.Verbose.Println("Warning: falling back to Masters config field: ", masters)
}
// try each listed mesos master before dying
for i, master := range masters {
ip, port, err := getProto(master)
if err != nil {
logging.Error.Println(err)
}
if sj, err = rg.loadWrap(ip, port); err == nil && sj.Leader == "" {
logging.VeryVerbose.Println("Warning: not a leader - trying next one")
if len(masters)-1 == i {
return sj, errors.New("no master")
}
} else {
return sj, nil
}
}
return sj, errors.New("no master")
}
// Loads state.json from mesos master
func (rg *RecordGenerator) loadFromMaster(ip string, port string) (state.State, error) {
// REFACTOR: state.json security
var sj state.State
u := url.URL{
Scheme: "http",
Host: net.JoinHostPort(ip, port),
Path: "/master/state.json",
}
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
logging.Error.Println(err)
return state.State{}, err
}
req.Header.Set("Content-Type", "application/json")
resp, err := rg.httpClient.Do(req)
if err != nil {
logging.Error.Println(err)
return state.State{}, err
}
defer errorutil.Ignore(resp.Body.Close)
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
logging.Error.Println(err)
return state.State{}, err
}
err = json.Unmarshal(body, &sj)
if err != nil {
logging.Error.Println(err)
return state.State{}, err
}
return sj, nil
}
// Catches an attempt to load state.json from a mesos master
// attempts can fail from down server or mesos master secondary
// it also reloads from a different master if the master it attempted to
// load from was not the leader
func (rg *RecordGenerator) loadWrap(ip string, port string) (state.State, error) {
var err error
var sj state.State
logging.VeryVerbose.Println("reloading from master " + ip)
sj, err = rg.loadFromMaster(ip, port)
if err != nil {
return state.State{}, err
}
if rip := leaderIP(sj.Leader); rip != ip {
logging.VeryVerbose.Println("Warning: master changed to " + ip)
sj, err = rg.loadFromMaster(rip, port)
return sj, err
}
return sj, nil
}
// hashes a given name using a truncated sha1 hash
// 5 characters extracted from the zbase32 encoded hash provides
// enough entropy to avoid collisions
// zbase32: http://philzimmermann.com/docs/human-oriented-base-32-encoding.txt
// is used to promote human-readable names
func hashString(s string) string {
hash := sha1.Sum([]byte(s))
return zbase32.EncodeToString(hash[:])[:5]
}
// attempt to translate the hostname into an IPv4 address. logs an error if IP
// lookup fails. if an IP address cannot be found, returns the same hostname
// that was given. upon success returns the IP address as a string.
func hostToIP4(hostname string) (string, bool) {
ip := net.ParseIP(hostname)
if ip == nil {
t, err := net.ResolveIPAddr("ip4", hostname)
if err != nil {
logging.Error.Printf("cannot translate hostname %q into an ip4 address", hostname)
return hostname, false
}
ip = t.IP
}
return ip.String(), true
}
// InsertState transforms a StateJSON into RecordGenerator RRs
func (rg *RecordGenerator) InsertState(sj state.State, domain, ns, listener string, masters, ipSources []string, spec labels.Func) error {
rg.SlaveIPs = map[string]string{}
rg.SRVs = rrs{}
rg.As = rrs{}
rg.frameworkRecords(sj, domain, spec)
rg.slaveRecords(sj, domain, spec)
rg.listenerRecord(listener, ns)
rg.masterRecord(domain, masters, sj.Leader)
rg.taskRecords(sj, domain, spec, ipSources)
return nil
}
// frameworkRecords injects A and SRV records into the generator store:
// frameworkname.domain. // resolves to IPs of each framework
// _framework._tcp.frameworkname.domain. // resolves to the driver port and IP of each framework
func (rg *RecordGenerator) frameworkRecords(sj state.State, domain string, spec labels.Func) {
for _, f := range sj.Frameworks {
fname := labels.DomainFrag(f.Name, labels.Sep, spec)
host, port := f.HostPort()
if address, ok := hostToIP4(host); ok {
a := fname + "." + domain + "."
rg.insertRR(a, address, A)
if port != "" {
srvAddress := net.JoinHostPort(a, port)
rg.insertRR("_framework._tcp."+a, srvAddress, SRV)
}
}
}
}
// slaveRecords injects A and SRV records into the generator store:
// slave.domain. // resolves to IPs of all slaves
// _slave._tc.domain. // resolves to the driver port and IP of all slaves
func (rg *RecordGenerator) slaveRecords(sj state.State, domain string, spec labels.Func) {
for _, slave := range sj.Slaves {
address, ok := hostToIP4(slave.PID.Host)
if ok {
a := "slave." + domain + "."
rg.insertRR(a, address, A)
srv := net.JoinHostPort(a, slave.PID.Port)
rg.insertRR("_slave._tcp."+domain+".", srv, SRV)
} else {
logging.VeryVerbose.Printf("string '%q' for slave with id %q is not a valid IP address", address, slave.ID)
address = labels.DomainFrag(address, labels.Sep, spec)
}
rg.SlaveIPs[slave.ID] = address
}
}
// masterRecord injects A and SRV records into the generator store:
// master.domain. // resolves to IPs of all masters
// masterN.domain. // one IP address for each master
// leader.domain. // one IP address for the leading master
//
// The current func implementation makes an assumption about the order of masters:
// it's the order in which you expect the enumerated masterN records to be created.
// This is probably important: if a new leader is elected, you may not want it to
// become master0 simply because it's the leader. You probably want your DNS records
// to change as little as possible. And this func should have the least impact on
// enumeration order, or name/IP mappings - it's just creating the records. So let
// the caller do the work of ordering/sorting (if desired) the masters list if a
// different outcome is desired.
//
// Another consequence of the current overall mesos-dns app implementation is that
// the leader may not even be in the masters list at some point in time. masters is
// really fallback-masters (only consider these to be masters if I can't find a
// leader via ZK). At some point in time, they may not actually be masters any more.
// Consider a cluster of 3 nodes that suffers the loss of a member, and gains a new
// member (VM crashed, was replaced by another VM). And the cycle repeats several
// times. You end up with a set of running masters (and leader) that's different
// than the set of statically configured fallback masters.
//
// So the func tries to index the masters as they're listed and begrudgingly assigns
// the leading master an index out-of-band if it's not actually listed in the masters
// list. There are probably better ways to do it.
func (rg *RecordGenerator) masterRecord(domain string, masters []string, leader string) {
// create records for leader
// A records
h := strings.Split(leader, "@")
if len(h) < 2 {
logging.Error.Println(leader)
return // avoid a panic later
}
leaderAddress := h[1]
ip, port, err := getProto(leaderAddress)
if err != nil {
logging.Error.Println(err)
return
}
arec := "leader." + domain + "."
rg.insertRR(arec, ip, A)
arec = "master." + domain + "."
rg.insertRR(arec, ip, A)
// SRV records
tcp := "_leader._tcp." + domain + "."
udp := "_leader._udp." + domain + "."
host := "leader." + domain + "." + ":" + port
rg.insertRR(tcp, host, SRV)
rg.insertRR(udp, host, SRV)
// if there is a list of masters, insert that as well
addedLeaderMasterN := false
idx := 0
for _, master := range masters {
masterIP, _, err := getProto(master)
if err != nil {
logging.Error.Println(err)
continue
}
// A records (master and masterN)
if master != leaderAddress {
arec := "master." + domain + "."
added := rg.insertRR(arec, masterIP, A)
if !added {
// duplicate master?!
continue
}
}
if master == leaderAddress && addedLeaderMasterN {
// duplicate leader in masters list?!
continue
}
arec := "master" + strconv.Itoa(idx) + "." + domain + "."
rg.insertRR(arec, masterIP, A)
idx++
if master == leaderAddress {
addedLeaderMasterN = true
}
}
// flake: we ended up with a leader that's not in the list of all masters?
if !addedLeaderMasterN {
// only a flake if there were fallback masters configured
if len(masters) > 0 {
logging.Error.Printf("warning: leader %q is not in master list", leader)
}
arec = "master" + strconv.Itoa(idx) + "." + domain + "."
rg.insertRR(arec, ip, A)
}
}
// A record for mesos-dns (the name is listed in SOA replies)
func (rg *RecordGenerator) listenerRecord(listener string, ns string) {
if listener == "0.0.0.0" {
rg.setFromLocal(listener, ns)
} else if listener == "127.0.0.1" {
rg.insertRR(ns, "127.0.0.1", A)
} else {
rg.insertRR(ns, listener, A)
}
}
func (rg *RecordGenerator) taskRecords(sj state.State, domain string, spec labels.Func, ipSources []string) {
for _, f := range sj.Frameworks {
enumerableFramework := &EnumerableFramework{
Name: f.Name,
Tasks: []*EnumerableTask{},
}
rg.EnumData.Frameworks = append(rg.EnumData.Frameworks, enumerableFramework)
for _, task := range f.Tasks {
var ok bool
task.SlaveIP, ok = rg.SlaveIPs[task.SlaveID]
// only do running and discoverable tasks
if ok && (task.State == "TASK_RUNNING") {
rg.taskRecord(task, f, domain, spec, ipSources, enumerableFramework)
}
}
}
}
type context struct {
taskName,
taskID,
slaveID,
taskIP,
slaveIP string
}
func (rg *RecordGenerator) taskRecord(task state.Task, f state.Framework, domain string, spec labels.Func, ipSources []string, enumFW *EnumerableFramework) {
newTask := &EnumerableTask{ID: task.ID, Name: task.Name}
enumFW.Tasks = append(enumFW.Tasks, newTask)
// define context
ctx := context{
spec(task.Name),
hashString(task.ID),
slaveIDTail(task.SlaveID),
task.IP(ipSources...),
task.SlaveIP,
}
// use DiscoveryInfo name if defined instead of task name
if task.HasDiscoveryInfo() {
// LEGACY TODO: REMOVE
ctx.taskName = task.DiscoveryInfo.Name
rg.taskContextRecord(ctx, task, f, domain, spec, newTask)
// LEGACY, TODO: REMOVE
ctx.taskName = spec(task.DiscoveryInfo.Name)
rg.taskContextRecord(ctx, task, f, domain, spec, newTask)
} else {
rg.taskContextRecord(ctx, task, f, domain, spec, newTask)
}
}
func (rg *RecordGenerator) taskContextRecord(ctx context, task state.Task, f state.Framework, domain string, spec labels.Func, enumTask *EnumerableTask) {
fname := labels.DomainFrag(f.Name, labels.Sep, spec)
tail := "." + domain + "."
// insert canonical A records
canonical := ctx.taskName + "-" + ctx.taskID + "-" + ctx.slaveID + "." + fname
arec := ctx.taskName + "." + fname
rg.insertTaskRR(arec+tail, ctx.taskIP, A, enumTask)
rg.insertTaskRR(canonical+tail, ctx.taskIP, A, enumTask)
rg.insertTaskRR(arec+".slave"+tail, ctx.slaveIP, A, enumTask)
rg.insertTaskRR(canonical+".slave"+tail, ctx.slaveIP, A, enumTask)
// recordName generates records for ctx.taskName, given some generation chain
recordName := func(gen chain) { gen("_" + ctx.taskName) }
// asSRV is always the last link in a chain, it must insert RR's
asSRV := func(target string) chain {
return func(records ...string) {
for i := range records {
name := records[i] + tail
rg.insertTaskRR(name, target, SRV, enumTask)
}
}
}
// Add RFC 2782 SRV records
var subdomains []string
if task.HasDiscoveryInfo() {
subdomains = []string{"slave"}
} else {
subdomains = []string{"slave", domainNone}
}
slaveHost := canonical + ".slave" + tail
for _, port := range task.Ports() {
slaveTarget := slaveHost + ":" + port
recordName(withProtocol(protocolNone, fname, spec,
withSubdomains(subdomains, asSRV(slaveTarget))))
}
if !task.HasDiscoveryInfo() {
return
}
for _, port := range task.DiscoveryInfo.Ports.DiscoveryPorts {
target := canonical + tail + ":" + strconv.Itoa(port.Number)
recordName(withProtocol(port.Protocol, fname, spec,
withNamedPort(port.Name, spec, asSRV(target))))
}
}
// A records for each local interface
// If this causes problems you should explicitly set the
// listener address in config.json
func (rg *RecordGenerator) setFromLocal(host string, ns string) {
ifaces, err := net.Interfaces()
if err != nil {
logging.Error.Println(err)
}
// handle err
for _, i := range ifaces {
addrs, err := i.Addrs()
if err != nil {
logging.Error.Println(err)
}
for _, addr := range addrs {
var ip net.IP
switch v := addr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
if ip == nil || ip.IsLoopback() {
continue
}
ip = ip.To4()
if ip == nil {
continue
}
rg.insertRR(ns, ip.String(), A)
}
}
}
// insertRR adds a record to the appropriate record map for the given name/host pair,
// but only if the pair is unique. returns true if added, false otherwise.
// TODO(???): REFACTOR when storage is updated
func (rg *RecordGenerator) insertTaskRR(name, host string, kind rrsKind, enumTask *EnumerableTask) bool {
if rg.insertRR(name, host, kind) {
enumRecord := EnumerableRecord{Name: name, Host: host, Rtype: string(kind)}
enumTask.Records = append(enumTask.Records, enumRecord)
return true
}
return false
}
func (rg *RecordGenerator) insertRR(name, host string, kind rrsKind) (added bool) {
if rrs := kind.rrs(rg); rrs != nil {
if added = rrs.add(name, host); added {
logging.VeryVerbose.Println("[" + string(kind) + "]\t" + name + ": " + host)
}
}
return
}
// leaderIP returns the ip for the mesos master
// input format master@ip:port
func leaderIP(leader string) string {
pair := strings.Split(leader, "@")[1]
return strings.Split(pair, ":")[0]
}
// return the slave number from a Mesos slave id
func slaveIDTail(slaveID string) string {
fields := strings.Split(slaveID, "-")
return strings.ToLower(fields[len(fields)-1])
}
// should be able to accept
// ip:port
// zk://host1:port1,host2:port2,.../path
// zk://username:password@host1:port1,host2:port2,.../path
// file:///path/to/file (where file contains one of the above)
func getProto(pair string) (string, string, error) {
h := strings.SplitN(pair, ":", 2)
if len(h) != 2 {
return "", "", fmt.Errorf("unable to parse proto from %q", pair)
}
return h[0], h[1], nil
}

View file

@ -0,0 +1,83 @@
package labels
import (
"bytes"
"strings"
)
// Sep is the default domain fragment separator.
const Sep = "."
// DomainFrag mangles the given name in order to produce a valid domain fragment.
// A valid domain fragment will consist of one or more host name labels
// concatenated by the given separator.
func DomainFrag(name, sep string, label Func) string {
var labels []string
for _, part := range strings.Split(name, sep) {
if lab := label(part); lab != "" {
labels = append(labels, lab)
}
}
return strings.Join(labels, sep)
}
// Func is a function type representing label functions.
type Func func(string) string
// RFC952 mangles a name to conform to the DNS label rules specified in RFC952.
// See http://www.rfc-base.org/txt/rfc-952.txt
func RFC952(name string) string {
return string(label([]byte(name), 24, "-0123456789", "-"))
}
// RFC1123 mangles a name to conform to the DNS label rules specified in RFC1123.
// See http://www.rfc-base.org/txt/rfc-1123.txt
func RFC1123(name string) string {
return string(label([]byte(name), 63, "-", "-"))
}
// label computes a label from the given name with maxlen length and the
// left and right cutsets trimmed from their respective ends.
func label(name []byte, maxlen int, left, right string) []byte {
return trimCut(bytes.Map(mapping, name), maxlen, left, right)
}
// mapping maps a given rune to its valid DNS label counterpart.
func mapping(r rune) rune {
switch {
case r >= 'A' && r <= 'Z':
return r - ('A' - 'a')
case r >= 'a' && r <= 'z':
fallthrough
case r >= '0' && r <= '9':
return r
case r == '-' || r == '.' || r == '_':
return '-'
default:
return -1
}
}
// trimCut cuts the given label at min(maxlen, len(label)) and ensures the left
// and right cutsets are trimmed from their respective ends.
func trimCut(label []byte, maxlen int, left, right string) []byte {
trim := bytes.TrimLeft(label, left)
size := min(len(trim), maxlen)
head := bytes.TrimRight(trim[:size], right)
if len(head) == size {
return head
}
tail := bytes.TrimLeft(trim[size:], right)
if len(tail) > 0 {
return append(head, tail[:size-len(head)]...)
}
return head
}
// min returns the minimum of two ints
func min(a, b int) int {
if a < b {
return a
}
return b
}

View file

@ -0,0 +1,271 @@
package state
import (
"bytes"
"github.com/mesos/mesos-go/upid"
"net"
"strconv"
"strings"
)
// Resources holds resources as defined in the /state.json Mesos HTTP endpoint.
type Resources struct {
PortRanges string `json:"ports"`
}
// Ports returns a slice of individual ports expanded from PortRanges.
func (r Resources) Ports() []string {
if r.PortRanges == "" || r.PortRanges == "[]" {
return []string{}
}
rhs := strings.Split(r.PortRanges, "[")[1]
lhs := strings.Split(rhs, "]")[0]
yports := []string{}
mports := strings.Split(lhs, ",")
for _, port := range mports {
tmp := strings.TrimSpace(port)
pz := strings.Split(tmp, "-")
lo, _ := strconv.Atoi(pz[0])
hi, _ := strconv.Atoi(pz[1])
for t := lo; t <= hi; t++ {
yports = append(yports, strconv.Itoa(t))
}
}
return yports
}
// Label holds a label as defined in the /state.json Mesos HTTP endpoint.
type Label struct {
Key string `json:"key"`
Value string `json:"value"`
}
// Status holds a task status as defined in the /state.json Mesos HTTP endpoint.
type Status struct {
Timestamp float64 `json:"timestamp"`
State string `json:"state"`
Labels []Label `json:"labels,omitempty"`
ContainerStatus ContainerStatus `json:"container_status,omitempty"`
Healthy *bool `json:"healthy"`
}
// ContainerStatus holds container metadata as defined in the /state.json
// Mesos HTTP endpoint.
type ContainerStatus struct {
NetworkInfos []NetworkInfo `json:"network_infos,omitempty"`
}
// NetworkInfo holds the network configuration for a single interface
// as defined in the /state.json Mesos HTTP endpoint.
type NetworkInfo struct {
IPAddresses []IPAddress `json:"ip_addresses,omitempty"`
// back-compat with 0.25 IPAddress format
IPAddress string `json:"ip_address,omitempty"`
}
// IPAddress holds a single IP address configured on an interface,
// as defined in the /state.json Mesos HTTP endpoint.
type IPAddress struct {
IPAddress string `json:"ip_address,omitempty"`
}
// Task holds a task as defined in the /state.json Mesos HTTP endpoint.
type Task struct {
FrameworkID string `json:"framework_id"`
ID string `json:"id"`
Name string `json:"name"`
SlaveID string `json:"slave_id"`
State string `json:"state"`
Statuses []Status `json:"statuses"`
Resources `json:"resources"`
DiscoveryInfo DiscoveryInfo `json:"discovery"`
SlaveIP string `json:"-"`
Labels []Label `json:"labels,omitempty"`
}
// HasDiscoveryInfo return whether the DiscoveryInfo was provided in the state.json
func (t *Task) HasDiscoveryInfo() bool {
return t.DiscoveryInfo.Name != ""
}
// IP returns the first Task IP found in the given sources.
func (t *Task) IP(srcs ...string) string {
if ips := t.IPs(srcs...); len(ips) > 0 {
return ips[0].String()
}
return ""
}
// IPs returns a slice of IPs sourced from the given sources with ascending
// priority.
func (t *Task) IPs(srcs ...string) (ips []net.IP) {
if t == nil {
return nil
}
for i := range srcs {
if src, ok := sources[srcs[i]]; ok {
for _, srcIP := range src(t) {
if ip := net.ParseIP(srcIP); len(ip) > 0 {
ips = append(ips, ip)
}
}
}
}
return ips
}
// sources maps the string representation of IP sources to their functions.
var sources = map[string]func(*Task) []string{
"host": hostIPs,
"mesos": mesosIPs,
"docker": dockerIPs,
"netinfo": networkInfoIPs,
}
// hostIPs is an IPSource which returns the IP addresses of the slave a Task
// runs on.
func hostIPs(t *Task) []string { return []string{t.SlaveIP} }
// networkInfoIPs returns IP addresses from a given Task's
// []Status.ContainerStatus.[]NetworkInfos.[]IPAddresses.IPAddress
func networkInfoIPs(t *Task) []string {
return statusIPs(t.Statuses, func(s *Status) []string {
ips := make([]string, len(s.ContainerStatus.NetworkInfos))
for _, netinfo := range s.ContainerStatus.NetworkInfos {
if len(netinfo.IPAddresses) > 0 {
// In v0.26, we use the IPAddresses field.
for _, ipAddress := range netinfo.IPAddresses {
ips = append(ips, ipAddress.IPAddress)
}
} else {
// Fall back to v0.25 syntax of single IPAddress if that's being used.
if netinfo.IPAddress != "" {
ips = append(ips, netinfo.IPAddress)
}
}
}
return ips
})
}
const (
// DockerIPLabel is the key of the Label which holds the Docker containerizer IP value.
DockerIPLabel = "Docker.NetworkSettings.IPAddress"
// MesosIPLabel is the key of the label which holds the Mesos containerizer IP value.
MesosIPLabel = "MesosContainerizer.NetworkSettings.IPAddress"
)
// dockerIPs returns IP addresses from the values of all
// Task.[]Status.[]Labels whose keys are equal to "Docker.NetworkSettings.IPAddress".
func dockerIPs(t *Task) []string {
return statusIPs(t.Statuses, labels(DockerIPLabel))
}
// mesosIPs returns IP addresses from the values of all
// Task.[]Status.[]Labels whose keys are equal to
// "MesosContainerizer.NetworkSettings.IPAddress".
func mesosIPs(t *Task) []string {
return statusIPs(t.Statuses, labels(MesosIPLabel))
}
// statusIPs returns the latest running status IPs extracted with the given src
func statusIPs(st []Status, src func(*Status) []string) []string {
// the state.json we extract from mesos makes no guarantees re: the order
// of the task statuses so we should check the timestamps to avoid problems
// down the line. we can't rely on seeing the same sequence. (@joris)
// https://github.com/apache/mesos/blob/0.24.0/src/slave/slave.cpp#L5226-L5238
ts, j := -1.0, -1
for i := range st {
if st[i].State == "TASK_RUNNING" && st[i].Timestamp > ts {
ts, j = st[i].Timestamp, i
}
}
if j >= 0 {
return src(&st[j])
}
return nil
}
// labels returns all given Status.[]Labels' values whose keys are equal
// to the given key
func labels(key string) func(*Status) []string {
return func(s *Status) []string {
vs := make([]string, 0, len(s.Labels))
for _, l := range s.Labels {
if l.Key == key {
vs = append(vs, l.Value)
}
}
return vs
}
}
// Framework holds a framework as defined in the /state.json Mesos HTTP endpoint.
type Framework struct {
Tasks []Task `json:"tasks"`
PID PID `json:"pid"`
Name string `json:"name"`
Hostname string `json:"hostname"`
}
// HostPort returns the hostname and port where a framework's scheduler is
// listening on.
func (f Framework) HostPort() (string, string) {
if f.PID.UPID != nil {
return f.PID.Host, f.PID.Port
}
return f.Hostname, ""
}
// Slave holds a slave as defined in the /state.json Mesos HTTP endpoint.
type Slave struct {
ID string `json:"id"`
Hostname string `json:"hostname"`
PID PID `json:"pid"`
}
// PID holds a Mesos PID and implements the json.Unmarshaler interface.
type PID struct{ *upid.UPID }
// UnmarshalJSON implements the json.Unmarshaler interface for PIDs.
func (p *PID) UnmarshalJSON(data []byte) (err error) {
p.UPID, err = upid.Parse(string(bytes.Trim(data, `" `)))
return err
}
// State holds the state defined in the /state.json Mesos HTTP endpoint.
type State struct {
Frameworks []Framework `json:"frameworks"`
Slaves []Slave `json:"slaves"`
Leader string `json:"leader"`
}
// DiscoveryInfo holds the discovery meta data for a task defined in the /state.json Mesos HTTP endpoint.
type DiscoveryInfo struct {
Visibilty string `json:"visibility"`
Version string `json:"version,omitempty"`
Name string `json:"name,omitempty"`
Location string `json:"location,omitempty"`
Environment string `json:"environment,omitempty"`
Labels struct {
Labels []Label `json:"labels"`
} `json:"labels"`
Ports Ports `json:"ports"`
}
// Ports holds a list of DiscoveryPort
type Ports struct {
DiscoveryPorts []DiscoveryPort `json:"ports"`
}
// DiscoveryPort holds a port for a task defined in the /state.json Mesos HTTP endpoint.
type DiscoveryPort struct {
Protocol string `json:"protocol"`
Number int `json:"number"`
Name string `json:"name"`
}

View file

@ -0,0 +1,84 @@
package records
import (
"fmt"
"net"
)
func validateEnabledServices(c *Config) error {
if !c.DNSOn && !c.HTTPOn {
return fmt.Errorf("Either DNS or HTTP server should be on")
}
if len(c.Masters) == 0 && c.Zk == "" {
return fmt.Errorf("specify mesos masters or zookeeper in config.json")
}
return nil
}
// validateMasters checks that each master in the list is a properly formatted host:ip pair.
// duplicate masters in the list are not allowed.
// returns nil if the masters list is empty, or else all masters in the list are valid.
func validateMasters(ms []string) error {
if len(ms) == 0 {
return nil
}
valid := make(map[string]struct{}, len(ms))
for i, m := range ms {
h, p, err := net.SplitHostPort(m)
if err != nil {
return fmt.Errorf("illegal host:port specified for master %q", ms[i])
}
// normalize ipv6 addresses
if ip := net.ParseIP(h); ip != nil {
h = ip.String()
m = h + "_" + p
}
//TODO(jdef) distinguish between intended hostnames and invalid ip addresses
if _, found := valid[m]; found {
return fmt.Errorf("duplicate master specified: %v", ms[i])
}
valid[m] = struct{}{}
}
return nil
}
// validateResolvers checks that each resolver in the list is a properly formatted IP address.
// duplicate resolvers in the list are not allowed.
// returns nil if the resolver list is empty, or else all resolvers in the list are valid.
func validateResolvers(rs []string) error {
if len(rs) == 0 {
return nil
}
ips := make(map[string]struct{}, len(rs))
for _, r := range rs {
ip := net.ParseIP(r)
if ip == nil {
return fmt.Errorf("illegal IP specified for resolver %q", r)
}
ipstr := ip.String()
if _, found := ips[ipstr]; found {
return fmt.Errorf("duplicate resolver IP specified: %v", r)
}
ips[ipstr] = struct{}{}
}
return nil
}
// validateIPSources checks validity of ip sources
func validateIPSources(srcs []string) error {
if len(srcs) == 0 {
return fmt.Errorf("empty ip sources")
}
if len(srcs) != len(unique(srcs)) {
return fmt.Errorf("duplicate ip source specified")
}
for _, src := range srcs {
switch src {
case "host", "docker", "mesos", "netinfo":
default:
return fmt.Errorf("invalid ip source %q", src)
}
}
return nil
}