Support libkv.WatchTree chan errors:

- libkv.WatchTree returns a channel whose messages represent changes
    to the watched tree. In situations where libkv cannot read from the
    underlying store, libkv will close the provided channel.
  - This PR handles this edge case and fixes #238.

provider/kv.go

@@ -35,6 +35,27 @@ type KvTLS struct {
 	InsecureSkipVerify bool
 }
 
+func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix string) {
+	for {
+		chanKeys, err := provider.kvclient.WatchTree(provider.Prefix, make(chan struct{}) /* stop chan */)
+		if err != nil {
+			log.Errorf("Failed to WatchTree %s", err)
+			continue
+		}
+
+		for range chanKeys {
+			configuration := provider.loadConfig()
+			if configuration != nil {
+				configurationChan <- types.ConfigMessage{
+					ProviderName:  string(provider.storeType),
+					Configuration: configuration,
+				}
+			}
+		}
+		log.Warnf("Intermittent failure to WatchTree KV. Retrying.")
+	}
+}
+
 func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage) error {
 	storeConfig := &store.Config{
 		ConnectionTimeout: 30 * time.Second,
@@ -80,24 +101,7 @@ func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage) error
 	}
 	provider.kvclient = kv
 	if provider.Watch {
-		stopCh := make(chan struct{})
-		chanKeys, err := kv.WatchTree(provider.Prefix, stopCh)
-		if err != nil {
-			return err
-		}
-		go func() {
-			for {
-				<-chanKeys
-				configuration := provider.loadConfig()
-				if configuration != nil {
-					configurationChan <- types.ConfigMessage{
-						ProviderName:  string(provider.storeType),
-						Configuration: configuration,
-					}
-				}
-				defer close(stopCh)
-			}
-		}()
+		go provider.watchKv(configurationChan, provider.Prefix)
 	}
 	configuration := provider.loadConfig()
 	configurationChan <- types.ConfigMessage{