Move code to pkg
This commit is contained in:
Ludovic Fernandez
Traefiker Bot
parent
bd4c822670
commit
f1b085fa36
465 changed files with 656 additions and 680 deletions
223
pkg/healthcheck/healthcheck.go
Normal file
223
pkg/healthcheck/healthcheck.go
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/containous/traefik/pkg/log"
|
||||
"github.com/containous/traefik/pkg/safe"
|
||||
"github.com/go-kit/kit/metrics"
|
||||
"github.com/vulcand/oxy/roundrobin"
|
||||
)
|
||||
|
||||
var singleton *HealthCheck
|
||||
var once sync.Once
|
||||
|
||||
// BalancerHandler includes functionality for load-balancing management.
|
||||
type BalancerHandler interface {
|
||||
ServeHTTP(w http.ResponseWriter, req *http.Request)
|
||||
Servers() []*url.URL
|
||||
RemoveServer(u *url.URL) error
|
||||
UpsertServer(u *url.URL, options ...roundrobin.ServerOption) error
|
||||
}
|
||||
|
||||
// metricsRegistry is a local interface in the health check package, exposing only the required metrics
|
||||
// necessary for the health check package. This makes it easier for the tests.
|
||||
type metricsRegistry interface {
|
||||
BackendServerUpGauge() metrics.Gauge
|
||||
}
|
||||
|
||||
// Options are the public health check options.
|
||||
type Options struct {
|
||||
Headers map[string]string
|
||||
Hostname string
|
||||
Scheme string
|
||||
Path string
|
||||
Port int
|
||||
Transport http.RoundTripper
|
||||
Interval time.Duration
|
||||
Timeout time.Duration
|
||||
LB BalancerHandler
|
||||
}
|
||||
|
||||
func (opt Options) String() string {
|
||||
return fmt.Sprintf("[Hostname: %s Headers: %v Path: %s Port: %d Interval: %s Timeout: %s]", opt.Hostname, opt.Headers, opt.Path, opt.Port, opt.Interval, opt.Timeout)
|
||||
}
|
||||
|
||||
// BackendConfig HealthCheck configuration for a backend
|
||||
type BackendConfig struct {
|
||||
Options
|
||||
name string
|
||||
disabledURLs []*url.URL
|
||||
}
|
||||
|
||||
func (b *BackendConfig) newRequest(serverURL *url.URL) (*http.Request, error) {
|
||||
u, err := serverURL.Parse(b.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(b.Scheme) > 0 {
|
||||
u.Scheme = b.Scheme
|
||||
}
|
||||
|
||||
if b.Port != 0 {
|
||||
u.Host = net.JoinHostPort(u.Hostname(), strconv.Itoa(b.Port))
|
||||
}
|
||||
|
||||
return http.NewRequest(http.MethodGet, u.String(), http.NoBody)
|
||||
}
|
||||
|
||||
// this function adds additional http headers and hostname to http.request
|
||||
func (b *BackendConfig) addHeadersAndHost(req *http.Request) *http.Request {
|
||||
if b.Options.Hostname != "" {
|
||||
req.Host = b.Options.Hostname
|
||||
}
|
||||
|
||||
for k, v := range b.Options.Headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// HealthCheck struct
|
||||
type HealthCheck struct {
|
||||
Backends map[string]*BackendConfig
|
||||
metrics metricsRegistry
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
// SetBackendsConfiguration set backends configuration
|
||||
func (hc *HealthCheck) SetBackendsConfiguration(parentCtx context.Context, backends map[string]*BackendConfig) {
|
||||
hc.Backends = backends
|
||||
if hc.cancel != nil {
|
||||
hc.cancel()
|
||||
}
|
||||
ctx, cancel := context.WithCancel(parentCtx)
|
||||
hc.cancel = cancel
|
||||
|
||||
for _, backend := range backends {
|
||||
currentBackend := backend
|
||||
safe.Go(func() {
|
||||
hc.execute(ctx, currentBackend)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (hc *HealthCheck) execute(ctx context.Context, backend *BackendConfig) {
|
||||
log.Debugf("Initial health check for backend: %q", backend.name)
|
||||
hc.checkBackend(backend)
|
||||
ticker := time.NewTicker(backend.Interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Debugf("Stopping current health check goroutines of backend: %s", backend.name)
|
||||
return
|
||||
case <-ticker.C:
|
||||
log.Debugf("Refreshing health check for backend: %s", backend.name)
|
||||
hc.checkBackend(backend)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (hc *HealthCheck) checkBackend(backend *BackendConfig) {
|
||||
enabledURLs := backend.LB.Servers()
|
||||
var newDisabledURLs []*url.URL
|
||||
// FIXME re enable metrics
|
||||
for _, disableURL := range backend.disabledURLs {
|
||||
// FIXME serverUpMetricValue := float64(0)
|
||||
if err := checkHealth(disableURL, backend); err == nil {
|
||||
log.Warnf("Health check up: Returning to server list. Backend: %q URL: %q", backend.name, disableURL.String())
|
||||
if err = backend.LB.UpsertServer(disableURL, roundrobin.Weight(1)); err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
// FIXME serverUpMetricValue = 1
|
||||
} else {
|
||||
log.Warnf("Health check still failing. Backend: %q URL: %q Reason: %s", backend.name, disableURL.String(), err)
|
||||
newDisabledURLs = append(newDisabledURLs, disableURL)
|
||||
}
|
||||
// FIXME labelValues := []string{"backend", backend.name, "url", disableURL.String()}
|
||||
// FIXME hc.metrics.BackendServerUpGauge().With(labelValues...).Set(serverUpMetricValue)
|
||||
}
|
||||
backend.disabledURLs = newDisabledURLs
|
||||
|
||||
// FIXME re enable metrics
|
||||
for _, enableURL := range enabledURLs {
|
||||
// FIXME serverUpMetricValue := float64(1)
|
||||
if err := checkHealth(enableURL, backend); err != nil {
|
||||
log.Warnf("Health check failed: Remove from server list. Backend: %q URL: %q Reason: %s", backend.name, enableURL.String(), err)
|
||||
if err := backend.LB.RemoveServer(enableURL); err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
backend.disabledURLs = append(backend.disabledURLs, enableURL)
|
||||
// FIXME serverUpMetricValue = 0
|
||||
}
|
||||
// FIXME labelValues := []string{"backend", backend.name, "url", enableURL.String()}
|
||||
// FIXME hc.metrics.BackendServerUpGauge().With(labelValues...).Set(serverUpMetricValue)
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME re add metrics
|
||||
//func GetHealthCheck(metrics metricsRegistry) *HealthCheck {
|
||||
|
||||
// GetHealthCheck returns the health check which is guaranteed to be a singleton.
|
||||
func GetHealthCheck() *HealthCheck {
|
||||
once.Do(func() {
|
||||
singleton = newHealthCheck()
|
||||
//singleton = newHealthCheck(metrics)
|
||||
})
|
||||
return singleton
|
||||
}
|
||||
|
||||
// FIXME re add metrics
|
||||
//func newHealthCheck(metrics metricsRegistry) *HealthCheck {
|
||||
func newHealthCheck() *HealthCheck {
|
||||
return &HealthCheck{
|
||||
Backends: make(map[string]*BackendConfig),
|
||||
//metrics: metrics,
|
||||
}
|
||||
}
|
||||
|
||||
// NewBackendConfig Instantiate a new BackendConfig
|
||||
func NewBackendConfig(options Options, backendName string) *BackendConfig {
|
||||
return &BackendConfig{
|
||||
Options: options,
|
||||
name: backendName,
|
||||
}
|
||||
}
|
||||
|
||||
// checkHealth returns a nil error in case it was successful and otherwise
|
||||
// a non-nil error with a meaningful description why the health check failed.
|
||||
func checkHealth(serverURL *url.URL, backend *BackendConfig) error {
|
||||
req, err := backend.newRequest(serverURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create HTTP request: %s", err)
|
||||
}
|
||||
|
||||
req = backend.addHeadersAndHost(req)
|
||||
|
||||
client := http.Client{
|
||||
Timeout: backend.Options.Timeout,
|
||||
Transport: backend.Options.Transport,
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("HTTP request failed: %s", err)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest {
|
||||
return fmt.Errorf("received error status code: %v", resp.StatusCode)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
429
pkg/healthcheck/healthcheck_test.go
Normal file
429
pkg/healthcheck/healthcheck_test.go
Normal file
|
|
@ -0,0 +1,429 @@
|
|||
package healthcheck
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/containous/traefik/pkg/testhelpers"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/vulcand/oxy/roundrobin"
|
||||
)
|
||||
|
||||
const healthCheckInterval = 200 * time.Millisecond
|
||||
const healthCheckTimeout = 100 * time.Millisecond
|
||||
|
||||
type testHandler struct {
|
||||
done func()
|
||||
healthSequence []int
|
||||
}
|
||||
|
||||
func TestSetBackendsConfiguration(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
startHealthy bool
|
||||
healthSequence []int
|
||||
expectedNumRemovedServers int
|
||||
expectedNumUpsertedServers int
|
||||
expectedGaugeValue float64
|
||||
}{
|
||||
{
|
||||
desc: "healthy server staying healthy",
|
||||
startHealthy: true,
|
||||
healthSequence: []int{http.StatusOK},
|
||||
expectedNumRemovedServers: 0,
|
||||
expectedNumUpsertedServers: 0,
|
||||
expectedGaugeValue: 1,
|
||||
},
|
||||
{
|
||||
desc: "healthy server staying healthy (StatusNoContent)",
|
||||
startHealthy: true,
|
||||
healthSequence: []int{http.StatusNoContent},
|
||||
expectedNumRemovedServers: 0,
|
||||
expectedNumUpsertedServers: 0,
|
||||
expectedGaugeValue: 1,
|
||||
},
|
||||
{
|
||||
desc: "healthy server staying healthy (StatusPermanentRedirect)",
|
||||
startHealthy: true,
|
||||
healthSequence: []int{http.StatusPermanentRedirect},
|
||||
expectedNumRemovedServers: 0,
|
||||
expectedNumUpsertedServers: 0,
|
||||
expectedGaugeValue: 1,
|
||||
},
|
||||
{
|
||||
desc: "healthy server becoming sick",
|
||||
startHealthy: true,
|
||||
healthSequence: []int{http.StatusServiceUnavailable},
|
||||
expectedNumRemovedServers: 1,
|
||||
expectedNumUpsertedServers: 0,
|
||||
expectedGaugeValue: 0,
|
||||
},
|
||||
{
|
||||
desc: "sick server becoming healthy",
|
||||
startHealthy: false,
|
||||
healthSequence: []int{http.StatusOK},
|
||||
expectedNumRemovedServers: 0,
|
||||
expectedNumUpsertedServers: 1,
|
||||
expectedGaugeValue: 1,
|
||||
},
|
||||
{
|
||||
desc: "sick server staying sick",
|
||||
startHealthy: false,
|
||||
healthSequence: []int{http.StatusServiceUnavailable},
|
||||
expectedNumRemovedServers: 0,
|
||||
expectedNumUpsertedServers: 0,
|
||||
expectedGaugeValue: 0,
|
||||
},
|
||||
{
|
||||
desc: "healthy server toggling to sick and back to healthy",
|
||||
startHealthy: true,
|
||||
healthSequence: []int{http.StatusServiceUnavailable, http.StatusOK},
|
||||
expectedNumRemovedServers: 1,
|
||||
expectedNumUpsertedServers: 1,
|
||||
expectedGaugeValue: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range testCases {
|
||||
test := test
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// The context is passed to the health check and canonically canceled by
|
||||
// the test server once all expected requests have been received.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
ts := newTestServer(cancel, test.healthSequence)
|
||||
defer ts.Close()
|
||||
|
||||
lb := &testLoadBalancer{RWMutex: &sync.RWMutex{}}
|
||||
backend := NewBackendConfig(Options{
|
||||
Path: "/path",
|
||||
Interval: healthCheckInterval,
|
||||
Timeout: healthCheckTimeout,
|
||||
LB: lb,
|
||||
}, "backendName")
|
||||
|
||||
serverURL := testhelpers.MustParseURL(ts.URL)
|
||||
if test.startHealthy {
|
||||
lb.servers = append(lb.servers, serverURL)
|
||||
} else {
|
||||
backend.disabledURLs = append(backend.disabledURLs, serverURL)
|
||||
}
|
||||
|
||||
collectingMetrics := testhelpers.NewCollectingHealthCheckMetrics()
|
||||
check := HealthCheck{
|
||||
Backends: make(map[string]*BackendConfig),
|
||||
metrics: collectingMetrics,
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(1)
|
||||
|
||||
go func() {
|
||||
check.execute(ctx, backend)
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
// Make test timeout dependent on number of expected requests, health
|
||||
// check interval, and a safety margin.
|
||||
timeout := time.Duration(len(test.healthSequence)*int(healthCheckInterval) + 500)
|
||||
select {
|
||||
case <-time.After(timeout):
|
||||
t.Fatal("test did not complete in time")
|
||||
case <-ctx.Done():
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
lb.Lock()
|
||||
defer lb.Unlock()
|
||||
|
||||
assert.Equal(t, test.expectedNumRemovedServers, lb.numRemovedServers, "removed servers")
|
||||
assert.Equal(t, test.expectedNumUpsertedServers, lb.numUpsertedServers, "upserted servers")
|
||||
// FIXME re add metrics
|
||||
//assert.Equal(t, test.expectedGaugeValue, collectingMetrics.Gauge.GaugeValue, "ServerUp Gauge")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRequest(t *testing.T) {
|
||||
type expected struct {
|
||||
err bool
|
||||
value string
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
desc string
|
||||
serverURL string
|
||||
options Options
|
||||
expected expected
|
||||
}{
|
||||
{
|
||||
desc: "no port override",
|
||||
serverURL: "http://backend1:80",
|
||||
options: Options{
|
||||
Path: "/test",
|
||||
Port: 0,
|
||||
},
|
||||
expected: expected{
|
||||
err: false,
|
||||
value: "http://backend1:80/test",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "port override",
|
||||
serverURL: "http://backend2:80",
|
||||
options: Options{
|
||||
Path: "/test",
|
||||
Port: 8080,
|
||||
},
|
||||
expected: expected{
|
||||
err: false,
|
||||
value: "http://backend2:8080/test",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "no port override with no port in server URL",
|
||||
serverURL: "http://backend1",
|
||||
options: Options{
|
||||
Path: "/health",
|
||||
Port: 0,
|
||||
},
|
||||
expected: expected{
|
||||
err: false,
|
||||
value: "http://backend1/health",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "port override with no port in server URL",
|
||||
serverURL: "http://backend2",
|
||||
options: Options{
|
||||
Path: "/health",
|
||||
Port: 8080,
|
||||
},
|
||||
expected: expected{
|
||||
err: false,
|
||||
value: "http://backend2:8080/health",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "scheme override",
|
||||
serverURL: "https://backend1:80",
|
||||
options: Options{
|
||||
Scheme: "http",
|
||||
Path: "/test",
|
||||
Port: 0,
|
||||
},
|
||||
expected: expected{
|
||||
err: false,
|
||||
value: "http://backend1:80/test",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "path with param",
|
||||
serverURL: "http://backend1:80",
|
||||
options: Options{
|
||||
Path: "/health?powpow=do",
|
||||
Port: 0,
|
||||
},
|
||||
expected: expected{
|
||||
err: false,
|
||||
value: "http://backend1:80/health?powpow=do",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "path with params",
|
||||
serverURL: "http://backend1:80",
|
||||
options: Options{
|
||||
Path: "/health?powpow=do&do=powpow",
|
||||
Port: 0,
|
||||
},
|
||||
expected: expected{
|
||||
err: false,
|
||||
value: "http://backend1:80/health?powpow=do&do=powpow",
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "path with invalid path",
|
||||
serverURL: "http://backend1:80",
|
||||
options: Options{
|
||||
Path: ":",
|
||||
Port: 0,
|
||||
},
|
||||
expected: expected{
|
||||
err: true,
|
||||
value: "",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range testCases {
|
||||
test := test
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
backend := NewBackendConfig(test.options, "backendName")
|
||||
|
||||
u := testhelpers.MustParseURL(test.serverURL)
|
||||
|
||||
req, err := backend.newRequest(u)
|
||||
|
||||
if test.expected.err {
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, nil)
|
||||
} else {
|
||||
require.NoError(t, err, "failed to create new backend request")
|
||||
require.NotNil(t, req)
|
||||
assert.Equal(t, test.expected.value, req.URL.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddHeadersAndHost(t *testing.T) {
|
||||
testCases := []struct {
|
||||
desc string
|
||||
serverURL string
|
||||
options Options
|
||||
expectedHostname string
|
||||
expectedHeader string
|
||||
}{
|
||||
{
|
||||
desc: "override hostname",
|
||||
serverURL: "http://backend1:80",
|
||||
options: Options{
|
||||
Hostname: "myhost",
|
||||
Path: "/",
|
||||
},
|
||||
expectedHostname: "myhost",
|
||||
expectedHeader: "",
|
||||
},
|
||||
{
|
||||
desc: "not override hostname",
|
||||
serverURL: "http://backend1:80",
|
||||
options: Options{
|
||||
Hostname: "",
|
||||
Path: "/",
|
||||
},
|
||||
expectedHostname: "backend1:80",
|
||||
expectedHeader: "",
|
||||
},
|
||||
{
|
||||
desc: "custom header",
|
||||
serverURL: "http://backend1:80",
|
||||
options: Options{
|
||||
Headers: map[string]string{"Custom-Header": "foo"},
|
||||
Hostname: "",
|
||||
Path: "/",
|
||||
},
|
||||
expectedHostname: "backend1:80",
|
||||
expectedHeader: "foo",
|
||||
},
|
||||
{
|
||||
desc: "custom header with hostname override",
|
||||
serverURL: "http://backend1:80",
|
||||
options: Options{
|
||||
Headers: map[string]string{"Custom-Header": "foo"},
|
||||
Hostname: "myhost",
|
||||
Path: "/",
|
||||
},
|
||||
expectedHostname: "myhost",
|
||||
expectedHeader: "foo",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range testCases {
|
||||
test := test
|
||||
t.Run(test.desc, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
backend := NewBackendConfig(test.options, "backendName")
|
||||
|
||||
u, err := url.Parse(test.serverURL)
|
||||
require.NoError(t, err)
|
||||
|
||||
req, err := backend.newRequest(u)
|
||||
require.NoError(t, err, "failed to create new backend request")
|
||||
|
||||
req = backend.addHeadersAndHost(req)
|
||||
|
||||
assert.Equal(t, "http://backend1:80/", req.URL.String())
|
||||
assert.Equal(t, test.expectedHostname, req.Host)
|
||||
assert.Equal(t, test.expectedHeader, req.Header.Get("Custom-Header"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type testLoadBalancer struct {
|
||||
// RWMutex needed due to parallel test execution: Both the system-under-test
|
||||
// and the test assertions reference the counters.
|
||||
*sync.RWMutex
|
||||
numRemovedServers int
|
||||
numUpsertedServers int
|
||||
servers []*url.URL
|
||||
}
|
||||
|
||||
func (lb *testLoadBalancer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
// noop
|
||||
}
|
||||
|
||||
func (lb *testLoadBalancer) RemoveServer(u *url.URL) error {
|
||||
lb.Lock()
|
||||
defer lb.Unlock()
|
||||
lb.numRemovedServers++
|
||||
lb.removeServer(u)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (lb *testLoadBalancer) UpsertServer(u *url.URL, options ...roundrobin.ServerOption) error {
|
||||
lb.Lock()
|
||||
defer lb.Unlock()
|
||||
lb.numUpsertedServers++
|
||||
lb.servers = append(lb.servers, u)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (lb *testLoadBalancer) Servers() []*url.URL {
|
||||
return lb.servers
|
||||
}
|
||||
|
||||
func (lb *testLoadBalancer) removeServer(u *url.URL) {
|
||||
var i int
|
||||
var serverURL *url.URL
|
||||
for i, serverURL = range lb.servers {
|
||||
if *serverURL == *u {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
lb.servers = append(lb.servers[:i], lb.servers[i+1:]...)
|
||||
}
|
||||
|
||||
func newTestServer(done func(), healthSequence []int) *httptest.Server {
|
||||
handler := &testHandler{
|
||||
done: done,
|
||||
healthSequence: healthSequence,
|
||||
}
|
||||
return httptest.NewServer(handler)
|
||||
}
|
||||
|
||||
// ServeHTTP returns HTTP response codes following a status sequences.
|
||||
// It calls the given 'done' function once all request health indicators have been depleted.
|
||||
func (th *testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if len(th.healthSequence) == 0 {
|
||||
panic("received unexpected request")
|
||||
}
|
||||
|
||||
w.WriteHeader(th.healthSequence[0])
|
||||
|
||||
th.healthSequence = th.healthSequence[1:]
|
||||
if len(th.healthSequence) == 0 {
|
||||
th.done()
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue