homelab/traefik
1
0
Fork 0
Code Activity

healthcheck: add support at the load-balancers of services level

Browse source 
Co-authored-by: Dmitry Sharshakov <d3dx12.xx@gmail.com>
Co-authored-by: Julien Salleyron <julien.salleyron@gmail.com>
Co-authored-by: Jean-Baptiste Doumenjou <925513+jbdoumenjou@users.noreply.github.com>
Co-authored-by: Romain <rtribotte@users.noreply.github.com>
Co-authored-by: Tom Moulard <tom.moulard@traefik.io>
This commit is contained in:
mpl 2021-06-25 21:08:11 +02:00 • committed by GitHub
parent 5e3e47b484
commit 838a8e18d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 1196 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

117
integration/fixtures/healthcheck/propagate.toml Normal file
View file

@ -0,0 +1,117 @@
[global]
checkNewVersion = false
sendAnonymousUsage = false
[log]
level = "DEBUG"
[entryPoints]
[entryPoints.web]
address = ":8000"
[api]
insecure = true
[providers.file]
filename = "{{ .SelfFilename }}"
## dynamic configuration ##
[http.routers]
[http.routers.wsp-router-1]
service = "wsp-service1"
rule = "Host(`root.localhost`)"
[http.routers.wsp-router-2]
service = "wsp-service2"
rule = "Host(`foo.localhost`)"
[http.routers.wsp-router-3]
service = "wsp-service3"
rule = "Host(`bar.localhost`)"
[http.services]
[http.services.wsp-service1.weighted]
[http.services.wsp-service1.weighted.healthcheck]
[[http.services.wsp-service1.weighted.services]]
name = "wsp12"
weight = 1
[[http.services.wsp-service1.weighted.services]]
name = "wsp34"
weight = 1
[http.services.wsp-service2.weighted]
[http.services.wsp-service2.weighted.healthcheck]
[[http.services.wsp-service2.weighted.services]]
name = "wsp13"
weight = 1
[[http.services.wsp-service2.weighted.services]]
name = "wsp12"
weight = 1
[http.services.wsp-service3.weighted]
[http.services.wsp-service3.weighted.healthcheck]
[[http.services.wsp-service3.weighted.services]]
name = "wsp13"
weight = 1
[[http.services.wsp-service3.weighted.services]]
name = "wsp12b"
weight = 1
[http.services.wsp12.weighted]
[http.services.wsp12.weighted.healthcheck]
[[http.services.wsp12.weighted.services]]
name = "wsp1"
weight = 1
[[http.services.wsp12.weighted.services]]
name = "wsp2"
weight = 1
[http.services.wsp34.weighted]
[http.services.wsp34.weighted.healthcheck]
[[http.services.wsp34.weighted.services]]
name = "wsp3"
weight = 1
[[http.services.wsp34.weighted.services]]
name = "wsp4"
weight = 1
[http.services.wsp13.weighted]
[http.services.wsp13.weighted.healthcheck]
[[http.services.wsp13.weighted.services]]
name = "wsp1"
weight = 1
[[http.services.wsp13.weighted.services]]
name = "wsp3"
weight = 1
[http.services.wsp1.loadBalancer]
[http.services.wsp1.loadBalancer.healthcheck]
path = "/health"
interval = "1s"
timeout = "0.9s"
[[http.services.wsp1.loadBalancer.servers]]
url = "http://{{.Server1}}:80"
[http.services.wsp2.loadBalancer]
[http.services.wsp2.loadBalancer.healthcheck]
path = "/health"
interval = "1s"
timeout = "0.9s"
[[http.services.wsp2.loadBalancer.servers]]
url = "http://{{.Server2}}:80"
[http.services.wsp3.loadBalancer]
[http.services.wsp3.loadBalancer.healthcheck]
path = "/health"
interval = "1s"
timeout = "0.9s"
[[http.services.wsp3.loadBalancer.servers]]
url = "http://{{.Server3}}:80"
[http.services.wsp4.loadBalancer]
[http.services.wsp4.loadBalancer.healthcheck]
path = "/health"
interval = "1s"
timeout = "0.9s"
[[http.services.wsp4.loadBalancer.servers]]
url = "http://{{.Server4}}:80"
[http.services.wsp12b.loadBalancer]
[http.services.wsp12b.loadBalancer.healthcheck]
path = "/health"
interval = "1s"
timeout = "0.9s"
[[http.services.wsp12b.loadBalancer.servers]]
url = "http://{{.Server1}}:80"
[[http.services.wsp12b.loadBalancer.servers]]
url = "http://{{.Server2}}:80"

38
integration/fixtures/healthcheck/propagate_no_healthcheck.toml Normal file
View file

@ -0,0 +1,38 @@
[global]
checkNewVersion = false
sendAnonymousUsage = false
[log]
level = "DEBUG"
[entryPoints]
[entryPoints.web]
address = ":8000"
[api]
insecure = true
[providers.file]
filename = "{{ .SelfFilename }}"
## dynamic configuration ##
[http.routers]
[http.routers.wsp-router-1]
service = "wsp-service1"
rule = "Host(`root.localhost`)"
[http.routers.noop]
service = "noop@internal"
rule = "Host(`noop.localhost`)"
[http.services]
[http.services.wsp-service1.weighted]
[http.services.wsp-service1.weighted.healthcheck]
[[http.services.wsp-service1.weighted.services]]
name = "wsp1"
weight = 1
[http.services.wsp1.loadBalancer]
[[http.services.wsp1.loadBalancer.servers]]
url = "http://{{.Server1}}:80"

46
integration/fixtures/healthcheck/reload_with_healthcheck.toml Normal file
View file

@ -0,0 +1,46 @@
[global]
checkNewVersion = false
sendAnonymousUsage = false
[log]
level = "DEBUG"
[entryPoints]
[entryPoints.web]
address = ":8000"
[api]
insecure = true
[providers.file]
filename = "{{ .SelfFilename }}"
[http.routers]
[http.routers.wsp-router-1]
service = "wsp-service1"
rule = "Host(`root.localhost`)"
[http.services]
[http.services.wsp-service1.weighted]
[http.services.wsp-service1.weighted.healthcheck]
[[http.services.wsp-service1.weighted.services]]
name = "wsp1"
weight = 1
[[http.services.wsp-service1.weighted.services]]
name = "wsp2"
weight = 1
[http.services.wsp1.loadBalancer]
[http.services.wsp1.loadBalancer.healthcheck]
path = "/health"
interval = "1s"
timeout = "0.9s"
[[http.services.wsp1.loadBalancer.servers]]
url = "http://{{.Server1}}:80"
[http.services.wsp2.loadBalancer]
[http.services.wsp2.loadBalancer.healthcheck]
path = "/health"
interval = "1s"
timeout = "0.9s"
[[http.services.wsp2.loadBalancer.servers]]
url = "http://{{.Server2}}:80"

46
integration/fixtures/healthcheck/reload_without_healthcheck.toml Normal file
View file

@ -0,0 +1,46 @@
[global]
checkNewVersion = false
sendAnonymousUsage = false
[log]
level = "DEBUG"
[entryPoints]
[entryPoints.web]
address = ":8000"
[api]
insecure = true
[providers.file]
filename = "{{ .SelfFilename }}"
[http.routers]
[http.routers.wsp-router-1]
service = "wsp-service1"
rule = "Host(`root.localhost`)"
[http.services]
[http.services.wsp-service1.weighted]
# [http.services.wsp-service1.weighted.healthcheck]
[[http.services.wsp-service1.weighted.services]]
name = "wsp1"
weight = 1
[[http.services.wsp-service1.weighted.services]]
name = "wsp2"
weight = 1
[http.services.wsp1.loadBalancer]
[http.services.wsp1.loadBalancer.healthcheck]
path = "/health"
interval = "1s"
timeout = "0.9s"
[[http.services.wsp1.loadBalancer.servers]]
url = "http://{{.Server1}}:80"
[http.services.wsp2.loadBalancer]
[http.services.wsp2.loadBalancer.healthcheck]
path = "/health"
interval = "1s"
timeout = "0.9s"
[[http.services.wsp2.loadBalancer.servers]]
url = "http://{{.Server2}}:80"

268
integration/healthcheck_test.go
View file

@ -2,6 +2,8 @@ package integration
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"os"
"time"
@ -16,6 +18,8 @@ type HealthCheckSuite struct {
BaseSuite
whoami1IP string
whoami2IP string
whoami3IP string
whoami4IP string
}
func (s *HealthCheckSuite) SetUpSuite(c *check.C) {
@ -24,6 +28,8 @@ func (s *HealthCheckSuite) SetUpSuite(c *check.C) {
s.whoami1IP = s.composeProject.Container(c, "whoami1").NetworkSettings.IPAddress
s.whoami2IP = s.composeProject.Container(c, "whoami2").NetworkSettings.IPAddress
s.whoami3IP = s.composeProject.Container(c, "whoami3").NetworkSettings.IPAddress
s.whoami4IP = s.composeProject.Container(c, "whoami4").NetworkSettings.IPAddress
}
func (s *HealthCheckSuite) TestSimpleConfiguration(c *check.C) {
@ -271,3 +277,265 @@ func (s *HealthCheckSuite) TestMultipleRoutersOnSameService(c *check.C) {
err = try.Request(healthReqWeb2, 3*time.Second, try.StatusCodeIs(http.StatusOK))
c.Assert(err, checker.IsNil)
}
func (s *HealthCheckSuite) TestPropagate(c *check.C) {
file := s.adaptFile(c, "fixtures/healthcheck/propagate.toml", struct {
Server1 string
Server2 string
Server3 string
Server4 string
}{s.whoami1IP, s.whoami2IP, s.whoami3IP, s.whoami4IP})
defer os.Remove(file)
cmd, display := s.traefikCmd(withConfigFile(file))
defer display(c)
err := cmd.Start()
c.Assert(err, checker.IsNil)
defer s.killCmd(cmd)
// wait for traefik
err = try.GetRequest("http://127.0.0.1:8080/api/rawdata", 60*time.Second, try.BodyContains("Host(`root.localhost`)"))
c.Assert(err, checker.IsNil)
rootReq, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8000", nil)
c.Assert(err, checker.IsNil)
rootReq.Host = "root.localhost"
err = try.Request(rootReq, 500*time.Millisecond, try.StatusCodeIs(http.StatusOK))
c.Assert(err, checker.IsNil)
// Bring whoami1 and whoami3 down
client := http.Client{
Timeout: 10 * time.Second,
}
whoamiHosts := []string{s.whoami1IP, s.whoami3IP}
for _, whoami := range whoamiHosts {
statusInternalServerErrorReq, err := http.NewRequest(http.MethodPost, "http://"+whoami+"/health", bytes.NewBuffer([]byte("500")))
c.Assert(err, checker.IsNil)
_, err = client.Do(statusInternalServerErrorReq)
c.Assert(err, checker.IsNil)
}
try.Sleep(time.Second)
// Verify load-balancing on root still works, and that we're getting wsp2, wsp4, wsp2, wsp4, etc.
var want string
for i := 0; i < 4; i++ {
if i%2 == 0 {
want = `IP: ` + s.whoami4IP
} else {
want = `IP: ` + s.whoami2IP
}
resp, err := client.Do(rootReq)
c.Assert(err, checker.IsNil)
body, err := ioutil.ReadAll(resp.Body)
c.Assert(err, checker.IsNil)
c.Assert(string(body), checker.Contains, want)
}
fooReq, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8000", nil)
c.Assert(err, checker.IsNil)
fooReq.Host = "foo.localhost"
// Verify load-balancing on foo still works, and that we're getting wsp2, wsp2, wsp2, wsp2, etc.
want = `IP: ` + s.whoami2IP
for i := 0; i < 4; i++ {
resp, err := client.Do(fooReq)
c.Assert(err, checker.IsNil)
body, err := ioutil.ReadAll(resp.Body)
c.Assert(err, checker.IsNil)
c.Assert(string(body), checker.Contains, want)
}
barReq, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8000", nil)
c.Assert(err, checker.IsNil)
barReq.Host = "bar.localhost"
// Verify load-balancing on bar still works, and that we're getting wsp2, wsp2, wsp2, wsp2, etc.
want = `IP: ` + s.whoami2IP
for i := 0; i < 4; i++ {
resp, err := client.Do(barReq)
c.Assert(err, checker.IsNil)
body, err := ioutil.ReadAll(resp.Body)
c.Assert(err, checker.IsNil)
c.Assert(string(body), checker.Contains, want)
}
// Bring whoami2 and whoami4 down
whoamiHosts = []string{s.whoami2IP, s.whoami4IP}
for _, whoami := range whoamiHosts {
statusInternalServerErrorReq, err := http.NewRequest(http.MethodPost, "http://"+whoami+"/health", bytes.NewBuffer([]byte("500")))
c.Assert(err, checker.IsNil)
_, err = client.Do(statusInternalServerErrorReq)
c.Assert(err, checker.IsNil)
}
try.Sleep(time.Second)
// Verify that everything is down, and that we get 503s everywhere.
for i := 0; i < 2; i++ {
resp, err := client.Do(rootReq)
c.Assert(err, checker.IsNil)
c.Assert(resp.StatusCode, checker.Equals, http.StatusServiceUnavailable)
resp, err = client.Do(fooReq)
c.Assert(err, checker.IsNil)
c.Assert(resp.StatusCode, checker.Equals, http.StatusServiceUnavailable)
resp, err = client.Do(barReq)
c.Assert(err, checker.IsNil)
c.Assert(resp.StatusCode, checker.Equals, http.StatusServiceUnavailable)
}
// Bring everything back up.
whoamiHosts = []string{s.whoami1IP, s.whoami2IP, s.whoami3IP, s.whoami4IP}
for _, whoami := range whoamiHosts {
statusOKReq, err := http.NewRequest(http.MethodPost, "http://"+whoami+"/health", bytes.NewBuffer([]byte("200")))
c.Assert(err, checker.IsNil)
_, err = client.Do(statusOKReq)
c.Assert(err, checker.IsNil)
}
try.Sleep(time.Second)
// Verify everything is up on root router.
wantIPs := []string{s.whoami3IP, s.whoami1IP, s.whoami4IP, s.whoami2IP}
for i := 0; i < 4; i++ {
want := `IP: ` + wantIPs[i]
resp, err := client.Do(rootReq)
c.Assert(err, checker.IsNil)
body, err := ioutil.ReadAll(resp.Body)
c.Assert(err, checker.IsNil)
c.Assert(string(body), checker.Contains, want)
}
// Verify everything is up on foo router.
wantIPs = []string{s.whoami1IP, s.whoami1IP, s.whoami3IP, s.whoami2IP}
for i := 0; i < 4; i++ {
want := `IP: ` + wantIPs[i]
resp, err := client.Do(fooReq)
c.Assert(err, checker.IsNil)
body, err := ioutil.ReadAll(resp.Body)
c.Assert(err, checker.IsNil)
c.Assert(string(body), checker.Contains, want)
}
// Verify everything is up on bar router.
wantIPs = []string{s.whoami1IP, s.whoami1IP, s.whoami3IP, s.whoami2IP}
for i := 0; i < 4; i++ {
want := `IP: ` + wantIPs[i]
resp, err := client.Do(barReq)
c.Assert(err, checker.IsNil)
body, err := ioutil.ReadAll(resp.Body)
c.Assert(err, checker.IsNil)
c.Assert(string(body), checker.Contains, want)
}
}
func (s *HealthCheckSuite) TestPropagateNoHealthCheck(c *check.C) {
file := s.adaptFile(c, "fixtures/healthcheck/propagate_no_healthcheck.toml", struct {
Server1 string
}{s.whoami1IP})
defer os.Remove(file)
cmd, display := s.traefikCmd(withConfigFile(file))
defer display(c)
err := cmd.Start()
c.Assert(err, checker.IsNil)
defer s.killCmd(cmd)
// wait for traefik
err = try.GetRequest("http://127.0.0.1:8080/api/rawdata", 60*time.Second, try.BodyContains("Host(`noop.localhost`)"), try.BodyNotContains("Host(`root.localhost`)"))
c.Assert(err, checker.IsNil)
rootReq, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8000", nil)
c.Assert(err, checker.IsNil)
rootReq.Host = "root.localhost"
err = try.Request(rootReq, 500*time.Millisecond, try.StatusCodeIs(http.StatusNotFound))
c.Assert(err, checker.IsNil)
}
func (s *HealthCheckSuite) TestPropagateReload(c *check.C) {
// Setup a WSP service without the healthcheck enabled (wsp-service1)
withoutHealthCheck := s.adaptFile(c, "fixtures/healthcheck/reload_without_healthcheck.toml", struct {
Server1 string
Server2 string
}{s.whoami1IP, s.whoami2IP})
defer os.Remove(withoutHealthCheck)
withHealthCheck := s.adaptFile(c, "fixtures/healthcheck/reload_with_healthcheck.toml", struct {
Server1 string
Server2 string
}{s.whoami1IP, s.whoami2IP})
defer os.Remove(withHealthCheck)
cmd, display := s.traefikCmd(withConfigFile(withoutHealthCheck))
defer display(c)
err := cmd.Start()
c.Assert(err, checker.IsNil)
defer s.killCmd(cmd)
// wait for traefik
err = try.GetRequest("http://127.0.0.1:8080/api/rawdata", 60*time.Second, try.BodyContains("Host(`root.localhost`)"))
c.Assert(err, checker.IsNil)
// Allow one of the underlying services on it to fail all servers HC (whoami2)
client := http.Client{
Timeout: 10 * time.Second,
}
statusOKReq, err := http.NewRequest(http.MethodPost, "http://"+s.whoami2IP+"/health", bytes.NewBuffer([]byte("500")))
c.Assert(err, checker.IsNil)
_, err = client.Do(statusOKReq)
c.Assert(err, checker.IsNil)
rootReq, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8000", nil)
c.Assert(err, checker.IsNil)
rootReq.Host = "root.localhost"
// Check the failed service (whoami2) is getting requests, but answer 500
err = try.Request(rootReq, 500*time.Millisecond, try.StatusCodeIs(http.StatusServiceUnavailable))
c.Assert(err, checker.IsNil)
// Enable the healthcheck on the root WSP (wsp-service1) and let Traefik reload the config
fr1, err := os.OpenFile(withoutHealthCheck, os.O_APPEND|os.O_WRONLY, 0o644)
c.Assert(fr1, checker.NotNil)
c.Assert(err, checker.IsNil)
err = fr1.Truncate(0)
c.Assert(err, checker.IsNil)
fr2, err := os.ReadFile(withHealthCheck)
c.Assert(err, checker.IsNil)
_, err = fmt.Fprint(fr1, string(fr2))
c.Assert(err, checker.IsNil)
err = fr1.Close()
c.Assert(err, checker.IsNil)
try.Sleep(1 * time.Second)
// Check the failed service (whoami2) is not getting requests
wantIPs := []string{s.whoami1IP, s.whoami1IP, s.whoami1IP, s.whoami1IP}
for _, ip := range wantIPs {
want := "IP: " + ip
resp, err := client.Do(rootReq)
c.Assert(err, checker.IsNil)
body, err := ioutil.ReadAll(resp.Body)
c.Assert(err, checker.IsNil)
c.Assert(string(body), checker.Contains, want)
}
}

6
integration/resources/compose/healthcheck.yml
View file

@ -3,3 +3,9 @@ whoami1:
whoami2:
image: traefik/whoami
whoami3:
image: traefik/whoami
whoami4:
image: traefik/whoami