Make the loadbalancers servers order random

Co-authored-by: Mathieu Lonjaret <mathieu.lonjaret@gmail.com>
Co-authored-by: Romain <rtribotte@users.noreply.github.com>
This commit is contained in:
Qi 2022-09-14 20:42:08 +08:00 committed by GitHub
parent 89dc466b23
commit 788f8fa951
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 177 additions and 46 deletions

View file

@ -6,6 +6,7 @@ import (
"io"
"net/http"
"os"
"strings"
"time"
"github.com/go-check/check"
@ -319,30 +320,51 @@ func (s *HealthCheckSuite) TestPropagate(c *check.C) {
try.Sleep(time.Second)
// Verify load-balancing on root still works, and that we're getting wsp2, wsp4, wsp2, wsp4, etc.
var want string
for i := 0; i < 4; i++ {
if i%2 == 0 {
want = `IP: ` + s.whoami4IP
} else {
want = `IP: ` + s.whoami2IP
}
want2 := `IP: ` + s.whoami2IP
want4 := `IP: ` + s.whoami4IP
// Verify load-balancing on root still works, and that we're getting an alternation between wsp2, and wsp4.
reachedServers := make(map[string]int)
for i := 0; i < 4; i++ {
resp, err := client.Do(rootReq)
c.Assert(err, checker.IsNil)
body, err := io.ReadAll(resp.Body)
c.Assert(err, checker.IsNil)
c.Assert(string(body), checker.Contains, want)
if reachedServers[s.whoami4IP] > reachedServers[s.whoami2IP] {
c.Assert(string(body), checker.Contains, want2)
reachedServers[s.whoami2IP]++
continue
}
if reachedServers[s.whoami2IP] > reachedServers[s.whoami4IP] {
c.Assert(string(body), checker.Contains, want4)
reachedServers[s.whoami4IP]++
continue
}
// First iteration, so we can't tell whether it's going to be wsp2, or wsp4.
if strings.Contains(string(body), `IP: `+s.whoami4IP) {
reachedServers[s.whoami4IP]++
continue
}
if strings.Contains(string(body), `IP: `+s.whoami2IP) {
reachedServers[s.whoami2IP]++
continue
}
}
c.Assert(reachedServers[s.whoami2IP], checker.Equals, 2)
c.Assert(reachedServers[s.whoami4IP], checker.Equals, 2)
fooReq, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8000", nil)
c.Assert(err, checker.IsNil)
fooReq.Host = "foo.localhost"
// Verify load-balancing on foo still works, and that we're getting wsp2, wsp2, wsp2, wsp2, etc.
want = `IP: ` + s.whoami2IP
want := `IP: ` + s.whoami2IP
for i := 0; i < 4; i++ {
resp, err := client.Do(fooReq)
c.Assert(err, checker.IsNil)
@ -407,43 +429,109 @@ func (s *HealthCheckSuite) TestPropagate(c *check.C) {
try.Sleep(time.Second)
// Verify everything is up on root router.
wantIPs := []string{s.whoami3IP, s.whoami1IP, s.whoami4IP, s.whoami2IP}
reachedServers = make(map[string]int)
for i := 0; i < 4; i++ {
want := `IP: ` + wantIPs[i]
resp, err := client.Do(rootReq)
c.Assert(err, checker.IsNil)
body, err := io.ReadAll(resp.Body)
c.Assert(err, checker.IsNil)
c.Assert(string(body), checker.Contains, want)
if strings.Contains(string(body), `IP: `+s.whoami1IP) {
reachedServers[s.whoami1IP]++
continue
}
if strings.Contains(string(body), `IP: `+s.whoami2IP) {
reachedServers[s.whoami2IP]++
continue
}
if strings.Contains(string(body), `IP: `+s.whoami3IP) {
reachedServers[s.whoami3IP]++
continue
}
if strings.Contains(string(body), `IP: `+s.whoami4IP) {
reachedServers[s.whoami4IP]++
continue
}
}
c.Assert(reachedServers[s.whoami1IP], checker.Equals, 1)
c.Assert(reachedServers[s.whoami2IP], checker.Equals, 1)
c.Assert(reachedServers[s.whoami3IP], checker.Equals, 1)
c.Assert(reachedServers[s.whoami4IP], checker.Equals, 1)
// Verify everything is up on foo router.
wantIPs = []string{s.whoami1IP, s.whoami1IP, s.whoami3IP, s.whoami2IP}
reachedServers = make(map[string]int)
for i := 0; i < 4; i++ {
want := `IP: ` + wantIPs[i]
resp, err := client.Do(fooReq)
c.Assert(err, checker.IsNil)
body, err := io.ReadAll(resp.Body)
c.Assert(err, checker.IsNil)
c.Assert(string(body), checker.Contains, want)
if strings.Contains(string(body), `IP: `+s.whoami1IP) {
reachedServers[s.whoami1IP]++
continue
}
if strings.Contains(string(body), `IP: `+s.whoami2IP) {
reachedServers[s.whoami2IP]++
continue
}
if strings.Contains(string(body), `IP: `+s.whoami3IP) {
reachedServers[s.whoami3IP]++
continue
}
if strings.Contains(string(body), `IP: `+s.whoami4IP) {
reachedServers[s.whoami4IP]++
continue
}
}
c.Assert(reachedServers[s.whoami1IP], checker.Equals, 2)
c.Assert(reachedServers[s.whoami2IP], checker.Equals, 1)
c.Assert(reachedServers[s.whoami3IP], checker.Equals, 1)
c.Assert(reachedServers[s.whoami4IP], checker.Equals, 0)
// Verify everything is up on bar router.
wantIPs = []string{s.whoami1IP, s.whoami1IP, s.whoami3IP, s.whoami2IP}
reachedServers = make(map[string]int)
for i := 0; i < 4; i++ {
want := `IP: ` + wantIPs[i]
resp, err := client.Do(barReq)
c.Assert(err, checker.IsNil)
body, err := io.ReadAll(resp.Body)
c.Assert(err, checker.IsNil)
c.Assert(string(body), checker.Contains, want)
if strings.Contains(string(body), `IP: `+s.whoami1IP) {
reachedServers[s.whoami1IP]++
continue
}
if strings.Contains(string(body), `IP: `+s.whoami2IP) {
reachedServers[s.whoami2IP]++
continue
}
if strings.Contains(string(body), `IP: `+s.whoami3IP) {
reachedServers[s.whoami3IP]++
continue
}
if strings.Contains(string(body), `IP: `+s.whoami4IP) {
reachedServers[s.whoami4IP]++
continue
}
}
c.Assert(reachedServers[s.whoami1IP], checker.Equals, 2)
c.Assert(reachedServers[s.whoami2IP], checker.Equals, 1)
c.Assert(reachedServers[s.whoami3IP], checker.Equals, 1)
c.Assert(reachedServers[s.whoami4IP], checker.Equals, 0)
}
func (s *HealthCheckSuite) TestPropagateNoHealthCheck(c *check.C) {

View file

@ -36,13 +36,11 @@ func (s *RetrySuite) TestRetry(c *check.C) {
err = try.GetRequest("http://127.0.0.1:8080/api/rawdata", 60*time.Second, try.BodyContains("PathPrefix(`/`)"))
c.Assert(err, checker.IsNil)
start := time.Now()
// This simulates a DialTimeout when connecting to the backend server.
response, err := http.Get("http://127.0.0.1:8000/")
duration, allowed := time.Since(start), time.Millisecond*250
c.Assert(err, checker.IsNil)
// The test only verifies that the retry middleware makes sure that the working service is eventually reached.
c.Assert(response.StatusCode, checker.Equals, http.StatusOK)
c.Assert(int64(duration), checker.LessThan, int64(allowed))
}
func (s *RetrySuite) TestRetryBackoff(c *check.C) {
@ -58,16 +56,11 @@ func (s *RetrySuite) TestRetryBackoff(c *check.C) {
err = try.GetRequest("http://127.0.0.1:8080/api/rawdata", 60*time.Second, try.BodyContains("PathPrefix(`/`)"))
c.Assert(err, checker.IsNil)
start := time.Now()
// This simulates a DialTimeout when connecting to the backend server.
response, err := http.Get("http://127.0.0.1:8000/")
duration := time.Since(start)
// test case delays: 500 + 700 + 1000ms with randomization. It should be safely > 1500ms
minAllowed := time.Millisecond * 1400
c.Assert(err, checker.IsNil)
// The test only verifies that the retry middleware allows finally to reach the working service.
c.Assert(response.StatusCode, checker.Equals, http.StatusOK)
c.Assert(int64(duration), checker.GreaterThan, int64(minAllowed))
}
func (s *RetrySuite) TestRetryWebsocket(c *check.C) {
@ -83,11 +76,12 @@ func (s *RetrySuite) TestRetryWebsocket(c *check.C) {
err = try.GetRequest("http://127.0.0.1:8080/api/rawdata", 60*time.Second, try.BodyContains("PathPrefix(`/`)"))
c.Assert(err, checker.IsNil)
// This simulates a DialTimeout when connecting to the backend server.
// The test only verifies that the retry middleware makes sure that the working service is eventually reached.
_, response, err := websocket.DefaultDialer.Dial("ws://127.0.0.1:8000/echo", nil)
c.Assert(err, checker.IsNil)
c.Assert(response.StatusCode, checker.Equals, http.StatusSwitchingProtocols)
// The test verifies a second time that the working service is eventually reached.
_, response, err = websocket.DefaultDialer.Dial("ws://127.0.0.1:8000/echo", nil)
c.Assert(err, checker.IsNil)
c.Assert(response.StatusCode, checker.Equals, http.StatusSwitchingProtocols)

View file

@ -76,7 +76,7 @@ func (s *UDPSuite) TestWRR(c *check.C) {
stop := make(chan struct{})
go func() {
call := map[string]int{}
for i := 0; i < 4; i++ {
for i := 0; i < 8; i++ {
out, err := guessWhoUDP("127.0.0.1:8093")
c.Assert(err, checker.IsNil)
switch {
@ -90,7 +90,7 @@ func (s *UDPSuite) TestWRR(c *check.C) {
call["unknown"]++
}
}
c.Assert(call, checker.DeepEquals, map[string]int{"whoami-a": 2, "whoami-b": 1, "whoami-c": 1})
c.Assert(call, checker.DeepEquals, map[string]int{"whoami-a": 3, "whoami-b": 2, "whoami-c": 3})
close(stop)
}()