Native Kubernetes service load-balancing

This commit is contained in:
Romain 2023-03-20 16:46:05 +01:00 committed by GitHub
parent 7af9d16208
commit 6e460cd652
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
37 changed files with 1013 additions and 67 deletions

View file

@ -252,3 +252,17 @@ apiVersion: v1
metadata:
name: whoami-without-endpoints-subsets
namespace: default
---
apiVersion: v1
kind: Service
metadata:
name: native-svc
namespace: default
spec:
ports:
- name: web
port: 80
type: ClusterIP
clusterIP: 10.10.0.1

View file

@ -261,3 +261,17 @@ apiVersion: v1
metadata:
name: whoamitcp-without-endpoints-subsets
namespace: default
---
apiVersion: v1
kind: Service
metadata:
name: native-svc
namespace: default
spec:
ports:
- name: myapp
port: 8000
type: ClusterIP
clusterIP: 10.10.0.1

View file

@ -0,0 +1,16 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRouteTCP
metadata:
name: test.route
namespace: default
spec:
entryPoints:
- foo
routes:
- match: HostSNI(`foo.com`)
services:
- name: native-svc
port: 8000
nativeLB: true

View file

@ -220,3 +220,17 @@ apiVersion: v1
metadata:
name: whoamiudp-without-endpoints-subsets
namespace: default
---
apiVersion: v1
kind: Service
metadata:
name: native-svc
namespace: default
spec:
ports:
- name: myapp
port: 8000
type: ClusterIP
clusterIP: 10.10.0.1

View file

@ -0,0 +1,15 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRouteUDP
metadata:
name: test.route
namespace: default
spec:
entryPoints:
- foo
routes:
- services:
- name: native-svc
port: 8000
nativeLB: true

View file

@ -0,0 +1,17 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: test.route
namespace: default
spec:
entryPoints:
- foo
routes:
- match: Host(`foo.com`)
kind: Rule
services:
- name: native-svc
port: 80
nativeLB: true

View file

@ -10,8 +10,10 @@ import (
"encoding/json"
"errors"
"fmt"
"net"
"os"
"sort"
"strconv"
"strings"
"time"
@ -395,6 +397,7 @@ func (p *Provider) loadConfigurationFromCRD(ctx context.Context, client Client)
return conf
}
// getServicePort always returns a valid port, an error otherwise.
func getServicePort(svc *corev1.Service, port intstr.IntOrString) (*corev1.ServicePort, error) {
if svc == nil {
return nil, errors.New("service is not defined")
@ -427,6 +430,18 @@ func getServicePort(svc *corev1.Service, port intstr.IntOrString) (*corev1.Servi
return &corev1.ServicePort{Port: port.IntVal}, nil
}
func getNativeServiceAddress(service corev1.Service, svcPort corev1.ServicePort) (string, error) {
if service.Spec.ClusterIP == "None" {
return "", fmt.Errorf("no clusterIP on headless service: %s/%s", service.Namespace, service.Name)
}
if service.Spec.ClusterIP == "" {
return "", fmt.Errorf("no clusterIP found for service: %s/%s", service.Namespace, service.Name)
}
return net.JoinHostPort(service.Spec.ClusterIP, strconv.Itoa(int(svcPort.Port))), nil
}
func createPluginMiddleware(k8sClient Client, ns string, plugins map[string]apiextensionv1.JSON) (map[string]dynamic.PluginConf, error) {
if plugins == nil {
return nil, nil

View file

@ -368,6 +368,20 @@ func (c configBuilder) loadServers(parentNamespace string, svc v1alpha1.LoadBala
return nil, err
}
if svc.NativeLB {
address, err := getNativeServiceAddress(*service, *svcPort)
if err != nil {
return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err)
}
protocol, err := parseServiceProtocol(svc.Scheme, svcPort.Name, svcPort.Port)
if err != nil {
return nil, err
}
return []dynamic.Server{{URL: fmt.Sprintf("%s://%s", protocol, address)}}, nil
}
var servers []dynamic.Server
if service.Spec.Type == corev1.ServiceTypeExternalName {
if !c.allowExternalNameServices {

View file

@ -226,6 +226,15 @@ func (p *Provider) loadTCPServers(client Client, namespace string, svc v1alpha1.
return nil, err
}
if svc.NativeLB {
address, err := getNativeServiceAddress(*service, *svcPort)
if err != nil {
return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err)
}
return []dynamic.TCPServer{{Address: address}}, nil
}
var servers []dynamic.TCPServer
if service.Spec.Type == corev1.ServiceTypeExternalName {
servers = append(servers, dynamic.TCPServer{

View file

@ -6242,6 +6242,214 @@ func TestExternalNameService(t *testing.T) {
}
}
func TestNativeLB(t *testing.T) {
testCases := []struct {
desc string
ingressClass string
paths []string
expected *dynamic.Configuration
}{
{
desc: "Empty",
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
TCP: &dynamic.TCPConfiguration{
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "HTTP with native Service LB",
paths: []string{"services.yml", "with_native_service_lb.yml"},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
TCP: &dynamic.TCPConfiguration{
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{
"default-test-route-6f97418635c7e18853da": {
EntryPoints: []string{"foo"},
Service: "default-test-route-6f97418635c7e18853da",
Rule: "Host(`foo.com`)",
Priority: 0,
},
},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{
"default-test-route-6f97418635c7e18853da": {
LoadBalancer: &dynamic.ServersLoadBalancer{
Servers: []dynamic.Server{
{
URL: "http://10.10.0.1:80",
},
},
PassHostHeader: Bool(true),
},
},
},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "TCP with native Service LB",
paths: []string{"tcp/services.yml", "tcp/with_native_service_lb.yml"},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
},
TCP: &dynamic.TCPConfiguration{
Routers: map[string]*dynamic.TCPRouter{
"default-test.route-fdd3e9338e47a45efefc": {
EntryPoints: []string{"foo"},
Service: "default-test.route-fdd3e9338e47a45efefc",
Rule: "HostSNI(`foo.com`)",
},
},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{
"default-test.route-fdd3e9338e47a45efefc": {
LoadBalancer: &dynamic.TCPServersLoadBalancer{
Servers: []dynamic.TCPServer{
{
Address: "10.10.0.1:8000",
Port: "",
},
},
},
},
},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "UDP with native Service LB",
paths: []string{"udp/services.yml", "udp/with_native_service_lb.yml"},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{
"default-test.route-0": {
EntryPoints: []string{"foo"},
Service: "default-test.route-0",
},
},
Services: map[string]*dynamic.UDPService{
"default-test.route-0": {
LoadBalancer: &dynamic.UDPServersLoadBalancer{
Servers: []dynamic.UDPServer{
{
Address: "10.10.0.1:8000",
Port: "",
},
},
},
},
},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
},
TCP: &dynamic.TCPConfiguration{
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
},
TLS: &dynamic.TLSConfiguration{},
},
},
}
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
var k8sObjects []runtime.Object
var crdObjects []runtime.Object
for _, path := range test.paths {
yamlContent, err := os.ReadFile(filepath.FromSlash("./fixtures/" + path))
if err != nil {
panic(err)
}
objects := k8s.MustParseYaml(yamlContent)
for _, obj := range objects {
switch o := obj.(type) {
case *corev1.Service, *corev1.Endpoints, *corev1.Secret:
k8sObjects = append(k8sObjects, o)
case *v1alpha1.IngressRoute:
crdObjects = append(crdObjects, o)
case *v1alpha1.IngressRouteTCP:
crdObjects = append(crdObjects, o)
case *v1alpha1.IngressRouteUDP:
crdObjects = append(crdObjects, o)
case *v1alpha1.Middleware:
crdObjects = append(crdObjects, o)
case *v1alpha1.TraefikService:
crdObjects = append(crdObjects, o)
case *v1alpha1.TLSOption:
crdObjects = append(crdObjects, o)
case *v1alpha1.TLSStore:
crdObjects = append(crdObjects, o)
default:
}
}
}
kubeClient := kubefake.NewSimpleClientset(k8sObjects...)
crdClient := crdfake.NewSimpleClientset(crdObjects...)
client := newClientImpl(kubeClient, crdClient)
stopCh := make(chan struct{})
eventCh, err := client.WatchAll([]string{"default", "cross-ns"}, stopCh)
require.NoError(t, err)
if k8sObjects != nil || crdObjects != nil {
// just wait for the first event
<-eventCh
}
p := Provider{}
conf := p.loadConfigurationFromCRD(context.Background(), client)
assert.Equal(t, test.expected, conf)
})
}
}
func TestCreateBasicAuthCredentials(t *testing.T) {
var k8sObjects []runtime.Object
var crdObjects []runtime.Object

View file

@ -120,6 +120,15 @@ func (p *Provider) loadUDPServers(client Client, namespace string, svc v1alpha1.
return nil, err
}
if svc.NativeLB {
address, err := getNativeServiceAddress(*service, *svcPort)
if err != nil {
return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err)
}
return []dynamic.UDPServer{{Address: address}}, nil
}
var servers []dynamic.UDPServer
if service.Spec.Type == corev1.ServiceTypeExternalName {
servers = append(servers, dynamic.UDPServer{

View file

@ -115,10 +115,14 @@ type LoadBalancerSpec struct {
// It allows to configure the transport between Traefik and your servers.
// Can only be used on a Kubernetes Service.
ServersTransport string `json:"serversTransport,omitempty"`
// Weight defines the weight and should only be specified when Name references a TraefikService object
// (and to be precise, one that embeds a Weighted Round Robin).
Weight *int `json:"weight,omitempty"`
// NativeLB controls, when creating the load-balancer,
// whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP.
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
}
// Service defines an upstream HTTP service to proxy traffic to.

View file

@ -78,6 +78,11 @@ type ServiceTCP struct {
// ProxyProtocol defines the PROXY protocol configuration.
// More info: https://doc.traefik.io/traefik/v2.9/routing/services/#proxy-protocol
ProxyProtocol *dynamic.ProxyProtocol `json:"proxyProtocol,omitempty"`
// NativeLB controls, when creating the load-balancer,
// whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP.
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
}
// +genclient

View file

@ -33,6 +33,11 @@ type ServiceUDP struct {
Port intstr.IntOrString `json:"port"`
// Weight defines the weight used when balancing requests between multiple Kubernetes Service.
Weight *int `json:"weight,omitempty"`
// NativeLB controls, when creating the load-balancer,
// whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP.
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
}
// +genclient

View file

@ -115,10 +115,14 @@ type LoadBalancerSpec struct {
// It allows to configure the transport between Traefik and your servers.
// Can only be used on a Kubernetes Service.
ServersTransport string `json:"serversTransport,omitempty"`
// Weight defines the weight and should only be specified when Name references a TraefikService object
// (and to be precise, one that embeds a Weighted Round Robin).
Weight *int `json:"weight,omitempty"`
// NativeLB controls, when creating the load-balancer,
// whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP.
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
}
// Service defines an upstream HTTP service to proxy traffic to.

View file

@ -78,6 +78,11 @@ type ServiceTCP struct {
// ProxyProtocol defines the PROXY protocol configuration.
// More info: https://doc.traefik.io/traefik/v2.9/routing/services/#proxy-protocol
ProxyProtocol *dynamic.ProxyProtocol `json:"proxyProtocol,omitempty"`
// NativeLB controls, when creating the load-balancer,
// whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP.
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
}
// +genclient

View file

@ -33,6 +33,11 @@ type ServiceUDP struct {
Port intstr.IntOrString `json:"port"`
// Weight defines the weight used when balancing requests between multiple Kubernetes Service.
Weight *int `json:"weight,omitempty"`
// NativeLB controls, when creating the load-balancer,
// whether the LB's children are directly the pods IPs or if the only child is the Kubernetes Service clusterIP.
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
}
// +genclient

View file

@ -45,6 +45,7 @@ type ServiceIng struct {
ServersTransport string `json:"serversTransport,omitempty"`
PassHostHeader *bool `json:"passHostHeader"`
Sticky *dynamic.Sticky `json:"sticky,omitempty" label:"allowEmpty"`
NativeLB bool `json:"nativeLB,omitempty"`
}
// SetDefaults sets the default values.

View file

@ -106,6 +106,7 @@ func Test_parseServiceConfig(t *testing.T) {
"traefik.ingress.kubernetes.io/service.serversscheme": "protocol",
"traefik.ingress.kubernetes.io/service.serverstransport": "foobar@file",
"traefik.ingress.kubernetes.io/service.passhostheader": "true",
"traefik.ingress.kubernetes.io/service.nativelb": "true",
"traefik.ingress.kubernetes.io/service.sticky.cookie": "true",
"traefik.ingress.kubernetes.io/service.sticky.cookie.httponly": "true",
"traefik.ingress.kubernetes.io/service.sticky.cookie.name": "foobar",
@ -125,6 +126,7 @@ func Test_parseServiceConfig(t *testing.T) {
ServersScheme: "protocol",
ServersTransport: "foobar@file",
PassHostHeader: Bool(true),
NativeLB: true,
},
},
},

View file

@ -0,0 +1,15 @@
kind: Ingress
apiVersion: networking.k8s.io/v1beta1
metadata:
name: ""
namespace: testing
spec:
rules:
- host: traefik.tchouk
http:
paths:
- path: /bar
backend:
serviceName: service1
servicePort: 8080

View file

@ -0,0 +1,15 @@
kind: Service
apiVersion: v1
metadata:
name: service1
namespace: testing
annotations:
traefik.ingress.kubernetes.io/service.nativelb: "true"
spec:
ports:
- port: 8080
clusterIP: 10.0.0.1
type: ClusterIP
externalName: traefik.wtf

View file

@ -538,6 +538,21 @@ func (p *Provider) loadService(client Client, namespace string, backend networki
if svcConfig.Service.ServersTransport != "" {
svc.LoadBalancer.ServersTransport = svcConfig.Service.ServersTransport
}
if svcConfig.Service.NativeLB {
address, err := getNativeServiceAddress(*service, portSpec)
if err != nil {
return nil, fmt.Errorf("getting native Kubernetes Service address: %w", err)
}
protocol := getProtocol(portSpec, portSpec.Name, svcConfig)
svc.LoadBalancer.Servers = []dynamic.Server{
{URL: fmt.Sprintf("%s://%s", protocol, address)},
}
return svc, nil
}
}
if service.Spec.Type == corev1.ServiceTypeExternalName {
@ -587,6 +602,18 @@ func (p *Provider) loadService(client Client, namespace string, backend networki
return svc, nil
}
func getNativeServiceAddress(service corev1.Service, svcPort corev1.ServicePort) (string, error) {
if service.Spec.ClusterIP == "None" {
return "", fmt.Errorf("no clusterIP on headless service: %s/%s", service.Namespace, service.Name)
}
if service.Spec.ClusterIP == "" {
return "", fmt.Errorf("no clusterIP found for service: %s/%s", service.Namespace, service.Name)
}
return net.JoinHostPort(service.Spec.ClusterIP, strconv.Itoa(int(svcPort.Port))), nil
}
func getProtocol(portSpec corev1.ServicePort, portName string, svcConfig *ServiceConfig) string {
if svcConfig != nil && svcConfig.Service != nil && svcConfig.Service.ServersScheme != "" {
return svcConfig.Service.ServersScheme

View file

@ -5,6 +5,7 @@ import (
"errors"
"math"
"os"
"path/filepath"
"strings"
"testing"
@ -1790,8 +1791,87 @@ func TestLoadConfigurationFromIngressesWithExternalNameServices(t *testing.T) {
}
}
func TestLoadConfigurationFromIngressesWithNativeLB(t *testing.T) {
testCases := []struct {
desc string
ingressClass string
serverVersion string
expected *dynamic.Configuration
}{
{
desc: "Ingress with native service lb",
expected: &dynamic.Configuration{
TCP: &dynamic.TCPConfiguration{},
HTTP: &dynamic.HTTPConfiguration{
Middlewares: map[string]*dynamic.Middleware{},
Routers: map[string]*dynamic.Router{
"testing-traefik-tchouk-bar": {
Rule: "Host(`traefik.tchouk`) && PathPrefix(`/bar`)",
Service: "testing-service1-8080",
},
},
Services: map[string]*dynamic.Service{
"testing-service1-8080": {
LoadBalancer: &dynamic.ServersLoadBalancer{
PassHostHeader: Bool(true),
Servers: []dynamic.Server{
{
URL: "http://10.0.0.1:8080",
},
},
},
},
},
},
},
},
}
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
var paths []string
_, err := os.Stat(generateTestFilename("_ingress", test.desc))
if err == nil {
paths = append(paths, generateTestFilename("_ingress", test.desc))
}
_, err = os.Stat(generateTestFilename("_endpoint", test.desc))
if err == nil {
paths = append(paths, generateTestFilename("_endpoint", test.desc))
}
_, err = os.Stat(generateTestFilename("_service", test.desc))
if err == nil {
paths = append(paths, generateTestFilename("_service", test.desc))
}
_, err = os.Stat(generateTestFilename("_secret", test.desc))
if err == nil {
paths = append(paths, generateTestFilename("_secret", test.desc))
}
_, err = os.Stat(generateTestFilename("_ingressclass", test.desc))
if err == nil {
paths = append(paths, generateTestFilename("_ingressclass", test.desc))
}
serverVersion := test.serverVersion
if serverVersion == "" {
serverVersion = "v1.17"
}
clientMock := newClientMock(serverVersion, paths...)
p := Provider{IngressClass: test.ingressClass}
conf := p.loadConfigurationFromIngresses(context.Background(), clientMock)
assert.Equal(t, test.expected, conf)
})
}
}
func generateTestFilename(suffix, desc string) string {
return "./fixtures/" + strings.ReplaceAll(desc, " ", "-") + suffix + ".yml"
return filepath.Join("fixtures", strings.ReplaceAll(desc, " ", "-")+suffix+".yml")
}
func TestGetCertificates(t *testing.T) {