homelab/traefik
1
0
Fork 0
Code Activity

Allow to use internal node IPs for NodePort services

Browse source
This commit is contained in:
Joris Vergeer 2024-02-27 10:54:04 +01:00 • committed by GitHub
parent 73769af0fe
commit c1ef742977
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 813 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

27
pkg/provider/kubernetes/crd/client.go
View file

@ -46,6 +46,7 @@ type Client interface {
GetService(namespace, name string) (*corev1.Service, bool, error)
GetSecret(namespace, name string) (*corev1.Secret, bool, error)
GetEndpoints(namespace, name string) (*corev1.Endpoints, bool, error)
GetNodes() ([]*corev1.Node, bool, error)
}
// TODO: add tests for the clientWrapper (and its methods) itself.
@ -53,9 +54,10 @@ type clientWrapper struct {
csCrd traefikclientset.Interface
csKube kclientset.Interface
factoriesCrd map[string]traefikinformers.SharedInformerFactory
factoriesKube map[string]kinformers.SharedInformerFactory
factoriesSecret map[string]kinformers.SharedInformerFactory
factoryClusterScope kinformers.SharedInformerFactory
factoriesCrd map[string]traefikinformers.SharedInformerFactory
factoriesKube map[string]kinformers.SharedInformerFactory
factoriesSecret map[string]kinformers.SharedInformerFactory
labelSelector string
@ -232,11 +234,18 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (<
c.factoriesSecret[ns] = factorySecret
}
c.factoryClusterScope = kinformers.NewSharedInformerFactory(c.csKube, resyncPeriod)
_, err := c.factoryClusterScope.Core().V1().Nodes().Informer().AddEventHandler(eventHandler)
if err != nil {
return nil, err
}
for _, ns := range namespaces {
c.factoriesCrd[ns].Start(stopCh)
c.factoriesKube[ns].Start(stopCh)
c.factoriesSecret[ns].Start(stopCh)
}
c.factoryClusterScope.Start(stopCh)
for _, ns := range namespaces {
for t, ok := range c.factoriesCrd[ns].WaitForCacheSync(stopCh) {
@ -258,6 +267,12 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (<
}
}
for t, ok := range c.factoryClusterScope.WaitForCacheSync(stopCh) {
if !ok {
return nil, fmt.Errorf("timed out waiting for controller caches to sync %s", t.String())
}
}
return eventCh, nil
}
@ -450,6 +465,12 @@ func (c *clientWrapper) GetSecret(namespace, name string) (*corev1.Secret, bool,
return secret, exist, err
}
func (c *clientWrapper) GetNodes() ([]*corev1.Node, bool, error) {
nodes, err := c.factoryClusterScope.Core().V1().Nodes().Lister().List(labels.Everything())
exist, err := translateNotFoundError(err)
return nodes, exist, err
}
// lookupNamespace returns the lookup namespace key for the given namespace.
// When listening on all namespaces, it returns the client-go identifier ("")
// for all-namespaces. Otherwise, it returns the given namespace.

15
pkg/provider/kubernetes/crd/fixtures/services.yml
View file

@ -266,3 +266,18 @@ spec:
port: 80
type: ClusterIP
clusterIP: 10.10.0.1
---
apiVersion: v1
kind: Service
metadata:
name: nodeport-svc
namespace: default
spec:
ports:
- name: web
port: 80
nodePort: 32456
type: NodePort
clusterIP: 10.10.0.1

15
pkg/provider/kubernetes/crd/fixtures/tcp/services.yml
View file

@ -262,3 +262,18 @@ spec:
port: 8000
type: ClusterIP
clusterIP: 10.10.0.1
---
apiVersion: v1
kind: Service
metadata:
name: nodeport-svc-tcp
namespace: default
spec:
ports:
- name: myapp
port: 8000
nodePort: 32456
type: NodePort
clusterIP: 10.10.0.1

26
pkg/provider/kubernetes/crd/fixtures/tcp/with_node_port_service_lb.yml Normal file
View file

@ -0,0 +1,26 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRouteTCP
metadata:
name: test.route
namespace: default
spec:
entryPoints:
- foo
routes:
- match: HostSNI(`foo.com`)
services:
- name: nodeport-svc-tcp
port: 8000
nodePortLB: true
---
kind: Node
apiVersion: v1
metadata:
name: traefik-node
status:
addresses:
- type: InternalIP
address: 172.16.4.4

15
pkg/provider/kubernetes/crd/fixtures/udp/services.yml
View file

@ -221,3 +221,18 @@ spec:
port: 8000
type: ClusterIP
clusterIP: 10.10.0.1
---
apiVersion: v1
kind: Service
metadata:
name: nodeport-svc-udp
namespace: default
spec:
ports:
- name: myapp
port: 8000
nodePort: 32456
type: NodePort
clusterIP: 10.10.0.1

25
pkg/provider/kubernetes/crd/fixtures/udp/with_node_port_service_lb.yml Normal file
View file

@ -0,0 +1,25 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRouteUDP
metadata:
name: test.route
namespace: default
spec:
entryPoints:
- foo
routes:
- services:
- name: nodeport-svc-udp
port: 8000
nodePortLB: true
---
kind: Node
apiVersion: v1
metadata:
name: traefik-node
status:
addresses:
- type: InternalIP
address: 172.16.4.4

27
pkg/provider/kubernetes/crd/fixtures/with_node_port_lb.yml Normal file
View file

@ -0,0 +1,27 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: test.route
namespace: default
spec:
entryPoints:
- foo
routes:
- match: Host(`foo.com`)
kind: Rule
services:
- name: nodeport-svc
port: 80
nodePortLB: true
---
kind: Node
apiVersion: v1
metadata:
name: traefik-node
status:
addresses:
- type: InternalIP
address: 172.16.4.4

33
pkg/provider/kubernetes/crd/kubernetes_http.go
View file

@ -409,6 +409,39 @@ func (c configBuilder) loadServers(parentNamespace string, svc traefikv1alpha1.L
}), nil
}
if service.Spec.Type == corev1.ServiceTypeNodePort && svc.NodePortLB {
nodes, nodesExists, nodesErr := c.client.GetNodes()
if nodesErr != nil {
return nil, nodesErr
}
if !nodesExists || len(nodes) == 0 {
return nil, fmt.Errorf("nodes not found for NodePort service %s/%s", namespace, sanitizedName)
}
protocol, err := parseServiceProtocol(svc.Scheme, svcPort.Name, svcPort.Port)
if err != nil {
return nil, err
}
for _, node := range nodes {
for _, addr := range node.Status.Addresses {
if addr.Type == corev1.NodeInternalIP {
hostPort := net.JoinHostPort(addr.Address, strconv.Itoa(int(svcPort.NodePort)))
servers = append(servers, dynamic.Server{
URL: fmt.Sprintf("%s://%s", protocol, hostPort),
})
}
}
}
if len(servers) == 0 {
return nil, fmt.Errorf("no servers were generated for service %s in namespace", sanitizedName)
}
return servers, nil
}
endpoints, endpointsExists, endpointsErr := c.client.GetEndpoints(namespace, sanitizedName)
if endpointsErr != nil {
return nil, endpointsErr

28
pkg/provider/kubernetes/crd/kubernetes_tcp.go
View file

@ -247,6 +247,34 @@ func (p *Provider) loadTCPServers(client Client, namespace string, svc traefikv1
}
var servers []dynamic.TCPServer
if service.Spec.Type == corev1.ServiceTypeNodePort && svc.NodePortLB {
nodes, nodesExists, nodesErr := client.GetNodes()
if nodesErr != nil {
return nil, nodesErr
}
if !nodesExists || len(nodes) == 0 {
return nil, fmt.Errorf("nodes not found for NodePort service %s/%s", svc.Namespace, svc.Name)
}
for _, node := range nodes {
for _, addr := range node.Status.Addresses {
if addr.Type == corev1.NodeInternalIP {
servers = append(servers, dynamic.TCPServer{
Address: net.JoinHostPort(addr.Address, strconv.Itoa(int(svcPort.NodePort))),
})
}
}
}
if len(servers) == 0 {
return nil, fmt.Errorf("no servers were generated for service %s/%s", svc.Namespace, svc.Name)
}
return servers, nil
}
if service.Spec.Type == corev1.ServiceTypeExternalName {
servers = append(servers, dynamic.TCPServer{
Address: net.JoinHostPort(service.Spec.ExternalName, strconv.Itoa(int(svcPort.Port))),

183
pkg/provider/kubernetes/crd/kubernetes_test.go
View file

@ -7020,6 +7020,189 @@ func TestNativeLB(t *testing.T) {
}
}
func TestNodePortLB(t *testing.T) {
testCases := []struct {
desc string
ingressClass string
paths []string
expected *dynamic.Configuration
}{
{
desc: "Empty",
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
TCP: &dynamic.TCPConfiguration{
ServersTransports: map[string]*dynamic.TCPServersTransport{},
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "HTTP with node port LB",
paths: []string{"services.yml", "with_node_port_lb.yml"},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
TCP: &dynamic.TCPConfiguration{
ServersTransports: map[string]*dynamic.TCPServersTransport{},
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{
"default-test-route-6f97418635c7e18853da": {
EntryPoints: []string{"foo"},
Service: "default-test-route-6f97418635c7e18853da",
Rule: "Host(`foo.com`)",
Priority: 0,
},
},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{
"default-test-route-6f97418635c7e18853da": {
LoadBalancer: &dynamic.ServersLoadBalancer{
ResponseForwarding: &dynamic.ResponseForwarding{FlushInterval: dynamic.DefaultFlushInterval},
Servers: []dynamic.Server{
{
URL: "http://172.16.4.4:32456",
},
},
PassHostHeader: Bool(true),
},
},
},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "TCP with native Service LB",
paths: []string{"tcp/services.yml", "tcp/with_node_port_service_lb.yml"},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{},
Services: map[string]*dynamic.UDPService{},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
},
TCP: &dynamic.TCPConfiguration{
ServersTransports: map[string]*dynamic.TCPServersTransport{},
Routers: map[string]*dynamic.TCPRouter{
"default-test.route-fdd3e9338e47a45efefc": {
EntryPoints: []string{"foo"},
Service: "default-test.route-fdd3e9338e47a45efefc",
Rule: "HostSNI(`foo.com`)",
},
},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{
"default-test.route-fdd3e9338e47a45efefc": {
LoadBalancer: &dynamic.TCPServersLoadBalancer{
Servers: []dynamic.TCPServer{
{
Address: "172.16.4.4:32456",
Port: "",
},
},
},
},
},
},
TLS: &dynamic.TLSConfiguration{},
},
},
{
desc: "UDP with native Service LB",
paths: []string{"udp/services.yml", "udp/with_node_port_service_lb.yml"},
expected: &dynamic.Configuration{
UDP: &dynamic.UDPConfiguration{
Routers: map[string]*dynamic.UDPRouter{
"default-test.route-0": {
EntryPoints: []string{"foo"},
Service: "default-test.route-0",
},
},
Services: map[string]*dynamic.UDPService{
"default-test.route-0": {
LoadBalancer: &dynamic.UDPServersLoadBalancer{
Servers: []dynamic.UDPServer{
{
Address: "172.16.4.4:32456",
Port: "",
},
},
},
},
},
},
HTTP: &dynamic.HTTPConfiguration{
ServersTransports: map[string]*dynamic.ServersTransport{},
Routers: map[string]*dynamic.Router{},
Middlewares: map[string]*dynamic.Middleware{},
Services: map[string]*dynamic.Service{},
},
TCP: &dynamic.TCPConfiguration{
ServersTransports: map[string]*dynamic.TCPServersTransport{},
Routers: map[string]*dynamic.TCPRouter{},
Middlewares: map[string]*dynamic.TCPMiddleware{},
Services: map[string]*dynamic.TCPService{},
},
TLS: &dynamic.TLSConfiguration{},
},
},
}
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
k8sObjects, crdObjects := readResources(t, test.paths)
kubeClient := kubefake.NewSimpleClientset(k8sObjects...)
crdClient := traefikcrdfake.NewSimpleClientset(crdObjects...)
client := newClientImpl(kubeClient, crdClient)
stopCh := make(chan struct{})
eventCh, err := client.WatchAll([]string{"default", "cross-ns"}, stopCh)
require.NoError(t, err)
if k8sObjects != nil || crdObjects != nil {
// just wait for the first event
<-eventCh
}
p := Provider{}
conf := p.loadConfigurationFromCRD(context.Background(), client)
assert.Equal(t, test.expected, conf)
})
}
}
func TestCreateBasicAuthCredentials(t *testing.T) {
var k8sObjects []runtime.Object
yamlContent, err := os.ReadFile(filepath.FromSlash("./fixtures/basic_auth_secrets.yml"))

28
pkg/provider/kubernetes/crd/kubernetes_udp.go
View file

@ -131,6 +131,34 @@ func (p *Provider) loadUDPServers(client Client, namespace string, svc traefikv1
}
var servers []dynamic.UDPServer
if service.Spec.Type == corev1.ServiceTypeNodePort && svc.NodePortLB {
nodes, nodesExists, nodesErr := client.GetNodes()
if nodesErr != nil {
return nil, nodesErr
}
if !nodesExists || len(nodes) == 0 {
return nil, fmt.Errorf("nodes not found for NodePort service %s/%s", svc.Namespace, svc.Name)
}
for _, node := range nodes {
for _, addr := range node.Status.Addresses {
if addr.Type == corev1.NodeInternalIP {
servers = append(servers, dynamic.UDPServer{
Address: net.JoinHostPort(addr.Address, strconv.Itoa(int(svcPort.NodePort))),
})
}
}
}
if len(servers) == 0 {
return nil, fmt.Errorf("no servers were generated for service %s/%s", svc.Namespace, svc.Name)
}
return servers, nil
}
if service.Spec.Type == corev1.ServiceTypeExternalName {
servers = append(servers, dynamic.UDPServer{
Address: net.JoinHostPort(service.Spec.ExternalName, strconv.Itoa(int(svcPort.Port))),

5
pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroute.go
View file

@ -126,6 +126,11 @@ type LoadBalancerSpec struct {
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
// NodePortLB controls, when creating the load-balancer,
// whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
// It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
// By default, NodePortLB is false.
NodePortLB bool `json:"nodePortLB,omitempty"`
}
type ResponseForwarding struct {

5
pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressroutetcp.go
View file

@ -93,6 +93,11 @@ type ServiceTCP struct {
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
// NodePortLB controls, when creating the load-balancer,
// whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
// It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
// By default, NodePortLB is false.
NodePortLB bool `json:"nodePortLB,omitempty"`
}
// +genclient

5
pkg/provider/kubernetes/crd/traefikio/v1alpha1/ingressrouteudp.go
View file

@ -38,6 +38,11 @@ type ServiceUDP struct {
// The Kubernetes Service itself does load-balance to the pods.
// By default, NativeLB is false.
NativeLB bool `json:"nativeLB,omitempty"`
// NodePortLB controls, when creating the load-balancer,
// whether the LB's children are directly the nodes internal IPs using the nodePort when the service type is NodePort.
// It allows services to be reachable when Traefik runs externally from the Kubernetes cluster but within the same network of the nodes.
// By default, NodePortLB is false.
NodePortLB bool `json:"nodePortLB,omitempty"`
}
// +genclient