From 1ebd12ff82394cc154bfa87c94847c2c5c1b49b1 Mon Sep 17 00:00:00 2001 From: Romain Date: Tue, 17 Sep 2024 10:50:04 +0200 Subject: [PATCH] Add support for Gateway API BackendTLSPolicies --- .../kubernetes-gateway-rbac.yml | 6 +- .../fixtures/k8s-conformance/01-rbac.yml | 3 + ...api-crd.yml => 00-experimental-v1.1.0.yml} | 0 pkg/provider/kubernetes/gateway/client.go | 199 +++++++++++++---- .../httproute/with_backend_tls_policy.yml | 78 +++++++ .../with_backend_tls_policy_system.yml | 66 ++++++ pkg/provider/kubernetes/gateway/httproute.go | 166 ++++++++++++-- .../kubernetes/gateway/kubernetes_test.go | 203 ++++++++++++++++++ pkg/provider/kubernetes/k8s/parser.go | 2 +- 9 files changed, 657 insertions(+), 66 deletions(-) rename integration/fixtures/k8s/{01-gateway-api-crd.yml => 00-experimental-v1.1.0.yml} (100%) create mode 100644 pkg/provider/kubernetes/gateway/fixtures/httproute/with_backend_tls_policy.yml create mode 100644 pkg/provider/kubernetes/gateway/fixtures/httproute/with_backend_tls_policy_system.yml diff --git a/docs/content/reference/dynamic-configuration/kubernetes-gateway-rbac.yml b/docs/content/reference/dynamic-configuration/kubernetes-gateway-rbac.yml index ed0c744df..31b0837c8 100644 --- a/docs/content/reference/dynamic-configuration/kubernetes-gateway-rbac.yml +++ b/docs/content/reference/dynamic-configuration/kubernetes-gateway-rbac.yml @@ -16,6 +16,7 @@ rules: resources: - services - secrets + - configmaps verbs: - get - list @@ -34,9 +35,10 @@ rules: - gateways - httproutes - grpcroutes - - referencegrants - tcproutes - tlsroutes + - referencegrants + - backendtlspolicies verbs: - get - list @@ -50,6 +52,8 @@ rules: - grpcroutes/status - tcproutes/status - tlsroutes/status + - referencegrants/status + - backendtlspolicies/status verbs: - update diff --git a/integration/fixtures/k8s-conformance/01-rbac.yml b/integration/fixtures/k8s-conformance/01-rbac.yml index 0e30b4f3b..40ff94c6d 100644 --- a/integration/fixtures/k8s-conformance/01-rbac.yml +++ b/integration/fixtures/k8s-conformance/01-rbac.yml @@ -16,6 +16,7 @@ rules: resources: - services - secrets + - configmaps verbs: - get - list @@ -38,6 +39,7 @@ rules: - tcproutes - tlsroutes - referencegrants + - backendtlspolicies verbs: - get - list @@ -52,6 +54,7 @@ rules: - tcproutes/status - tlsroutes/status - referencegrants/status + - backendtlspolicies/status verbs: - update diff --git a/integration/fixtures/k8s/01-gateway-api-crd.yml b/integration/fixtures/k8s/00-experimental-v1.1.0.yml similarity index 100% rename from integration/fixtures/k8s/01-gateway-api-crd.yml rename to integration/fixtures/k8s/00-experimental-v1.1.0.yml diff --git a/pkg/provider/kubernetes/gateway/client.go b/pkg/provider/kubernetes/gateway/client.go index b318411d3..638760762 100644 --- a/pkg/provider/kubernetes/gateway/client.go +++ b/pkg/provider/kubernetes/gateway/client.go @@ -25,6 +25,7 @@ import ( "k8s.io/client-go/util/retry" gatev1 "sigs.k8s.io/gateway-api/apis/v1" gatev1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gatev1alpha3 "sigs.k8s.io/gateway-api/apis/v1alpha3" gatev1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" gateclientset "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" gateinformers "sigs.k8s.io/gateway-api/pkg/client/informers/externalversions" @@ -48,30 +49,6 @@ func (reh *resourceEventHandler) OnDelete(obj interface{}) { eventHandlerFunc(reh.ev, obj) } -// Client is a client for the Provider master. -// WatchAll starts the watch of the Provider resources and updates the stores. -// The stores can then be accessed via the Get* functions. -type Client interface { - WatchAll(namespaces []string, stopCh <-chan struct{}) (<-chan interface{}, error) - UpdateGatewayStatus(ctx context.Context, gateway ktypes.NamespacedName, status gatev1.GatewayStatus) error - UpdateGatewayClassStatus(ctx context.Context, name string, status gatev1.GatewayClassStatus) error - UpdateHTTPRouteStatus(ctx context.Context, route ktypes.NamespacedName, status gatev1.HTTPRouteStatus) error - UpdateGRPCRouteStatus(ctx context.Context, route ktypes.NamespacedName, status gatev1.GRPCRouteStatus) error - UpdateTCPRouteStatus(ctx context.Context, route ktypes.NamespacedName, status gatev1alpha2.TCPRouteStatus) error - UpdateTLSRouteStatus(ctx context.Context, route ktypes.NamespacedName, status gatev1alpha2.TLSRouteStatus) error - ListGatewayClasses() ([]*gatev1.GatewayClass, error) - ListGateways() []*gatev1.Gateway - ListHTTPRoutes() ([]*gatev1.HTTPRoute, error) - ListGRPCRoutes() ([]*gatev1.GRPCRoute, error) - ListTCPRoutes() ([]*gatev1alpha2.TCPRoute, error) - ListTLSRoutes() ([]*gatev1alpha2.TLSRoute, error) - ListNamespaces(selector labels.Selector) ([]string, error) - ListReferenceGrants(namespace string) ([]*gatev1beta1.ReferenceGrant, error) - ListEndpointSlicesForService(namespace, serviceName string) ([]*discoveryv1.EndpointSlice, error) - GetService(namespace, name string) (*corev1.Service, bool, error) - GetSecret(namespace, name string) (*corev1.Secret, bool, error) -} - type clientWrapper struct { csGateway gateclientset.Interface csKube kclientset.Interface @@ -198,6 +175,16 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (< } for _, ns := range namespaces { + factoryKube := kinformers.NewSharedInformerFactoryWithOptions(c.csKube, resyncPeriod, kinformers.WithNamespace(ns)) + _, err = factoryKube.Core().V1().Services().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + _, err = factoryKube.Discovery().V1().EndpointSlices().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + factoryGateway := gateinformers.NewSharedInformerFactoryWithOptions(c.csGateway, resyncPeriod, gateinformers.WithNamespace(ns)) _, err = factoryGateway.Gateway().V1().Gateways().Informer().AddEventHandler(eventHandler) if err != nil { @@ -225,16 +212,14 @@ func (c *clientWrapper) WatchAll(namespaces []string, stopCh <-chan struct{}) (< if err != nil { return nil, err } - } - - factoryKube := kinformers.NewSharedInformerFactoryWithOptions(c.csKube, resyncPeriod, kinformers.WithNamespace(ns)) - _, err = factoryKube.Core().V1().Services().Informer().AddEventHandler(eventHandler) - if err != nil { - return nil, err - } - _, err = factoryKube.Discovery().V1().EndpointSlices().Informer().AddEventHandler(eventHandler) - if err != nil { - return nil, err + _, err = factoryGateway.Gateway().V1alpha3().BackendTLSPolicies().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } + _, err = factoryKube.Core().V1().ConfigMaps().Informer().AddEventHandler(eventHandler) + if err != nil { + return nil, err + } } factorySecret := kinformers.NewSharedInformerFactoryWithOptions(c.csKube, resyncPeriod, kinformers.WithNamespace(ns), kinformers.WithTweakListOptions(notOwnedByHelm)) @@ -367,8 +352,6 @@ func (c *clientWrapper) ListTLSRoutes() ([]*gatev1alpha2.TLSRoute, error) { func (c *clientWrapper) ListReferenceGrants(namespace string) ([]*gatev1beta1.ReferenceGrant, error) { if !c.isWatchedNamespace(namespace) { - log.Warn().Msgf("Failed to get ReferenceGrants: %q is not within watched namespaces", namespace) - return nil, fmt.Errorf("failed to get ReferenceGrants: namespace %s is not within watched namespaces", namespace) } @@ -424,7 +407,7 @@ func (c *clientWrapper) UpdateGatewayClassStatus(ctx context.Context, name strin return nil }) if err != nil { - return fmt.Errorf("failed to update GatewayClass %q status: %w", name, err) + return fmt.Errorf("failed to update GatewayClass %s status: %w", name, err) } return nil @@ -459,7 +442,7 @@ func (c *clientWrapper) UpdateGatewayStatus(ctx context.Context, gateway ktypes. return nil }) if err != nil { - return fmt.Errorf("failed to update Gateway %q status: %w", gateway.Name, err) + return fmt.Errorf("failed to update Gateway %s/%s status: %w", gateway.Namespace, gateway.Name, err) } return nil @@ -486,7 +469,6 @@ func (c *clientWrapper) UpdateHTTPRouteStatus(ctx context.Context, route ktypes. for _, parentStatus := range currentRoute.Status.Parents { if parentStatus.ControllerName != controllerName { parentStatuses = append(parentStatuses, parentStatus) - continue } } @@ -511,7 +493,7 @@ func (c *clientWrapper) UpdateHTTPRouteStatus(ctx context.Context, route ktypes. return nil }) if err != nil { - return fmt.Errorf("failed to update HTTPRoute %q status: %w", route.Name, err) + return fmt.Errorf("failed to update HTTPRoute %s/%s status: %w", route.Namespace, route.Name, err) } return nil @@ -538,7 +520,6 @@ func (c *clientWrapper) UpdateGRPCRouteStatus(ctx context.Context, route ktypes. for _, parentStatus := range currentRoute.Status.Parents { if parentStatus.ControllerName != controllerName { parentStatuses = append(parentStatuses, parentStatus) - continue } } @@ -590,7 +571,6 @@ func (c *clientWrapper) UpdateTCPRouteStatus(ctx context.Context, route ktypes.N for _, parentStatus := range currentRoute.Status.Parents { if parentStatus.ControllerName != controllerName { parentStatuses = append(parentStatuses, parentStatus) - continue } } @@ -615,7 +595,7 @@ func (c *clientWrapper) UpdateTCPRouteStatus(ctx context.Context, route ktypes.N return nil }) if err != nil { - return fmt.Errorf("failed to update TCPRoute %q status: %w", route.Name, err) + return fmt.Errorf("failed to update TCPRoute %s/%s status: %w", route.Namespace, route.Name, err) } return nil @@ -642,7 +622,6 @@ func (c *clientWrapper) UpdateTLSRouteStatus(ctx context.Context, route ktypes.N for _, parentStatus := range currentRoute.Status.Parents { if parentStatus.ControllerName != controllerName { parentStatuses = append(parentStatuses, parentStatus) - continue } } @@ -667,7 +646,69 @@ func (c *clientWrapper) UpdateTLSRouteStatus(ctx context.Context, route ktypes.N return nil }) if err != nil { - return fmt.Errorf("failed to update TLSRoute %q status: %w", route.Name, err) + return fmt.Errorf("failed to update TLSRoute %s/%s status: %w", route.Namespace, route.Name, err) + } + + return nil +} + +func (c *clientWrapper) UpdateBackendTLSPolicyStatus(ctx context.Context, policy ktypes.NamespacedName, status gatev1alpha2.PolicyStatus) error { + if !c.isWatchedNamespace(policy.Namespace) { + return fmt.Errorf("updating BackendTLSPolicy status %s/%s: namespace is not within watched namespaces", policy.Namespace, policy.Name) + } + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + currentPolicy, err := c.factoriesGateway[c.lookupNamespace(policy.Namespace)].Gateway().V1alpha3().BackendTLSPolicies().Lister().BackendTLSPolicies(policy.Namespace).Get(policy.Name) + if err != nil { + // We have to return err itself here (not wrapped inside another error) + // so that RetryOnConflict can identify it correctly. + return err + } + + ancestorStatuses := make([]gatev1alpha2.PolicyAncestorStatus, len(status.Ancestors)) + copy(ancestorStatuses, status.Ancestors) + + // keep statuses added by other gateway controllers, + // and statuses for Traefik gateway controller but not for the same Gateway as the one in parameter (AncestorRef). + for _, ancestorStatus := range currentPolicy.Status.Ancestors { + if ancestorStatus.ControllerName != controllerName { + ancestorStatuses = append(ancestorStatuses, ancestorStatus) + continue + } + + if slices.ContainsFunc(status.Ancestors, func(status gatev1alpha2.PolicyAncestorStatus) bool { + return reflect.DeepEqual(ancestorStatus.AncestorRef, status.AncestorRef) + }) { + continue + } + + ancestorStatuses = append(ancestorStatuses, ancestorStatus) + } + + if len(ancestorStatuses) > 16 { + return fmt.Errorf("failed to update BackendTLSPolicy %s/%s status: PolicyAncestor statuses count exceeds 16", policy.Namespace, policy.Name) + } + + // do not update status when nothing has changed. + if policyAncestorStatusesEqual(currentPolicy.Status.Ancestors, ancestorStatuses) { + return nil + } + + currentPolicy = currentPolicy.DeepCopy() + currentPolicy.Status = gatev1alpha2.PolicyStatus{ + Ancestors: ancestorStatuses, + } + + if _, err = c.csGateway.GatewayV1alpha3().BackendTLSPolicies(policy.Namespace).UpdateStatus(ctx, currentPolicy, metav1.UpdateOptions{}); err != nil { + // We have to return err itself here (not wrapped inside another error) + // so that RetryOnConflict can identify it correctly. + return err + } + + return nil + }) + if err != nil { + return fmt.Errorf("failed to update BackendTLSPolicy %s/%s status: %w", policy.Namespace, policy.Name, err) } return nil @@ -701,6 +742,32 @@ func (c *clientWrapper) ListEndpointSlicesForService(namespace, serviceName stri return c.factoriesKube[c.lookupNamespace(namespace)].Discovery().V1().EndpointSlices().Lister().EndpointSlices(namespace).List(serviceSelector) } +// ListBackendTLSPoliciesForService returns the BackendTLSPolicy for the given service name in the given namespace. +func (c *clientWrapper) ListBackendTLSPoliciesForService(namespace, serviceName string) ([]*gatev1alpha3.BackendTLSPolicy, error) { + if !c.isWatchedNamespace(namespace) { + return nil, fmt.Errorf("failed to get BackendTLSPolicies for service %s/%s: namespace is not within watched namespaces", namespace, serviceName) + } + + policies, err := c.factoriesGateway[c.lookupNamespace(namespace)].Gateway().V1alpha3().BackendTLSPolicies().Lister().BackendTLSPolicies(namespace).List(labels.Everything()) + if err != nil { + return nil, fmt.Errorf("failed to list BackendTLSPolicies in namespace %s", namespace) + } + + var servicePolicies []*gatev1alpha3.BackendTLSPolicy + for _, policy := range policies { + for _, ref := range policy.Spec.TargetRefs { + // The policy does not target the service. + if ref.Group != groupCore || ref.Kind != kindService || string(ref.Name) != serviceName { + continue + } + + servicePolicies = append(servicePolicies, policy) + } + } + + return servicePolicies, nil +} + // GetSecret returns the named secret from the given namespace. func (c *clientWrapper) GetSecret(namespace, name string) (*corev1.Secret, bool, error) { if !c.isWatchedNamespace(namespace) { @@ -713,6 +780,18 @@ func (c *clientWrapper) GetSecret(namespace, name string) (*corev1.Secret, bool, return secret, exist, err } +// GetConfigMap returns the named configMap from the given namespace. +func (c *clientWrapper) GetConfigMap(namespace, name string) (*corev1.ConfigMap, bool, error) { + if !c.isWatchedNamespace(namespace) { + return nil, false, fmt.Errorf("failed to get configMap %s/%s: namespace is not within watched namespaces", namespace, name) + } + + configMap, err := c.factoriesKube[c.lookupNamespace(namespace)].Core().V1().ConfigMaps().Lister().ConfigMaps(namespace).Get(name) + exist, err := translateNotFoundError(err) + + return configMap, exist, err +} + // lookupNamespace returns the lookup namespace key for the given namespace. // When listening on all namespaces, it returns the client-go identifier ("") // for all-namespaces. Otherwise, it returns the given namespace. @@ -761,6 +840,36 @@ func gatewayStatusEqual(statusA, statusB gatev1.GatewayStatus) bool { conditionsEqual(statusA.Conditions, statusB.Conditions) } +func policyAncestorStatusesEqual(policyAncestorStatusesA, policyAncestorStatusesB []gatev1alpha2.PolicyAncestorStatus) bool { + if len(policyAncestorStatusesA) != len(policyAncestorStatusesB) { + return false + } + + for _, sA := range policyAncestorStatusesA { + if !slices.ContainsFunc(policyAncestorStatusesB, func(sB gatev1alpha2.PolicyAncestorStatus) bool { + return policyAncestorStatusEqual(sB, sA) + }) { + return false + } + } + + for _, sB := range policyAncestorStatusesB { + if !slices.ContainsFunc(policyAncestorStatusesA, func(sA gatev1alpha2.PolicyAncestorStatus) bool { + return policyAncestorStatusEqual(sA, sB) + }) { + return false + } + } + + return true +} + +func policyAncestorStatusEqual(sA, sB gatev1alpha2.PolicyAncestorStatus) bool { + return sA.ControllerName == sB.ControllerName && + reflect.DeepEqual(sA.AncestorRef, sB.AncestorRef) && + conditionsEqual(sA.Conditions, sB.Conditions) +} + func routeParentStatusesEqual(routeParentStatusesA, routeParentStatusesB []gatev1alpha2.RouteParentStatus) bool { if len(routeParentStatusesA) != len(routeParentStatusesB) { return false diff --git a/pkg/provider/kubernetes/gateway/fixtures/httproute/with_backend_tls_policy.yml b/pkg/provider/kubernetes/gateway/fixtures/httproute/with_backend_tls_policy.yml new file mode 100644 index 000000000..e64a341b6 --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/httproute/with_backend_tls_policy.yml @@ -0,0 +1,78 @@ +--- +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway-class +spec: + controllerName: traefik.io/gateway-controller + +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + kinds: + - kind: HTTPRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +kind: HTTPRoute +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: http-app-1 + namespace: default +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + hostnames: + - "foo.com" + rules: + - matches: + - path: + type: Exact + value: /bar + backendRefs: + - name: whoami + port: 80 + weight: 1 + kind: Service + group: "" + +--- +kind: BackendTLSPolicy +apiVersion: gateway.networking.k8s.io/v1alpha3 +metadata: + name: policy-1 + namespace: default +spec: + targetRefs: + - group: core + kind: Service + name: whoami + validation: + hostname: whoami + caCertificateRefs: + - group: core + kind: ConfigMap + name: ca-file + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ca-file + namespace: default +data: + ca.crt: "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0=" diff --git a/pkg/provider/kubernetes/gateway/fixtures/httproute/with_backend_tls_policy_system.yml b/pkg/provider/kubernetes/gateway/fixtures/httproute/with_backend_tls_policy_system.yml new file mode 100644 index 000000000..cc36c0468 --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/httproute/with_backend_tls_policy_system.yml @@ -0,0 +1,66 @@ +--- +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway-class +spec: + controllerName: traefik.io/gateway-controller + +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + kinds: + - kind: HTTPRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +kind: HTTPRoute +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: http-app-1 + namespace: default +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + hostnames: + - "foo.com" + rules: + - matches: + - path: + type: Exact + value: /bar + backendRefs: + - name: whoami + port: 80 + weight: 1 + kind: Service + group: "" + +--- +kind: BackendTLSPolicy +apiVersion: gateway.networking.k8s.io/v1alpha3 +metadata: + name: policy-1 + namespace: default +spec: + targetRefs: + - group: core + kind: Service + name: whoami + validation: + hostname: whoami + wellKnownCACertificates: System diff --git a/pkg/provider/kubernetes/gateway/httproute.go b/pkg/provider/kubernetes/gateway/httproute.go index c18dc48d3..d6c5a4c08 100644 --- a/pkg/provider/kubernetes/gateway/httproute.go +++ b/pkg/provider/kubernetes/gateway/httproute.go @@ -13,11 +13,14 @@ import ( "github.com/rs/zerolog/log" "github.com/traefik/traefik/v3/pkg/config/dynamic" "github.com/traefik/traefik/v3/pkg/provider" + "github.com/traefik/traefik/v3/pkg/types" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ktypes "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" gatev1 "sigs.k8s.io/gateway-api/apis/v1" + gatev1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gatev1alpha3 "sigs.k8s.io/gateway-api/apis/v1alpha3" ) func (p *Provider) loadHTTPRoutes(ctx context.Context, gatewayListeners []gatewayListener, conf *dynamic.Configuration) { @@ -158,7 +161,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, listener gatewayListener, default: var serviceCondition *metav1.Condition - router.Service, serviceCondition = p.loadWRRService(conf, routerName, routeRule, route) + router.Service, serviceCondition = p.loadWRRService(ctx, listener, conf, routerName, routeRule, route) if serviceCondition != nil { condition = *serviceCondition } @@ -173,7 +176,7 @@ func (p *Provider) loadHTTPRoute(ctx context.Context, listener gatewayListener, return conf, condition } -func (p *Provider) loadWRRService(conf *dynamic.Configuration, routeKey string, routeRule gatev1.HTTPRouteRule, route *gatev1.HTTPRoute) (string, *metav1.Condition) { +func (p *Provider) loadWRRService(ctx context.Context, listener gatewayListener, conf *dynamic.Configuration, routeKey string, routeRule gatev1.HTTPRouteRule, route *gatev1.HTTPRoute) (string, *metav1.Condition) { name := routeKey + "-wrr" if _, ok := conf.HTTP.Services[name]; ok { return name, nil @@ -182,7 +185,7 @@ func (p *Provider) loadWRRService(conf *dynamic.Configuration, routeKey string, var wrr dynamic.WeightedRoundRobin var condition *metav1.Condition for _, backendRef := range routeRule.BackendRefs { - svcName, svc, errCondition := p.loadService(route, backendRef) + svcName, errCondition := p.loadService(ctx, listener, conf, route, backendRef) weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1))) if errCondition != nil { condition = errCondition @@ -194,10 +197,6 @@ func (p *Provider) loadWRRService(conf *dynamic.Configuration, routeKey string, continue } - if svc != nil { - conf.HTTP.Services[svcName] = svc - } - wrr.Services = append(wrr.Services, dynamic.WRRService{ Name: svcName, Weight: weight, @@ -210,7 +209,7 @@ func (p *Provider) loadWRRService(conf *dynamic.Configuration, routeKey string, // loadService returns a dynamic.Service config corresponding to the given gatev1.HTTPBackendRef. // Note that the returned dynamic.Service config can be nil (for cross-provider, internal services, and backendFunc). -func (p *Provider) loadService(route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (string, *dynamic.Service, *metav1.Condition) { +func (p *Provider) loadService(ctx context.Context, listener gatewayListener, conf *dynamic.Configuration, route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (string, *metav1.Condition) { kind := ptr.Deref(backendRef.Kind, kindService) group := groupCore @@ -226,7 +225,7 @@ func (p *Provider) loadService(route *gatev1.HTTPRoute, backendRef gatev1.HTTPBa serviceName := provider.Normalize(namespace + "-" + string(backendRef.Name)) if err := p.isReferenceGranted(kindHTTPRoute, route.Namespace, group, string(kind), string(backendRef.Name), namespace); err != nil { - return serviceName, nil, &metav1.Condition{ + return serviceName, &metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), Status: metav1.ConditionFalse, ObservedGeneration: route.Generation, @@ -239,7 +238,7 @@ func (p *Provider) loadService(route *gatev1.HTTPRoute, backendRef gatev1.HTTPBa if group != groupCore || kind != kindService { name, service, err := p.loadHTTPBackendRef(namespace, backendRef) if err != nil { - return serviceName, nil, &metav1.Condition{ + return serviceName, &metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), Status: metav1.ConditionFalse, ObservedGeneration: route.Generation, @@ -249,12 +248,16 @@ func (p *Provider) loadService(route *gatev1.HTTPRoute, backendRef gatev1.HTTPBa } } - return name, service, nil + if service != nil { + conf.HTTP.Services[name] = service + } + + return name, nil } port := ptr.Deref(backendRef.Port, gatev1.PortNumber(0)) if port == 0 { - return serviceName, nil, &metav1.Condition{ + return serviceName, &metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), Status: metav1.ConditionFalse, ObservedGeneration: route.Generation, @@ -267,12 +270,97 @@ func (p *Provider) loadService(route *gatev1.HTTPRoute, backendRef gatev1.HTTPBa portStr := strconv.FormatInt(int64(port), 10) serviceName = provider.Normalize(serviceName + "-" + portStr) - lb, errCondition := p.loadHTTPServers(namespace, route, backendRef) + lb, svcPort, errCondition := p.loadHTTPServers(namespace, route, backendRef) if errCondition != nil { - return serviceName, nil, errCondition + return serviceName, errCondition } - return serviceName, &dynamic.Service{LoadBalancer: lb}, nil + if !p.ExperimentalChannel { + conf.HTTP.Services[serviceName] = &dynamic.Service{LoadBalancer: lb} + + return serviceName, nil + } + + servicePolicies, err := p.client.ListBackendTLSPoliciesForService(namespace, string(backendRef.Name)) + if err != nil { + return serviceName, &metav1.Condition{ + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + ObservedGeneration: route.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.RouteReasonRefNotPermitted), + Message: fmt.Sprintf("Cannot list BackendTLSPolicies for Service %s/%s: %s", namespace, string(backendRef.Name), err), + } + } + + var matchedPolicy *gatev1alpha3.BackendTLSPolicy + for _, policy := range servicePolicies { + matched := false + for _, targetRef := range policy.Spec.TargetRefs { + if targetRef.SectionName == nil || svcPort.Name == string(*targetRef.SectionName) { + matchedPolicy = policy + matched = true + break + } + } + + // If the policy targets the service, but doesn't match any port. + if !matched { + // update policy status + status := gatev1alpha2.PolicyStatus{ + Ancestors: []gatev1alpha2.PolicyAncestorStatus{{ + AncestorRef: gatev1alpha2.ParentReference{ + Group: ptr.To(gatev1.Group(groupGateway)), + Kind: ptr.To(gatev1.Kind(kindGateway)), + Namespace: ptr.To(gatev1.Namespace(namespace)), + Name: gatev1.ObjectName(listener.GWName), + SectionName: ptr.To(gatev1.SectionName(listener.Name)), + }, + ControllerName: controllerName, + Conditions: []metav1.Condition{{ + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + ObservedGeneration: route.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.RouteReasonBackendNotFound), + Message: fmt.Sprintf("BackendTLSPolicy has no valid TargetRef for Service %s/%s", namespace, string(backendRef.Name)), + }}, + }}, + } + + if err := p.client.UpdateBackendTLSPolicyStatus(ctx, ktypes.NamespacedName{Namespace: policy.Namespace, Name: policy.Name}, status); err != nil { + logger := log.Ctx(ctx).With(). + Str("http_route", route.Name). + Str("namespace", route.Namespace).Logger() + logger.Warn(). + Err(err). + Msg("Unable to update TLSRoute status") + } + } + } + + if matchedPolicy != nil { + st, err := p.loadServersTransport(namespace, *matchedPolicy) + if err != nil { + return serviceName, &metav1.Condition{ + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + ObservedGeneration: route.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.RouteReasonRefNotPermitted), + Message: fmt.Sprintf("Cannot apply BackendTLSPolicy for Service %s/%s: %s", namespace, string(backendRef.Name), err), + } + } + + if st != nil { + lb.ServersTransport = serviceName + conf.HTTP.ServersTransports[serviceName] = st + } + } + + conf.HTTP.Services[serviceName] = &dynamic.Service{LoadBalancer: lb} + + return serviceName, nil } func (p *Provider) loadHTTPBackendRef(namespace string, backendRef gatev1.HTTPBackendRef) (string, *dynamic.Service, error) { @@ -365,10 +453,10 @@ func (p *Provider) loadHTTPRouteFilterExtensionRef(namespace string, extensionRe return filterFunc(string(extensionRef.Name), namespace) } -func (p *Provider) loadHTTPServers(namespace string, route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (*dynamic.ServersLoadBalancer, *metav1.Condition) { +func (p *Provider) loadHTTPServers(namespace string, route *gatev1.HTTPRoute, backendRef gatev1.HTTPBackendRef) (*dynamic.ServersLoadBalancer, corev1.ServicePort, *metav1.Condition) { backendAddresses, svcPort, err := p.getBackendAddresses(namespace, backendRef.BackendRef) if err != nil { - return nil, &metav1.Condition{ + return nil, corev1.ServicePort{}, &metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), Status: metav1.ConditionFalse, ObservedGeneration: route.Generation, @@ -380,7 +468,7 @@ func (p *Provider) loadHTTPServers(namespace string, route *gatev1.HTTPRoute, ba protocol, err := getProtocol(svcPort) if err != nil { - return nil, &metav1.Condition{ + return nil, corev1.ServicePort{}, &metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), Status: metav1.ConditionFalse, ObservedGeneration: route.Generation, @@ -398,7 +486,40 @@ func (p *Provider) loadHTTPServers(namespace string, route *gatev1.HTTPRoute, ba URL: fmt.Sprintf("%s://%s", protocol, net.JoinHostPort(ba.Address, strconv.Itoa(int(ba.Port)))), }) } - return lb, nil + return lb, svcPort, nil +} + +func (p *Provider) loadServersTransport(namespace string, policy gatev1alpha3.BackendTLSPolicy) (*dynamic.ServersTransport, error) { + st := &dynamic.ServersTransport{ + ServerName: string(policy.Spec.Validation.Hostname), + } + + if policy.Spec.Validation.WellKnownCACertificates != nil { + return st, nil + } + + for _, caCertRef := range policy.Spec.Validation.CACertificateRefs { + if caCertRef.Group != groupCore || caCertRef.Kind != "ConfigMap" { + continue + } + + configMap, exists, err := p.client.GetConfigMap(namespace, string(caCertRef.Name)) + if err != nil { + return nil, fmt.Errorf("getting configmap: %w", err) + } + if !exists { + return nil, fmt.Errorf("configmap %s/%s not found", namespace, string(caCertRef.Name)) + } + + caCRT, ok := configMap.Data["ca.crt"] + if !ok { + return nil, fmt.Errorf("configmap %s/%s does not have ca.crt", namespace, string(caCertRef.Name)) + } + + st.RootCAs = append(st.RootCAs, types.FileOrContent(caCRT)) + } + + return st, nil } func buildHostRule(hostnames []gatev1.Hostname) (string, int) { @@ -715,4 +836,11 @@ func mergeHTTPConfiguration(from, to *dynamic.Configuration) { for serviceName, service := range from.HTTP.Services { to.HTTP.Services[serviceName] = service } + + if to.HTTP.ServersTransports == nil { + to.HTTP.ServersTransports = map[string]*dynamic.ServersTransport{} + } + for name, serversTransport := range from.HTTP.ServersTransports { + to.HTTP.ServersTransports[name] = serversTransport + } } diff --git a/pkg/provider/kubernetes/gateway/kubernetes_test.go b/pkg/provider/kubernetes/gateway/kubernetes_test.go index c5cdb413b..211381d75 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes_test.go +++ b/pkg/provider/kubernetes/gateway/kubernetes_test.go @@ -26,6 +26,7 @@ import ( "k8s.io/utils/ptr" gatev1 "sigs.k8s.io/gateway-api/apis/v1" gatev1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gatev1alpha3 "sigs.k8s.io/gateway-api/apis/v1alpha3" gatev1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" gatefake "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned/fake" ) @@ -43,6 +44,9 @@ func init() { if err := gatev1alpha2.AddToScheme(kscheme.Scheme); err != nil { panic(err) } + if err := gatev1alpha3.AddToScheme(kscheme.Scheme); err != nil { + panic(err) + } } func TestLoadHTTPRoutes(t *testing.T) { @@ -2132,6 +2136,204 @@ func TestLoadHTTPRoutes(t *testing.T) { TLS: &dynamic.TLSConfiguration{}, }, }, + { + desc: "Simple HTTPRoute and BackendTLSPolicy, experimental channel disabled", + paths: []string{"services.yml", "httproute/with_backend_tls_policy.yml"}, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr", + Rule: "Host(`foo.com`) && Path(`/bar`)", + Priority: 100008, + RuleSyntax: "v3", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "default-whoami-80", + Weight: ptr.To(1), + }, + }, + }, + }, + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: ptypes.Duration(100 * time.Millisecond), + }, + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{}, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Simple HTTPRoute and BackendTLSPolicy with CA certificate, experimental channel enabled", + paths: []string{"services.yml", "httproute/with_backend_tls_policy.yml"}, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + experimentalChannel: true, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr", + Rule: "Host(`foo.com`) && Path(`/bar`)", + Priority: 100008, + RuleSyntax: "v3", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "default-whoami-80", + Weight: ptr.To(1), + }, + }, + }, + }, + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: ptypes.Duration(100 * time.Millisecond), + }, + ServersTransport: "default-whoami-80", + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{ + "default-whoami-80": { + ServerName: "whoami", + RootCAs: []types.FileOrContent{ + "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0=", + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Simple HTTPRoute and BackendTLSPolicy with System CA, experimental channel enabled", + paths: []string{"services.yml", "httproute/with_backend_tls_policy_system.yml"}, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + experimentalChannel: true, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06": { + EntryPoints: []string{"web"}, + Service: "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr", + Rule: "Host(`foo.com`) && Path(`/bar`)", + Priority: 100008, + RuleSyntax: "v3", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "default-http-app-1-my-gateway-web-0-1c0cf64bde37d9d0df06-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "default-whoami-80", + Weight: ptr.To(1), + }, + }, + }, + }, + "default-whoami-80": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Servers: []dynamic.Server{ + { + URL: "http://10.10.0.1:80", + }, + { + URL: "http://10.10.0.2:80", + }, + }, + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: ptypes.Duration(100 * time.Millisecond), + }, + ServersTransport: "default-whoami-80", + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{ + "default-whoami-80": { + ServerName: "whoami", + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, } for _, test := range testCases { @@ -2148,6 +2350,7 @@ func TestLoadHTTPRoutes(t *testing.T) { gwClient := newGatewaySimpleClientSet(t, gwObjects...) client := newClientImpl(kubeClient, gwClient) + client.experimentalChannel = test.experimentalChannel eventCh, err := client.WatchAll(nil, make(chan struct{})) require.NoError(t, err) diff --git a/pkg/provider/kubernetes/k8s/parser.go b/pkg/provider/kubernetes/k8s/parser.go index 0c50ab61e..422513ade 100644 --- a/pkg/provider/kubernetes/k8s/parser.go +++ b/pkg/provider/kubernetes/k8s/parser.go @@ -12,7 +12,7 @@ import ( // MustParseYaml parses a YAML to objects. func MustParseYaml(content []byte) []runtime.Object { - acceptedK8sTypes := regexp.MustCompile(`^(Namespace|Deployment|EndpointSlice|Node|Service|Ingress|IngressRoute|IngressRouteTCP|IngressRouteUDP|Middleware|MiddlewareTCP|Secret|TLSOption|TLSStore|TraefikService|IngressClass|ServersTransport|ServersTransportTCP|GatewayClass|Gateway|HTTPRoute|TCPRoute|TLSRoute|ReferenceGrant)$`) + acceptedK8sTypes := regexp.MustCompile(`^(Namespace|Deployment|EndpointSlice|Node|Service|ConfigMap|Ingress|IngressRoute|IngressRouteTCP|IngressRouteUDP|Middleware|MiddlewareTCP|Secret|TLSOption|TLSStore|TraefikService|IngressClass|ServersTransport|ServersTransportTCP|GatewayClass|Gateway|HTTPRoute|TCPRoute|TLSRoute|ReferenceGrant|BackendTLSPolicy)$`) files := strings.Split(string(content), "---\n") retVal := make([]runtime.Object, 0, len(files))