homelab/traefik
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

In cluster Gateway API Conformance Tests

Browse source 
Co-authored-by: Kevin Pollet <pollet.kevin@gmail.com>
This commit is contained in:
Romain 2024-04-17 15:22:04 +02:00 • committed by GitHub
parent da7bb5fc25
commit 52e95deee3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 12094 additions and 142 deletions
Download patch file Download diff file Expand all files Collapse all files

189
integration/k8s_conformance_test.go
View file

@ -3,24 +3,27 @@ package integration
import (
"context"
"fmt"
"net"
"io"
"os"
"path/filepath"
"strings"
"slices"
"testing"
"time"
"github.com/rs/zerolog/log"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/modules/k3s"
"github.com/testcontainers/testcontainers-go/network"
"github.com/traefik/traefik/v3/integration/try"
"github.com/traefik/traefik/v3/pkg/version"
"gopkg.in/yaml.v3"
ktypes "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
kclientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"sigs.k8s.io/controller-runtime/pkg/client"
klog "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
gatev1 "sigs.k8s.io/gateway-api/apis/v1"
gatev1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
gatev1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
@ -30,106 +33,146 @@ import (
ksuite "sigs.k8s.io/gateway-api/conformance/utils/suite"
)
const (
k3sImage = "docker.io/rancher/k3s:v1.29.3-k3s1"
traefikImage = "traefik/traefik:latest"
traefikDeployment = "deployments/traefik"
traefikNamespace = "traefik"
)
// K8sConformanceSuite tests suite.
type K8sConformanceSuite struct{ BaseSuite }
type K8sConformanceSuite struct {
BaseSuite
k3sContainer *k3s.K3sContainer
kubeClient client.Client
clientSet *kclientset.Clientset
}
func TestK8sConformanceSuite(t *testing.T) {
suite.Run(t, new(K8sConformanceSuite))
}
func (s *K8sConformanceSuite) SetupSuite() {
s.BaseSuite.SetupSuite()
s.createComposeProject("k8s")
s.composeUp()
abs, err := filepath.Abs("./fixtures/k8s/config.skip/kubeconfig.yaml")
require.NoError(s.T(), err)
err = try.Do(60*time.Second, func() error {
_, err := os.Stat(abs)
return err
})
require.NoError(s.T(), err)
data, err := os.ReadFile(abs)
require.NoError(s.T(), err)
content := strings.ReplaceAll(string(data), "https://server:6443", fmt.Sprintf("https://%s", net.JoinHostPort(s.getComposeServiceIP("server"), "6443")))
err = os.WriteFile(abs, []byte(content), 0o644)
require.NoError(s.T(), err)
err = os.Setenv("KUBECONFIG", abs)
require.NoError(s.T(), err)
}
func (s *K8sConformanceSuite) TearDownSuite() {
s.BaseSuite.TearDownSuite()
generatedFiles := []string{
"./fixtures/k8s/config.skip/kubeconfig.yaml",
"./fixtures/k8s/config.skip/k3s.log",
"./fixtures/k8s/rolebindings.yaml",
"./fixtures/k8s/ccm.yaml",
}
for _, filename := range generatedFiles {
if err := os.Remove(filename); err != nil {
log.Warn().Err(err).Send()
}
}
}
func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
if !*k8sConformance {
s.T().Skip("Skip because it can take a long time to execute. To enable pass the `k8sConformance` flag.")
}
configFromFlags, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG"))
s.BaseSuite.SetupSuite()
// Avoid panic.
klog.SetLogger(zap.New())
provider, err := testcontainers.ProviderDocker.GetProvider()
if err != nil {
s.T().Fatal(err)
}
kClient, err := client.New(configFromFlags, client.Options{})
ctx := context.Background()
// Ensure image is available locally.
images, err := provider.ListImages(ctx)
if err != nil {
s.T().Fatal(err)
}
if !slices.ContainsFunc(images, func(img testcontainers.ImageInfo) bool {
return img.Name == traefikImage
}) {
s.T().Fatal("Traefik image is not present")
}
s.k3sContainer, err = k3s.RunContainer(ctx,
testcontainers.WithImage(k3sImage),
k3s.WithManifest("./fixtures/k8s-conformance/00-experimental-v1.0.0.yml"),
k3s.WithManifest("./fixtures/k8s-conformance/01-rbac.yml"),
k3s.WithManifest("./fixtures/k8s-conformance/02-traefik.yml"),
network.WithNetwork(nil, s.network),
)
if err != nil {
s.T().Fatal(err)
}
if err = s.k3sContainer.LoadImages(ctx, traefikImage); err != nil {
s.T().Fatal(err)
}
exitCode, _, err := s.k3sContainer.Exec(ctx, []string{"kubectl", "wait", "-n", traefikNamespace, traefikDeployment, "--for=condition=Available", "--timeout=30s"})
if err != nil || exitCode > 0 {
s.T().Fatalf("Traefik pod is not ready: %v", err)
}
kubeConfigYaml, err := s.k3sContainer.GetKubeConfig(ctx)
if err != nil {
s.T().Fatal(err)
}
restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeConfigYaml)
if err != nil {
s.T().Fatalf("Error loading Kubernetes config: %v", err)
}
s.kubeClient, err = client.New(restConfig, client.Options{})
if err != nil {
s.T().Fatalf("Error initializing Kubernetes client: %v", err)
}
kClientSet, err := kclientset.NewForConfig(configFromFlags)
s.clientSet, err = kclientset.NewForConfig(restConfig)
if err != nil {
s.T().Fatalf("Error initializing Kubernetes REST client: %v", err)
}
if err = gatev1alpha2.AddToScheme(s.kubeClient.Scheme()); err != nil {
s.T().Fatal(err)
}
err = gatev1alpha2.AddToScheme(kClient.Scheme())
require.NoError(s.T(), err)
err = gatev1beta1.AddToScheme(kClient.Scheme())
require.NoError(s.T(), err)
err = gatev1.AddToScheme(kClient.Scheme())
require.NoError(s.T(), err)
if err = gatev1beta1.AddToScheme(s.kubeClient.Scheme()); err != nil {
s.T().Fatal(err)
}
s.traefikCmd(withConfigFile("fixtures/k8s_gateway_conformance.toml"))
if err = gatev1.AddToScheme(s.kubeClient.Scheme()); err != nil {
s.T().Fatal(err)
}
}
// Wait for traefik to start
err = try.GetRequest("http://127.0.0.1:8080/api/entrypoints", 10*time.Second, try.BodyContains(`"name":"web"`))
require.NoError(s.T(), err)
func (s *K8sConformanceSuite) TearDownSuite() {
ctx := context.Background()
err = try.Do(10*time.Second, func() error {
gwc := &gatev1.GatewayClass{}
err := kClient.Get(context.Background(), ktypes.NamespacedName{Name: "my-gateway-class"}, gwc)
if err != nil {
return fmt.Errorf("error fetching GatewayClass: %w", err)
if s.T().Failed() || *showLog {
k3sLogs, err := s.k3sContainer.Logs(ctx)
if err == nil {
if res, err := io.ReadAll(k3sLogs); err == nil {
s.T().Log(string(res))
}
}
return nil
})
exitCode, result, err := s.k3sContainer.Exec(ctx, []string{"kubectl", "logs", "-n", traefikNamespace, traefikDeployment})
if err == nil || exitCode == 0 {
if res, err := io.ReadAll(result); err == nil {
s.T().Log(string(res))
}
}
}
if err := s.k3sContainer.Terminate(ctx); err != nil {
s.T().Fatal(err)
}
s.BaseSuite.TearDownSuite()
}
func (s *K8sConformanceSuite) TestK8sGatewayAPIConformance() {
// Wait for traefik to start
k3sContainerIP, err := s.k3sContainer.ContainerIP(context.Background())
require.NoError(s.T(), err)
err = try.GetRequest("http://"+k3sContainerIP+":8080/api/entrypoints", 10*time.Second, try.BodyContains(`"name":"web"`))
require.NoError(s.T(), err)
opts := ksuite.Options{
Client: kClient,
RestConfig: configFromFlags,
Clientset: kClientSet,
GatewayClassName: "my-gateway-class",
Client: s.kubeClient,
Clientset: s.clientSet,
GatewayClassName: "traefik",
Debug: true,
CleanupBaseResources: true,
TimeoutConfig: config.TimeoutConfig{