[1.17] Fix route delegation flakes (#10705)

Author: jenshu

changelog/v1.17.28/delegation-flakes.yaml

@@ -0,0 +1,5 @@
+changelog:
+- type: NON_USER_FACING
+  issueLink: https://github.com/kgateway-dev/kgateway/issues/10114
+  resolvesIssue: false
+  description: Make the route delegation e2e tests more robust against flakes.

test/kubernetes/e2e/defaults/defaults.go

@@ -26,6 +26,8 @@ var (
 
 	CurlPodManifest = filepath.Join(util.MustGetThisDir(), "testdata", "curl_pod.yaml")
 
+	CurlPodLabelSelector = "app.kubernetes.io/name=curl"
+
 	HttpEchoPod = &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "http-echo",

test/kubernetes/e2e/features/route_delegation/suite.go

@@ -2,12 +2,14 @@ package route_delegation
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"time"
 
 	. "github.com/onsi/gomega"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	gwv1 "sigs.k8s.io/gateway-api/apis/v1"
@@ -16,7 +18,6 @@ import (
 	testmatchers "github.com/solo-io/gloo/test/gomega/matchers"
 	"github.com/solo-io/gloo/test/kubernetes/e2e"
 	"github.com/solo-io/gloo/test/kubernetes/e2e/defaults"
-	"github.com/solo-io/gloo/test/kubernetes/testutils/gloogateway"
 )
 
 var _ e2e.NewSuiteFunc = NewTestingSuite
@@ -33,6 +34,9 @@ type tsuite struct {
 	manifests map[string][]string
 
 	manifestObjects map[string][]client.Object
+
+	// resources from common manifest
+	commonResources []client.Object
 }
 
 func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite {
@@ -44,21 +48,20 @@ func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.
 
 func (s *tsuite) SetupSuite() {
 	s.manifests = map[string][]string{
-		"TestBasic":                       {commonManifest, basicRoutesManifest},
-		"TestRecursive":                   {commonManifest, recursiveRoutesManifest},
-		"TestCyclic":                      {commonManifest, cyclicRoutesManifest},
-		"TestInvalidChild":                {commonManifest, invalidChildRoutesManifest},
-		"TestHeaderQueryMatch":            {commonManifest, headerQueryMatchRoutesManifest},
-		"TestMultipleParents":             {commonManifest, multipleParentsManifest},
-		"TestInvalidChildValidStandalone": {commonManifest, invalidChildValidStandaloneManifest},
-		"TestUnresolvedChild":             {commonManifest, unresolvedChildManifest},
-		"TestRouteOptions":                {commonManifest, routeOptionsManifest},
-		"TestMatcherInheritance":          {commonManifest, matcherInheritanceManifest},
+		"TestBasic":                       {basicRoutesManifest},
+		"TestRecursive":                   {recursiveRoutesManifest},
+		"TestCyclic":                      {cyclicRoutesManifest},
+		"TestInvalidChild":                {invalidChildRoutesManifest},
+		"TestHeaderQueryMatch":            {headerQueryMatchRoutesManifest},
+		"TestMultipleParents":             {multipleParentsManifest},
+		"TestInvalidChildValidStandalone": {invalidChildValidStandaloneManifest},
+		"TestUnresolvedChild":             {unresolvedChildManifest},
+		"TestRouteOptions":                {routeOptionsManifest},
+		"TestMatcherInheritance":          {matcherInheritanceManifest},
 	}
 	// Not every resource that is applied needs to be verified. We are not testing `kubectl apply`,
 	// but the below code demonstrates how it can be done if necessary
 	s.manifestObjects = map[string][]client.Object{
-		commonManifest:                      {proxyService, proxyDeployment, defaults.CurlPod, httpbinTeam1, httpbinTeam2, gateway},
 		basicRoutesManifest:                 {routeRoot, routeTeam1, routeTeam2},
 		cyclicRoutesManifest:                {routeRoot, routeTeam1, routeTeam2},
 		recursiveRoutesManifest:             {routeRoot, routeTeam1, routeTeam2},
@@ -70,13 +73,56 @@ func (s *tsuite) SetupSuite() {
 		routeOptionsManifest:                {routeRoot, routeTeam1, routeTeam2},
 		matcherInheritanceManifest:          {routeParent1, routeParent2, routeTeam1},
 	}
-	clients, err := gloogateway.NewResourceClients(s.ctx, s.ti.ClusterContext)
-	s.Require().NoError(err)
-	s.ti.ResourceClients = clients
+
+	s.commonResources = []client.Object{
+		// resources from manifest
+		httpbinTeam1Service, httpbinTeam1Deployment, httpbinTeam2Service, httpbinTeam2Deployment, gateway,
+		// deployer-generated resources
+		proxyDeployment, proxyService,
+	}
+
+	// set up common resources once
+	err := s.ti.Actions.Kubectl().ApplyFile(s.ctx, commonManifest)
+	s.Require().NoError(err, "can apply common manifest")
+	s.ti.Assertions.EventuallyObjectsExist(s.ctx, s.commonResources...)
+	// make sure pods are running
+	s.ti.Assertions.EventuallyPodsRunning(s.ctx, httpbinTeam1Deployment.GetNamespace(), metav1.ListOptions{
+		LabelSelector: "app=httpbin,version=v1",
+	})
+	s.ti.Assertions.EventuallyPodsRunning(s.ctx, httpbinTeam2Deployment.GetNamespace(), metav1.ListOptions{
+		LabelSelector: "app=httpbin,version=v2",
+	})
+	s.ti.Assertions.EventuallyPodsRunning(s.ctx, proxyMeta.GetNamespace(), metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("app.kubernetes.io/name=%s", proxyMeta.GetName()),
+	})
+
+	// set up curl once
+	err = s.ti.Actions.Kubectl().ApplyFile(s.ctx, defaults.CurlPodManifest)
+	s.Require().NoError(err, "can apply curl pod manifest")
+	s.ti.Assertions.EventuallyPodsRunning(s.ctx, defaults.CurlPod.GetNamespace(), metav1.ListOptions{
+		LabelSelector: defaults.CurlPodLabelSelector,
+	})
 }
 
 func (s *tsuite) TearDownSuite() {
-	// nothing at the moment
+	// clean up curl
+	err := s.ti.Actions.Kubectl().DeleteFileSafe(s.ctx, defaults.CurlPodManifest)
+	s.Require().NoError(err, "can delete curl pod manifest")
+	s.ti.Assertions.EventuallyObjectsNotExist(s.ctx, defaults.CurlPod)
+
+	// clean up common resources
+	err = s.ti.Actions.Kubectl().DeleteFileSafe(s.ctx, commonManifest)
+	s.Require().NoError(err, "can delete common manifest")
+	s.ti.Assertions.EventuallyObjectsNotExist(s.ctx, s.commonResources...)
+	s.ti.Assertions.EventuallyPodsNotExist(s.ctx, httpbinTeam1Deployment.GetNamespace(), metav1.ListOptions{
+		LabelSelector: "app=httpbin,version=v1",
+	})
+	s.ti.Assertions.EventuallyPodsNotExist(s.ctx, httpbinTeam2Deployment.GetNamespace(), metav1.ListOptions{
+		LabelSelector: "app=httpbin,version=v2",
+	})
+	s.ti.Assertions.EventuallyPodsNotExist(s.ctx, proxyMeta.GetNamespace(), metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("app.kubernetes.io/name=%s", proxyMeta.GetName()),
+	})
 }
 
 func (s *tsuite) BeforeTest(suiteName, testName string) {

test/kubernetes/e2e/features/route_delegation/testdata/common.yaml

@@ -4,9 +4,6 @@
 # - team1/httpbin
 # - team2/httpbin
 #
-# Pods:
-# - curl/curl
-#
 # Services:
 # - team1/svc1
 # - team2/svc2
@@ -126,27 +123,6 @@ spec:
 ---
 apiVersion: v1
 kind: Namespace
-metadata:
-  name: curl
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: curl
-  namespace: curl
-  labels:
-    app: curl
-spec:
-  containers:
-    - name: curl
-      image: curlimages/curl:7.83.1
-      imagePullPolicy: IfNotPresent
-      command:
-        - "sleep"
-        - "365d"
----
-apiVersion: v1
-kind: Namespace
 metadata:
   name: infra
 ---

test/kubernetes/e2e/features/route_delegation/types.go

@@ -12,28 +12,33 @@ import (
 )
 
 const (
-	// ref: test/kubernetes/e2e/features/delegation/testdata/common.yaml
 	gatewayPort = 8080
 )
 
-// ref: test/kubernetes/e2e/features/delegation/testdata/common.yaml
+// ref: common.yaml
 var (
 	commonManifest = filepath.Join(util.MustGetThisDir(), "testdata", "common.yaml")
-	proxyMeta      = metav1.ObjectMeta{
-		Name:      "gloo-proxy-http-gateway",
-		Namespace: "infra",
-	}
-	proxyDeployment = &appsv1.Deployment{ObjectMeta: proxyMeta}
-	proxyService    = &corev1.Service{ObjectMeta: proxyMeta}
-	proxyHostPort   = fmt.Sprintf("%s.%s.svc:%d", proxyService.Name, proxyService.Namespace, gatewayPort)
 
-	httpbinTeam1 = &appsv1.Deployment{
+	// resources from common manifest
+	httpbinTeam1Service = &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "svc1",
+			Namespace: "team1",
+		},
+	}
+	httpbinTeam1Deployment = &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "httpbin",
 			Namespace: "team1",
 		},
 	}
-	httpbinTeam2 = &appsv1.Deployment{
+	httpbinTeam2Service = &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "svc2",
+			Namespace: "team2",
+		},
+	}
+	httpbinTeam2Deployment = &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "httpbin",
 			Namespace: "team2",
@@ -45,9 +50,18 @@ var (
 			Namespace: "infra",
 		},
 	}
+
+	// resources produced by deployer when Gateway is applied
+	proxyMeta = metav1.ObjectMeta{
+		Name:      "gloo-proxy-http-gateway",
+		Namespace: "infra",
+	}
+	proxyDeployment = &appsv1.Deployment{ObjectMeta: proxyMeta}
+	proxyService    = &corev1.Service{ObjectMeta: proxyMeta}
+	proxyHostPort   = fmt.Sprintf("%s.%s.svc:%d", proxyService.Name, proxyService.Namespace, gatewayPort)
 )
 
-// ref: test/kubernetes/e2e/features/delegation/testdata/basic.yaml
+// ref: basic.yaml
 var (
 	routeRoot = &gwv1.HTTPRoute{
 		ObjectMeta: metav1.ObjectMeta{
@@ -85,7 +99,7 @@ var (
 	pathTeam2 = "anything/team2/foo"
 )
 
-// ref: test/kubernetes/e2e/features/route_delegation/inputs/invalid_child_valid_standalone.yaml
+// ref: invalid_child_valid_standalone.yaml
 var (
 	gatewayTestPort = 8090
 

test/kubernetes/testutils/assertions/pods.go

@@ -45,3 +45,22 @@ func (p *Provider) EventuallyPodsMatches(
 		WithPolling(pollingInterval).
 		Should(gomega.Succeed(), fmt.Sprintf("Failed to match pod in namespace %s", podNamespace))
 }
+
+// EventuallyPodsNotExist asserts that the pod(s) are no longer present
+func (p *Provider) EventuallyPodsNotExist(
+	ctx context.Context,
+	podNamespace string,
+	listOpt metav1.ListOptions,
+	timeout ...time.Duration,
+) {
+	currentTimeout, pollingInterval := helper.GetTimeouts(timeout...)
+
+	p.Gomega.Eventually(func(g gomega.Gomega) {
+		pods, err := p.clusterContext.Clientset.CoreV1().Pods(podNamespace).List(ctx, listOpt)
+		g.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to list pods")
+		g.Expect(pods.Items).To(gomega.BeEmpty(), "Pods found")
+	}).
+		WithTimeout(currentTimeout).
+		WithPolling(pollingInterval).
+		Should(gomega.Succeed(), fmt.Sprintf("Failed to match pod in namespace %s", podNamespace))
+}