re-enable deployer e2e tests (#10705)

Signed-off-by: Jenny Shu <[email protected]>
Co-authored-by: Nathan Fudenberg <[email protected]>

Author: jenshu

.github/workflows/pr-kubernetes-tests.yaml

@@ -28,12 +28,12 @@ jobs:
         # Feb 27, 2025: 13 minutes
         - cluster-name: 'cluster-one'
           go-test-args: '-v -timeout=25m'
-          go-test-run-regex: '^TestKgateway$$/^BasicRouting$$|^TestKgateway$$/^HTTPRouteServices$$|^TestKgateway$$/^TCPRouteServices$$'
+          go-test-run-regex: '^TestKgateway$$/^BasicRouting$$|^TestKgateway$$/^Deployer$$|^TestKgateway$$/^HTTPRouteServices$$|^TestKgateway$$/^TCPRouteServices$$'
 
 #         # Dec 4, 2024: 23 minutes
 #         - cluster-name: 'cluster-two'
 #           go-test-args: '-v -timeout=25m'
-#           go-test-run-regex: '^TestKgateway$$/^RouteDelegation$$|^TestKgatewayIstioRevision$$|^TestRevisionIstioRegression$$|^TestKgateway$$/^Deployer$$|^TestKgateway$$/^RouteOptions$$|^TestKgateway$$/^VirtualHostOptions$$|^TestKgateway$$/^Upstreams$$|^TestKgateway$$/^HeadlessSvc$$|^TestKgateway$$/^PortRouting$$|^TestKgatewayMinimalDefaultGatewayParameters$$|^TestKgateway$$/^DirectResponse$$|^TestKgateway$$/^HttpListenerOptions$$|^TestKgateway$$/^ListenerOptions$$|^TestKgateway$$/^GlooAdminServer$$'
+#           go-test-run-regex: '^TestKgateway$$/^RouteDelegation$$|^TestKgatewayIstioRevision$$|^TestRevisionIstioRegression$$|^TestKgateway$$/^RouteOptions$$|^TestKgateway$$/^VirtualHostOptions$$|^TestKgateway$$/^Upstreams$$|^TestKgateway$$/^HeadlessSvc$$|^TestKgateway$$/^PortRouting$$|^TestKgatewayMinimalDefaultGatewayParameters$$|^TestKgateway$$/^DirectResponse$$|^TestKgateway$$/^HttpListenerOptions$$|^TestKgateway$$/^ListenerOptions$$|^TestKgateway$$/^GlooAdminServer$$'
 
 #         # Dec 4, 2024: 24 minutes
 #         - cluster-name: 'cluster-three'
@@ -53,7 +53,7 @@ jobs:
 #         # Dec 4, 2024: 26 minutes
 #         - cluster-name: 'cluster-six'
 #           go-test-args: '-v -timeout=30m'
-#           go-test-run-regex: '^TestDiscoveryWatchlabels$$|^TestKgatewayNoValidation$$|^TestHelm$$|^TestHelmSettings$$|^TestKgatewayAws$$|^TestKgateway$$/^HTTPRouteServices$$|^TestKgateway$$/^TCPRouteServices$$|^TestZeroDowntimeRollout$$'
+#           go-test-run-regex: '^TestDiscoveryWatchlabels$$|^TestKgatewayNoValidation$$|^TestHelm$$|^TestHelmSettings$$|^TestKgatewayAws$$|^TestZeroDowntimeRollout$$'
 
 #         # Dec 4, 2024: 13 minutes
 #         - cluster-name: 'cluster-seven'

internal/kgateway/admin/server.go

@@ -14,10 +14,7 @@ import (
 	"istio.io/istio/pkg/kube/krt"
 
 	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/controller"
-)
-
-const (
-	AdminPort = 9097
+	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/wellknown"
 )
 
 func RunAdminServer(ctx context.Context, setupOpts *controller.SetupOpts) error {
@@ -89,7 +86,7 @@ func startHandlers(ctx context.Context, addHandlers ...func(mux *http.ServeMux,
 	mux.HandleFunc("/", idx)
 	mux.HandleFunc("/snapshots/", idx)
 	server := &http.Server{
-		Addr:    fmt.Sprintf("localhost:%d", AdminPort),
+		Addr:    fmt.Sprintf("localhost:%d", wellknown.KgatewayAdminPort),
 		Handler: mux,
 	}
 	contextutils.LoggerFrom(ctx).Infof("Admin server starting at %s", server.Addr)

internal/kgateway/extensions2/settings/settings_test.go

@@ -8,6 +8,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/extensions2/settings"
+	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/wellknown"
 )
 
 func TestSettings(t *testing.T) {
@@ -32,8 +33,8 @@ func TestSettings(t *testing.T) {
 				EnableAutoMtls:         false,
 				StsClusterName:         "",
 				StsUri:                 "",
-				XdsServiceName:         "kgateway",
-				XdsServicePort:         9977,
+				XdsServiceName:         wellknown.DefaultXdsService,
+				XdsServicePort:         wellknown.DefaultXdsPort,
 			},
 		},
 		{
@@ -78,8 +79,8 @@ func TestSettings(t *testing.T) {
 			},
 			expectedSettings: &settings.Settings{
 				EnableAutoMtls: true,
-				XdsServiceName: "kgateway",
-				XdsServicePort: 9977,
+				XdsServiceName: wellknown.DefaultXdsService,
+				XdsServicePort: wellknown.DefaultXdsPort,
 			},
 		},
 	}

internal/kgateway/wellknown/ports.go

@@ -0,0 +1,18 @@
+package wellknown
+
+// DefaultXdsService is the default name of the Kubernetes Service that serves xDS config.
+// This value should stay in sync with:
+// - the default value of `XdsServiceName` in internal/kgateway/extensions2/settings/settings.go
+// - the default Service name in install/helm/kgateway/templates/service.yaml
+const DefaultXdsService = "kgateway"
+
+// DefaultXdsPort is the default xDS port. This value should stay in sync with:
+// - the default value of `XdsServicePort` in internal/kgateway/extensions2/settings/settings.go
+// - the `controller.service.ports.grpc` value in install/helm/kgateway/values.yaml
+var DefaultXdsPort uint32 = 9977
+
+// EnvoyAdminPort is the default envoy admin port
+var EnvoyAdminPort uint32 = 19000
+
+// KgatewayAdminPort is the kgateway admin server port
+var KgatewayAdminPort uint32 = 9097

pkg/utils/controllerutils/admincli/client.go

@@ -8,7 +8,7 @@ import (
 	"github.com/rotisserie/eris"
 	"github.com/solo-io/go-utils/threadsafe"
 
-	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/admin"
+	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/wellknown"
 	"github.com/kgateway-dev/kgateway/v2/pkg/utils/cmdutils"
 	"github.com/kgateway-dev/kgateway/v2/pkg/utils/requestutils/curl"
 )
@@ -36,7 +36,7 @@ func NewClient() *Client {
 		curlOptions: []curl.Option{
 			curl.WithScheme("http"),
 			curl.WithHost("127.0.0.1"),
-			curl.WithPort(admin.AdminPort),
+			curl.WithPort(int(wellknown.KgatewayAdminPort)),
 			// 3 retries, exponential back-off, 10 second max
 			curl.WithRetries(3, 0, 10),
 		},

pkg/utils/envoyutils/admincli/client.go

@@ -12,12 +12,15 @@ import (
 	listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	"github.com/solo-io/go-utils/threadsafe"
 
+	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/wellknown"
 	"github.com/kgateway-dev/kgateway/v2/pkg/utils/cmdutils"
+	"github.com/kgateway-dev/kgateway/v2/pkg/utils/kubeutils/kubectl"
+	"github.com/kgateway-dev/kgateway/v2/pkg/utils/kubeutils/portforward"
 	"github.com/kgateway-dev/kgateway/v2/pkg/utils/protoutils"
 	"github.com/kgateway-dev/kgateway/v2/pkg/utils/requestutils/curl"
 
-	"github.com/kgateway-dev/kgateway/v2/pkg/utils/kubeutils/kubectl"
-	"github.com/kgateway-dev/kgateway/v2/pkg/utils/kubeutils/portforward"
+	// import for side effects; this is needed so we can unmarshal envoy types
+	_ "github.com/kgateway-dev/kgateway/v2/internal/envoyinit/hack/filter_types"
 )
 
 const (
@@ -32,8 +35,6 @@ const (
 	ServerInfoPath     = "server_info"
 )
 
-var envoyAdminPort uint32 = 19000
-
 // DumpOptions should have flags for any kind of underlying optional
 // filtering or inclusion of Envoy dump data, such as including EDS, filters, etc.
 type DumpOptions struct {
@@ -58,7 +59,7 @@ func NewClient() *Client {
 		curlOptions: []curl.Option{
 			curl.WithScheme("http"),
 			curl.WithHost("127.0.0.1"),
-			curl.WithPort(int(envoyAdminPort)),
+			curl.WithPort(int(wellknown.EnvoyAdminPort)),
 			// 3 retries, exponential back-off, 10 second max
 			curl.WithRetries(3, 0, 10),
 		},
@@ -77,7 +78,7 @@ func NewPortForwardedClient(ctx context.Context, proxySelector, namespace string
 	// 1. Open a port-forward to the Kubernetes Deployment, so that we can query the Envoy Admin API directly
 	portForwarder, err := kubectl.NewCli().StartPortForward(ctx,
 		selector,
-		portforward.WithRemotePort(int(envoyAdminPort)))
+		portforward.WithRemotePort(int(wellknown.EnvoyAdminPort)))
 	if err != nil {
 		return nil, nil, err
 	}

pkg/utils/kubeutils/kubectl/cli.go

@@ -218,9 +218,9 @@ func (c *Cli) StartPortForward(ctx context.Context, options ...portforward.Optio
 	err := portForwarder.Start(
 		ctx,
 		retry.LastErrorOnly(true),
-		retry.Delay(100*time.Millisecond),
-		retry.DelayType(retry.BackOffDelay),
-		retry.Attempts(5),
+		retry.Delay(250*time.Millisecond),
+		retry.DelayType(retry.FixedDelay),
+		retry.Attempts(60),
 	)
 	return portForwarder, err
 }

pkg/utils/kubeutils/portforward/cli_portforwarder.go

@@ -56,7 +56,6 @@ func (c *cliPortForwarder) startOnce(ctx context.Context) error {
 	}
 
 	cmdCtx, cmdCancel := context.WithCancel(ctx)
-	defer cmdCancel() // paranoia for vet
 	c.cmd = exec.CommandContext(
 		cmdCtx,
 		"kubectl",

test/helpers/kube_dump.go

@@ -15,7 +15,7 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	"github.com/solo-io/go-utils/threadsafe"
 
-	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/admin"
+	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/wellknown"
 	kgatewayAdminCli "github.com/kgateway-dev/kgateway/v2/pkg/utils/controllerutils/admincli"
 	"github.com/kgateway-dev/kgateway/v2/pkg/utils/envoyutils/admincli"
 	"github.com/kgateway-dev/kgateway/v2/pkg/utils/kubeutils/kubectl"
@@ -344,7 +344,7 @@ func ControllerDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, outLog i
 				// Open a port-forward to the controller pod's admin port
 				portForwarder, err := kubectlCli.StartPortForward(ctx,
 					portforward.WithPod(podName, ns),
-					portforward.WithPorts(int(admin.AdminPort), int(admin.AdminPort)),
+					portforward.WithPorts(int(wellknown.KgatewayAdminPort), int(wellknown.KgatewayAdminPort)),
 				)
 				if err != nil {
 					fmt.Printf("error starting port forward to controller admin port: %f\n", err)
@@ -359,7 +359,7 @@ func ControllerDumpOnFail(ctx context.Context, kubectlCli *kubectl.Cli, outLog i
 					WithReceiver(io.Discard).
 					WithCurlOptions(
 						curl.WithRetries(3, 0, 10),
-						curl.WithPort(int(admin.AdminPort)),
+						curl.WithPort(int(wellknown.KgatewayAdminPort)),
 					)
 
 				krtSnapshotFile := fileAtPath(filepath.Join(namespaceOutDir, fmt.Sprintf("%s.krt_snapshot.log", podName)))

test/kubernetes/e2e/features/deployer/minimal_default_gatewayparameters_suite.go

@@ -1,5 +1,3 @@
-//go:build ignore
-
 package deployer
 
 import (

test/kubernetes/e2e/features/deployer/suite.go

@@ -1,13 +1,10 @@
-//go:build ignore
-
 package deployer
 
 import (
 	"context"
 	"fmt"
 	"time"
 
-	tlsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	"github.com/onsi/gomega"
 	"github.com/onsi/gomega/gstruct"
 	"github.com/stretchr/testify/assert"
@@ -20,10 +17,8 @@ import (
 	gwv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	"github.com/kgateway-dev/kgateway/v2/api/v1alpha1"
-	"github.com/kgateway-dev/kgateway/v2/internal/gloo/pkg/utils"
 	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/wellknown"
 	"github.com/kgateway-dev/kgateway/v2/pkg/utils/envoyutils/admincli"
-	"github.com/kgateway-dev/kgateway/v2/pkg/utils/kubeutils"
 	"github.com/kgateway-dev/kgateway/v2/test/kubernetes/e2e"
 	testdefaults "github.com/kgateway-dev/kgateway/v2/test/kubernetes/e2e/defaults"
 )
@@ -38,7 +33,7 @@ type testingSuite struct {
 	ctx context.Context
 
 	// testInstallation contains all the metadata/utilities necessary to execute a series of tests
-	// against an installation of Gloo Gateway
+	// against an installation of kgateway
 	testInstallation *e2e.TestInstallation
 
 	// manifests maps test name to a list of manifests to apply before the test
@@ -77,10 +72,6 @@ func (s *testingSuite) SetupSuite() {
 		"TestSelfManagedGateway": {
 			selfManagedGateway,
 		},
-		"TestConfigureAwsLambda": {
-			testdefaults.NginxPodManifest,
-			gatewayWithoutParameters,
-		},
 	}
 	s.manifestObjects = map[string][]client.Object{
 		testdefaults.NginxPodManifest: {testdefaults.NginxPod, testdefaults.NginxSvc},
@@ -111,6 +102,14 @@ func (s *testingSuite) AfterTest(suiteName, testName string) {
 		s.Require().NoError(err)
 		s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, s.manifestObjects[manifest]...)
 	}
+
+	// make sure the proxy pods are gone before the next test starts
+	s.testInstallation.Assertions.EventuallyPodsNotExist(
+		s.ctx,
+		s.testInstallation.Metadata.InstallNamespace,
+		metav1.ListOptions{
+			LabelSelector: "app.kubernetes.io/name=gw",
+		})
 }
 
 func (s *testingSuite) TestProvisionDeploymentAndService() {
@@ -123,7 +122,7 @@ func (s *testingSuite) TestConfigureProxiesFromGatewayParameters() {
 	// check that the labels and annotations got passed through from GatewayParameters to the ServiceAccount
 	sa := &corev1.ServiceAccount{}
 	err := s.testInstallation.ClusterContext.Client.Get(s.ctx,
-		types.NamespacedName{Name: glooProxyObjectMeta.Name, Namespace: glooProxyObjectMeta.Namespace},
+		types.NamespacedName{Name: proxyObjectMeta.Name, Namespace: proxyObjectMeta.Namespace},
 		sa)
 	s.Require().NoError(err)
 	s.testInstallation.Assertions.Gomega.Expect(sa.GetLabels()).To(
@@ -134,7 +133,7 @@ func (s *testingSuite) TestConfigureProxiesFromGatewayParameters() {
 	// check that the labels and annotations got passed through from GatewayParameters to the Service
 	svc := &corev1.Service{}
 	err = s.testInstallation.ClusterContext.Client.Get(s.ctx,
-		types.NamespacedName{Name: glooProxyObjectMeta.Name, Namespace: glooProxyObjectMeta.Namespace},
+		types.NamespacedName{Name: proxyObjectMeta.Name, Namespace: proxyObjectMeta.Namespace},
 		svc)
 	s.Require().NoError(err)
 	s.testInstallation.Assertions.Gomega.Expect(svc.GetLabels()).To(
@@ -155,22 +154,12 @@ func (s *testingSuite) TestConfigureProxiesFromGatewayParameters() {
 
 	s.testInstallation.Assertions.AssertEnvoyAdminApi(
 		s.ctx,
-		proxyDeployment.ObjectMeta,
+		proxyObjectMeta,
 		serverInfoLogLevelAssertion(s.testInstallation, "debug", "connection:trace,upstream:debug"),
 		xdsClusterAssertion(s.testInstallation),
 	)
 }
 
-func (s *testingSuite) TestConfigureAwsLambda() {
-	s.testInstallation.Assertions.EventuallyReadyReplicas(s.ctx, proxyDeployment.ObjectMeta, gomega.Equal(1))
-
-	s.testInstallation.Assertions.AssertEnvoyAdminApi(
-		s.ctx,
-		proxyDeployment.ObjectMeta,
-		awsStsClusterAssertion(s.testInstallation),
-	)
-}
-
 func (s *testingSuite) TestProvisionResourcesUpdatedWithValidParameters() {
 	s.testInstallation.Assertions.EventuallyReadyReplicas(s.ctx, proxyDeployment.ObjectMeta, gomega.Equal(1))
 
@@ -224,14 +213,13 @@ func (s *testingSuite) TestProvisionResourcesNotUpdatedWithInvalidParameters() {
 		g.Expect(proxyDeployment.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(origPrivileged)
 		g.Expect(proxyDeployment.Spec.Replicas).To(gstruct.PointTo(gomega.Equal(int32(1))))
 	}, "30s", "1s").Should(gomega.Succeed())
-
 }
 
 func (s *testingSuite) TestSelfManagedGateway() {
 	s.Require().EventuallyWithT(func(c *assert.CollectT) {
 		gw := &gwv1.Gateway{}
 		err := s.testInstallation.ClusterContext.Client.Get(s.ctx,
-			types.NamespacedName{Name: glooProxyObjectMeta.Name, Namespace: glooProxyObjectMeta.Namespace},
+			types.NamespacedName{Name: proxyObjectMeta.Name, Namespace: proxyObjectMeta.Namespace},
 			gw)
 		assert.NoError(c, err, "gateway not found")
 
@@ -314,59 +302,12 @@ func xdsClusterAssertion(testInstallation *e2e.TestInstallation) func(ctx contex
 			xdsSocketAddress := xdsCluster.GetLoadAssignment().GetEndpoints()[0].GetLbEndpoints()[0].GetEndpoint().GetAddress().GetSocketAddress()
 			g.Expect(xdsSocketAddress).NotTo(gomega.BeNil())
 
-			g.Expect(xdsSocketAddress.GetAddress()).To(gomega.Equal(kubeutils.ServiceFQDN(metav1.ObjectMeta{
-				Name:      "kgateway",
-				Namespace: testInstallation.Metadata.InstallNamespace,
-			})), "xds socket address points to gloo service, in installation namespace")
-
-			g.Expect(xdsSocketAddress.GetPortValue()).To(gomega.Equal(9977), "xds socket port points to gloo service, in installation namespace")
-		}).
-			WithContext(ctx).
-			WithTimeout(time.Second * 10).
-			WithPolling(time.Millisecond * 200).
-			Should(gomega.Succeed())
-	}
-}
-
-// awsStsClusterAssertion asserts that:
-// - if the installation is configured to use aws with service account creds, then the proxy contains an aws sts cluster with the expected sts uri
-// - if the installation is NOT configured to use aws with service account creds, then no aws sts cluster should exist on the proxy
-func awsStsClusterAssertion(testInstallation *e2e.TestInstallation) func(ctx context.Context, adminClient *admincli.Client) {
-	// get aws values from installation
-	awsOpts := testInstallation.Metadata.AwsOptions
-	shouldHaveStsCluster := awsOpts != nil && awsOpts.EnableServiceAccountCredentials
-	var expectedStsUri string
-	if shouldHaveStsCluster {
-		expectedStsUri = fmt.Sprintf("sts.%s.amazonaws.com", awsOpts.StsCredentialsRegion)
-	}
-
-	return func(ctx context.Context, adminClient *admincli.Client) {
-		testInstallation.Assertions.Gomega.Eventually(func(g gomega.Gomega) {
-			clusters, err := adminClient.GetStaticClusters(ctx)
-			g.Expect(err).NotTo(gomega.HaveOccurred(), "can get static clusters from config dump")
-
-			awsStsCluster, ok := clusters["aws_sts_cluster"]
-			if shouldHaveStsCluster {
-				g.Expect(ok).To(gomega.BeTrue(), "should contain cluster aws_sts_cluster")
-			} else {
-				g.Expect(ok).To(gomega.BeFalse(), "should not contain cluster aws_sts_cluster")
-				// nothing else to test, so return here
-				return
-			}
+			g.Expect(xdsSocketAddress.GetAddress()).To(gomega.Equal(
+				fmt.Sprintf("%s.%s.svc.cluster.local", wellknown.DefaultXdsService, testInstallation.Metadata.InstallNamespace),
+			), "xds socket address points to kgateway service, in installation namespace")
 
-			// check that transport socket has expected values
-			msg, err := utils.AnyToMessage(awsStsCluster.GetTransportSocket().GetTypedConfig())
-			g.Expect(err).NotTo(gomega.HaveOccurred())
-			tlsCtx, ok := msg.(*tlsv3.UpstreamTlsContext)
-			g.Expect(ok).To(gomega.BeTrue(), "should be able to get UpstreamTlsContext")
-			g.Expect(tlsCtx.GetSni()).To(gomega.Equal(expectedStsUri))
-
-			// check that load assignment has expected values
-			g.Expect(awsStsCluster.GetLoadAssignment().GetEndpoints()).To(gomega.HaveLen(1))
-			g.Expect(awsStsCluster.GetLoadAssignment().GetEndpoints()[0].GetLbEndpoints()).To(gomega.HaveLen(1))
-			socketAddr := awsStsCluster.GetLoadAssignment().GetEndpoints()[0].GetLbEndpoints()[0].GetEndpoint().GetAddress().GetSocketAddress()
-			g.Expect(socketAddr).NotTo(gomega.BeNil())
-			g.Expect(socketAddr.GetAddress()).To(gomega.Equal(expectedStsUri))
+			g.Expect(xdsSocketAddress.GetPortValue()).To(gomega.Equal(wellknown.DefaultXdsPort),
+				"xds socket port points to kgateway service, in installation namespace")
 		}).
 			WithContext(ctx).
 			WithTimeout(time.Second * 10).