Pass xds host/port to controller (#10688)

Signed-off-by: Jenny Shu <[email protected]>

Author: jenshu

install/helm/kgateway/templates/deployment.yaml

@@ -34,7 +34,7 @@ spec:
           image: "{{ .Values.controller.image.registry | default .Values.image.registry }}/{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Values.image.tag | default .Chart.Version }}"
           imagePullPolicy: {{ .Values.controller.image.pullPolicy | default .Values.image.pullPolicy }}
           ports:
-            - containerPort: {{ .Values.controller.service.ports.grpc | default 9977 }}
+            - containerPort: {{ .Values.controller.service.ports.grpc }}
               name: grpc-xds
               protocol: TCP
           readinessProbe:
@@ -43,7 +43,7 @@ spec:
             periodSeconds: 10
             successThreshold: 1
             tcpSocket:
-              port: 9977
+              port: {{ .Values.controller.service.ports.grpc }}
             timeoutSeconds: 1
           env:
             - name: GOMEMLIMIT
@@ -58,6 +58,10 @@ spec:
                   resource: limits.cpu
             - name: LOG_LEVEL
               value: {{ .Values.controller.logLevel | quote }}
+            - name: KGW_XDS_SERVICE_NAME
+              value: {{ include "kgateway.fullname" . }}
+            - name: KGW_XDS_SERVICE_PORT
+              value: {{ .Values.controller.service.ports.grpc | quote }}
             # TODO: Remove this once the cleanup is done. Required as the gloo-system
             # namespace is the default namespace and conformance will fail as a result.
             - name: POD_NAMESPACE

install/test/helm_test.go

@@ -39,6 +39,7 @@ import (
 	"github.com/kgateway-dev/kgateway/v2/internal/gateway/pkg/defaults"
 	"github.com/kgateway-dev/kgateway/v2/pkg/utils/kubeutils"
 	"github.com/kgateway-dev/kgateway/v2/test/gomega/matchers"
+	"github.com/kgateway-dev/kgateway/v2/test/helpers"
 	glootestutils "github.com/kgateway-dev/kgateway/v2/test/testutils"
 )
 
@@ -5037,7 +5038,7 @@ metadata:
 						})
 
 						// make sure the resource requests and limits are set in the pod template
-						deploy := getStructuredDeployment(testManifest, kubeutils.GlooDeploymentName)
+						deploy := getStructuredDeployment(testManifest, helpers.DefaultKgatewayDeploymentName)
 						glooContainer := deploy.Spec.Template.Spec.Containers[0]
 						Expect(glooContainer.Resources).To(Equal(corev1.ResourceRequirements{
 							Limits: corev1.ResourceList{
@@ -5078,7 +5079,7 @@ metadata:
 						})
 
 						// make sure the resource requests are set in the pod template
-						deploy := getStructuredDeployment(testManifest, kubeutils.GlooDeploymentName)
+						deploy := getStructuredDeployment(testManifest, helpers.DefaultKgatewayDeploymentName)
 						glooContainer := deploy.Spec.Template.Spec.Containers[0]
 						Expect(glooContainer.Resources).To(Equal(corev1.ResourceRequirements{
 							Requests: corev1.ResourceList{

install/test/k8sgateway_test.go

@@ -16,8 +16,8 @@ import (
 	"github.com/kgateway-dev/kgateway/v2/api/v1alpha1"
 	"github.com/kgateway-dev/kgateway/v2/install/utils/kuberesource"
 	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/wellknown"
-	"github.com/kgateway-dev/kgateway/v2/pkg/utils/kubeutils"
 	"github.com/kgateway-dev/kgateway/v2/test/gomega/matchers"
+	"github.com/kgateway-dev/kgateway/v2/test/helpers"
 	glootestutils "github.com/kgateway-dev/kgateway/v2/test/testutils"
 )
 
@@ -46,7 +46,7 @@ var _ = Describe("Kubernetes Gateway API integration", func() {
 			})
 
 			It("relevant resources are rendered", func() {
-				deployment := getDeployment(testManifest, namespace, kubeutils.GlooDeploymentName)
+				deployment := getDeployment(testManifest, namespace, helpers.DefaultKgatewayDeploymentName)
 				Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(1), "should have exactly 1 container")
 
 				// make sure the GatewayClass and RBAC resources exist (note, since they are all cluster-scoped, they do not have a namespace)
@@ -487,7 +487,7 @@ var _ = Describe("Kubernetes Gateway API integration", func() {
 			})
 
 			It("relevant resources are not rendered", func() {
-				deployment := getDeployment(testManifest, namespace, kubeutils.GlooDeploymentName)
+				deployment := getDeployment(testManifest, namespace, helpers.DefaultKgatewayDeploymentName)
 				Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(1), "should have exactly 1 container")
 
 				// the RBAC resources should not be rendered

internal/kgateway/controller/start.go

@@ -3,26 +3,21 @@ package controller
 import (
 	"context"
 
-	"k8s.io/client-go/rest"
-	"k8s.io/utils/ptr"
-	"sigs.k8s.io/controller-runtime/pkg/config"
-
+	envoycache "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
 	"github.com/solo-io/go-utils/contextutils"
-
-	glooschemes "github.com/kgateway-dev/kgateway/v2/pkg/schemes"
-
+	uzap "go.uber.org/zap"
+	istiokube "istio.io/istio/pkg/kube"
+	"istio.io/istio/pkg/kube/krt"
+	istiolog "istio.io/istio/pkg/log"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/rest"
+	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
-
+	"sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	czap "sigs.k8s.io/controller-runtime/pkg/log/zap"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
-
-	envoycache "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
-	uzap "go.uber.org/zap"
-	istiokube "istio.io/istio/pkg/kube"
-	"istio.io/istio/pkg/kube/krt"
-	istiolog "istio.io/istio/pkg/log"
 	apiv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/deployer"
@@ -37,6 +32,9 @@ import (
 	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/utils/krtutil"
 	"github.com/kgateway-dev/kgateway/v2/internal/kgateway/wellknown"
 	"github.com/kgateway-dev/kgateway/v2/pkg/client/clientset/versioned"
+	glooschemes "github.com/kgateway-dev/kgateway/v2/pkg/schemes"
+	"github.com/kgateway-dev/kgateway/v2/pkg/utils/kubeutils"
+	"github.com/kgateway-dev/kgateway/v2/pkg/utils/namespaces"
 )
 
 const (
@@ -51,8 +49,8 @@ type SetupOpts struct {
 
 	KrtDebugger *krt.DebugHandler
 
-	XdsHost string
-	XdsPort int32
+	// static set of global Settings
+	GlobalSettings *settings.Settings
 }
 
 var setupLog = ctrl.Log.WithName("setup")
@@ -81,7 +79,6 @@ type ControllerBuilder struct {
 	cfg         StartConfig
 	mgr         ctrl.Manager
 	isOurGw     func(gw *apiv1.Gateway) bool
-	settings    settings.Settings
 }
 
 func NewControllerBuilder(ctx context.Context, cfg StartConfig) (*ControllerBuilder, error) {
@@ -139,6 +136,7 @@ func NewControllerBuilder(ctx context.Context, cfg StartConfig) (*ControllerBuil
 		cfg.Client,
 		cli,
 		setupLog,
+		*cfg.SetupOpts.GlobalSettings,
 	)
 	gwClasses := sets.New(append(cfg.SetupOpts.ExtraGatewayClasses, wellknown.GatewayClassName)...)
 	isOurGw := func(gw *apiv1.Gateway) bool {
@@ -169,7 +167,6 @@ func NewControllerBuilder(ctx context.Context, cfg StartConfig) (*ControllerBuil
 		cfg:         cfg,
 		mgr:         mgr,
 		isOurGw:     isOurGw,
-		settings:    commoncol.Settings,
 	}, nil
 }
 
@@ -185,21 +182,22 @@ func pluginFactoryWithBuiltin(extraPlugins []extensionsplug.Plugin) extensions2.
 func (c *ControllerBuilder) Start(ctx context.Context) error {
 	logger := contextutils.LoggerFrom(ctx).Desugar()
 	logger.Info("starting gateway controller")
-	// GetXdsAddress waits for gloo-edge to populate the xds address of the server.
-	// in the future this logic may move here and be duplicated.
-	xdsHost, xdsPort := c.cfg.SetupOpts.XdsHost, c.cfg.SetupOpts.XdsPort
-	if xdsHost == "" {
-		return ctx.Err()
-	}
 
-	logger.Info("got xds address for deployer", uzap.String("xds_host", xdsHost), uzap.Int32("xds_port", xdsPort))
+	globalSettings := c.cfg.SetupOpts.GlobalSettings
+
+	xdsHost := kubeutils.ServiceFQDN(metav1.ObjectMeta{
+		Name:      globalSettings.XdsServiceName,
+		Namespace: namespaces.GetPodNamespace(),
+	})
+	xdsPort := globalSettings.XdsServicePort
+	logger.Info("got xds address for deployer", uzap.String("xds_host", xdsHost), uzap.Uint32("xds_port", xdsPort))
 
-	integrationEnabled := c.settings.EnableIstioIntegration
+	integrationEnabled := globalSettings.EnableIstioIntegration
 
 	// copy over relevant aws options (if any) from Settings
 	var awsInfo *deployer.AwsInfo
-	stsCluster := c.settings.StsClusterName
-	stsUri := c.settings.StsUri
+	stsCluster := globalSettings.StsClusterName
+	stsUri := globalSettings.StsUri
 	if stsCluster != "" && stsUri != "" {
 		awsInfo = &deployer.AwsInfo{
 			EnableServiceAccountCredentials: true,

internal/kgateway/deployer/deployer.go

@@ -53,7 +53,7 @@ type Deployer struct {
 
 type ControlPlaneInfo struct {
 	XdsHost string
-	XdsPort int32
+	XdsPort uint32
 }
 
 type AwsInfo struct {

internal/kgateway/deployer/values.go

@@ -101,7 +101,7 @@ type helmServiceAccount struct {
 // to receive xds config updates
 type helmXds struct {
 	Host *string `json:"host,omitempty"`
-	Port *int32  `json:"port,omitempty"`
+	Port *uint32 `json:"port,omitempty"`
 }
 
 type helmAutoscaling struct {

internal/kgateway/extensions2/common/krt.go

@@ -1,8 +1,6 @@
 package common
 
 import (
-	"fmt"
-
 	"github.com/go-logr/logr"
 	"istio.io/istio/pkg/kube"
 	istiokube "istio.io/istio/pkg/kube"
@@ -44,6 +42,7 @@ func NewCommonCollections(
 	client istiokube.Client,
 	ourClient versioned.Interface,
 	logger logr.Logger,
+	settings settings.Settings,
 ) *CommonCollections {
 	secretClient := kclient.New[*corev1.Secret](client)
 	k8sSecretsRaw := krt.WrapClient(secretClient, krt.WithStop(krtOptions.Stop), krt.WithName("Secrets") /* no debug here - we don't want raw secrets printed*/)
@@ -67,18 +66,13 @@ func NewCommonCollections(
 	refgrantsCol := krt.WrapClient(kclient.New[*gwv1beta1.ReferenceGrant](client), krtOptions.ToOptions("RefGrants")...)
 	refgrants := krtcollections.NewRefGrantIndex(refgrantsCol)
 
-	st, err := settings.BuildSettings()
-	if err != nil {
-		logger.Error(err, "got err while parsing Settings from env")
-	}
-	logger.Info(fmt.Sprintf("got settings from env: %+v", *st))
 	return &CommonCollections{
 		OurClient: ourClient,
 		Client:    client,
 		KrtOpts:   krtOptions,
 		Secrets:   krtcollections.NewSecretIndex(secrets, refgrants),
 		Pods:      krtcollections.NewPodsCollection(client, krtOptions),
 		RefGrants: refgrants,
-		Settings:  *st,
+		Settings:  settings,
 	}
 }

internal/kgateway/extensions2/settings/settings.go

@@ -9,6 +9,14 @@ type Settings struct {
 	EnableAutoMtls         bool   `split_words:"true"`
 	StsClusterName         string `split_words:"true"`
 	StsUri                 string `split_words:"true"`
+
+	// XdsServiceName is the name of the Kubernetes Service that serves xDS config.
+	// It it assumed to be in the kgateway install namespace.
+	XdsServiceName string `split_words:"true" default:"kgateway"`
+
+	// XdsServicePort is the port of the Kubernetes Service that serves xDS config.
+	// This corresponds to the value of the `grpc-xds` port in the service.
+	XdsServicePort uint32 `split_words:"true" default:"9977"`
 }
 
 // BuildSettings returns a zero-valued Settings obj if error is encountered when parsing env

internal/kgateway/extensions2/settings/settings_test.go

@@ -25,13 +25,15 @@ func TestSettings(t *testing.T) {
 		expectedErrorStr string
 	}{
 		{
-			name:    "defaults to empty values",
+			name:    "defaults to empty or default values",
 			envVars: map[string]string{},
 			expectedSettings: &settings.Settings{
 				EnableIstioIntegration: false,
 				EnableAutoMtls:         false,
 				StsClusterName:         "",
 				StsUri:                 "",
+				XdsServiceName:         "kgateway",
+				XdsServicePort:         9977,
 			},
 		},
 		{
@@ -41,12 +43,16 @@ func TestSettings(t *testing.T) {
 				"KGW_ENABLE_AUTO_MTLS":         "true",
 				"KGW_STS_CLUSTER_NAME":         "my-cluster",
 				"KGW_STS_URI":                  "my.sts.uri",
+				"KGW_XDS_SERVICE_NAME":         "custom-svc",
+				"KGW_XDS_SERVICE_PORT":         "1234",
 			},
 			expectedSettings: &settings.Settings{
 				EnableIstioIntegration: true,
 				EnableAutoMtls:         true,
 				StsClusterName:         "my-cluster",
 				StsUri:                 "my.sts.uri",
+				XdsServiceName:         "custom-svc",
+				XdsServicePort:         1234,
 			},
 		},
 		{
@@ -56,6 +62,13 @@ func TestSettings(t *testing.T) {
 			},
 			expectedErrorStr: "invalid syntax",
 		},
+		{
+			name: "errors on invalid port",
+			envVars: map[string]string{
+				"KGW_XDS_SERVICE_PORT": "a123",
+			},
+			expectedErrorStr: "invalid syntax",
+		},
 		{
 			name: "ignores other env vars",
 			envVars: map[string]string{
@@ -65,6 +78,8 @@ func TestSettings(t *testing.T) {
 			},
 			expectedSettings: &settings.Settings{
 				EnableAutoMtls: true,
+				XdsServiceName: "kgateway",
+				XdsServicePort: 9977,
 			},
 		},
 	}