edge/regression: enable experimental k8s gateway controller (#9174)

* start gg controller from setup syncer

* use same helmignore values, deleted unused file

* copy helm templates over

* debug

* auto-provision

* run codegen

* comment out v2 install/uninstall

* gloo-gateway: re-use ControlPlane server and cache  (#9166)

* First pass at re-uusing xds cache

* scale down glood

* fix connection with control plane

* update NodeHash impelentations

* fix test

* update log line

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>

* add a changelog

* unit tests for AggregateNodeHash

* comment out

* add comment about alpha api

* add desc

* deployer: fix compile errors'

* clean up old xds code

* protect against npe

* fix lint errors

* deployer_test: simplify assertion

* remove v2 glooctl check

* boostrap: support turning on/off the k8s gateway controller

* fmt

* fix comment

* control feature by env variable, turn off for all existing tests

* disable gateway2 in helm tests for now

* add gg crds

* remove gw crds from chart

* apply crds in setup-kind

* re-enable EnableK8sGatewayController

* conformance test script

* Revert "re-enable EnableK8sGatewayController"

This reverts commit f41133b.

* use values file

* create ns

* try with prefixes

* only delete own snapshots

* try to re-enable gg

* Revert "try to re-enable gg"

This reverts commit b1f005c.

* [DNM] Helm and other merge cleanup (#9171)

* remove extra options

* remove readyz

* use same context

* Revert "use same context"

This reverts commit 71f6801.

* revert changes

* check err

* controlplane: remove unused helm templates, values, helper functions, use gloo serviceaccount

* delete unused deployer templates

* remove controlPlane helpers

* use right service account name

---------

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>

* clean up helm values and scripts

* delete gateway2 cmd

* remove glood check

* re-enable gg in most tests

* get translator name from proxy

* enable experimental k8s controller in gloo kube2e tests

* Add kubeutils for port-fowarding, not yet used

* enable experimental k8s controller in kube2e tests

* fix typo in env.go

* create StartFunc concept

* cleanup gw controller

* remove extra logs

* fall back to GE, add port back to logs

* Revert "fall back to GE, add port back to logs"

This reverts commit f4e1f59.

* Revert "get translator name from proxy"

This reverts commit 16c7d76.

* add back log, use existing const

* enable experimental k8s controller ALL kube2e tests

* goimports

* undo local change for gloo kube tests

* cleanup

* casing

* cleanup controller code

* add changelog

* fix builder

* small cleanup

* fix func signature

* undo context changes

* move contants to controller package

* move to the builder pattern entirely

* define constants externally, try to avoid import cycle

* pass context explicitly

* goimports

* improve logging in start_func

* log at end of ExecuteAsynchronousStartFuncs

* add log line at end of RunGloo

* remove kubeutils - dead code

* prefer property object over builder pattern

---------

Co-authored-by: Jenny Shu <[email protected]>
Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: npolshakova <[email protected]>
Co-authored-by: Jenny Shu <[email protected]>

Author: 4 people

changelog/v1.17.0-beta9/run-edge-and-gateway.yaml

@@ -0,0 +1,7 @@
+changelog:
+  - type: NON_USER_FACING
+    issueLink: https://github.com/solo-io/solo-projects/issues/5663
+    resolvesIssue: false
+    description: >-
+      Ensure that the k8s gateway controller can be run at the same time as the rest of the classic edge setup code.
+      Enable the experimental controller API to be executed during edge regression tests to prove that this works.

projects/gateway2/controller/controller.go

@@ -87,7 +87,11 @@ func (c *controllerBuilder) watchGw(ctx context.Context) error {
 	log := log.FromContext(ctx)
 
 	log.Info("creating deployer", "ctrlname", c.cfg.ControllerName, "server", c.cfg.ControlPlane.GetBindAddress(), "port", c.cfg.ControlPlane.GetBindPort())
-	d, err := deployer.NewDeployer(c.cfg.Mgr.GetScheme(), c.cfg.Dev, c.cfg.ControllerName, c.cfg.ControlPlane.GetBindPort())
+	d, err := deployer.NewDeployer(c.cfg.Mgr.GetScheme(), &deployer.Inputs{
+		ControllerName: c.cfg.ControllerName,
+		Dev:            c.cfg.Dev,
+		Port:           c.cfg.ControlPlane.GetBindPort(),
+	})
 	if err != nil {
 		return err
 	}
@@ -124,15 +128,15 @@ func (c *controllerBuilder) watchGw(ctx context.Context) error {
 		buildr.Owns(clientObj, opts...)
 	}
 
-	gwreconciler := &gatewayReconciler{
+	gwReconciler := &gatewayReconciler{
 		cli:           c.cfg.Mgr.GetClient(),
 		scheme:        c.cfg.Mgr.GetScheme(),
 		className:     c.cfg.GWClass,
 		autoProvision: c.cfg.AutoProvision,
 		deployer:      d,
 		kick:          c.cfg.Kick,
 	}
-	err = buildr.Complete(gwreconciler)
+	err = buildr.Complete(gwReconciler)
 	if err != nil {
 		return err
 	}

projects/gateway2/controller/main.go projects/gateway2/controller/start.go

@@ -1,47 +1,53 @@
 package controller
 
 import (
-	"os"
+	"context"
 
-	"github.com/solo-io/gloo/projects/gloo/pkg/bootstrap"
+	"github.com/solo-io/gloo/projects/gateway2/wellknown"
+
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	"github.com/solo-io/gloo/projects/gateway2/controller/scheme"
 	"github.com/solo-io/gloo/projects/gateway2/discovery"
 	"github.com/solo-io/gloo/projects/gateway2/secrets"
 	"github.com/solo-io/gloo/projects/gateway2/xds"
+	"github.com/solo-io/gloo/projects/gloo/pkg/bootstrap"
 	"github.com/solo-io/gloo/projects/gloo/pkg/syncer/sanitizer"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	apiv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
+const (
+	// AutoProvision controls whether the controller will be responsible for provisioning dynamic
+	// infrastructure for the Gateway API.
+	AutoProvision = true
+)
+
 var (
+	gatewayClass = apiv1.ObjectName(wellknown.GatewayClassName)
+
 	setupLog = ctrl.Log.WithName("setup")
 )
 
-type ControllerConfig struct {
-	// The name of the GatewayClass to watch for
-	GatewayClassName      string
-	GatewayControllerName string
-	Release               string
-	AutoProvision         bool
-	Dev                   bool
-
+type StartConfig struct {
+	Dev          bool
 	ControlPlane bootstrap.ControlPlane
 }
 
-func Start(cfg ControllerConfig) {
-	setupLog.Info("xxxxx starting gw2 controller xxxxxx")
+// Start runs the controllers responsible for processing the K8s Gateway API objects
+// It is intended to be run in a goroutine as the function will block until the supplied
+// context is cancelled
+func Start(ctx context.Context, cfg StartConfig) error {
 	var opts []zap.Opts
 	if cfg.Dev {
 		setupLog.Info("starting log in dev mode")
 		opts = append(opts, zap.UseDevMode(true))
 	}
 	ctrl.SetLogger(zap.New(opts...))
+
 	mgrOpts := ctrl.Options{
 		Scheme:           scheme.NewScheme(),
 		PprofBindAddress: "127.0.0.1:9099",
@@ -54,19 +60,17 @@ func Start(cfg ControllerConfig) {
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOpts)
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
-		os.Exit(1)
+		return err
 	}
 
 	// TODO: replace this with something that checks that we have xds snapshot ready (or that we don't need one).
 	mgr.AddReadyzCheck("ready-ping", healthz.Ping)
 
-	ctx := signals.SetupSignalHandler()
-
 	glooTranslator := newGlooTranslator(ctx)
 	var sanz sanitizer.XdsSanitizers
 	inputChannels := xds.NewXdsInputChannels()
 	xdsSyncer := xds.NewXdsSyncer(
-		cfg.GatewayControllerName,
+		wellknown.GatewayControllerName,
 		glooTranslator,
 		sanz,
 		cfg.ControlPlane.SnapshotCache,
@@ -77,48 +81,31 @@ func Start(cfg ControllerConfig) {
 	)
 	if err := mgr.Add(xdsSyncer); err != nil {
 		setupLog.Error(err, "unable to add xdsSyncer runnable")
-		os.Exit(1)
+		return err
 	}
 
-	// sam-heilbron: I don't think this is necessary, as we should have a shared cache
-	if cfg.Dev {
-		go xdsSyncer.ServeXdsSnapshots()
-	}
-
-	var gatewayClassName apiv1.ObjectName = apiv1.ObjectName(cfg.GatewayClassName)
-
-	gwcfg := GatewayConfig{
+	gwCfg := GatewayConfig{
 		Mgr:            mgr,
-		GWClass:        gatewayClassName,
-		Dev:            cfg.Dev,
-		ControllerName: cfg.GatewayControllerName,
-		AutoProvision:  cfg.AutoProvision,
+		GWClass:        gatewayClass,
+		ControllerName: wellknown.GatewayControllerName,
+		AutoProvision:  AutoProvision,
 		ControlPlane:   cfg.ControlPlane,
 		Kick:           inputChannels.Kick,
 	}
-	err = NewBaseGatewayController(ctx, gwcfg)
-
-	if err != nil {
+	if err = NewBaseGatewayController(ctx, gwCfg); err != nil {
 		setupLog.Error(err, "unable to create controller")
-		os.Exit(1)
+		return err
 	}
 
-	err = discovery.NewDiscoveryController(ctx, mgr, inputChannels)
-	if err != nil {
+	if err = discovery.NewDiscoveryController(ctx, mgr, inputChannels); err != nil {
 		setupLog.Error(err, "unable to create controller")
-		os.Exit(1)
+		return err
 	}
 
-	err = secrets.NewSecretsController(ctx, mgr, inputChannels)
-	if err != nil {
+	if err = secrets.NewSecretsController(ctx, mgr, inputChannels); err != nil {
 		setupLog.Error(err, "unable to create controller")
-		os.Exit(1)
-	}
-
-	setupLog.Info("starting manager")
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "problem running manager")
-		os.Exit(1)
+		return err
 	}
 
+	return mgr.Start(ctx)
 }

projects/gateway2/deployer/builder.go

@@ -9,9 +9,11 @@ import (
 	"io/fs"
 	"path/filepath"
 
+	"github.com/solo-io/gloo/pkg/version"
+	"github.com/solo-io/gloo/projects/gateway2/helm"
+
 	"github.com/solo-io/gloo/projects/gloo/pkg/defaults"
 
-	"github.com/solo-io/gloo/projects/gateway2/helm"
 	"github.com/solo-io/gloo/projects/gateway2/ports"
 	"golang.org/x/exp/slices"
 	"helm.sh/helm/v3/pkg/action"
@@ -36,28 +38,38 @@ type gatewayPort struct {
 	TargetPort uint16 `json:"targetPort"`
 }
 
+// A Deployer is responsible for deploying proxies
 type Deployer struct {
-	dev            bool
-	chart          *chart.Chart
-	scheme         *runtime.Scheme
-	controllerName string
-	port           int
+	chart  *chart.Chart
+	scheme *runtime.Scheme
+
+	inputs *Inputs
 }
 
-// NewDeployer builds a Deployer or returns an error if one could not be constructed
-// A Deployer is responsible for deploying proxies
-// NOTE: This constructor is flawed, as it will fall subject to the telescoping constructor anti-pattern as we
-// add more properties. We should migrate to using just the builder
-func NewDeployer(scheme *runtime.Scheme, dev bool, controllerName string, xdsPort int) (*Deployer, error) {
-	deployerOptions := []Option{
-		WithScheme(scheme),
-		WithChartFs(helm.GlooGatewayHelmChart),
-		WithControllerName(controllerName),
-		WithXdsServer(xdsPort),
-		WithDevMode(dev),
+// Inputs is the set of options used to configure the gateway deployer deployment
+type Inputs struct {
+	ControllerName string
+	Dev            bool
+	Port           int
+}
+
+// NewDeployer creates a new gateway deployer
+func NewDeployer(scheme *runtime.Scheme, inputs *Inputs) (*Deployer, error) {
+	helmChart, err := loadFs(helm.GlooGatewayHelmChart)
+	if err != nil {
+		return nil, err
+	}
+	// simulate what `helm package` in the Makefile does
+	if version.Version != version.UndefinedVersion {
+		helmChart.Metadata.AppVersion = version.Version
+		helmChart.Metadata.Version = version.Version
 	}
 
-	return BuildDeployer(deployerOptions...)
+	return &Deployer{
+		scheme: scheme,
+		chart:  helmChart,
+		inputs: inputs,
+	}, nil
 }
 
 func (d *Deployer) GetGvksToWatch(ctx context.Context) ([]schema.GroupVersionKind, error) {
@@ -128,14 +140,19 @@ func (d *Deployer) renderChartToObjects(ctx context.Context, gw *api.Gateway) ([
 				"type": "LoadBalancer",
 			},
 			"xds": map[string]any{
-				// This creates a limitation that the Deployer can only work when the ControlPlane is installed to the gloo-system
-				// namespace. We can address this in a follow-up, and we should be able to identify the namespace of the control plane programmatically
+				// The xds host/port MUST map to the Service definition for the Control Plane
+				// This is the socket address that the Proxy will connect to on startup, to receive xds updates
+				//
+				// NOTE: The current implementation in flawed in multiple ways:
+				//	1 - This assumes that the Control Plane is installed in `gloo-system`
+				//	2 - The port is the bindAddress of the Go server, but there is not a strong guarantee that that port
+				//		will always be what is exposed by the Kubernetes Service.
 				"host": fmt.Sprintf("gloo.%s.svc.%s", defaults.GlooSystem, "cluster.local"),
-				"port": d.port,
+				"port": d.inputs.Port,
 			},
 		},
 	}
-	if d.dev {
+	if d.inputs.Dev {
 		vals["develop"] = true
 	}
 	log := log.FromContext(ctx)
@@ -200,7 +217,7 @@ func (d *Deployer) GetObjsToDeploy(ctx context.Context, gw *api.Gateway) ([]clie
 
 func (d *Deployer) DeployObjs(ctx context.Context, objs []client.Object, cli client.Client) error {
 	for _, obj := range objs {
-		if err := cli.Patch(ctx, obj, client.Apply, client.ForceOwnership, client.FieldOwner(d.controllerName)); err != nil {
+		if err := cli.Patch(ctx, obj, client.Apply, client.ForceOwnership, client.FieldOwner(d.inputs.ControllerName)); err != nil {
 			return fmt.Errorf("failed to apply object %s %s: %w", obj.GetObjectKind().GroupVersionKind().String(), obj.GetName(), err)
 		}
 	}