Refactors initial commit

Signed-off-by: Daneyon Hansen <[email protected]>

Author: danehans

changelog/v1.18.0-beta26/issue_10092.yaml changelog/v1.18.0-beta34/issue_10092.yaml

@@ -1,9 +1,9 @@
 changelog:
-  - type: NEW_FEATURE
+  - type: FIX
     issueLink: https://github.com/solo-io/gloo/issues/10092
     resolvesIssue: true
     description: >-
       Adds support for the GatewayClass `SupportedVersion` status condition and updates the ReconcileGatewayClasses()
-      method to manage status conditions according to the Gateway API specification. The gateway controller will now
-      reconcile supported Gateway API CRDs and set the `SupportedVersion` status condition based on the value of the
-      `gateway.networking.k8s.io/bundle-version` annotation.
+      method to manage status conditions according to the Gateway API specification. The gateway controller will
+      reconcile supported Gateway API CRDs and set the GatewayClass `SupportedVersion` status condition based on the
+      value of the `gateway.networking.k8s.io/bundle-version` annotation.

pkg/schemes/extended_scheme.go

@@ -1,20 +1,21 @@
 package schemes
 
 import (
+	"context"
 	"fmt"
 
-	"github.com/solo-io/gloo/projects/gateway2/wellknown"
-	"k8s.io/apimachinery/pkg/api/meta"
+	istiokube "istio.io/istio/pkg/kube"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/client-go/discovery"
-	"k8s.io/client-go/rest"
-
 	gwv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
+
+	"github.com/solo-io/gloo/projects/gateway2/crds"
+	"github.com/solo-io/gloo/projects/gateway2/wellknown"
 )
 
 // AddGatewayV1A2Scheme adds the gwv1a2 scheme to the provided scheme if the TCPRoute CRD exists.
-func AddGatewayV1A2Scheme(restConfig *rest.Config, scheme *runtime.Scheme) error {
-	exists, err := CRDExists(restConfig, gwv1a2.GroupVersion.Group, gwv1a2.GroupVersion.Version, wellknown.TCPRouteKind)
+func AddGatewayV1A2Scheme(ctx context.Context, cli istiokube.Client, scheme *runtime.Scheme) error {
+	exists, err := crdExists(ctx, cli, crds.TCPRoute)
 	if err != nil {
 		return fmt.Errorf("error checking if %s CRD exists: %w", wellknown.TCPRouteKind, err)
 	}
@@ -28,27 +29,11 @@ func AddGatewayV1A2Scheme(restConfig *rest.Config, scheme *runtime.Scheme) error
 	return nil
 }
 
-// Helper function to check if a CRD exists
-func CRDExists(restConfig *rest.Config, group, version, kind string) (bool, error) {
-	discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig)
+// crdExists queries the Kubernetes API for the provided CRD name.
+func crdExists(ctx context.Context, cli istiokube.Client, name string) (bool, error) {
+	_, err := cli.Dynamic().Resource(crds.GVR).Get(ctx, name, metav1.GetOptions{})
 	if err != nil {
 		return false, err
 	}
-
-	groupVersion := fmt.Sprintf("%s/%s", group, version)
-	apiResourceList, err := discoveryClient.ServerResourcesForGroupVersion(groupVersion)
-	if err != nil {
-		if discovery.IsGroupDiscoveryFailedError(err) || meta.IsNoMatchError(err) {
-			return false, nil
-		}
-		return false, err
-	}
-
-	for _, apiResource := range apiResourceList.APIResources {
-		if apiResource.Kind == kind {
-			return true, nil
-		}
-	}
-
-	return false, nil
+	return true, nil
 }

projects/controller/cli/pkg/cmd/install/kubegateway/install.go

@@ -7,7 +7,7 @@ import (
 	"github.com/solo-io/gloo/pkg/utils/kubeutils"
 	"github.com/solo-io/gloo/projects/controller/cli/pkg/cmd/options"
 	"github.com/solo-io/gloo/projects/controller/cli/pkg/kubegatewayutils"
-	"github.com/solo-io/gloo/projects/gateway2/crds"
+	"github.com/solo-io/gloo/projects/gateway2/crds/experimental"
 	"github.com/solo-io/gloo/projects/gateway2/deployer"
 	"helm.sh/helm/v3/pkg/cli"
 	"helm.sh/helm/v3/pkg/cli/values"
@@ -64,7 +64,7 @@ func install(opts *options.Options, installOpts *Options) error {
 
 	if !crdsExist {
 		fmt.Printf("Applying Gateway CRDs... ")
-		crds, err := deployer.ConvertYAMLToObjects(cli.Scheme(), crds.GatewayCrds)
+		crds, err := deployer.ConvertYAMLToObjects(cli.Scheme(), experimental.GatewayCrds)
 		if err != nil {
 			fmt.Printf("Failed\n")
 			return err

projects/controller/cli/pkg/cmd/install/kubegateway/uninstall.go

@@ -7,7 +7,7 @@ import (
 	"github.com/solo-io/gloo/pkg/utils/kubeutils"
 
 	"github.com/solo-io/gloo/projects/controller/cli/pkg/cmd/options"
-	"github.com/solo-io/gloo/projects/gateway2/crds"
+	"github.com/solo-io/gloo/projects/gateway2/crds/experimental"
 	"github.com/solo-io/gloo/projects/gateway2/deployer"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -51,7 +51,7 @@ func uninstall(opts *options.Options, installOpts *Options) error {
 	}
 
 	fmt.Printf("Deleting Gateway CRDs... ")
-	crds, err := deployer.ConvertYAMLToObjects(cli.Scheme(), crds.GatewayCrds)
+	crds, err := deployer.ConvertYAMLToObjects(cli.Scheme(), experimental.GatewayCrds)
 	if err != nil {
 		fmt.Printf("Failed\n")
 	} else {

projects/gateway2/controller/controller.go

@@ -4,12 +4,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"maps"
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	discoveryv1 "k8s.io/api/discovery/v1"
 	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
-	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -61,8 +61,8 @@ type GatewayConfig struct {
 	Aws                     *deployer.AwsInfo
 
 	Extensions extensions.K8sGatewayExtensions
-	// CRDs defines the set of discovered Gateway API CRDs
-	CRDs sets.Set[string]
+	// CRDs is a map of supported Gateway API CRDs with the discovered annotations for each.
+	CRDs *crds.CrdToAnnotation
 }
 
 func NewBaseGatewayController(ctx context.Context, cfg GatewayConfig) error {
@@ -76,6 +76,7 @@ func NewBaseGatewayController(ctx context.Context, cfg GatewayConfig) error {
 			cli:            cfg.Mgr.GetClient(),
 			scheme:         cfg.Mgr.GetScheme(),
 			kick:           cfg.Kick,
+			crds:           cfg.CRDs,
 		},
 	}
 
@@ -135,9 +136,11 @@ func (c *controllerBuilder) addIndexes(ctx context.Context) error {
 	}
 
 	// Conditionally index for TCPRoute
-	if c.cfg.CRDs.Has(wellknown.TCPRouteCRD) {
-		if err := c.cfg.Mgr.GetFieldIndexer().IndexField(ctx, &apiv1a2.TCPRoute{}, query.TcpRouteTargetField, query.IndexerByObjType); err != nil {
-			errs = append(errs, err)
+	if c.cfg.CRDs != nil {
+		if _, exists := (*c.cfg.CRDs)[crds.TCPRoute]; exists {
+			if err := c.cfg.Mgr.GetFieldIndexer().IndexField(ctx, &apiv1a2.TCPRoute{}, query.TcpRouteTargetField, query.IndexerByObjType); err != nil {
+				errs = append(errs, err)
+			}
 		}
 	}
 
@@ -297,8 +300,8 @@ func (c *controllerBuilder) watchGwClass(_ context.Context) error {
 		Complete(reconcile.Func(c.reconciler.ReconcileGatewayClasses))
 }
 
-// watchCustomResourceDefinitions sets up a controller to watch for changes to specific Gateway API
-// CRDs and triggers GatewayClass reconciliation if generation or annotations change.
+// watchCRDs sets up a controller to watch for changes to supported Gateway API CRDs
+// and triggers GatewayClass reconciliation if generation or annotations change.
 func (c *controllerBuilder) watchCRDs(_ context.Context) error {
 	return ctrl.NewControllerManagedBy(c.cfg.Mgr).
 		For(&apiextv1.CustomResourceDefinition{}).
@@ -308,7 +311,20 @@ func (c *controllerBuilder) watchCRDs(_ context.Context) error {
 				return false
 			}
 			// Check if the CRD is one we care about
-			return c.cfg.CRDs.Has(crd.Name)
+			if !crds.IsSupported(crd.Name) {
+				return false
+			}
+			// Maintain the set of supported CRDs
+			if c.cfg.CRDs != nil {
+				annotations, exist := (*c.cfg.CRDs)[crd.Name]
+				if !exist {
+					(*c.cfg.CRDs)[crd.Name] = make(map[string]string)
+				}
+				if !maps.Equal(annotations, crd.Annotations) {
+					(*c.cfg.CRDs)[crd.Name] = crd.Annotations
+				}
+			}
+			return true
 		})).
 		WithEventFilter(predicate.Or(
 			predicate.AnnotationChangedPredicate{},
@@ -325,8 +341,13 @@ func (c *controllerBuilder) watchHttpRoute(_ context.Context) error {
 }
 
 func (c *controllerBuilder) watchTcpRoute(ctx context.Context) error {
-	if !c.cfg.CRDs.Has(wellknown.TCPRouteCRD) {
-		log.FromContext(ctx).Info("TCPRoute type not registered in scheme; skipping TCPRoute controller setup")
+	log := log.FromContext(ctx)
+
+	if c.cfg.CRDs == nil {
+		log.Info("CRD annotations map not initialized; skipping TCPRoute controller setup")
+	}
+	if _, exists := (*c.cfg.CRDs)[crds.TCPRoute]; !exists {
+		log.Info("TCPRoute type not registered in scheme; skipping TCPRoute controller setup")
 		return nil
 	}
 
@@ -424,6 +445,8 @@ type controllerReconciler struct {
 	cli            client.Client
 	scheme         *runtime.Scheme
 	kick           func(ctx context.Context)
+	// crds is a map of supported Gateway API CRDs with the discovered annotations for each.
+	crds *crds.CrdToAnnotation
 }
 
 func (r *controllerReconciler) ReconcileHttpListenerOptions(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -576,32 +599,25 @@ func (r *controllerReconciler) ReconcileGatewayClasses(ctx context.Context, req
 	log.Info("Reconciling GatewayClass")
 
 	// Initialize the status conditions. No need to set LastTransitionTime since it's handled by SetStatusCondition.
-	msg := "Gateway API CRDs are a supported version"
 	acceptedCondition := metav1.Condition{
 		Type:               string(apiv1.GatewayClassConditionStatusAccepted),
 		Status:             metav1.ConditionTrue,
 		Reason:             string(apiv1.GatewayClassReasonAccepted),
-		Message:            msg,
+		Message:            "All dependencies have been met",
 		ObservedGeneration: gwclass.Generation,
 	}
 	supportedCondition := metav1.Condition{
 		Type:               string(apiv1.GatewayClassConditionStatusSupportedVersion),
 		Status:             metav1.ConditionTrue,
 		Reason:             string(apiv1.GatewayClassReasonSupportedVersion),
-		Message:            msg,
+		Message:            "Gateway API CRDs are a supported version",
 		ObservedGeneration: gwclass.Generation,
 	}
 
-	// Check CRD versions
-	supported, err := r.checkCRDVersions(ctx)
-	if err != nil {
-		log.Error(err, "Failed to check CRD versions")
-		return ctrl.Result{}, err
-	}
-
-	if !supported {
+	// Set status conditions based on observed CRD versions
+	if !r.crdsSupportedVersion(ctx) {
 		// Update the values of status conditions
-		msg = fmt.Sprintf("Unsupported Gateway API CRDs detected. Supported versions are: %s",
+		msg := fmt.Sprintf("Unsupported Gateway API CRDs detected. Supported versions are: %s",
 			strings.Join(wellknown.SupportedVersions, ", "))
 		acceptedCondition.Status = metav1.ConditionFalse
 		acceptedCondition.Reason = string(apiv1.GatewayClassReasonUnsupportedVersion)
@@ -625,22 +641,31 @@ func (r *controllerReconciler) ReconcileGatewayClasses(ctx context.Context, req
 	return ctrl.Result{}, nil
 }
 
-// checkCRDVersions checks that the "gateway.networking.k8s.io/bundle-version" annotation key is set
-// to a supported version for each required Gateway API CRD.
-func (r *controllerReconciler) checkCRDVersions(ctx context.Context) (bool, error) {
-	for _, crdName := range crds.Required {
-		crd := &apiextv1.CustomResourceDefinition{}
-		if err := r.cli.Get(ctx, client.ObjectKey{Name: crdName}, crd); err != nil {
-			if kerrors.IsNotFound(err) {
-				return false, nil
-			}
-			return false, err
+// crdsSupportedVersion returns true if the "gateway.networking.k8s.io/bundle-version" annotation key is
+// set to a supported version for each managed Gateway API CRD.
+func (r *controllerReconciler) crdsSupportedVersion(ctx context.Context) bool {
+	log := log.FromContext(ctx)
+
+	if r.crds == nil {
+		return false
+	}
+
+	for crdName, annotations := range *r.crds {
+		if annotations == nil || len(annotations) == 0 {
+			log.Info("no annotations found for CRD", "name", crdName)
+			return false
 		}
 
-		bundleVersion, exists := crd.Annotations[consts.BundleVersionAnnotation]
-		if !exists || !crds.IsSupportedVersion(bundleVersion) {
-			return false, nil
+		bundleVersion, exists := annotations[consts.BundleVersionAnnotation]
+		if !exists {
+			log.Info("bundle version annotation does not exist for CRD", "name", crdName)
+			return false
+		}
+		if !crds.IsSupportedVersion(bundleVersion) {
+			log.Info("bundle version annotation not a supported version for CRD", "name", crdName, "version", bundleVersion)
+			return false
 		}
 	}
-	return true, nil
+
+	return true
 }

projects/gateway2/controller/controller_suite_test.go

@@ -17,8 +17,11 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/solo-io/gloo/projects/gateway2/api/v1alpha1"
 	"github.com/solo-io/gloo/projects/gateway2/controller"
+	"github.com/solo-io/gloo/projects/gateway2/crds"
 	"github.com/solo-io/gloo/projects/gateway2/extensions"
+	"github.com/solo-io/gloo/projects/gateway2/setup"
 	"github.com/solo-io/gloo/projects/gateway2/wellknown"
+	istiokube "istio.io/istio/pkg/kube"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -38,6 +41,7 @@ import (
 var (
 	cfg       *rest.Config
 	k8sClient client.Client
+	i8sClient istiokube.Client
 	testEnv   *envtest.Environment
 	ctx       context.Context
 	cancel    context.CancelFunc
@@ -86,11 +90,16 @@ var _ = BeforeSuite(func() {
 	cfg, err = testEnv.Start()
 	Expect(err).NotTo(HaveOccurred())
 	Expect(cfg).NotTo(BeNil())
-	scheme := schemes.DefaultScheme()
+
+	scheme := schemes.TestingScheme()
 	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme})
 	Expect(err).NotTo(HaveOccurred())
 	Expect(k8sClient).NotTo(BeNil())
 
+	i8sClient, err = setup.CreateKubeClient(cfg)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(i8sClient).NotTo(BeNil())
+
 	webhookInstallOptions := &testEnv.WebhookInstallOptions
 	mgrOpts := ctrl.Options{
 		Scheme: scheme,
@@ -117,13 +126,19 @@ var _ = BeforeSuite(func() {
 		Mgr: mgr,
 	})
 	Expect(err).ToNot(HaveOccurred())
+
+	crdMap, err := crds.GetGatewayCRDs(ctx, i8sClient)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(crdMap).NotTo(BeNil())
+
 	cfg := controller.GatewayConfig{
 		Mgr:            mgr,
 		ControllerName: gatewayControllerName,
 		GWClasses:      gwClasses,
 		AutoProvision:  true,
 		Kick:           func(ctx context.Context) { return },
 		Extensions:     exts,
+		CRDs:           crdMap,
 	}
 	err = controller.NewBaseGatewayController(ctx, cfg)
 	Expect(err).ToNot(HaveOccurred())