Mutations of `HTTPRoute`s belonging to other controllers

[imax9000]

Ingress controller is unable to co-exist with any other Gateway API controller in the same cluster: it updates status of all routes regardless of their Gateway's class name. This results in two controllers hammering the HTTPRoute's status with constant updates, with ingress controller complaining about overloaded envoy and OOMing after a few minutes.

Exhibit A:

ingress-controller/controllers/gateway/fetch.go

Lines 69 to 83 in 0c6d986

	o.HTTPRoutesByGateway = make(map[refKey][]httpRouteInfo)
	for i := range hrl.Items {
	hr := &hrl.Items[i]
	o.OriginalHTTPRouteStatus = append(o.OriginalHTTPRouteStatus,
	httpRouteAndOriginalStatus{route: hr, originalStatus: hr.Status.DeepCopy()})
	ensureRouteParentStatusExists(hr, c.ControllerName)
	for j := range hr.Spec.ParentRefs {
	pr := &hr.Spec.ParentRefs[j]
	key := refKeyForParentRef(hr, pr)
	if _, ok := o.Gateways[key]; ok {
	o.HTTPRoutesByGateway[key] = append(o.HTTPRoutesByGateway[key],
	httpRouteInfo{hr, pr, &hr.Status.Parents[j]})
	}
	}
	}

On line 74 it adds an incomplete status to the HTTPRoute struct, after adding a pointer to it to OriginalHTTPRouteStatus. That loop is supposed to filter out routes that belong to other controllers, but, unfortunately, that is done later on line 78.

Exhibit B:

ingress-controller/controllers/gateway/gateway.go

Line 33 in 0c6d986

if err := c.updateModifiedHTTPRouteStatus(ctx, o.OriginalHTTPRouteStatus); err != nil {

That happens in the function processGateways that is called right after fetchObject (which is Exhibit A), overwriting route's status with an incomplete stub.

[imax9000]

Potential fix:

--- a/controllers/gateway/fetch.go
+++ b/controllers/gateway/fetch.go
@@ -69,17 +69,38 @@ func (c *gatewayController) fetchObjects(ctx context.Context) (*objects, error)
 	o.HTTPRoutesByGateway = make(map[refKey][]httpRouteInfo)
 	for i := range hrl.Items {
 		hr := &hrl.Items[i]
-		o.OriginalHTTPRouteStatus = append(o.OriginalHTTPRouteStatus,
-			httpRouteAndOriginalStatus{route: hr, originalStatus: hr.Status.DeepCopy()})
-		ensureRouteParentStatusExists(hr, c.ControllerName)
+		matchingParents := map[int]bool{}
 		for j := range hr.Spec.ParentRefs {
 			pr := &hr.Spec.ParentRefs[j]
 			key := refKeyForParentRef(hr, pr)
 			if _, ok := o.Gateways[key]; ok {
-				o.HTTPRoutesByGateway[key] = append(o.HTTPRoutesByGateway[key],
-					httpRouteInfo{hr, pr, &hr.Status.Parents[j]})
+				matchingParents[j] = true
 			}
 		}
+		if len(matchingParents) == 0 {
+			continue
+		}
+
+		o.OriginalHTTPRouteStatus = append(o.OriginalHTTPRouteStatus,
+			httpRouteAndOriginalStatus{route: hr, originalStatus: hr.Status.DeepCopy()})
+
+		// Ensure that the output slice is long enough.
+		if len(hr.Status.Parents) < len(hr.Spec.ParentRefs) {
+			s := make([]gateway_v1.RouteParentStatus, len(hr.Spec.ParentRefs))
+			copy(s, hr.Status.Parents)
+			hr.Status.Parents = s
+		}
+
+		// Initialize status entries and store them in HTTPRoutesByGateway.
+		for j := range matchingParents {
+			pr := &hr.Spec.ParentRefs[j]
+			hr.Status.Parents[j].ParentRef = *pr
+			hr.Status.Parents[j].ControllerName = gateway_v1.GatewayController(c.ControllerName)
+
+			key := refKeyForParentRef(hr, pr)
+			o.HTTPRoutesByGateway[key] = append(o.HTTPRoutesByGateway[key],
+				httpRouteInfo{hr, pr, &hr.Status.Parents[j]})
+		}
 	}
 
 	// Fetch all Namespaces (the labels may be needed for the allowedRoutes restrictions).
diff --git a/controllers/gateway/httproute.go b/controllers/gateway/httproute.go
index bebec45..3026924 100644
--- a/controllers/gateway/httproute.go
+++ b/controllers/gateway/httproute.go
@@ -5,7 +5,6 @@ import (
 
 	"github.com/hashicorp/go-set/v3"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
@@ -276,28 +275,3 @@ func (b backendRefSet) Valid(obj client.Object, r *gateway_v1.BackendRef) bool {
 	}
 	return b.c.Contains(refKeyForBackendRef(obj, &r.BackendObjectReference))
 }
-
-// ensureRouteParentStatusExists ensures that the elements of r.Status.Parents correspond to the
-// elements of r.Spec.ParentRef.
-func ensureRouteParentStatusExists(r *gateway_v1.HTTPRoute, controllerName string) {
-	// Check to see if the parent status items already match.
-	if len(r.Status.Parents) == len(r.Spec.ParentRefs) {
-		ok := true
-		for i := range r.Spec.ParentRefs {
-			if !equality.Semantic.DeepEqual(r.Spec.ParentRefs[i], r.Status.Parents[i].ParentRef) {
-				ok = false
-				break
-			}
-		}
-		if ok {
-			return
-		}
-	}
-
-	// Allocate new parent status items.
-	r.Status.Parents = make([]gateway_v1.RouteParentStatus, len(r.Spec.ParentRefs))
-	for i := range r.Status.Parents {
-		r.Status.Parents[i].ParentRef = r.Spec.ParentRefs[i]
-		r.Status.Parents[i].ControllerName = gateway_v1.GatewayController(controllerName)
-	}
-}

---

UPDATE: also this, to stop it from constantly triggering no-op config changes

--- a/pomerium/sync.go
+++ b/pomerium/sync.go
@@ -2,8 +2,10 @@
 package pomerium
 
 import (
+       "bytes"
        "context"
        "fmt"
+       "slices"
        "sort"
 
        "github.com/hashicorp/go-multierror"
@@ -198,6 +200,12 @@ func (r *DataBrokerReconciler) saveConfig(ctx context.Context, prev, next *pb.Co
        // envoy matches according to the order routes are present in the configuration
        sort.Sort(routeList(next.Routes))
 
+       if next.GetSettings().GetCertificates() != nil {
+               slices.SortFunc(next.Settings.Certificates, func(a, b *pb.Settings_Certificate) int {
+                       return bytes.Compare(a.CertBytes, b.CertBytes)
+               })
+       }
+
        if err := validate(ctx, next, id); err != nil {
                return false, fmt.Errorf("config validation: %w", err)
        }