fix(sync): avoid O(n^2) envoy validation during initial sync

DataBrokerReconciler.Set() validated the full merged config with the embedded
envoy subprocess once per ingress inside the merge loop, so an initial sync of
N ingresses performed N full-config validations of a growing config (plus a
final one in saveConfig). With 300+ ingresses this made pod-restart resync take
over an hour.

Build the merged config using only cheap, in-process policy/option validation
and let saveConfig perform the single full envoy validation. If that batch
validation fails, fall back to the previous incremental per-ingress full
validation to isolate and skip only the offending ingress(es).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: danielwatrous-zendesk

pomerium/sync_databroker.go

@@ -107,8 +107,53 @@ func (r *DataBrokerReconciler) Set(ctx context.Context, ics []*model.IngressConf
 	if err != nil {
 		return false, fmt.Errorf("get config: %w", err)
 	}
+
+	// Build the merged config using only cheap (non-envoy) per-ingress validation.
+	// The expensive full-config envoy validation runs a single time, in saveConfig.
+	// This avoids the O(n^2) behavior of validating a growing config once per ingress.
+	next := buildConfigCheap(ctx, ics)
+
+	changed, err := r.saveConfig(ctx, prev, next, r.ConfigID)
+	if err == nil {
+		return changed, nil
+	}
+
+	// The single full-config envoy validation failed. Rather than reject every
+	// ingress, fall back to validating ingresses incrementally to isolate and skip
+	// only the offending one(s). This path performs O(n) envoy validations, but is
+	// only reached when a batch fails to validate as a whole.
+	logger.Error(err, "batch config validation failed; falling back to incremental validation to isolate invalid ingress(es)")
+	next = r.buildConfigIncremental(ctx, ics)
+	return r.saveConfig(ctx, prev, next, r.ConfigID)
+}
+
+// buildConfigCheap merges the routes for all ingresses into a single config, using
+// only cheap (non-envoy) validation to skip individually-invalid ingresses.
+func buildConfigCheap(ctx context.Context, ics []*model.IngressConfig) *pb.Config {
+	logger := log.FromContext(ctx)
 	next := new(pb.Config)
+	for _, ic := range ics {
+		cfg := proto.Clone(next).(*pb.Config)
+		if err := multierror.Append(
+			upsertRoutes(ctx, cfg, ic),
+			validateCheap(ctx, cfg),
+		).ErrorOrNil(); err != nil {
+			logger.Error(err, "skip ingress", "ingress", fmt.Sprintf("%s/%s", ic.Namespace, ic.Name))
+			continue
+		}
+		addCerts(cfg, ic.Secrets)
+		next = cfg
+	}
+	return next
+}
 
+// buildConfigIncremental merges the routes for all ingresses into a single config,
+// running the full envoy validation after each ingress so that an ingress which
+// only fails full validation (not cheap validation) is isolated and skipped. This
+// is the pre-caching behavior, retained as a fallback for the rare batch-failure case.
+func (r *DataBrokerReconciler) buildConfigIncremental(ctx context.Context, ics []*model.IngressConfig) *pb.Config {
+	logger := log.FromContext(ctx)
+	next := new(pb.Config)
 	for _, ic := range ics {
 		cfg := proto.Clone(next).(*pb.Config)
 		if err := multierror.Append(
@@ -121,8 +166,7 @@ func (r *DataBrokerReconciler) Set(ctx context.Context, ics []*model.IngressConf
 		addCerts(cfg, ic.Secrets)
 		next = cfg
 	}
-
-	return r.saveConfig(ctx, prev, next, r.ConfigID)
+	return next
 }
 
 // SetConfig updates just the shared config settings

pomerium/sync_test.go

@@ -0,0 +1,101 @@
+package pomerium
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	"github.com/pomerium/ingress-controller/model"
+
+	_ "github.com/pomerium/ingress-controller/internal"
+)
+
+// validIngressConfig returns a minimal, valid IngressConfig for the given name/host.
+func validIngressConfig(name, host string) *model.IngressConfig {
+	pathTypePrefix := networkingv1.PathTypePrefix
+	return &model.IngressConfig{
+		AnnotationPrefix: "ingress.pomerium.io",
+		Ingress: &networkingv1.Ingress{
+			ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"},
+			Spec: networkingv1.IngressSpec{
+				Rules: []networkingv1.IngressRule{{
+					Host: host,
+					IngressRuleValue: networkingv1.IngressRuleValue{
+						HTTP: &networkingv1.HTTPIngressRuleValue{
+							Paths: []networkingv1.HTTPIngressPath{{
+								Path:     "/",
+								PathType: &pathTypePrefix,
+								Backend: networkingv1.IngressBackend{
+									Service: &networkingv1.IngressServiceBackend{
+										Name: "service",
+										Port: networkingv1.ServiceBackendPort{Name: "http"},
+									},
+								},
+							}},
+						},
+					},
+				}},
+			},
+		},
+		Services: map[types.NamespacedName]*corev1.Service{
+			{Name: "service", Namespace: "default"}: {
+				ObjectMeta: metav1.ObjectMeta{Name: "service", Namespace: "default"},
+				Spec: corev1.ServiceSpec{
+					Ports: []corev1.ServicePort{{
+						Name:       "http",
+						Protocol:   corev1.ProtocolTCP,
+						Port:       80,
+						TargetPort: intstr.FromInt(80),
+					}},
+				},
+			},
+		},
+	}
+}
+
+// invalidIngressConfig returns an IngressConfig that fails route generation
+// (empty host without the allow-empty-host annotation).
+func invalidIngressConfig(name string) *model.IngressConfig {
+	ic := validIngressConfig(name, "")
+	return ic
+}
+
+func TestBuildConfigCheap_AllValid(t *testing.T) {
+	ctx := context.Background()
+	ics := []*model.IngressConfig{
+		validIngressConfig("a", "a.localhost.pomerium.io"),
+		validIngressConfig("b", "b.localhost.pomerium.io"),
+	}
+
+	cfg := buildConfigCheap(ctx, ics)
+	require.NotNil(t, cfg)
+	assert.Len(t, cfg.GetRoutes(), 2, "both valid ingresses should produce a route")
+}
+
+func TestBuildConfigCheap_SkipsInvalid(t *testing.T) {
+	ctx := context.Background()
+	ics := []*model.IngressConfig{
+		validIngressConfig("a", "a.localhost.pomerium.io"),
+		invalidIngressConfig("bad"), // empty host -> route generation fails
+		validIngressConfig("c", "c.localhost.pomerium.io"),
+	}
+
+	cfg := buildConfigCheap(ctx, ics)
+	require.NotNil(t, cfg)
+	// The invalid ingress is skipped; the two valid ones remain.
+	assert.Len(t, cfg.GetRoutes(), 2, "invalid ingress should be skipped, valid ones kept")
+
+	hosts := map[string]bool{}
+	for _, r := range cfg.GetRoutes() {
+		hosts[r.GetFrom()] = true
+	}
+	assert.True(t, hosts["https://a.localhost.pomerium.io"], "route a should be present")
+	assert.True(t, hosts["https://c.localhost.pomerium.io"], "route c should be present")
+}

pomerium/validate.go

@@ -15,25 +15,45 @@ import (
 	"github.com/pomerium/ingress-controller/pomerium/envoy"
 )
 
-// validate validates pomerium config.
-func validate(ctx context.Context, cfg *pb.Config, id string) error {
+// validateOptions builds and validates pomerium options from the config without
+// invoking the (expensive) embedded envoy subprocess. It catches route/policy and
+// option-level errors cheaply.
+func validateOptions(ctx context.Context, cfg *pb.Config) (*config.Options, error) {
 	options := config.NewDefaultOptions()
 	options.ApplySettings(ctx, cryptutil.NewCertificatesIndex(), cfg.GetSettings())
 	options.InsecureServer = true
 
 	for _, r := range cfg.GetRoutes() {
 		p, err := config.NewPolicyFromProto(r)
 		if err != nil {
-			return err
+			return nil, err
 		}
 		err = p.Validate()
 		if err != nil {
-			return err
+			return nil, err
 		}
 		options.Policies = append(options.Policies, *p)
 	}
 
-	err := options.Validate()
+	if err := options.Validate(); err != nil {
+		return nil, err
+	}
+
+	return options, nil
+}
+
+// validateCheap validates a pomerium config without invoking the embedded envoy
+// subprocess. It is used to catch most invalid ingresses quickly, deferring the
+// single expensive full-config envoy validation to saveConfig.
+func validateCheap(ctx context.Context, cfg *pb.Config) error {
+	_, err := validateOptions(ctx, cfg)
+	return err
+}
+
+// validate validates pomerium config, including a full bootstrap validation via
+// the embedded envoy binary.
+func validate(ctx context.Context, cfg *pb.Config, id string) error {
+	options, err := validateOptions(ctx, cfg)
 	if err != nil {
 		return err
 	}