feat: basic ingress controller health checks implementation (#1227)

## Summary

Injects the bootstrap config health checks into pomerium core. This will
force the health probes to wait to succeed until the bootstrap
controller has run successfully at least once. This has the intended
side of effect of always failing on startup if the ingress controller
fails to construct the pomerium config from the pomerium CR - but that
would be expected.

Adds the health check probes to deployment and the probes kustomize
file.

Requires pomerium/pomerium#5822

## Related issues


[ENG-2850](https://linear.app/pomerium/issue/ENG-2850/implement-ingress-controller-health-checks)
<!-- For example...
Fixes #159
-->


## Checklist

- [X] reference any related issues
- [X] updated docs 
  -  docs PR will follow to pomerium/documentation
- [X] updated unit tests
- [X] updated UPGRADING.md
- [X] add appropriate tag (`improvement` / `bug` / etc)
- [X] ready for review

Author: alexandreLamarre

cmd/all_in_one.go

@@ -25,6 +25,7 @@ import (
 	"github.com/pomerium/pomerium/config"
 	"github.com/pomerium/pomerium/pkg/grpc/databroker"
 	"github.com/pomerium/pomerium/pkg/grpcutil"
+	"github.com/pomerium/pomerium/pkg/health"
 	"github.com/pomerium/pomerium/pkg/netutil"
 	"github.com/pomerium/pomerium/pkg/telemetry/trace"
 
@@ -35,6 +36,7 @@ import (
 	"github.com/pomerium/ingress-controller/pomerium"
 	pomerium_ctrl "github.com/pomerium/ingress-controller/pomerium/ctrl"
 	"github.com/pomerium/ingress-controller/util"
+	health_ctrl "github.com/pomerium/ingress-controller/util/health"
 )
 
 type allCmdOptions struct {
@@ -47,10 +49,12 @@ type allCmdOptions struct {
 	configControllerShutdownTimeout time.Duration
 	// metricsBindAddress must be externally accessible host:port
 	metricsBindAddress string `validate:"required,hostname_port"`
-	serverAddr         string `validate:"required,hostname_port"`
-	sshAddr            string
-	httpRedirectAddr   string `validate:"required,hostname_port"`
-	deriveTLS          string `validate:"required,hostname"`
+	// healthProbeBindAddress must be externally accessible host:port
+	healthProbeBindAddress string
+	serverAddr             string `validate:"required,hostname_port"`
+	sshAddr                string
+	httpRedirectAddr       string `validate:"required,hostname_port"`
+	deriveTLS              string `validate:"required,hostname"`
 }
 
 type allCmdParam struct {
@@ -118,6 +122,7 @@ func (s *allCmd) setupFlags() error {
 	flags.BoolVar(&s.debugPomerium, debugPomerium, false, "enable debug logging for pomerium")
 	flags.BoolVar(&s.debugEnvoy, debugEnvoy, false, "enable debug logging for envoy")
 	flags.StringVar(&s.metricsBindAddress, metricsBindAddress, "", "host:port for aggregate metrics. host is mandatory")
+	flags.StringVar(&s.healthProbeBindAddress, healthProbeBindAddress, "127.0.0.1:28080", "host:port for http health probes")
 	flags.StringVar(&s.adminBindAddr, debugAdminBindAddr, "", "host:port for admin server")
 	flags.StringVar(&s.serverAddr, "server-addr", ":8443", "the address the HTTPS server would bind to")
 	flags.StringVar(&s.sshAddr, "ssh-addr", "", "the address the SSH server would bind to")
@@ -195,6 +200,10 @@ func (s *allCmdParam) run(ctx context.Context) error {
 	defer cancel()
 
 	ctx = trace.NewContext(ctx, trace.NewSyncClient(nil))
+	ctx = health.Context(
+		ctx,
+		health_ctrl.SettingsBootstrapReconciler,
+	)
 
 	eg, ctx := errgroup.WithContext(ctx)
 	cfgCtl := util.NewRestartOnChange[*config.Config]()
@@ -244,6 +253,7 @@ func (s *allCmdParam) makeBootstrapConfig(opt allCmdOptions) error {
 	s.ingressMetricsAddr = fmt.Sprintf("localhost:%s", ports[7])
 
 	s.cfg.Options.MetricsAddr = opt.metricsBindAddress
+	s.cfg.Options.HealthCheckAddr = opt.healthProbeBindAddress
 
 	s.cfg.MetricsScrapeEndpoints = []config.MetricsScrapeEndpoint{
 		{
@@ -383,11 +393,11 @@ func (s *allCmdParam) runBootstrapConfigController(ctx context.Context, reconcil
 	if err != nil {
 		return fmt.Errorf("manager: %w", err)
 	}
-	name := "bootstrap"
+	name := settings.ControllerNameBootstrap
 	if host, err := os.Hostname(); err == nil {
 		name = fmt.Sprintf("%s pod/%s", name, host)
 	}
-	if err := settings.NewSettingsController(mgr, reconciler, s.settings, name, false); err != nil {
+	if err := settings.NewSettingsController(mgr, reconciler, s.settings, name, false, health_ctrl.SettingsBootstrapReconciler); err != nil {
 		return fmt.Errorf("settings controller: %w", err)
 	}
 	return mgr.Start(ctx)

config/pomerium/deployment/args.yaml

@@ -12,6 +12,7 @@ spec:
             - --pomerium-config=global
             - --update-status-from-service=$(POMERIUM_NAMESPACE)/pomerium-proxy
             - --metrics-bind-address=$(POD_IP):9090
+            - --health-probe-bind-address=$(POD_IP):28080
           env:
             - name: POMERIUM_NAMESPACE
               valueFrom:

config/pomerium/deployment/healthcheck.yaml

@@ -7,15 +7,21 @@ spec:
     spec:
       containers:
         - name: pomerium
+          startupProbe:
+            httpGet:
+              path: /startupz
+              port : 28080
           livenessProbe:
             httpGet:
               path: /healthz
-              port: 8081
+              port: 28080
             initialDelaySeconds: 15
-            periodSeconds: 20
+            periodSeconds: 60
+            failureThreshold: 10
           readinessProbe:
             httpGet:
               path: /readyz
-              port: 8081
-            initialDelaySeconds: 5
-            periodSeconds: 10
+              port: 28080
+            initialDelaySeconds: 15
+            periodSeconds: 60
+            failureThreshold: 5

controllers/config_controller.go

@@ -20,6 +20,7 @@ import (
 	"github.com/pomerium/ingress-controller/controllers/reporter"
 	"github.com/pomerium/ingress-controller/controllers/settings"
 	"github.com/pomerium/ingress-controller/pomerium"
+	health_ctrl "github.com/pomerium/ingress-controller/util/health"
 )
 
 const (
@@ -78,7 +79,7 @@ func (c *Controller) RunLeased(ctx context.Context) (err error) {
 		return fmt.Errorf("create ingress controller: %w", err)
 	}
 	if c.GlobalSettings != nil {
-		if err = settings.NewSettingsController(mgr, c.ConfigReconciler, *c.GlobalSettings, "pomerium-crd", true); err != nil {
+		if err = settings.NewSettingsController(mgr, c.ConfigReconciler, *c.GlobalSettings, "pomerium-crd", true, health_ctrl.SettingsReconciler); err != nil {
 			return fmt.Errorf("create settings controller: %w", err)
 		}
 	} else {

controllers/settings/controller.go

@@ -15,6 +15,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
 	pom_cfg "github.com/pomerium/pomerium/config"
+	"github.com/pomerium/pomerium/pkg/health"
 
 	icsv1 "github.com/pomerium/ingress-controller/apis/ingress/v1"
 	"github.com/pomerium/ingress-controller/controllers/deps"
@@ -25,6 +26,12 @@ import (
 	"github.com/pomerium/ingress-controller/util/generic"
 )
 
+const (
+	// ControllerNameBootstrap is a default name for the bootstrap config settings
+	// controller
+	ControllerNameBootstrap = "bootstrap"
+)
+
 type settingsController struct {
 	// key kind/name of a settings object to watch, all others would be ignored
 	key model.Key
@@ -38,6 +45,8 @@ type settingsController struct {
 	reporter.MultiPomeriumStatusReporter
 	// emitWarnings related to configuration. as there are multiple controllers running, not all should report
 	emitWarnings bool
+
+	ctrlCheck health.Check
 }
 
 // NewSettingsController creates and registers a new controller for
@@ -48,6 +57,7 @@ func NewSettingsController(
 	name types.NamespacedName,
 	controllerName string,
 	emitWarnings bool,
+	check health.Check,
 ) error {
 	if name.Namespace != "" {
 		return fmt.Errorf("pomerium CRD is cluster-scoped")
@@ -80,8 +90,8 @@ func NewSettingsController(
 			&reporter.SettingsLogReporter{},
 		},
 		emitWarnings: emitWarnings,
+		ctrlCheck:    check,
 	}
-
 	secretKind := generic.GVKForType[*corev1.Secret](mgr.GetScheme()).Kind
 	err := ctrl.NewControllerManagedBy(mgr).
 		Named(controllerName).
@@ -120,6 +130,8 @@ func (c *settingsController) Reconcile(ctx context.Context, req ctrl.Request) (c
 		c.SettingsRejected(ctx, &cfg.Pomerium, err)
 		return ctrl.Result{Requeue: true}, fmt.Errorf("get settings: %w", err)
 	}
+	// bootstrap config must at least construct a valid config once to be considered running
+	health.ReportRunning(c.ctrlCheck)
 
 	if deprecations, err := icsv1.GetDeprecations(&cfg.Pomerium.Spec); err != nil {
 		logger.Error(err, "checking config for deprecations")

controllers/settings/integration_test.go

@@ -26,6 +26,7 @@ import (
 	controllers_mock "github.com/pomerium/ingress-controller/controllers/mock"
 	"github.com/pomerium/ingress-controller/controllers/settings"
 	"github.com/pomerium/ingress-controller/pomerium"
+	health_ctrl "github.com/pomerium/ingress-controller/util/health"
 )
 
 var (
@@ -97,7 +98,7 @@ func (s *ControllerTestSuite) createTestController(ctx context.Context, reconcil
 		Scheme: s.Environment.Scheme,
 	})
 	s.NoError(err)
-	s.NoError(settings.NewSettingsController(mgr, reconciler, name, "test", false))
+	s.NoError(settings.NewSettingsController(mgr, reconciler, name, "test", false, health_ctrl.SettingsReconciler))
 
 	go func() {
 		if err = mgr.Start(ctx); err != nil && ctx.Err() == nil {

deployment.yaml

@@ -962,6 +962,7 @@ spec:
         - --pomerium-config=global
         - --update-status-from-service=$(POMERIUM_NAMESPACE)/pomerium-proxy
         - --metrics-bind-address=$(POD_IP):9090
+        - --health-probe-bind-address=$(POD_IP):28080
         env:
         - name: TMPDIR
           value: /tmp

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/martinlindhe/base36 v1.1.1
 	github.com/open-policy-agent/opa v1.8.0
 	github.com/pomerium/csrf v1.7.0
-	github.com/pomerium/pomerium v0.28.1-0.20250911195602-b3eb7b5ee438
+	github.com/pomerium/pomerium v0.28.1-0.20250915161701-b1114f9fb1d7
 	github.com/rs/zerolog v1.34.0
 	github.com/sergi/go-diff v1.4.0
 	github.com/spf13/cobra v1.10.1

go.sum

@@ -651,8 +651,8 @@ github.com/pomerium/datasource v0.18.2-0.20221108160055-c6134b5ed524 h1:3YQY1sb5
 github.com/pomerium/datasource v0.18.2-0.20221108160055-c6134b5ed524/go.mod h1:7fGbUYJnU8RcxZJvUvhukOIBv1G7LWDAHMfDxAf5+Y0=
 github.com/pomerium/envoy-custom v1.34.5-p1 h1:6ES5N96BnztDhgeTbsvvCTacHrzZkrrqeR/c2JZMb7E=
 github.com/pomerium/envoy-custom v1.34.5-p1/go.mod h1:+wpbZvum83bq/OD4cp9/8IZiMV6boBkwDhlFPLOoWoI=
-github.com/pomerium/pomerium v0.28.1-0.20250911195602-b3eb7b5ee438 h1:d5CfZCT5pFh7QAW3hOvkQNFrV3GwG9j0+hFLdbn44SQ=
-github.com/pomerium/pomerium v0.28.1-0.20250911195602-b3eb7b5ee438/go.mod h1:q1/EiUkvJ6GouvqkaY1lSOX4dKjYkze2AGA5Fz8gr2A=
+github.com/pomerium/pomerium v0.28.1-0.20250915161701-b1114f9fb1d7 h1:86oP9xj2gW0uh0Y3gAcHxMeu+GMb5x3EDmQsoW6lifk=
+github.com/pomerium/pomerium v0.28.1-0.20250915161701-b1114f9fb1d7/go.mod h1:q1/EiUkvJ6GouvqkaY1lSOX4dKjYkze2AGA5Fz8gr2A=
 github.com/pomerium/protoutil v0.0.0-20240813175624-47b7ac43ff46 h1:NRTg8JOXCxcIA1lAgD74iYud0rbshbWOB3Ou4+Huil8=
 github.com/pomerium/protoutil v0.0.0-20240813175624-47b7ac43ff46/go.mod h1:QqZmx6ZgPxz18va7kqoT4t/0yJtP7YFIDiT/W2n2fZ4=
 github.com/pomerium/webauthn v0.0.0-20240603205124-0428df511172 h1:TqoPqRgXSHpn+tEJq6H72iCS5pv66j3rPprThUEZg0E=

util/health/check.go

@@ -0,0 +1,13 @@
+// Package health encapsulates extensions to pomerium core's pkg/health
+package health
+
+import (
+	"github.com/pomerium/pomerium/pkg/health"
+)
+
+const (
+	// SettingsBootstrapReconciler checks that the bootstrap reconciler has run once successfully
+	SettingsBootstrapReconciler = health.Check("controller.settings.reconciler.bootstrap")
+	// SettingsReconciler checks that the leased settings reconciler has run
+	SettingsReconciler = health.Check("controller.settings.reconciler")
+)