add support for k8s leader election (#1386)

kenjenkins · web-flow · commit 05ce0e8a4581 · 2026-04-30T10:55:59.000-07:00
In principle, when ingress-controller runs in the standalone mode and 
uses the new unified API sync, it does not need a direct connection to
the Core databroker. However, ingress-controller does currently rely on
Core for leader election.

Add support to instead use Kubernetes leader election in this case. Add
new command-line flags to allow customizing the lease ID and namespace.
Rename the `acquire-databroker-lease` health check to just
`acquire-lease` to reflect that the lease being acquired is not
necessarily a databroker lease. Refactor the kustomize configuration so
that we can add permissions for acquiring a k8s lease object only within
the 'pomerium' namespace.
diff --git a/cmd/controller.go b/cmd/controller.go
@@ -34,6 +34,9 @@ type controllerCmd struct {
 	tlsInsecureSkipVerify      bool
 	tlsOverrideCertificateName string
 
+	leaderElectionID        string
+	leaderElectionNamespace string
+
 	sharedSecret string
 
 	debug bool
@@ -62,6 +65,8 @@ const (
 	databrokerTLSCA            = "databroker-tls-ca"
 	tlsInsecureSkipVerify      = "databroker-tls-insecure-skip-verify"
 	tlsOverrideCertificateName = "databroker-tls-override-certificate-name"
+	leaderElectionID           = "leader-election-id"
+	leaderElectionNamespace    = "leader-election-namespace"
 )
 
 func (s *controllerCmd) setupFlags() error {
@@ -76,6 +81,8 @@ func (s *controllerCmd) setupFlags() error {
 		"disable remote hosts TLS certificate chain and hostname check for the databroker connection")
 	flags.StringVar(&s.tlsOverrideCertificateName, tlsOverrideCertificateName, "",
 		"override the certificate name used for the databroker connection")
+	flags.StringVar(&s.leaderElectionID, leaderElectionID, "pomerium-ingress-controller", "leader election lease name")
+	flags.StringVar(&s.leaderElectionNamespace, leaderElectionNamespace, "", "leader election lease namespace")
 
 	flags.StringVar(&s.sharedSecret, sharedSecret, "",
 		"base64-encoded shared secret for signing JWTs")
@@ -99,7 +106,7 @@ func (s *controllerCmd) exec(*cobra.Command, []string) error {
 
 	eg, ctx := errgroup.WithContext(ctx)
 	eg.Go(func() error {
-		return runHealthz(ctx, s.probeAddr, healthz.NamedCheck("acquire-databroker-lease", c.ReadyzCheck))
+		return runHealthz(ctx, s.probeAddr, healthz.NamedCheck("acquire-lease", c.ReadyzCheck))
 	})
 	eg.Go(func() error { return c.Run(ctx) })
 
@@ -141,37 +148,40 @@ func (s *controllerCmd) buildController(ctx context.Context) (*controllers.Contr
 		return nil, fmt.Errorf("get scheme: %w", err)
 	}
 
-	conn, err := s.getDataBrokerConnection(ctx)
+	globalSettings, err := s.getGlobalSettings()
 	if err != nil {
-		return nil, fmt.Errorf("databroker connection: %w", err)
-	}
-	client := databroker.NewDataBrokerServiceClient(conn)
-	var reconciler pomerium.Reconciler
-	if s.SyncAPIURL != "" {
-		reconciler = pomerium.NewAPIReconciler(s.SyncAPIURL, s.SyncAPIToken)
-	} else {
-		reconciler = pomerium.NewDataBrokerReconciler(client, s.debug)
+		return nil, err
 	}
 
 	c := &controllers.Controller{
-		Reconciler:              reconciler,
-		DataBrokerServiceClient: client,
 		MgrOpts: ctrl.Options{
-			Scheme:         scheme,
-			Metrics:        metricsserver.Options{BindAddress: s.metricsAddr},
-			LeaderElection: false,
+			Scheme:  scheme,
+			Metrics: metricsserver.Options{BindAddress: s.metricsAddr},
 			Controller: config.Controller{
 				SkipNameValidation: ptr.To(true),
 			},
 		},
 		IngressCtrlOpts:         opts,
 		GatewayControllerConfig: gatewayConfig,
+		GlobalSettings:          globalSettings,
 	}
 
-	c.GlobalSettings, err = s.getGlobalSettings()
+	if s.SyncAPIURL != "" {
+		c.Reconciler = pomerium.NewAPIReconciler(s.SyncAPIURL, s.SyncAPIToken)
+		c.MgrOpts.LeaderElection = true
+		c.MgrOpts.LeaderElectionID = s.leaderElectionID
+		c.MgrOpts.LeaderElectionNamespace = s.leaderElectionNamespace
+		return c, nil
+	} else if f := s.Flags(); f.Changed(leaderElectionID) || f.Changed(leaderElectionNamespace) {
+		return nil, fmt.Errorf("kubernetes leader election can be used only with sync API")
+	}
+
+	conn, err := s.getDataBrokerConnection(ctx)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("databroker connection: %w", err)
 	}
 
+	c.DataBrokerServiceClient = databroker.NewDataBrokerServiceClient(conn)
+	c.Reconciler = pomerium.NewDataBrokerReconciler(c.DataBrokerServiceClient, s.debug)
 	return c, nil
 }
diff --git a/config/pomerium/rbac/cluster_role.yaml b/config/pomerium/rbac/cluster_role.yaml
@@ -0,0 +1,73 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: pomerium-controller
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+      - endpoints
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - services/status
+      - secrets/status
+      - endpoints/status
+    verbs:
+      - get
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses
+      - ingressclasses
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses/status
+    verbs:
+      - get
+      - patch
+      - update
+  - apiGroups:
+      - ingress.pomerium.io
+    resources:
+      - pomerium
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ingress.pomerium.io
+    resources:
+      - pomerium/status
+    verbs:
+      - get
+      - update
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
diff --git a/config/pomerium/rbac/cluster_role_binding.yaml b/config/pomerium/rbac/cluster_role_binding.yaml
@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: pomerium-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: pomerium-controller
+subjects:
+  - kind: ServiceAccount
+    name: pomerium-controller
diff --git a/config/pomerium/rbac/kustomization.yaml b/config/pomerium/rbac/kustomization.yaml
@@ -1,6 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+- cluster_role.yaml
+- cluster_role_binding.yaml
 - role.yaml
 - role_binding.yaml
 - service_account.yaml
diff --git a/config/pomerium/rbac/role.yaml b/config/pomerium/rbac/role.yaml
@@ -1,73 +1,17 @@
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
+kind: Role
 metadata:
   name: pomerium-controller
 rules:
   - apiGroups:
-      - ""
+      - "coordination.k8s.io"
     resources:
-      - services
-      - endpoints
+      - leases
     verbs:
       - get
       - list
       - watch
-  - apiGroups:
-      - ""
-    resources:
-      - secrets
-    verbs:
-      - get
-      - list
-      - watch
-      - patch
-  - apiGroups:
-      - ""
-    resources:
-      - services/status
-      - secrets/status
-      - endpoints/status
-    verbs:
-      - get
-  - apiGroups:
-      - networking.k8s.io
-    resources:
-      - ingresses
-      - ingressclasses
-    verbs:
-      - get
-      - list
-      - watch
-      - patch
-  - apiGroups:
-      - networking.k8s.io
-    resources:
-      - ingresses/status
-    verbs:
-      - get
-      - patch
-      - update
-  - apiGroups:
-      - ingress.pomerium.io
-    resources:
-      - pomerium
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups:
-      - ingress.pomerium.io
-    resources:
-      - pomerium/status
-    verbs:
-      - get
-      - update
-      - patch
-  - apiGroups:
-      - ""
-    resources:
-      - events
-    verbs:
       - create
+      - update
       - patch
diff --git a/config/pomerium/rbac/role_binding.yaml b/config/pomerium/rbac/role_binding.yaml
@@ -1,10 +1,10 @@
 apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
+kind: RoleBinding
 metadata:
   name: pomerium-controller
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
+  kind: Role
   name: pomerium-controller
 subjects:
   - kind: ServiceAccount
diff --git a/controllers/config_controller.go b/controllers/config_controller.go
@@ -52,6 +52,11 @@ type Controller struct {
 
 // Run runs controller using lease
 func (c *Controller) Run(ctx context.Context) error {
+	if c.MgrOpts.LeaderElection {
+		// If we're using k8s leader election, we don't need to acquire a
+		// databroker lease.
+		return c.RunLeased(ctx)
+	}
 	leaser := databroker.NewLeaser("ingress-controller", leaseDuration, c)
 	return leaser.Run(ctx)
 }
diff --git a/deployment.yaml b/deployment.yaml
@@ -825,6 +825,26 @@ metadata:
   namespace: pomerium
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/name: pomerium
+  name: pomerium-controller
+  namespace: pomerium
+rules:
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   labels:
@@ -915,6 +935,22 @@ rules:
   - get
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: pomerium
+  name: pomerium-controller
+  namespace: pomerium
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pomerium-controller
+subjects:
+- kind: ServiceAccount
+  name: pomerium-controller
+  namespace: pomerium
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   labels:
