feat: MaxConcurrentReconciles can be configured

cmd/app/app.go

@@ -168,6 +168,7 @@ func NewCommand(ctx context.Context) *cobra.Command {
 				TLS:                        tls,
 				Manager:                    mgr,
 				ConfigMapNamespaceSelector: opts.Controller.ConfigMapNamespaceSelector,
+				MaxConcurrentReconciles:    opts.Controller.MaxConcurrentReconciles,
 			}); err != nil {
 				return fmt.Errorf("failed to add CA root controller: %w", err)
 			}

cmd/app/options/options.go

@@ -82,6 +82,11 @@ type OptionsController struct {
 	// if the Kubernetes API server supports
 	// [API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/).
 	DisableKubernetesClientRateLimiter bool
+
+	// MaxConcurrentReconciles is the maximum number of concurrent reconciles
+	// that can be run for the controllers.
+	// The higher the number, the more goroutines get scheduled to handle queued reconciliations
+	MaxConcurrentReconciles int
 }
 
 func New() *Options {
@@ -157,6 +162,12 @@ func (o *Options) Complete() error {
 		log.Info("WARNING: --preserve-certificate-requests is enabled. Do not enable this option in production, or environments with any non-trivial number of workloads for an extended period of time. Doing so will balloon the resource consumption of ETCD, the API server, and istio-csr, leading to errors and slowdown. This option is intended for debugging purposes only, for limited periods of time.")
 	}
 
+	if o.Controller.MaxConcurrentReconciles < 1 {
+		return fmt.Errorf("max-concurrent-reconciles must be at least 1, got %d", o.Controller.MaxConcurrentReconciles)
+	}
+
+	o.IstiodCert.MaxConcurrentReconciles = o.Controller.MaxConcurrentReconciles
+
 	err = o.IstiodCert.Validate()
 	if err != nil {
 		return err
@@ -329,4 +340,8 @@ func (o *Options) addControllerFlags(fs *pflag.FlagSet) {
 		"disable-kubernetes-client-rate-limiter", false,
 		"Allows the default client-go rate limiter to be disabled if the Kubernetes API server supports "+
 			"[API Priority and Fairness](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/)")
+
+	fs.IntVar(&o.Controller.MaxConcurrentReconciles,
+		"max-concurrent-reconciles", 1,
+		"Maximum number of concurrent reconciles for controllers.")
 }

pkg/controller/configmap.go

@@ -30,6 +30,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -54,6 +55,9 @@ type Options struct {
 
 	// NamespaceSelector filters the namespace to creates the istio-ca-root-cert ConfigMap
 	ConfigMapNamespaceSelector string
+
+	// MaxConcurrentReconciles is the maximum number of concurrent reconciles.
+	MaxConcurrentReconciles int
 }
 
 // configmap is the controller that is responsible for ensuring that all
@@ -108,6 +112,9 @@ func AddConfigMapController(ctx context.Context, log logr.Logger, opts Options)
 	}
 
 	return ctrl.NewControllerManagedBy(opts.Manager).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: opts.MaxConcurrentReconciles,
+		}).
 		// Reconcile ConfigMaps but only cache metadata
 		For(new(corev1.ConfigMap), builder.OnlyMetadata, builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool {
 			// Only process ConfigMaps with the istio configmap name

pkg/istiodcert/options.go

@@ -51,6 +51,9 @@ type Options struct {
 	AdditionalAnnotations map[string]string
 
 	IstioRevisions []string
+
+	// MaxConcurrentReconciles is the maximum number of concurrent reconciles.
+	MaxConcurrentReconciles int
 }
 
 // Validate confirms that the given istiod cert options are valid
@@ -62,6 +65,10 @@ func (o *Options) Validate() error {
 
 	var errs []error
 
+	if o.MaxConcurrentReconciles < 1 {
+		errs = append(errs, fmt.Errorf("max-concurrent-reconciles must be at least 1, got %d", o.MaxConcurrentReconciles))
+	}
+
 	if o.RenewBefore.Nanoseconds() >= o.Duration.Nanoseconds() {
 		errs = append(errs, fmt.Errorf("istiod certificate renew-before %s must be smaller than the requested duration %s", o.RenewBefore.String(), o.Duration.String()))
 	}

pkg/istiodcert/worker.go

@@ -40,6 +40,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -151,7 +152,10 @@ func (dicp *DynamicIstiodCertProvisioner) handleNewIssuer(issuerRef *cmmeta.Obje
 // 1. Handle provisioning and updating the dynamic istiod cert
 // 2. Handle listening for updates to the active issuer ref and re-issuing
 func (dicp *DynamicIstiodCertProvisioner) AddControllersToManager(mgr manager.Manager) error {
-	b := ctrl.NewControllerManagedBy(mgr)
+	b := ctrl.NewControllerManagedBy(mgr).
+		WithOptions(controller.Options{
+			MaxConcurrentReconciles: dicp.opts.MaxConcurrentReconciles,
+		})
 
 	b.For(
 		new(cmapi.Certificate), builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool {