fix(operator): reconcile on Prometheus and ServiceMonitor changes (#1539)

Signed-off-by: gshaibi <gshaibi@nvidia.com>

Author: gshaibi

CHANGELOG.md

@@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Fixed `skipTopOwnerGrouper` not propagating per-type defaults (priority class and preemptibility) for skipped owners (e.g. `DynamoGraphDeployment`), causing PodGroup spec to retain stale values after defaults ConfigMap updates.
 - Fixed binder DRA detection on clusters where the upstream `DynamicResourceAllocation` feature gate does not reflect server-side DRA availability. The binder now probes the API server during init (matching the scheduler) so the DRA plugin is gated on the same authoritative decision. [#1481](https://github.com/kai-scheduler/KAI-Scheduler/issues/1481)
 - Suppressed noisy `Reconciler error` logs and `PodGrouperWarning` events on transient PodGroup update conflicts. The podgrouper now treats `IsConflict` errors as expected and silently requeues the reconcile instead of surfacing the apiserver's "object has been modified" message.
+- Fixed kai-operator not reconciling on Prometheus and ServiceMonitor changes. The Config controller now watches owned `Prometheus` and `ServiceMonitor` resources, so deletions and drift trigger reconciliation. CRD presence is checked at startup against the API server (the scheme-only check used previously could not detect missing CRDs), and the watch is registered only when the CRDs are installed. [#877](https://github.com/kai-scheduler/KAI-Scheduler/issues/877)
 - Added `before-hook-creation` to the `crd-upgrader` Helm hook delete policy so failed hook Jobs no longer block subsequent `helm upgrade --install` retries. Aligns with the policy already used by the chart's other hook resources. [#1404](https://github.com/kai-scheduler/KAI-Scheduler/issues/1404)
 
 ## [v0.14.0] - 2026-03-30

pkg/operator/operands/known_types/prometheus.go

@@ -26,46 +26,65 @@ func prometheusIndexer(object client.Object) []string {
 }
 
 func registerPrometheus() {
-	// Only register Prometheus collectable if CRD is available
-	// We'll check this at runtime during manager initialization
+	var prometheusCRDAvailable bool
 	collectable := &Collectable{
 		Collect: getCurrentPrometheusState,
 		InitWithManager: func(ctx context.Context, mgr manager.Manager) error {
-			// Try to register the indexer, but don't fail if the CRD is not available
 			log.FromContext(ctx).Info("Attempting to register Prometheus resource management")
-			err := mgr.GetFieldIndexer().IndexField(ctx, &monitoringv1.Prometheus{}, CollectableOwnerKey, prometheusIndexer)
+			available, err := crdAvailable(ctx, mgr, "prometheus")
 			if err != nil {
-				log.FromContext(ctx).Info("Prometheus CRD not available, skipping field indexer registration", "error", err)
-				return nil // Don't fail the test if CRD is not available
+				log.FromContext(ctx).Info("Failed to check Prometheus CRD availability, skipping registration", "error", err)
+				return nil
 			}
+			if !available {
+				log.FromContext(ctx).Info("Prometheus CRD not available, skipping registration")
+				return nil
+			}
+			if err := mgr.GetFieldIndexer().IndexField(ctx, &monitoringv1.Prometheus{}, CollectableOwnerKey, prometheusIndexer); err != nil {
+				return err
+			}
+			prometheusCRDAvailable = true
 			log.FromContext(ctx).Info("Successfully registered Prometheus resource management")
 			return nil
 		},
-		InitWithBuilder: func(builder *builder.Builder) *builder.Builder {
-			return builder
+		InitWithBuilder: func(b *builder.Builder) *builder.Builder {
+			if !prometheusCRDAvailable {
+				return b
+			}
+			return b.Owns(&monitoringv1.Prometheus{})
 		},
 		InitWithFakeClientBuilder: func(fakeClientBuilder *fake.ClientBuilder) {
 			fakeClientBuilder.WithIndex(&monitoringv1.Prometheus{}, CollectableOwnerKey, prometheusIndexer)
 		},
 	}
 	SetupKAIConfigOwned(collectable)
 
-	// Register ServiceMonitor collectable if CRD is available
+	var serviceMonitorCRDAvailable bool
 	serviceMonitorCollectable := &Collectable{
 		Collect: getCurrentServiceMonitorState,
 		InitWithManager: func(ctx context.Context, mgr manager.Manager) error {
-			// Try to register the indexer, but don't fail if the CRD is not available
 			log.FromContext(ctx).Info("Attempting to register ServiceMonitor resource management")
-			err := mgr.GetFieldIndexer().IndexField(ctx, &monitoringv1.ServiceMonitor{}, CollectableOwnerKey, serviceMonitorIndexer)
+			available, err := crdAvailable(ctx, mgr, "serviceMonitor")
 			if err != nil {
-				log.FromContext(ctx).Info("ServiceMonitor CRD not available, skipping field indexer registration", "error", err)
-				return nil // Don't fail the test if CRD is not available
+				log.FromContext(ctx).Info("Failed to check ServiceMonitor CRD availability, skipping registration", "error", err)
+				return nil
+			}
+			if !available {
+				log.FromContext(ctx).Info("ServiceMonitor CRD not available, skipping registration")
+				return nil
+			}
+			if err := mgr.GetFieldIndexer().IndexField(ctx, &monitoringv1.ServiceMonitor{}, CollectableOwnerKey, serviceMonitorIndexer); err != nil {
+				return err
 			}
+			serviceMonitorCRDAvailable = true
 			log.FromContext(ctx).Info("Successfully registered ServiceMonitor resource management")
 			return nil
 		},
-		InitWithBuilder: func(builder *builder.Builder) *builder.Builder {
-			return builder
+		InitWithBuilder: func(b *builder.Builder) *builder.Builder {
+			if !serviceMonitorCRDAvailable {
+				return b
+			}
+			return b.Owns(&monitoringv1.ServiceMonitor{})
 		},
 		InitWithFakeClientBuilder: func(fakeClientBuilder *fake.ClientBuilder) {
 			fakeClientBuilder.WithIndex(&monitoringv1.ServiceMonitor{}, CollectableOwnerKey, serviceMonitorIndexer)
@@ -74,6 +93,17 @@ func registerPrometheus() {
 	SetupKAIConfigOwned(serviceMonitorCollectable)
 }
 
+// crdAvailable performs a live API-server check for the given Prometheus-family CRD.
+// The manager scheme registers monitoringv1 unconditionally, so a scheme-only check
+// would falsely report availability on clusters without prometheus-operator installed.
+func crdAvailable(ctx context.Context, mgr manager.Manager, target string) (bool, error) {
+	tempClient, err := client.New(mgr.GetConfig(), client.Options{Scheme: mgr.GetScheme()})
+	if err != nil {
+		return false, err
+	}
+	return common.CheckPrometheusCRDsAvailable(ctx, tempClient, target)
+}
+
 func getCurrentPrometheusState(ctx context.Context, runtimeClient client.Client, reconciler client.Object) (map[string]client.Object, error) {
 	result := map[string]client.Object{}