feat(operator): reflect GPU sharing readiness status

Signed-off-by: davidLif <davidshani12@gmail.com>

Author: davidLif

.changes/unreleased/added-20260810-111710.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: |-
+  Gate NvFractions on GPU sharing readiness

deployments/kai-scheduler/crds/scheduling.run.ai_bindrequests.yaml

@@ -1333,6 +1333,7 @@ spec:
                 description: |-
                   SelectedGPUGroups is the name of the selected GPU groups for fractional GPU resources.
                   Only if the RecievedResourceType is "Fraction"
+                  Deprecated: Use SelectedFractionalGpuGroups instead
                 items:
                   type: string
                 type: array

deployments/kai-scheduler/templates/rbac/operator.yaml

@@ -136,6 +136,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - gpu-sharing.kai.scheduler
+  resources:
+  - gpusharingconfigs
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - kai.scheduler
   resources:

docs/operator/README.md

@@ -71,6 +71,49 @@ spec:
 
 - [Scheduling Shards](./scheduling-shards.md) - Advanced cluster partitioning
 
+## Status Conditions
+
+The KAI operator reports installation health on `Config.status.conditions`. The most relevant condition types are:
+
+- `Deployed`: the operator created the Kubernetes resources for KAI services.
+- `Available`: the KAI service controllers report availability.
+- `DependenciesFulfilled`: external dependencies required by the selected configuration are present and healthy.
+- `Ready`: the KAI services are available.
+
+When `global.gpuSharingMode` is `NvFractions`, the KAI operator also checks the cluster-scoped `GpuSharingConfig` resource from the GPU-sharing operator, as running fractional GPU pods depends on it.
+
+If all KAI services are deployed and available, but the GPU-sharing operator reports a dependency failure, the KAI `Config` can look like this:
+
+```yaml
+apiVersion: kai.scheduler/v1
+kind: Config
+metadata:
+  name: kai-config
+spec:
+  global:
+    gpuSharingMode: NvFractions
+status:
+  conditions:
+    - type: Deployed
+      status: "True"
+      reason: deployed
+      message: Resources deployed
+    - type: Available
+      status: "True"
+      reason: available
+      message: System available
+    - type: DependenciesFulfilled
+      status: "False"
+      reason: dependencies_missing
+      message: Gpu Sharing is not ready. Ready=False, reason=GPUOperatorNotReady, message=ClusterPolicy is not ready
+    - type: Ready
+      status: "True"
+      reason: ready
+      message: System is ready
+```
+
+In this state the KAI pods are healthy, but NvFractions should not be considered fully operational until `DependenciesFulfilled=True`. The dependency message carries the GPU-sharing operator's config-level status reason and message so the failing external dependency is visible directly from the KAI `Config`.
+
 ## Logging
 
 By default all KAI services use development-mode logging with colored, human-readable console

pkg/apis/scheduling/v1alpha2/zz_generated.deepcopy.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 
 	admissionv1 "k8s.io/api/admissionregistration/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	vpav1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
@@ -39,6 +40,7 @@ import (
 	"github.com/kai-scheduler/KAI-scheduler/pkg/operator/operands/binder"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/operator/operands/common"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/operator/operands/deployable"
+	"github.com/kai-scheduler/KAI-scheduler/pkg/operator/operands/gpu_sharing"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/operator/operands/known_types"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/operator/operands/node_scale_adjuster"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/operator/operands/numa_placement_exporter"
@@ -57,6 +59,7 @@ var ConfigReconcilerOperands = []operands.Operand{
 	&node_scale_adjuster.NodeScaleAdjuster{},
 	&admission.Admission{},
 	&prometheus.Prometheus{},
+	&gpu_sharing.GpuSharing{},
 	&scheduler.SchedulerForConfig{},
 	&numa_placement_exporter.NumaPlacementExporter{},
 }
@@ -83,6 +86,7 @@ func (r *ConfigReconciler) SetOperands(ops []operands.Operand) {
 // +kubebuilder:rbac:groups="admissionregistration.k8s.io",resources=mutatingwebhookconfigurations;validatingwebhookconfigurations,verbs=get;list;watch;create
 // +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,resourceNames=queues.scheduling.run.ai,verbs=delete;update;patch
 // +kubebuilder:rbac:groups="apiextensions.k8s.io",resources=customresourcedefinitions,verbs=get;list;watch;create
+// +kubebuilder:rbac:groups="gpu-sharing.kai.scheduler",resources=gpusharingconfigs,verbs=get;list;watch
 // +kubebuilder:rbac:groups="nvidia.com",resources=clusterpolicies,verbs=get;list;watch
 // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheuses;servicemonitors,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="scheduling.run.ai",resources=queues,verbs=get;list;watch
@@ -141,6 +145,7 @@ func (r *ConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *ConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	logger := log.FromContext(context.Background())
 	for _, collectable := range known_types.KAIConfigRegisteredCollectible {
 		if err := collectable.InitWithManager(context.Background(), mgr); err != nil {
 			return err
@@ -159,10 +164,26 @@ func (r *ConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	builder := ctrl.NewControllerManagedBy(mgr).
 		For(&kaiv1.Config{})
 
-	if checkForClusterPolicy(mgr) {
+	clusterPolicyExists, err := common.CheckCRDsAvailable(
+		context.Background(), mgr.GetAPIReader(), "clusterpolicies.nvidia.com",
+	)
+	if err != nil {
+		logger.Info("Failed to check for ClusterPolicy CRD existence", "error", err)
+	} else if clusterPolicyExists {
 		builder = builder.Watches(&nvidiav1.ClusterPolicy{}, handler.EnqueueRequestsFromMapFunc(enqueueWatched))
 	}
 
+	gpuSharingConfigExists, err := common.CheckCRDsAvailable(
+		context.Background(), mgr.GetAPIReader(), gpu_sharing.GpuSharingConfigCRDName,
+	)
+	if err != nil {
+		logger.Info("Failed to check for GpuSharingConfig CRD existence", "error", err)
+	} else if gpuSharingConfigExists {
+		gpuSharingConfig := &unstructured.Unstructured{}
+		gpuSharingConfig.SetGroupVersionKind(gpu_sharing.GpuSharingConfigGVK)
+		builder = builder.Watches(gpuSharingConfig, handler.EnqueueRequestsFromMapFunc(enqueueWatched))
+	}
+
 	builder = builder.Watches(&kaiv1.SchedulingShard{}, handler.EnqueueRequestsFromMapFunc(enqueueWatched))
 
 	for _, collectable := range known_types.KAIConfigRegisteredCollectible {
@@ -181,22 +202,3 @@ func enqueueWatched(_ context.Context, _ client.Object) []ctrl.Request {
 		},
 	}
 }
-
-func checkForClusterPolicy(mgr ctrl.Manager) bool {
-	logger := log.FromContext(context.Background())
-	tempClient, err := client.New(mgr.GetConfig(), client.Options{Scheme: mgr.GetScheme()})
-	if err != nil {
-		logger.Info("Failed to create temporary client to check for cluster policy", "error", err)
-		return false
-	}
-
-	clusterPolicyExists, err := common.CheckCRDsAvailable(
-		context.Background(), tempClient, "clusterpolicies.nvidia.com",
-	)
-	if err != nil {
-		logger.Info("Failed to check for ClusterPolicy CRD existence", "error", err)
-		return false
-	}
-
-	return clusterPolicyExists
-}

pkg/operator/controller/config_controller.go

@@ -0,0 +1,107 @@
+// Copyright 2025 NVIDIA CORPORATION
+// SPDX-License-Identifier: Apache-2.0
+
+package gpu_sharing
+
+import (
+	"context"
+	"fmt"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kaiv1 "github.com/kai-scheduler/KAI-scheduler/pkg/apis/kai/v1"
+	kaiv1common "github.com/kai-scheduler/KAI-scheduler/pkg/apis/kai/v1/common"
+)
+
+const GpuSharingConfigCRDName = "gpusharingconfigs.gpu-sharing.kai.scheduler"
+
+var GpuSharingConfigGVK = schema.GroupVersionKind{
+	Group:   "gpu-sharing.kai.scheduler",
+	Version: "v1alpha1",
+	Kind:    "GpuSharingConfig",
+}
+
+type GpuSharing struct{}
+
+func (g *GpuSharing) DesiredState(context.Context, client.Reader, *kaiv1.Config) ([]client.Object, error) {
+	return nil, nil
+}
+
+func (g *GpuSharing) IsDeployed(context.Context, client.Reader) (bool, error) {
+	return true, nil
+}
+
+func (g *GpuSharing) IsAvailable(context.Context, client.Reader) (bool, error) {
+	return true, nil
+}
+
+func (g *GpuSharing) Monitor(context.Context, client.Reader, *kaiv1.Config) error {
+	return nil
+}
+
+func (g *GpuSharing) HasMissingDependencies(
+	ctx context.Context, readerClient client.Reader, kaiConfig *kaiv1.Config,
+) (string, error) {
+	if !isNvFractionsConfigured(kaiConfig) {
+		return "", nil
+	}
+
+	gpuSharingConfig := &unstructured.Unstructured{}
+	gpuSharingConfig.SetGroupVersionKind(GpuSharingConfigGVK)
+	if err := readerClient.Get(ctx, types.NamespacedName{Name: "default"}, gpuSharingConfig); err != nil {
+		if apierrors.IsNotFound(err) || meta.IsNoMatchError(err) {
+			return "Gpu Sharing is not ready. GpuSharingConfig not found", nil
+		}
+		return "", err
+	}
+
+	readyConditions, found, err := unstructured.NestedSlice(gpuSharingConfig.Object, "status", "conditions")
+	if err != nil {
+		return "", err
+	}
+	for _, condition := range readyConditions {
+		conditionMap, ok := condition.(map[string]interface{})
+		if !ok {
+			continue
+		}
+		if conditionMap["type"] != "Ready" {
+			continue
+		}
+		if conditionMap["status"] == "True" {
+			return "", nil
+		}
+		return fmt.Sprintf(
+			"Gpu Sharing is not ready. Ready=%s, reason=%s, message=%s",
+			conditionFieldString(conditionMap, "status"),
+			conditionFieldString(conditionMap, "reason"),
+			conditionFieldString(conditionMap, "message"),
+		), nil
+	}
+
+	if !found {
+		return "Gpu Sharing is not ready. GpuSharingConfig conditions not found", nil
+	}
+	return "Gpu Sharing is not ready. Ready condition not found", nil
+}
+
+func (g *GpuSharing) Name() string {
+	return "GPU-sharing"
+}
+
+func isNvFractionsConfigured(kaiConfig *kaiv1.Config) bool {
+	return kaiConfig != nil && kaiConfig.Spec.Global != nil && kaiConfig.Spec.Global.GpuSharingMode != nil &&
+		*kaiConfig.Spec.Global.GpuSharingMode == kaiv1common.GpuSharingModeNvFractions
+}
+
+func conditionFieldString(condition map[string]interface{}, field string) string {
+	value, ok := condition[field].(string)
+	if !ok {
+		return ""
+	}
+	return value
+}