[Bug][API] Make kubectl get modeladapter show a truthful status

api/model/v1alpha1/modeladapter_types.go

@@ -132,8 +132,12 @@ const (
 // +kubebuilder:printcolumn:name="Desired",type=integer,JSONPath=`.status.desiredReplicas`
 // +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=`.status.readyReplicas`
 // +kubebuilder:printcolumn:name="Candidates",type=integer,JSONPath=`.status.candidates`
+// +kubebuilder:printcolumn:name="Reason",type=string,JSONPath=`.status.conditions[?(@.type=='Ready')].reason`
+// +kubebuilder:printcolumn:name="Base Model",type=string,JSONPath=`.spec.baseModel`,priority=1
 // +kubebuilder:printcolumn:name="Model Path",type=string,JSONPath=`.spec.artifactURL`
 // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
+// +kubebuilder:printcolumn:name="Instances",type=string,JSONPath=`.status.instances`,priority=1
+// +kubebuilder:printcolumn:name="Message",type=string,JSONPath=`.status.conditions[?(@.type=='Ready')].message`,priority=1
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // ModelAdapter is the Schema for the modeladapters API

config/crd/model/model.aibrix.ai_modeladapters.yaml

@@ -27,12 +27,27 @@ spec:
     - jsonPath: .status.candidates
       name: Candidates
       type: integer
+    - jsonPath: .status.conditions[?(@.type=='Ready')].reason
+      name: Reason
+      type: string
+    - jsonPath: .spec.baseModel
+      name: Base Model
+      priority: 1
+      type: string
     - jsonPath: .spec.artifactURL
       name: Model Path
       type: string
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - jsonPath: .status.instances
+      name: Instances
+      priority: 1
+      type: string
+    - jsonPath: .status.conditions[?(@.type=='Ready')].message
+      name: Message
+      priority: 1
+      type: string
     name: v1alpha1
     schema:
       openAPIV3Schema:

dist/chart/crds/model.aibrix.ai_modeladapters.yaml

@@ -27,12 +27,27 @@ spec:
     - jsonPath: .status.candidates
       name: Candidates
       type: integer
+    - jsonPath: .status.conditions[?(@.type=='Ready')].reason
+      name: Reason
+      type: string
+    - jsonPath: .spec.baseModel
+      name: Base Model
+      priority: 1
+      type: string
     - jsonPath: .spec.artifactURL
       name: Model Path
       type: string
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - jsonPath: .status.instances
+      name: Instances
+      priority: 1
+      type: string
+    - jsonPath: .status.conditions[?(@.type=='Ready')].message
+      name: Message
+      priority: 1
+      type: string
     name: v1alpha1
     schema:
       openAPIV3Schema:

docs/source/features/lora-dynamic-loading.rst

@@ -33,26 +33,30 @@ The ModelAdapter goes through several distinct phases during its lifecycle. Unde
 
 ::
 
-    Pending → Scheduled → Loading → Bound → Running
-       ↓         ↓         ↓        ↓        ↓
-    Starting  Pod      Adapter   Service   Ready for
-    reconcile Selected  Loading   Created   Inference
+    Pending → Scheduled → Running
+       ↓         ↓          ↓
+    Waiting   Pod        Adapter loaded,
+    for pods  selected   ready for inference
+
+    Loading fails on every candidate pod → Failed (retried until one load succeeds)
 
 **Phase Details:**
 
-1. **Pending**: Initial state when the ModelAdapter is first created. The controller starts reconciliation and validates the configuration.
+1. **Pending**: The controller has started reconciliation but no adapter instance is loaded yet. The ``Ready`` condition says why: reason ``NoReadyPods`` means no ready pod matches the ``podSelector`` (with ``replicas: 1`` it also covers pods that became ready too recently to be scheduled); reason ``ModelAdapterUnavailable`` means candidate pods exist but the adapter is not loaded on any of them yet.
 
-2. **Scheduled**: The controller has successfully identified and selected suitable pods that match the ``podSelector`` criteria. Pods are validated for readiness and stability before selection.
+2. **Scheduled**: Only with ``replicas: 1``. The controller has selected a pod that matches the ``podSelector`` and is loading the adapter on it. Pods are validated for readiness and stability before selection. Loading includes:
 
-3. **Loading**: The controller is actively loading the LoRA adapter onto the selected pods. This includes:
-
-   - Downloading the adapter from the specified ``artifactURL`` 
+   - Downloading the adapter from the specified ``artifactURL``
    - Registering the adapter with the vLLM engine
-   - Handling retry mechanisms with exponential backoff if loading fails
+   - Retrying with exponential backoff if loading fails
+
+   Without ``replicas`` (load on all matching pods) the adapter moves from ``Pending`` straight to ``Running``.
 
-4. **Bound**: The LoRA adapter has been successfully loaded onto the pods and the controller is creating the associated Kubernetes Service and EndpointSlice resources for service discovery.
+3. **Running**: The adapter is loaded on at least one pod and the Kubernetes Service and EndpointSlice for service discovery exist. The LoRA model is accessible through the gateway using the adapter name. ``readyReplicas`` out of ``desiredReplicas`` in the status shows how many candidate pods have the adapter loaded.
 
-5. **Running**: The ModelAdapter is fully operational and ready to serve inference requests. The LoRA model is accessible through the gateway using the adapter name.
+4. **Failed**: Loading failed on every candidate pod. The ``Ready`` condition carries reason ``ModelAdapterLoadingError`` and the per-pod errors in its message. The controller keeps retrying and the adapter returns to ``Running`` as soon as one load succeeds.
+
+``ResourceCreated`` is not a step of the normal flow. It shows up as the phase only when creating the Service or EndpointSlice fails; the condition of the same name carries the error details, and the controller keeps retrying. ``Bound`` is a condition, not a phase: it is ``True`` (together with the ``Scheduled`` condition) while the adapter is loaded, and turns ``False`` with reason ``ModelAdapterLoadingError`` when a loading pass fails, while the phase stays ``Pending`` or ``Failed``.
 
 **Error Handling and Reliability Features:**
 
@@ -63,13 +67,35 @@ The ModelAdapter goes through several distinct phases during its lifecycle. Unde
 
 **Monitoring Phase Transitions:**
 
-You can monitor the current phase and detailed status using:
+``kubectl get modeladapter`` shows the phase, the replica counters and the reason of the ``Ready`` condition at a glance. While the base model pod is still starting:
 
 .. code-block:: bash
 
-    kubectl describe modeladapter <adapter-name>
+    $ kubectl get modeladapter
+    NAME             PHASE     DESIRED   READY   CANDIDATES   REASON        MODEL PATH                                                     AGE
+    qwen-code-lora   Pending                                  NoReadyPods   huggingface://ai-blond/Qwen-Qwen2.5-Coder-1.5B-Instruct-lora   5s
+
+Once the adapter is loaded:
+
+.. code-block:: bash
+
+    $ kubectl get modeladapter
+    NAME             PHASE     DESIRED   READY   CANDIDATES   REASON                  MODEL PATH                                                     AGE
+    qwen-code-lora   Running   1         1       1            ModelAdapterAvailable   huggingface://ai-blond/Qwen-Qwen2.5-Coder-1.5B-Instruct-lora   2m
 
-The status section will show the current phase and transition history with timestamps and reasons for each state change.
+Blank counter cells mean zero. ``-o wide`` adds the base model, the pods hosting the adapter and the full ``Ready`` condition message:
+
+.. code-block:: bash
+
+    $ kubectl get modeladapter -o wide
+    NAME             PHASE     DESIRED   READY   CANDIDATES   REASON                  BASE MODEL                 MODEL PATH                                                     AGE   INSTANCES                                       MESSAGE
+    qwen-code-lora   Running   1         1       1            ModelAdapterAvailable   qwen-coder-1-5b-instruct   huggingface://ai-blond/Qwen-Qwen2.5-Coder-1.5B-Instruct-lora   2m    ["qwen-coder-1-5b-instruct-5587f4c57d-kml6s"]   ModelAdapter default/qwen-code-lora is ready
+
+For the full condition history with timestamps and reasons for each state change, use:
+
+.. code-block:: bash
+
+    kubectl describe modeladapter <adapter-name>
 
 Model Adapter Service Discovery
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -149,7 +175,7 @@ If you run ```kubectl describe modeladapter qwen-code-lora``, you will see the s
         Type:                  Initialized
       Phase:  Pending
 
-**Phase 2: Scheduled**
+**Phase 2: Scheduled** (only with ``replicas: 1``)
 
 .. code-block:: bash
 
@@ -168,12 +194,13 @@ If you run ```kubectl describe modeladapter qwen-code-lora``, you will see the s
         Type:                  Scheduled
       Phase:  Scheduled
 
-**Phase 3-5: Loading → Bound → Running**
+**Phase 3: Running**
 
 .. code-block:: bash
 
     $ kubectl describe modeladapter qwen-code-lora
     Status:
+      Candidates:  1
       Conditions:
         Last Transition Time:  2025-02-16T19:14:50Z
         Message:               Starting reconciliation
@@ -190,9 +217,11 @@ If you run ```kubectl describe modeladapter qwen-code-lora``, you will see the s
         Reason:                ModelAdapterAvailable
         Status:                True
         Type:                  Ready
+      Desired Replicas:  1
       Instances:
         qwen-coder-1-5b-instruct-5587f4c57d-kml6s
-      Phase:  Running
+      Phase:           Running
+      Ready Replicas:  1
 
 Send request using lora model name to the gateway.
 
@@ -244,7 +273,7 @@ If your ModelAdapter gets stuck in a particular phase, here are common issues an
 - Check controller logs for any loading errors: ``kubectl logs -n aibrix-system deployment/aibrix-controller-manager``
 - Verify vLLM pods have ``VLLM_ALLOW_RUNTIME_LORA_UPDATING`` enabled
 
-**Stuck in Loading Phase:**
+**Stuck in Failed Phase (loading errors):**
 
 - Check if the ``artifactURL`` is accessible and valid
 - For Hugging Face models, ensure the model path exists

pkg/controller/modeladapter/modeladapter_controller.go

@@ -31,6 +31,7 @@ import (
 	"github.com/vllm-project/aibrix/pkg/utils"
 	corev1 "k8s.io/api/core/v1"
 	discoveryv1 "k8s.io/api/discovery/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -444,15 +445,28 @@ func (r *ModelAdapterReconciler) DoReconcile(ctx context.Context, req ctrl.Reque
 	copy(oldInstances, instance.Status.Instances)
 
 	// Step 1: Reconcile Pod instances for ModelAdapter based on desired replicas
-	if ctrlResult, err := r.reconcileReplicas(ctx, instance); err != nil || ctrlResult.Requeue || ctrlResult.RequeueAfter > 0 {
-		return ctrlResult, err
+	replicasResult, err := r.reconcileReplicas(ctx, instance)
+	if err != nil {
+		return replicasResult, err
+	}
+	if replicasResult.Requeue || replicasResult.RequeueAfter > 0 {
+		// Waiting for pods to become schedulable. reconcileLoadOnSinglePod has recorded
+		// why in the Ready condition; derive the phase from the instance set and persist
+		// it with the refreshed Candidates and DesiredReplicas so that `kubectl get` does
+		// not keep showing a stale phase while the adapter waits.
+		if err := r.syncReadinessStatus(ctx, oldInstance, instance); err != nil {
+			return ctrl.Result{}, err
+		}
+		return replicasResult, nil
 	}
 
 	// Step 2: Reconcile Loading (pass oldInstances to detect pod removal)
 	if err := r.reconcileLoading(ctx, instance, oldInstances); err != nil {
 		// Don't overwrite Failed status - it should be preserved for visibility
 		if instance.Status.Phase != modelv1alpha1.ModelAdapterFailed {
-			instance.Status.Phase = modelv1alpha1.ModelAdapterBound
+			// reconcileLoading also fails when no active pod backs the adapter at all, so
+			// derive the phase from the instance set instead of reporting Bound.
+			recomputeReadiness(instance)
 			condition := NewCondition(string(modelv1alpha1.ModelAdapterConditionTypeBound), metav1.ConditionFalse,
 				ModelAdapterLoadingErrorReason, fmt.Sprintf("ModelAdapter %s loading failed", klog.KObj(instance)))
 			if err := r.updateStatus(ctx, instance, condition); err != nil {
@@ -489,31 +503,11 @@ func (r *ModelAdapterReconciler) DoReconcile(ctx context.Context, req ctrl.Reque
 		return ctrlResult, err
 	}
 
-	// Check if we need to update the status. Besides the usual field-level diff, also repair
-	// Bound/Scheduled if either is stuck False from an earlier failure/migration whose recovery
-	// path didn't clear it (see reconcileLoading and the Step 2 error path) -- otherwise, once
-	// the adapter has been stably healthy for a cycle, oldInstance/instance stop differing and
-	// the stale condition would never get another chance to be corrected.
-	conditionUnhealthy := func(condType string) bool {
-		cond := meta.FindStatusCondition(instance.Status.Conditions, condType)
-		return cond == nil || cond.Status != metav1.ConditionTrue
-	}
-	if r.inconsistentModelAdapterStatus(oldInstance.Status, instance.Status) ||
-		conditionUnhealthy(string(modelv1alpha1.ModelAdapterConditionTypeBound)) ||
-		conditionUnhealthy(string(modelv1alpha1.ModelAdapterConditionTypeScheduled)) {
-		readyCondition := NewCondition(string(modelv1alpha1.ModelAdapterConditionReady), metav1.ConditionTrue,
-			ModelAdapterAvailable, fmt.Sprintf("ModelAdapter %s is ready", klog.KObj(instance)))
-		// Reaching here means reconcileLoading returned no error, i.e. the adapter is
-		// actually loaded on at least one pod. Reassert Bound/Scheduled as healthy too,
-		// so a stale False left over from an earlier failure or migration (reconcileLoading's
-		// podRemoved branch, or the Step 2 error path) doesn't linger once the adapter recovers.
-		boundCondition := NewCondition(string(modelv1alpha1.ModelAdapterConditionTypeBound), metav1.ConditionTrue,
-			ModelAdapterBoundReason, fmt.Sprintf("ModelAdapter %s is bound to %d pod(s)", klog.KObj(instance), instance.Status.ReadyReplicas))
-		scheduledCondition := NewCondition(string(modelv1alpha1.ModelAdapterConditionTypeScheduled), metav1.ConditionTrue,
-			ModelAdapterScheduledReason, fmt.Sprintf("ModelAdapter %s is scheduled on %d pod(s)", klog.KObj(instance), instance.Status.ReadyReplicas))
-		if err := r.updateStatus(ctx, instance, readyCondition, boundCondition, scheduledCondition); err != nil {
-			return reconcile.Result{}, fmt.Errorf("update modelAdapter status error: %v", err)
-		}
+	// Derive ReadyReplicas, Phase and the Ready condition from the final instance set,
+	// reassert Bound and Scheduled while an instance is loaded, and persist only when
+	// something observable changed.
+	if err := r.syncReadinessStatus(ctx, oldInstance, instance); err != nil {
+		return reconcile.Result{}, fmt.Errorf("update modelAdapter status error: %v", err)
 	}
 
 	return ctrl.Result{}, nil
@@ -1061,7 +1055,6 @@ func (r *ModelAdapterReconciler) reconcileEndpointSlice(ctx context.Context, ins
 		if err := r.Create(ctx, eps); err != nil {
 			return ctrl.Result{}, err
 		}
-		instance.Status.Phase = modelv1alpha1.ModelAdapterRunning
 		return ctrl.Result{}, nil
 	}
 
@@ -1073,7 +1066,6 @@ func (r *ModelAdapterReconciler) reconcileEndpointSlice(ctx context.Context, ins
 	if err := r.Update(ctx, found); err != nil {
 		return ctrl.Result{}, err
 	}
-	instance.Status.Phase = modelv1alpha1.ModelAdapterRunning
 	return ctrl.Result{}, nil
 }
 
@@ -1090,9 +1082,72 @@ func (r *ModelAdapterReconciler) inconsistentModelAdapterStatus(oldStatus, newSt
 		return true
 	}
 
+	// A changed Ready condition (for example a new reason) must be persisted too.
+	if !apiequality.Semantic.DeepEqual(oldStatus.Conditions, newStatus.Conditions) {
+		return true
+	}
+
 	return false
 }
 
+// recomputeReadiness derives ReadyReplicas, Phase and the Ready condition from the
+// current Instances and Candidates so that `kubectl get modeladapter` never shows
+// Running or Ready=True without a loaded instance. While an instance is loaded it also
+// reasserts Bound and Scheduled, so a False left behind by an earlier failure or pod
+// migration does not linger once the adapter recovers. It only mutates the in-memory
+// object; the caller decides whether to persist. A Failed phase (recorded by
+// reconcileLoading together with its Ready=False loading-error condition) is kept
+// until an instance loads successfully, and a Ready=False scheduling reason recorded
+// by reconcileLoadOnSinglePod is kept because it is more specific.
+func recomputeReadiness(instance *modelv1alpha1.ModelAdapter) {
+	status := &instance.Status
+	status.ReadyReplicas = int32(len(status.Instances))
+
+	switch {
+	case status.ReadyReplicas > 0:
+		status.Phase = modelv1alpha1.ModelAdapterRunning
+		meta.SetStatusCondition(&status.Conditions, NewCondition(string(modelv1alpha1.ModelAdapterConditionReady), metav1.ConditionTrue,
+			ModelAdapterAvailable, fmt.Sprintf("ModelAdapter %s is ready", klog.KObj(instance))))
+		meta.SetStatusCondition(&status.Conditions, NewCondition(string(modelv1alpha1.ModelAdapterConditionTypeBound), metav1.ConditionTrue,
+			ModelAdapterBoundReason, fmt.Sprintf("ModelAdapter %s is bound to %d pod(s)", klog.KObj(instance), status.ReadyReplicas)))
+		meta.SetStatusCondition(&status.Conditions, NewCondition(string(modelv1alpha1.ModelAdapterConditionTypeScheduled), metav1.ConditionTrue,
+			ModelAdapterScheduledReason, fmt.Sprintf("ModelAdapter %s is scheduled on %d pod(s)", klog.KObj(instance), status.ReadyReplicas)))
+	case status.Phase == modelv1alpha1.ModelAdapterFailed:
+		// Keep the loading error recorded by reconcileLoading visible.
+	case hasSchedulingWaitReason(status.Conditions):
+		// reconcileLoadOnSinglePod already explained why no pod could be scheduled.
+		status.Phase = modelv1alpha1.ModelAdapterPending
+	case status.Candidates == 0:
+		status.Phase = modelv1alpha1.ModelAdapterPending
+		meta.SetStatusCondition(&status.Conditions, NewCondition(string(modelv1alpha1.ModelAdapterConditionReady), metav1.ConditionFalse,
+			NoReadyPodsReason, "no ready pods match the pod selector"))
+	default:
+		status.Phase = modelv1alpha1.ModelAdapterPending
+		meta.SetStatusCondition(&status.Conditions, NewCondition(string(modelv1alpha1.ModelAdapterConditionReady), metav1.ConditionFalse,
+			ModelAdapterUnavailable, fmt.Sprintf("adapter is not loaded on any of the %d candidate pods", status.Candidates)))
+	}
+}
+
+// hasSchedulingWaitReason reports whether the Ready condition is False with one of the
+// reasons reconcileLoadOnSinglePod records while it waits for a schedulable pod.
+func hasSchedulingWaitReason(conditions []metav1.Condition) bool {
+	cond := meta.FindStatusCondition(conditions, string(modelv1alpha1.ModelAdapterConditionReady))
+	if cond == nil || cond.Status != metav1.ConditionFalse {
+		return false
+	}
+	return cond.Reason == NoReadyPodsReason || cond.Reason == InsufficientReadyPodsReason
+}
+
+// syncReadinessStatus recomputes the derived readiness fields and persists the status
+// only when something observable changed during this reconcile.
+func (r *ModelAdapterReconciler) syncReadinessStatus(ctx context.Context, oldInstance, instance *modelv1alpha1.ModelAdapter) error {
+	recomputeReadiness(instance)
+	if !r.inconsistentModelAdapterStatus(oldInstance.Status, instance.Status) {
+		return nil
+	}
+	return r.updateStatus(ctx, instance)
+}
+
 // isPodReadyForScheduling checks if a pod is ready and stable for scheduling
 func (r *ModelAdapterReconciler) isPodReadyForScheduling(ctx context.Context, instance *modelv1alpha1.ModelAdapter, pod *corev1.Pod) bool {
 	if !utils.IsPodReady(pod) {