fix(providers): refresh status message when ModelDeployment reaches Running (#326)

Signed-off-by: robert-cronin <robert.owen.cronin@gmail.com>

Author: robert-cronin

providers/dynamo/controller.go

@@ -438,6 +438,10 @@ func (r *DynamoProviderReconciler) syncStatus(ctx context.Context, md *airunwayv
 	md.Status.Phase = statusResult.Phase
 	if statusResult.Message != "" {
 		md.Status.Message = statusResult.Message
+	} else if statusResult.Phase == airunwayv1alpha1.DeploymentPhaseRunning {
+		// The translator reports no message for a healthy DynamoGraphDeployment;
+		// replace the stale "waiting for pods" message so status reflects Running.
+		md.Status.Message = "DynamoGraphDeployment created, pods are ready"
 	}
 	md.Status.Replicas = statusResult.Replicas
 	md.Status.Endpoint = statusResult.Endpoint

providers/dynamo/controller_test.go

@@ -3,6 +3,7 @@ package dynamo
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -691,6 +692,7 @@ func TestSyncStatusRunning(t *testing.T) {
 	r := NewDynamoProviderReconciler(c, scheme, "")
 
 	md := &airunwayv1alpha1.ModelDeployment{}
+	md.Status.Message = "DynamoGraphDeployment created, waiting for pods to be ready"
 	desired := &unstructured.Unstructured{}
 	setDGDGVK(desired)
 	desired.SetName("test")
@@ -703,6 +705,13 @@ func TestSyncStatusRunning(t *testing.T) {
 	if md.Status.Phase != airunwayv1alpha1.DeploymentPhaseRunning {
 		t.Errorf("expected Running phase, got %s", md.Status.Phase)
 	}
+	// Issue #289: a Running deployment must not keep the "waiting for pods" message.
+	if strings.Contains(md.Status.Message, "waiting for pods") {
+		t.Errorf("status message still claims waiting for pods while Running: %q", md.Status.Message)
+	}
+	if md.Status.Message == "" {
+		t.Errorf("expected a non-empty status message in Running phase")
+	}
 }
 
 func TestSyncStatusFailed(t *testing.T) {

providers/kaito/controller.go

@@ -667,6 +667,10 @@ func (r *KaitoProviderReconciler) syncStatus(ctx context.Context, md *airunwayv1
 	md.Status.Phase = statusResult.Phase
 	if statusResult.Message != "" {
 		md.Status.Message = statusResult.Message
+	} else if statusResult.Phase == airunwayv1alpha1.DeploymentPhaseRunning {
+		// The translator reports no message for a healthy Workspace; replace the
+		// stale "waiting for pods" message so status reflects the Running phase.
+		md.Status.Message = "Workspace created, pods are ready"
 	}
 	md.Status.Replicas = statusResult.Replicas
 	md.Status.Endpoint = statusResult.Endpoint

providers/kaito/controller_test.go

@@ -469,6 +469,71 @@ func TestReconcileAlreadyRunning(t *testing.T) {
 	}
 }
 
+// TestReconcileRunningUpdatesMessage reproduces issue #289: once the Workspace
+// is ready the phase flips to Running, but the status message must no longer
+// claim it is "waiting for pods to be ready".
+func TestReconcileRunningUpdatesMessage(t *testing.T) {
+	scheme := newScheme()
+	md := newMDForController("test", "default")
+	md.UID = "test-uid"
+	controllerutil.AddFinalizer(md, FinalizerName)
+	// Simulate a prior reconcile loop that left the deploying-phase message.
+	md.Status.Phase = airunwayv1alpha1.DeploymentPhaseDeploying
+	md.Status.Message = "Workspace created, waiting for pods to be ready"
+
+	ws := &unstructured.Unstructured{}
+	setWorkspaceGVK(ws)
+	ws.SetName("test")
+	ws.SetNamespace("default")
+	ws.SetOwnerReferences([]metav1.OwnerReference{
+		{UID: "test-uid", APIVersion: "airunway.ai/v1alpha1", Kind: "ModelDeployment", Name: "test"},
+	})
+	ws.Object["resource"] = map[string]interface{}{
+		"count": int64(1),
+		"labelSelector": map[string]interface{}{
+			"matchLabels": map[string]interface{}{
+				"kubernetes.io/os": "linux",
+			},
+		},
+	}
+	ws.Object["inference"] = map[string]interface{}{
+		"preset": map[string]interface{}{
+			"name": "test-model",
+		},
+	}
+	ws.Object["status"] = map[string]interface{}{
+		"conditions": []interface{}{
+			map[string]interface{}{
+				"type":   "WorkspaceSucceeded",
+				"status": "True",
+			},
+		},
+	}
+
+	deploy := newReadyKaitoDeployment()
+	directC := probeClientBuilderWithWorkspace(t).WithObjects(deploy).Build()
+	c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(md, ws).WithStatusSubresource(md).Build()
+	r := NewKaitoProviderReconciler(c, scheme, directC, record.NewFakeRecorder(10))
+
+	if _, err := r.Reconcile(context.Background(), ctrl.Request{
+		NamespacedName: types.NamespacedName{Name: "test", Namespace: "default"},
+	}); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var updated airunwayv1alpha1.ModelDeployment
+	_ = c.Get(context.Background(), types.NamespacedName{Name: "test", Namespace: "default"}, &updated)
+	if updated.Status.Phase != airunwayv1alpha1.DeploymentPhaseRunning {
+		t.Fatalf("expected Running phase, got %s", updated.Status.Phase)
+	}
+	if strings.Contains(updated.Status.Message, "waiting for pods") {
+		t.Errorf("status message still claims waiting for pods while Running: %q", updated.Status.Message)
+	}
+	if updated.Status.Message == "" {
+		t.Errorf("expected a non-empty status message in Running phase")
+	}
+}
+
 func TestReconcileHandleDeletion(t *testing.T) {
 	scheme := newScheme()
 	md := newMDForController("test", "default")

providers/kuberay/controller.go

@@ -309,6 +309,10 @@ func (r *KubeRayProviderReconciler) syncStatus(ctx context.Context, md *airunway
 	md.Status.Phase = statusResult.Phase
 	if statusResult.Message != "" {
 		md.Status.Message = statusResult.Message
+	} else if statusResult.Phase == airunwayv1alpha1.DeploymentPhaseRunning {
+		// The translator reports no message for a healthy RayService; replace the
+		// stale "waiting for pods" message so status reflects the Running phase.
+		md.Status.Message = "RayService created, pods are ready"
 	}
 	md.Status.Replicas = statusResult.Replicas
 	md.Status.Endpoint = statusResult.Endpoint

providers/kuberay/controller_test.go

@@ -2,6 +2,7 @@ package kuberay
 
 import (
 	"context"
+	"strings"
 	"testing"
 	"time"
 
@@ -610,6 +611,7 @@ func TestSyncStatusRunning(t *testing.T) {
 	r := NewKubeRayProviderReconciler(c, scheme)
 
 	md := &airunwayv1alpha1.ModelDeployment{}
+	md.Status.Message = "RayService created, waiting for pods to be ready"
 	desired := &unstructured.Unstructured{}
 	setRayServiceGVK(desired)
 	desired.SetName("test")
@@ -622,6 +624,13 @@ func TestSyncStatusRunning(t *testing.T) {
 	if md.Status.Phase != airunwayv1alpha1.DeploymentPhaseRunning {
 		t.Errorf("expected Running, got %s", md.Status.Phase)
 	}
+	// Issue #289: a Running deployment must not keep the "waiting for pods" message.
+	if strings.Contains(md.Status.Message, "waiting for pods") {
+		t.Errorf("status message still claims waiting for pods while Running: %q", md.Status.Message)
+	}
+	if md.Status.Message == "" {
+		t.Errorf("expected a non-empty status message in Running phase")
+	}
 }
 
 func TestSyncStatusFailed(t *testing.T) {

providers/llmd/controller.go

@@ -305,6 +305,10 @@ func (r *LLMDProviderReconciler) syncStatus(ctx context.Context, md *airunwayv1a
 	md.Status.Phase = statusResult.Phase
 	if statusResult.Message != "" {
 		md.Status.Message = statusResult.Message
+	} else if statusResult.Phase == airunwayv1alpha1.DeploymentPhaseRunning {
+		// The translator reports no message for a healthy Deployment; replace the
+		// stale "waiting for pods" message so status reflects the Running phase.
+		md.Status.Message = "Deployments created, pods are ready"
 	}
 	md.Status.Replicas = statusResult.Replicas
 	md.Status.Endpoint = statusResult.Endpoint

providers/llmd/controller_test.go

@@ -2,10 +2,12 @@ package llmd
 
 import (
 	"context"
+	"strings"
 	"testing"
 
 	airunwayv1alpha1 "github.com/kaito-project/airunway/controller/api/v1alpha1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -107,7 +109,7 @@ func TestValidateCompatibility(t *testing.T) {
 			name: "disaggregated without prefill is incompatible",
 			md: &airunwayv1alpha1.ModelDeployment{
 				Spec: airunwayv1alpha1.ModelDeploymentSpec{
-					Engine: airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
+					Engine:  airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
 					Serving: &airunwayv1alpha1.ServingSpec{Mode: airunwayv1alpha1.ServingModeDisaggregated},
 					Scaling: &airunwayv1alpha1.ScalingSpec{
 						Decode: &airunwayv1alpha1.ComponentScalingSpec{
@@ -123,7 +125,7 @@ func TestValidateCompatibility(t *testing.T) {
 			name: "disaggregated without decode is incompatible",
 			md: &airunwayv1alpha1.ModelDeployment{
 				Spec: airunwayv1alpha1.ModelDeploymentSpec{
-					Engine: airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
+					Engine:  airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
 					Serving: &airunwayv1alpha1.ServingSpec{Mode: airunwayv1alpha1.ServingModeDisaggregated},
 					Scaling: &airunwayv1alpha1.ScalingSpec{
 						Prefill: &airunwayv1alpha1.ComponentScalingSpec{
@@ -139,7 +141,7 @@ func TestValidateCompatibility(t *testing.T) {
 			name: "disaggregated with both prefill and decode is compatible",
 			md: &airunwayv1alpha1.ModelDeployment{
 				Spec: airunwayv1alpha1.ModelDeploymentSpec{
-					Engine: airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
+					Engine:  airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
 					Serving: &airunwayv1alpha1.ServingSpec{Mode: airunwayv1alpha1.ServingModeDisaggregated},
 					Scaling: &airunwayv1alpha1.ScalingSpec{
 						Prefill: &airunwayv1alpha1.ComponentScalingSpec{
@@ -159,7 +161,7 @@ func TestValidateCompatibility(t *testing.T) {
 			name: "disaggregated without GPU on prefill is incompatible",
 			md: &airunwayv1alpha1.ModelDeployment{
 				Spec: airunwayv1alpha1.ModelDeploymentSpec{
-					Engine: airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
+					Engine:  airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
 					Serving: &airunwayv1alpha1.ServingSpec{Mode: airunwayv1alpha1.ServingModeDisaggregated},
 					Scaling: &airunwayv1alpha1.ScalingSpec{
 						Prefill: &airunwayv1alpha1.ComponentScalingSpec{
@@ -178,7 +180,7 @@ func TestValidateCompatibility(t *testing.T) {
 			name: "disaggregated without GPU on decode is incompatible",
 			md: &airunwayv1alpha1.ModelDeployment{
 				Spec: airunwayv1alpha1.ModelDeploymentSpec{
-					Engine: airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
+					Engine:  airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
 					Serving: &airunwayv1alpha1.ServingSpec{Mode: airunwayv1alpha1.ServingModeDisaggregated},
 					Scaling: &airunwayv1alpha1.ScalingSpec{
 						Prefill: &airunwayv1alpha1.ComponentScalingSpec{
@@ -197,7 +199,7 @@ func TestValidateCompatibility(t *testing.T) {
 			name: "disaggregated without top-level resources is compatible",
 			md: &airunwayv1alpha1.ModelDeployment{
 				Spec: airunwayv1alpha1.ModelDeploymentSpec{
-					Engine: airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
+					Engine:  airunwayv1alpha1.EngineSpec{Type: airunwayv1alpha1.EngineTypeVLLM},
 					Serving: &airunwayv1alpha1.ServingSpec{Mode: airunwayv1alpha1.ServingModeDisaggregated},
 					Scaling: &airunwayv1alpha1.ScalingSpec{
 						Prefill: &airunwayv1alpha1.ComponentScalingSpec{
@@ -273,3 +275,46 @@ func TestReconcileIgnoresNoProvider(t *testing.T) {
 		t.Error("expected no requeue when no provider assigned")
 	}
 }
+
+// TestSyncStatusRunningUpdatesMessage reproduces issue #289: once the upstream
+// Deployment is Available the phase flips to Running, but the status message
+// must no longer claim it is "waiting for pods to be ready".
+func TestSyncStatusRunningUpdatesMessage(t *testing.T) {
+	scheme := newScheme()
+
+	deploy := &unstructured.Unstructured{}
+	deploy.SetAPIVersion("apps/v1")
+	deploy.SetKind("Deployment")
+	deploy.SetName("test")
+	deploy.SetNamespace("default")
+	deploy.Object["status"] = map[string]interface{}{
+		"conditions": []interface{}{
+			map[string]interface{}{"type": "Available", "status": "True"},
+		},
+	}
+
+	c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(deploy).Build()
+	r := NewLLMDProviderReconciler(c, scheme)
+
+	md := &airunwayv1alpha1.ModelDeployment{}
+	md.Status.Message = "Deployments created, waiting for pods to be ready"
+
+	desired := &unstructured.Unstructured{}
+	desired.SetAPIVersion("apps/v1")
+	desired.SetKind("Deployment")
+	desired.SetName("test")
+	desired.SetNamespace("default")
+
+	if err := r.syncStatus(context.Background(), md, desired); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if md.Status.Phase != airunwayv1alpha1.DeploymentPhaseRunning {
+		t.Fatalf("expected Running phase, got %s", md.Status.Phase)
+	}
+	if strings.Contains(md.Status.Message, "waiting for pods") {
+		t.Errorf("status message still claims waiting for pods while Running: %q", md.Status.Message)
+	}
+	if md.Status.Message == "" {
+		t.Errorf("expected a non-empty status message in Running phase")
+	}
+}

providers/vllm/controller.go

@@ -347,6 +347,10 @@ func (r *VLLMProviderReconciler) syncStatus(ctx context.Context, md *airunwayv1a
 	md.Status.Phase = statusResult.Phase
 	if statusResult.Message != "" {
 		md.Status.Message = statusResult.Message
+	} else if statusResult.Phase == airunwayv1alpha1.DeploymentPhaseRunning {
+		// The translator reports no message for a healthy Deployment; replace the
+		// stale "waiting for pods" message so status reflects the Running phase.
+		md.Status.Message = "Deployments created, pods are ready"
 	}
 	md.Status.Replicas = statusResult.Replicas
 	md.Status.Endpoint = statusResult.Endpoint

providers/vllm/controller_test.go

@@ -495,3 +495,42 @@ func TestRemoteImageResolverRejectsEmptyAndInvalidRefs(t *testing.T) {
 		t.Error("expected parse error for malformed image reference")
 	}
 }
+
+func TestSyncStatusRunningUpdatesMessage(t *testing.T) {
+	scheme := newScheme()
+
+	deploy := &unstructured.Unstructured{}
+	deploy.SetGroupVersionKind(deploymentGVK)
+	deploy.SetName("test")
+	deploy.SetNamespace("default")
+	deploy.Object["status"] = map[string]interface{}{
+		"conditions": []interface{}{
+			map[string]interface{}{"type": "Available", "status": "True"},
+		},
+	}
+
+	c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(deploy).Build()
+	r := NewVLLMProviderReconciler(c, scheme)
+
+	md := &airunwayv1alpha1.ModelDeployment{}
+	// Simulate a prior reconcile loop that left the deploying-phase message.
+	md.Status.Message = "Deployments created, waiting for pods to be ready"
+
+	desired := &unstructured.Unstructured{}
+	desired.SetGroupVersionKind(deploymentGVK)
+	desired.SetName("test")
+	desired.SetNamespace("default")
+
+	if err := r.syncStatus(context.Background(), md, desired); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if md.Status.Phase != airunwayv1alpha1.DeploymentPhaseRunning {
+		t.Fatalf("expected Running phase, got %s", md.Status.Phase)
+	}
+	if strings.Contains(md.Status.Message, "waiting for pods") {
+		t.Errorf("status message still claims waiting for pods while Running: %q", md.Status.Message)
+	}
+	if md.Status.Message == "" {
+		t.Errorf("expected a non-empty status message in Running phase")
+	}
+}