nimservice should be notready while nimcache is populating

Signed-off-by: Varun Ramachandra Sekar <vsekar@nvidia.com>

Author: varunrsekar

internal/conditions/conditions.go

@@ -61,6 +61,8 @@ const (
 	ReasonStatefulSetFailed = "StatefulsetFailed"
 	// ReasonSecretFailed indicates that the creation of secret has failed.
 	ReasonSecretFailed = "SecretFailed"
+	// ReasonNIMCacheFailed indicates that the NIMCache is in failed state.
+	ReasonNIMCacheFailed = "NIMCacheFailed"
 )
 
 // Updater is the condition updater.

internal/controller/platform/standalone/nimservice.go

@@ -30,6 +30,7 @@ import (
 	networkingv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	apiResource "k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -185,8 +186,35 @@ func (r *NIMServiceReconciler) reconcileNIMService(ctx context.Context, nimServi
 
 	deploymentParams.OrchestratorType = string(r.GetOrchestratorType())
 
-	// Select PVC for model store
-	if nimService.GetNIMCacheName() != "" { // nolint:gocritic
+	// Select PVC for model storea
+	nimCacheName := nimService.GetNIMCacheName()
+	if len(nimCacheName) > 0 {
+		nimCache := appsv1alpha1.NIMCache{}
+		if err := r.Get(ctx, types.NamespacedName{Name: nimCacheName, Namespace: nimService.GetNamespace()}, &nimCache); err != nil {
+			statusError := r.updater.SetConditionsFailed(ctx, nimService, conditions.ReasonNIMCacheFailed, err.Error())
+			r.GetEventRecorder().Eventf(nimService, corev1.EventTypeWarning, conditions.Failed, err.Error())
+			if statusError != nil {
+				logger.Error(statusError, "failed to update status", "nimservice", nimService.Name)
+			}
+			return ctrl.Result{}, err
+		}
+		switch nimCache.Status.State {
+		case appsv1alpha1.NimCacheStatusReady:
+			logger.V(4).Info("NIMCache is ready", "nimcache", nimCacheName)
+		case appsv1alpha1.NimCacheStatusFailed:
+			msg := r.getNIMCacheFailedMessage(&nimCache)
+			err = r.updater.SetConditionsFailed(ctx, nimService, conditions.ReasonNIMCacheFailed, msg)
+			r.GetEventRecorder().Eventf(nimService, corev1.EventTypeWarning, conditions.Failed, msg)
+		default:
+			msg := fmt.Sprintf("NIMCache %s not ready", nimCacheName)
+			err = r.updater.SetConditionsNotReady(ctx, nimService, conditions.NotReady, msg)
+			r.GetEventRecorder().Eventf(nimService, corev1.EventTypeNormal, conditions.NotReady,
+				"NIMService %s not ready yet, msg: %s", nimService.Name, msg)
+			if err != nil {
+				logger.Error(err, "failed to ", "nimservice", nimService.Name)
+			}
+			return ctrl.Result{RequeueAfter: 5 * time.Second}, err
+		}
 		// Fetch PVC for the associated NIMCache instance and mount it
 		nimCachePVC, err := r.getNIMCachePVC(ctx, nimService)
 		if err != nil {
@@ -724,3 +752,11 @@ func (r *NIMServiceReconciler) assignGPUResources(ctx context.Context, nimServic
 
 	return nil
 }
+
+func (r *NIMServiceReconciler) getNIMCacheFailedMessage(nimCache *appsv1alpha1.NIMCache) string {
+	cond := meta.FindStatusCondition(nimCache.Status.Conditions, conditions.Failed)
+	if cond != nil {
+		return cond.Message
+	}
+	return ""
+}

internal/controller/platform/standalone/nimservice_test.go

@@ -27,6 +27,7 @@ import (
 	"path"
 	"sort"
 	"strings"
+	"time"
 
 	"os"
 
@@ -611,6 +612,26 @@ var _ = Describe("NIMServiceReconciler for a standalone platform", func() {
 
 	})
 
+	It("should be NotReady when nimcache is not ready", func() {
+		nimCache.Status = appsv1alpha1.NIMCacheStatus{
+			State: appsv1alpha1.NimCacheStatusNotReady,
+		}
+		Expect(client.Status().Update(context.TODO(), nimCache)).To(Succeed())
+		err := client.Create(context.TODO(), nimService)
+		Expect(err).NotTo(HaveOccurred())
+
+		result, err := reconciler.reconcileNIMService(context.TODO(), nimService)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result).To(Equal(ctrl.Result{RequeueAfter: 5 * time.Second}))
+
+		// Check that the NIMService is not ready.
+		namespacedName := types.NamespacedName{Name: nimService.Name, Namespace: nimService.Namespace}
+		obj := &appsv1alpha1.NIMService{}
+		err = client.Get(context.TODO(), namespacedName, obj)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(obj.Status.State).To(Equal(appsv1alpha1.NIMServiceStatusNotReady))
+	})
+
 	Describe("isDeploymentReady for setting status on NIMService", func() {
 		AfterEach(func() {
 			// Clean up the Deployment instance