Stop and resume a model by adding a new annotation [Serverless] (kserve#4337)

Signed-off-by: Hannah DeFazio <h2defazio@gmail.com>
Signed-off-by: Jooho Lee <jlee@redhat.com>
Co-authored-by: Jooho Lee <ljhiyh@gmail.com>

Author: hdefazio

pkg/apis/serving/v1beta1/inference_service.go

@@ -17,7 +17,11 @@ limitations under the License.
 package v1beta1
 
 import (
+	"strings"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/kserve/kserve/pkg/constants"
 )
 
 // InferenceServiceSpec is the top level type for this resource
@@ -136,3 +140,15 @@ type InferenceServiceList struct {
 func init() {
 	SchemeBuilder.Register(&InferenceService{}, &InferenceServiceList{})
 }
+
+func (isvc *InferenceService) GetForceStopRuntime() bool {
+	forceStopRuntime := false
+	if isvc == nil || isvc.Annotations == nil {
+		return forceStopRuntime
+	}
+	if val, exist := isvc.Annotations[constants.StopAnnotationKey]; exist {
+		forceStopRuntime = strings.EqualFold(val, "true")
+	}
+
+	return forceStopRuntime
+}

pkg/apis/serving/v1beta1/inference_service_status.go

@@ -124,6 +124,8 @@ const (
 	RoutesReady apis.ConditionType = "RoutesReady"
 	// LatestDeploymentReady is set when underlying configurations for all components have reported readiness.
 	LatestDeploymentReady apis.ConditionType = "LatestDeploymentReady"
+	// Stopped is set when the inference service has been stopped and all related objects are deleted
+	Stopped apis.ConditionType = "Stopped"
 )
 
 type ModelStatus struct {

pkg/constants/constants.go

@@ -94,6 +94,7 @@ var (
 	TargetUtilizationPercentage                 = KServeAPIGroupName + "/targetUtilizationPercentage"
 	MinScaleAnnotationKey                       = KnativeAutoscalingAPIGroupName + "/min-scale"
 	MaxScaleAnnotationKey                       = KnativeAutoscalingAPIGroupName + "/max-scale"
+	StopAnnotationKey                           = KServeAPIGroupName + "/stop"
 	RollOutDurationAnnotationKey                = KnativeServingAPIGroupName + "/rollout-duration"
 	KnativeOpenshiftEnablePassthroughKey        = "serving.knative.openshift.io/enablePassthrough"
 	EnableMetricAggregation                     = KServeAPIGroupName + "/enable-metric-aggregation"

pkg/controller/v1beta1/inferenceservice/components/predictor.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/utils/ptr"
+	"knative.dev/pkg/apis"
 	knservingv1 "knative.dev/serving/pkg/apis/serving/v1"
 
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -189,9 +190,47 @@ func (p *Predictor) Reconcile(ctx context.Context, isvc *v1beta1.InferenceServic
 	} else {
 		var err error
 		podLabelKey = constants.RevisionLabel
+
 		if kstatus, err = p.reconcileKnativeDeployment(ctx, isvc, &objectMeta, &podSpec); err != nil {
 			return ctrl.Result{}, err
 		}
+		if isvc.GetForceStopRuntime() {
+			// Exit early if we have already set the status to stopped
+			existing_stopped_condition := isvc.Status.GetCondition(v1beta1.Stopped)
+			if existing_stopped_condition != nil && existing_stopped_condition.Status == corev1.ConditionTrue {
+				return ctrl.Result{}, nil
+			}
+
+			deployMode := isvc.Status.DeploymentMode
+
+			// Clear all statuses
+			isvc.Status = v1beta1.InferenceServiceStatus{}
+
+			// Preserve the deployment mode value
+			isvc.Status.DeploymentMode = deployMode
+
+			// Set the ready condition
+			predictor_ready_condition := &apis.Condition{
+				Type:   v1beta1.PredictorReady,
+				Status: corev1.ConditionFalse,
+			}
+			isvc.Status.SetCondition(v1beta1.PredictorReady, predictor_ready_condition)
+
+			// Add the stopped condition
+			stopped_condition := &apis.Condition{
+				Type:   v1beta1.Stopped,
+				Status: corev1.ConditionTrue,
+			}
+			isvc.Status.SetCondition(v1beta1.Stopped, stopped_condition)
+
+			return ctrl.Result{}, nil
+		} else {
+			resume_condition := &apis.Condition{
+				Type:   v1beta1.Stopped,
+				Status: corev1.ConditionFalse,
+			}
+			isvc.Status.SetCondition(v1beta1.Stopped, resume_condition)
+		}
 	}
 
 	statusSpec := isvc.Status.Components[v1beta1.PredictorComponent]
@@ -204,6 +243,7 @@ func (p *Predictor) Reconcile(ctx context.Context, isvc *v1beta1.InferenceServic
 	if err != nil {
 		return ctrl.Result{}, errors.Wrapf(err, "fails to list inferenceservice pods by label")
 	}
+
 	if isvc.Status.PropagateModelStatus(statusSpec, predictorPods, rawDeployment, kstatus) {
 		return ctrl.Result{}, nil
 	} else {
@@ -693,6 +733,8 @@ func (p *Predictor) reconcileKnativeDeployment(ctx context.Context, isvc *v1beta
 	if err != nil {
 		return nil, errors.Wrapf(err, "fails to reconcile predictor")
 	}
-	isvc.Status.PropagateStatus(v1beta1.PredictorComponent, kstatus)
+	if !isvc.GetForceStopRuntime() {
+		isvc.Status.PropagateStatus(v1beta1.PredictorComponent, kstatus)
+	}
 	return kstatus, nil
 }

pkg/controller/v1beta1/inferenceservice/controller.go

@@ -254,8 +254,10 @@ func (r *InferenceServiceReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		if isvc.Spec.Explainer != nil {
 			componentList = append(componentList, v1beta1.ExplainerComponent)
 		}
-		isvc.Status.PropagateCrossComponentStatus(componentList, v1beta1.RoutesReady)
-		isvc.Status.PropagateCrossComponentStatus(componentList, v1beta1.LatestDeploymentReady)
+		if !isvc.GetForceStopRuntime() {
+			isvc.Status.PropagateCrossComponentStatus(componentList, v1beta1.RoutesReady)
+			isvc.Status.PropagateCrossComponentStatus(componentList, v1beta1.LatestDeploymentReady)
+		}
 	}
 	// Reconcile ingress
 	ingressConfig, err := v1beta1.NewIngressConfig(isvcConfigMap)