Support KServe platform for NIMService

Signed-off-by: Xieshen Zhang <xiezhang@redhat.com>

Author: xieshenzh

api/apps/v1alpha1/nimservice_types.go

@@ -1011,6 +1011,184 @@ func (n *NIMService) GetProxySpec() *ProxySpec {
 	return n.Spec.Proxy
 }
 
+// GetInferenceServiceParams returns params to render InferenceService from templates.
+func (n *NIMService) GetInferenceServiceParams() *rendertypes.InferenceServiceParams {
+	params := &rendertypes.InferenceServiceParams{}
+
+	// Set metadata
+	params.Name = n.GetName()
+	params.Namespace = n.GetNamespace()
+	params.Labels = n.GetServiceLabels()
+	params.Annotations = n.GetNIMServiceAnnotations()
+	params.PodAnnotations = n.GetNIMServiceAnnotations()
+	delete(params.PodAnnotations, utils.NvidiaAnnotationParentSpecHashKey)
+
+	// Set template spec
+	if !n.IsAutoScalingEnabled() {
+		params.MinReplicas = int32(n.GetReplicas())
+	} else {
+		hpa := n.GetHPA()
+
+		if hpa.MinReplicas != nil {
+			params.MinReplicas = *hpa.MinReplicas
+		}
+		params.MaxReplicas = hpa.MaxReplicas
+
+		for _, m := range hpa.Metrics {
+			if m.Type == autoscalingv2.ResourceMetricSourceType && m.Resource != nil {
+				if m.Resource.Name == corev1.ResourceCPU || m.Resource.Name == corev1.ResourceMemory {
+					params.ScaleMetric = string(m.Resource.Name)
+					params.ScaleMetricType = string(m.Resource.Target.Type)
+
+					switch m.Resource.Target.Type {
+					case autoscalingv2.UtilizationMetricType:
+						if m.Resource.Target.AverageUtilization != nil {
+							params.ScaleTarget = *m.Resource.Target.AverageUtilization
+						}
+					case autoscalingv2.ValueMetricType:
+						if m.Resource.Target.Value != nil {
+							params.ScaleTarget = int32((*m.Resource.Target.Value).Value())
+						}
+					case autoscalingv2.AverageValueMetricType:
+						if m.Resource.Target.AverageValue != nil {
+							params.ScaleTarget = int32((*m.Resource.Target.AverageValue).Value())
+						}
+					}
+					break
+				}
+			}
+		}
+	}
+
+	params.NodeSelector = n.GetNodeSelector()
+	params.Tolerations = n.GetTolerations()
+	params.Affinity = n.GetPodAffinity()
+	params.ImagePullSecrets = n.GetImagePullSecrets()
+	params.ImagePullPolicy = n.GetImagePullPolicy()
+
+	// Set labels and selectors
+	params.SelectorLabels = n.GetSelectorLabels()
+
+	// Set container spec
+	params.ContainerName = n.GetContainerName()
+	params.Env = n.GetEnv()
+	params.Args = n.GetArgs()
+	params.Command = n.GetCommand()
+	params.Resources = n.GetResources()
+	params.Image = n.GetImage()
+
+	// Set container probes
+	if IsProbeEnabled(n.Spec.LivenessProbe) {
+		if n.Spec.LivenessProbe.Probe == nil {
+			params.LivenessProbe = &corev1.Probe{
+				InitialDelaySeconds: 15,
+				TimeoutSeconds:      1,
+				PeriodSeconds:       10,
+				SuccessThreshold:    1,
+				FailureThreshold:    3,
+			}
+			if n.Spec.Expose.Service.GRPCPort != nil {
+				params.LivenessProbe.ProbeHandler = corev1.ProbeHandler{
+					GRPC: &corev1.GRPCAction{
+						Port: *n.Spec.Expose.Service.GRPCPort,
+					},
+				}
+			} else {
+				params.LivenessProbe.ProbeHandler = corev1.ProbeHandler{
+					HTTPGet: &corev1.HTTPGetAction{
+						Path: "/v1/health/live",
+						Port: intstr.FromInt32(*n.Spec.Expose.Service.Port),
+					},
+				}
+			}
+		} else {
+			params.LivenessProbe = n.Spec.LivenessProbe.Probe
+		}
+	}
+	if IsProbeEnabled(n.Spec.ReadinessProbe) {
+		if n.Spec.ReadinessProbe.Probe == nil {
+			params.ReadinessProbe = &corev1.Probe{
+				InitialDelaySeconds: 15,
+				TimeoutSeconds:      1,
+				PeriodSeconds:       10,
+				SuccessThreshold:    1,
+				FailureThreshold:    3,
+			}
+			if n.Spec.Expose.Service.GRPCPort != nil {
+				params.ReadinessProbe.ProbeHandler = corev1.ProbeHandler{
+					GRPC: &corev1.GRPCAction{
+						Port: *n.Spec.Expose.Service.GRPCPort,
+					},
+				}
+			} else {
+				params.ReadinessProbe.ProbeHandler = corev1.ProbeHandler{
+					HTTPGet: &corev1.HTTPGetAction{
+						Path: "/v1/health/ready",
+						Port: intstr.FromInt32(*n.Spec.Expose.Service.Port),
+					},
+				}
+			}
+		} else {
+			params.ReadinessProbe = n.Spec.ReadinessProbe.Probe
+		}
+	}
+	if IsProbeEnabled(n.Spec.StartupProbe) {
+		if n.Spec.StartupProbe.Probe == nil {
+			params.StartupProbe = &corev1.Probe{
+				InitialDelaySeconds: 30,
+				TimeoutSeconds:      1,
+				PeriodSeconds:       10,
+				SuccessThreshold:    1,
+				FailureThreshold:    30,
+			}
+			if n.Spec.Expose.Service.GRPCPort != nil {
+				params.StartupProbe.ProbeHandler = corev1.ProbeHandler{
+					GRPC: &corev1.GRPCAction{
+						Port: *n.Spec.Expose.Service.GRPCPort,
+					},
+				}
+			} else {
+				params.StartupProbe.ProbeHandler = corev1.ProbeHandler{
+					HTTPGet: &corev1.HTTPGetAction{
+						Path: "/v1/health/ready",
+						Port: intstr.FromInt32(*n.Spec.Expose.Service.Port),
+					},
+				}
+			}
+		} else {
+			params.StartupProbe = n.Spec.StartupProbe.Probe
+		}
+	}
+
+	params.UserID = n.GetUserID()
+	params.GroupID = n.GetGroupID()
+
+	// Set service account
+	params.ServiceAccountName = n.GetServiceAccountName()
+
+	// Set runtime class
+	params.RuntimeClassName = n.GetRuntimeClassName()
+
+	// Set scheduler
+	params.SchedulerName = n.GetSchedulerName()
+
+	// Setup container ports for nimservice
+	if n.Spec.Expose.Service.GRPCPort != nil {
+		params.Ports = append(params.Ports, corev1.ContainerPort{
+			Protocol:      corev1.ProtocolTCP,
+			ContainerPort: *n.Spec.Expose.Service.GRPCPort,
+		})
+	} else {
+		params.Ports = []corev1.ContainerPort{
+			{
+				Protocol:      corev1.ProtocolTCP,
+				ContainerPort: *n.Spec.Expose.Service.Port,
+			},
+		}
+	}
+	return params
+}
+
 func init() {
 	SchemeBuilder.Register(&NIMService{}, &NIMServiceList{})
 }

cmd/main.go

@@ -21,7 +21,9 @@ import (
 	"flag"
 	"os"
 
+	kservev1beta1 "github.com/kserve/kserve/pkg/apis/serving/v1beta1"
 	monitoring "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	"k8s.io/apimachinery/pkg/runtime"
@@ -55,6 +57,7 @@ func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(appsv1alpha1.AddToScheme(scheme))
 	utilruntime.Must(monitoring.AddToScheme(scheme))
+	utilruntime.Must(kservev1beta1.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
 }
 

config/manager/manager.yaml

@@ -90,7 +90,7 @@ spec:
         resources:
           limits:
             cpu: 500m
-            memory: 128Mi
+            memory: 512Mi
           requests:
             cpu: 10m
             memory: 64Mi

config/rbac/role.yaml

@@ -286,6 +286,18 @@ rules:
   - securitycontextconstraints
   verbs:
   - use
+- apiGroups:
+  - serving.kserve.io
+  resources:
+  - inferenceservices
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - storage.k8s.io
   resources: