Skip to content

Commit 4481b44

Browse files
visheshtanksalevisheshtanksale
authored
Adding support to set inferencing platform on NIMService CRD (#601)
* Adding support to set inferencing platform on NIMService CRD Signed-off-by: Vishesh Tanksale <vtanksale@nvidia.com> * Fixing build issue Signed-off-by: Vishesh Tanksale <vtanksale@nvidia.com> --------- Signed-off-by: Vishesh Tanksale <vtanksale@nvidia.com>
1 parent 581daa0 commit 4481b44

File tree

13 files changed

+120
-42
lines changed

13 files changed

+120
-42
lines changed

‎api/apps/v1alpha1/nimservice_types.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,16 @@ const (
7777
NIMBackendTypeLWS NIMBackendType = "lws"
7878
)
7979

80+
// PlatformType defines the supported inference platform types.
81+
type PlatformType string
82+
83+
const (
84+
// PlatformTypeStandalone represents standalone deployment platform.
85+
PlatformTypeStandalone PlatformType = "standalone"
86+
// PlatformTypeKServe represents KServe deployment platform.
87+
PlatformTypeKServe PlatformType = "kserve"
88+
)
89+
8090
// NIMServiceSpec defines the desired state of NIMService.
8191
// +kubebuilder:validation:XValidation:rule="!(has(self.multiNode) && has(self.scale) && has(self.scale.enabled) && self.scale.enabled)", message="autoScaling must be nil or disabled when multiNode is set"
8292
type NIMServiceSpec struct {
@@ -115,6 +125,11 @@ type NIMServiceSpec struct {
115125
RuntimeClassName string `json:"runtimeClassName,omitempty"`
116126
Proxy *ProxySpec `json:"proxy,omitempty"`
117127
MultiNode *NimServiceMultiNodeConfig `json:"multiNode,omitempty"`
128+
// InferencePlatform specifies the inference platform to use for this NIMService.
129+
// Valid values are "standalone" (default) and "kserve".
130+
// +kubebuilder:validation:Enum=standalone;kserve
131+
// +kubebuilder:default:="standalone"
132+
InferencePlatform PlatformType `json:"inferencePlatform,omitempty"`
118133
}
119134

120135
// NimServiceMultiNodeConfig defines the configuration for multi-node NIMService.

‎bundle/manifests/apps.nvidia.com_nimpipelines.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,15 @@ spec:
629629
- repository
630630
- tag
631631
type: object
632+
inferencePlatform:
633+
default: standalone
634+
description: |-
635+
InferencePlatform specifies the inference platform to use for this NIMService.
636+
Valid values are "standalone" (default) and "kserve".
637+
enum:
638+
- standalone
639+
- kserve
640+
type: string
632641
labels:
633642
additionalProperties:
634643
type: string

‎bundle/manifests/apps.nvidia.com_nimservices.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,15 @@ spec:
578578
- repository
579579
- tag
580580
type: object
581+
inferencePlatform:
582+
default: standalone
583+
description: |-
584+
InferencePlatform specifies the inference platform to use for this NIMService.
585+
Valid values are "standalone" (default) and "kserve".
586+
enum:
587+
- standalone
588+
- kserve
589+
type: string
581590
labels:
582591
additionalProperties:
583592
type: string

‎cmd/main.go‎

Lines changed: 13 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,6 @@ import (
4444
appsv1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1"
4545
"github.com/NVIDIA/k8s-nim-operator/internal/conditions"
4646
"github.com/NVIDIA/k8s-nim-operator/internal/controller"
47-
"github.com/NVIDIA/k8s-nim-operator/internal/controller/platform"
48-
"github.com/NVIDIA/k8s-nim-operator/internal/controller/platform/kserve"
49-
"github.com/NVIDIA/k8s-nim-operator/internal/controller/platform/standalone"
5047
"github.com/NVIDIA/k8s-nim-operator/internal/render"
5148
webhookappsv1alpha1 "github.com/NVIDIA/k8s-nim-operator/internal/webhook/apps/v1alpha1"
5249
// +kubebuilder:scaffold:imports
@@ -74,8 +71,9 @@ func main() {
7471
var enableHTTP2 bool
7572
var platformType string
7673

77-
flag.StringVar(&platformType, "platform", "standalone", "The model-serving inference platform to use."+
78-
"E.g., 'standalone (default)', 'kserve'.")
74+
flag.StringVar(&platformType, "platform", "", "DEPRECATED: Default platform for all NIMServices. "+
75+
"Use the 'platform' field in NIMService CR instead. If specified, this value is used as fallback "+
76+
"when NIMService doesn't specify a platform. Valid values: 'standalone', 'kserve'.")
7977
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+
8078
"Use the port :8080. If not set, it will be 0 in order to disable the metrics server")
8179
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -94,15 +92,16 @@ func main() {
9492

9593
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
9694

97-
var platformImpl platform.Platform
98-
switch platformType {
99-
case "standalone":
100-
platformImpl = &standalone.Standalone{}
101-
case "kserve":
102-
platformImpl = &kserve.KServe{}
103-
default:
104-
setupLog.Error(nil, "unsupported model-serving platform type", "platformType", platformType)
105-
os.Exit(1)
95+
// Validate the deprecated global platform flag if provided
96+
if platformType != "" {
97+
setupLog.Info("DEPRECATED: Global platform flag is deprecated. Use 'platform' field in NIMService CR instead.", "platform", platformType)
98+
switch platformType {
99+
case "standalone", "kserve":
100+
// Valid platform types for the deprecated global platform flag. No need to throw an error.
101+
default:
102+
setupLog.Error(nil, "unsupported model-serving platform type", "platformType", platformType)
103+
os.Exit(1)
104+
}
106105
}
107106

108107
// if the enable-http2 flag is false (the default), http/2 should be disabled
@@ -166,7 +165,6 @@ func main() {
166165
mgr.GetClient(),
167166
mgr.GetScheme(),
168167
ctrl.Log.WithName("controllers").WithName("NIMCache"),
169-
platformImpl,
170168
).SetupWithManager(mgr); err != nil {
171169
setupLog.Error(err, "unable to create controller", "controller", "NIMCache")
172170
os.Exit(1)
@@ -179,7 +177,6 @@ func main() {
179177
discoveryClient,
180178
render.NewRenderer("/manifests"),
181179
ctrl.Log.WithName("controllers").WithName("NIMService"),
182-
platformImpl,
183180
).SetupWithManager(mgr); err != nil {
184181
setupLog.Error(err, "unable to create controller", "controller", "NIMService")
185182
os.Exit(1)
@@ -197,7 +194,6 @@ func main() {
197194
mgr.GetClient(),
198195
mgr.GetScheme(),
199196
ctrl.Log.WithName("controllers").WithName("NIMBuild"),
200-
platformImpl,
201197
).SetupWithManager(mgr); err != nil {
202198
setupLog.Error(err, "unable to create controller", "controller", "NIMBuild")
203199
os.Exit(1)

‎config/crd/bases/apps.nvidia.com_nimpipelines.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,15 @@ spec:
629629
- repository
630630
- tag
631631
type: object
632+
inferencePlatform:
633+
default: standalone
634+
description: |-
635+
InferencePlatform specifies the inference platform to use for this NIMService.
636+
Valid values are "standalone" (default) and "kserve".
637+
enum:
638+
- standalone
639+
- kserve
640+
type: string
632641
labels:
633642
additionalProperties:
634643
type: string

‎config/crd/bases/apps.nvidia.com_nimservices.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,15 @@ spec:
578578
- repository
579579
- tag
580580
type: object
581+
inferencePlatform:
582+
default: standalone
583+
description: |-
584+
InferencePlatform specifies the inference platform to use for this NIMService.
585+
Valid values are "standalone" (default) and "kserve".
586+
enum:
587+
- standalone
588+
- kserve
589+
type: string
581590
labels:
582591
additionalProperties:
583592
type: string

‎deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nimpipelines.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,15 @@ spec:
629629
- repository
630630
- tag
631631
type: object
632+
inferencePlatform:
633+
default: standalone
634+
description: |-
635+
InferencePlatform specifies the inference platform to use for this NIMService.
636+
Valid values are "standalone" (default) and "kserve".
637+
enum:
638+
- standalone
639+
- kserve
640+
type: string
632641
labels:
633642
additionalProperties:
634643
type: string

‎deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nimservices.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,15 @@ spec:
578578
- repository
579579
- tag
580580
type: object
581+
inferencePlatform:
582+
default: standalone
583+
description: |-
584+
InferencePlatform specifies the inference platform to use for this NIMService.
585+
Valid values are "standalone" (default) and "kserve".
586+
enum:
587+
- standalone
588+
- kserve
589+
type: string
581590
labels:
582591
additionalProperties:
583592
type: string

‎internal/controller/nimbuild_controller.go‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ import (
5050

5151
appsv1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1"
5252
"github.com/NVIDIA/k8s-nim-operator/internal/conditions"
53-
"github.com/NVIDIA/k8s-nim-operator/internal/controller/platform"
5453
"github.com/NVIDIA/k8s-nim-operator/internal/k8sutil"
5554
"github.com/NVIDIA/k8s-nim-operator/internal/render"
5655
"github.com/NVIDIA/k8s-nim-operator/internal/shared"
@@ -72,7 +71,6 @@ type NIMBuildReconciler struct {
7271
client.Client
7372
scheme *runtime.Scheme
7473
log logr.Logger
75-
Platform platform.Platform
7674
orchestratorType k8sutil.OrchestratorType
7775
updater conditions.Updater
7876
recorder record.EventRecorder
@@ -82,12 +80,11 @@ type NIMBuildReconciler struct {
8280
var _ shared.Reconciler = &NIMBuildReconciler{}
8381

8482
// NewNIMBuildReconciler creates a new reconciler for NIMBuild with the given platform.
85-
func NewNIMBuildReconciler(client client.Client, scheme *runtime.Scheme, log logr.Logger, platform platform.Platform) *NIMBuildReconciler {
83+
func NewNIMBuildReconciler(client client.Client, scheme *runtime.Scheme, log logr.Logger) *NIMBuildReconciler {
8684
return &NIMBuildReconciler{
87-
Client: client,
88-
scheme: scheme,
89-
log: log,
90-
Platform: platform,
85+
Client: client,
86+
scheme: scheme,
87+
log: log,
9188
}
9289
}
9390

‎internal/controller/nimcache_controller.go‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ import (
4848

4949
appsv1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1"
5050
"github.com/NVIDIA/k8s-nim-operator/internal/conditions"
51-
platform "github.com/NVIDIA/k8s-nim-operator/internal/controller/platform"
5251
"github.com/NVIDIA/k8s-nim-operator/internal/k8sutil"
5352
"github.com/NVIDIA/k8s-nim-operator/internal/nimparser"
5453
nimparserutils "github.com/NVIDIA/k8s-nim-operator/internal/nimparser/utils"
@@ -86,7 +85,6 @@ type NIMCacheReconciler struct {
8685
client.Client
8786
scheme *runtime.Scheme
8887
log logr.Logger
89-
Platform platform.Platform
9088
orchestratorType k8sutil.OrchestratorType
9189
updater conditions.Updater
9290
recorder record.EventRecorder
@@ -96,12 +94,11 @@ type NIMCacheReconciler struct {
9694
var _ shared.Reconciler = &NIMCacheReconciler{}
9795

9896
// NewNIMCacheReconciler creates a new reconciler for NIMCache with the given platform.
99-
func NewNIMCacheReconciler(client client.Client, scheme *runtime.Scheme, log logr.Logger, platform platform.Platform) *NIMCacheReconciler {
97+
func NewNIMCacheReconciler(client client.Client, scheme *runtime.Scheme, log logr.Logger) *NIMCacheReconciler {
10098
return &NIMCacheReconciler{
101-
Client: client,
102-
scheme: scheme,
103-
log: log,
104-
Platform: platform,
99+
Client: client,
100+
scheme: scheme,
101+
log: log,
105102
}
106103
}
107104

0 commit comments

Comments
 (0)