Skip to content

Commit 8a91dd0

Browse files
visheshtanksalevisheshtanksale
committed
Adding support to set inferencing platform on NIMService CRD
Signed-off-by: Vishesh Tanksale <vtanksale@nvidia.com>
1 parent e3c87c0 commit 8a91dd0

13 files changed

Lines changed: 153 additions & 58 deletions

‎api/apps/v1alpha1/nimservice_types.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,16 @@ const (
7777
NIMBackendTypeLWS NIMBackendType = "lws"
7878
)
7979

80+
// PlatformType defines the supported inference platform types.
81+
type PlatformType string
82+
83+
const (
84+
// PlatformTypeStandalone represents standalone deployment platform.
85+
PlatformTypeStandalone PlatformType = "standalone"
86+
// PlatformTypeKServe represents KServe deployment platform.
87+
PlatformTypeKServe PlatformType = "kserve"
88+
)
89+
8090
// NIMServiceSpec defines the desired state of NIMService.
8191
// +kubebuilder:validation:XValidation:rule="!(has(self.multiNode) && has(self.scale) && has(self.scale.enabled) && self.scale.enabled)", message="autoScaling must be nil or disabled when multiNode is set"
8292
type NIMServiceSpec struct {
@@ -115,6 +125,11 @@ type NIMServiceSpec struct {
115125
RuntimeClassName string `json:"runtimeClassName,omitempty"`
116126
Proxy *ProxySpec `json:"proxy,omitempty"`
117127
MultiNode *NimServiceMultiNodeConfig `json:"multiNode,omitempty"`
128+
// InferencePlatform specifies the inference platform to use for this NIMService.
129+
// Valid values are "standalone" (default) and "kserve".
130+
// +kubebuilder:validation:Enum=standalone;kserve
131+
// +kubebuilder:default:="standalone"
132+
InferencePlatform PlatformType `json:"inferencePlatform,omitempty"`
118133
}
119134

120135
// NimServiceMultiNodeConfig defines the configuration for multi-node NIMService.

‎bundle/manifests/apps.nvidia.com_nimpipelines.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,15 @@ spec:
629629
- repository
630630
- tag
631631
type: object
632+
inferencePlatform:
633+
default: standalone
634+
description: |-
635+
InferencePlatform specifies the inference platform to use for this NIMService.
636+
Valid values are "standalone" (default) and "kserve".
637+
enum:
638+
- standalone
639+
- kserve
640+
type: string
632641
labels:
633642
additionalProperties:
634643
type: string

‎bundle/manifests/apps.nvidia.com_nimservices.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,15 @@ spec:
578578
- repository
579579
- tag
580580
type: object
581+
inferencePlatform:
582+
default: standalone
583+
description: |-
584+
InferencePlatform specifies the inference platform to use for this NIMService.
585+
Valid values are "standalone" (default) and "kserve".
586+
enum:
587+
- standalone
588+
- kserve
589+
type: string
581590
labels:
582591
additionalProperties:
583592
type: string

‎cmd/main.go‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,6 @@ import (
4444
"github.com/NVIDIA/k8s-nim-operator/internal/conditions"
4545
"github.com/NVIDIA/k8s-nim-operator/internal/controller"
4646
"github.com/NVIDIA/k8s-nim-operator/internal/controller/platform"
47-
"github.com/NVIDIA/k8s-nim-operator/internal/controller/platform/kserve"
48-
"github.com/NVIDIA/k8s-nim-operator/internal/controller/platform/standalone"
4947
"github.com/NVIDIA/k8s-nim-operator/internal/render"
5048
// +kubebuilder:scaffold:imports
5149
)
@@ -72,8 +70,9 @@ func main() {
7270
var enableHTTP2 bool
7371
var platformType string
7472

75-
flag.StringVar(&platformType, "platform", "standalone", "The model-serving inference platform to use."+
76-
"E.g., 'standalone (default)', 'kserve'.")
73+
flag.StringVar(&platformType, "platform", "", "DEPRECATED: Default platform for all NIMServices. "+
74+
"Use the 'platform' field in NIMService CR instead. If specified, this value is used as fallback "+
75+
"when NIMService doesn't specify a platform. Valid values: 'standalone', 'kserve'.")
7776
flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+
7877
"Use the port :8080. If not set, it will be 0 in order to disable the metrics server")
7978
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
@@ -92,17 +91,21 @@ func main() {
9291

9392
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
9493

95-
var platformImpl platform.Platform
96-
switch platformType {
97-
case "standalone":
98-
platformImpl = &standalone.Standalone{}
99-
case "kserve":
100-
platformImpl = &kserve.KServe{}
101-
default:
102-
setupLog.Error(nil, "unsupported model-serving platform type", "platformType", platformType)
103-
os.Exit(1)
94+
// Validate the deprecated global platform flag if provided
95+
if platformType != "" {
96+
setupLog.Info("DEPRECATED: Global platform flag is deprecated. Use 'platform' field in NIMService CR instead.", "platform", platformType)
97+
switch platformType {
98+
case "standalone", "kserve":
99+
// Valid platform types
100+
default:
101+
setupLog.Error(nil, "unsupported model-serving platform type", "platformType", platformType)
102+
os.Exit(1)
103+
}
104104
}
105105

106+
// Create platform factory
107+
platformFactory := platform.NewFactory()
108+
106109
// if the enable-http2 flag is false (the default), http/2 should be disabled
107110
// due to its vulnerabilities. More specifically, disabling http/2 will
108111
// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
@@ -164,7 +167,6 @@ func main() {
164167
mgr.GetClient(),
165168
mgr.GetScheme(),
166169
ctrl.Log.WithName("controllers").WithName("NIMCache"),
167-
platformImpl,
168170
).SetupWithManager(mgr); err != nil {
169171
setupLog.Error(err, "unable to create controller", "controller", "NIMCache")
170172
os.Exit(1)
@@ -177,7 +179,7 @@ func main() {
177179
discoveryClient,
178180
render.NewRenderer("/manifests"),
179181
ctrl.Log.WithName("controllers").WithName("NIMService"),
180-
platformImpl,
182+
platformFactory,
181183
).SetupWithManager(mgr); err != nil {
182184
setupLog.Error(err, "unable to create controller", "controller", "NIMService")
183185
os.Exit(1)
@@ -195,7 +197,6 @@ func main() {
195197
mgr.GetClient(),
196198
mgr.GetScheme(),
197199
ctrl.Log.WithName("controllers").WithName("NIMBuild"),
198-
platformImpl,
199200
).SetupWithManager(mgr); err != nil {
200201
setupLog.Error(err, "unable to create controller", "controller", "NIMBuild")
201202
os.Exit(1)

‎config/crd/bases/apps.nvidia.com_nimpipelines.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,15 @@ spec:
629629
- repository
630630
- tag
631631
type: object
632+
inferencePlatform:
633+
default: standalone
634+
description: |-
635+
InferencePlatform specifies the inference platform to use for this NIMService.
636+
Valid values are "standalone" (default) and "kserve".
637+
enum:
638+
- standalone
639+
- kserve
640+
type: string
632641
labels:
633642
additionalProperties:
634643
type: string

‎config/crd/bases/apps.nvidia.com_nimservices.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,15 @@ spec:
578578
- repository
579579
- tag
580580
type: object
581+
inferencePlatform:
582+
default: standalone
583+
description: |-
584+
InferencePlatform specifies the inference platform to use for this NIMService.
585+
Valid values are "standalone" (default) and "kserve".
586+
enum:
587+
- standalone
588+
- kserve
589+
type: string
581590
labels:
582591
additionalProperties:
583592
type: string

‎deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nimpipelines.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -629,6 +629,15 @@ spec:
629629
- repository
630630
- tag
631631
type: object
632+
inferencePlatform:
633+
default: standalone
634+
description: |-
635+
InferencePlatform specifies the inference platform to use for this NIMService.
636+
Valid values are "standalone" (default) and "kserve".
637+
enum:
638+
- standalone
639+
- kserve
640+
type: string
632641
labels:
633642
additionalProperties:
634643
type: string

‎deployments/helm/k8s-nim-operator/crds/apps.nvidia.com_nimservices.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,15 @@ spec:
578578
- repository
579579
- tag
580580
type: object
581+
inferencePlatform:
582+
default: standalone
583+
description: |-
584+
InferencePlatform specifies the inference platform to use for this NIMService.
585+
Valid values are "standalone" (default) and "kserve".
586+
enum:
587+
- standalone
588+
- kserve
589+
type: string
581590
labels:
582591
additionalProperties:
583592
type: string

‎internal/controller/nimbuild_controller.go‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ import (
5050

5151
appsv1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1"
5252
"github.com/NVIDIA/k8s-nim-operator/internal/conditions"
53-
"github.com/NVIDIA/k8s-nim-operator/internal/controller/platform"
5453
"github.com/NVIDIA/k8s-nim-operator/internal/k8sutil"
5554
"github.com/NVIDIA/k8s-nim-operator/internal/render"
5655
"github.com/NVIDIA/k8s-nim-operator/internal/shared"
@@ -72,7 +71,6 @@ type NIMBuildReconciler struct {
7271
client.Client
7372
scheme *runtime.Scheme
7473
log logr.Logger
75-
Platform platform.Platform
7674
orchestratorType k8sutil.OrchestratorType
7775
updater conditions.Updater
7876
recorder record.EventRecorder
@@ -82,12 +80,11 @@ type NIMBuildReconciler struct {
8280
var _ shared.Reconciler = &NIMBuildReconciler{}
8381

8482
// NewNIMBuildReconciler creates a new reconciler for NIMBuild with the given platform.
85-
func NewNIMBuildReconciler(client client.Client, scheme *runtime.Scheme, log logr.Logger, platform platform.Platform) *NIMBuildReconciler {
83+
func NewNIMBuildReconciler(client client.Client, scheme *runtime.Scheme, log logr.Logger) *NIMBuildReconciler {
8684
return &NIMBuildReconciler{
87-
Client: client,
88-
scheme: scheme,
89-
log: log,
90-
Platform: platform,
85+
Client: client,
86+
scheme: scheme,
87+
log: log,
9188
}
9289
}
9390

‎internal/controller/nimcache_controller.go‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ import (
4848

4949
appsv1alpha1 "github.com/NVIDIA/k8s-nim-operator/api/apps/v1alpha1"
5050
"github.com/NVIDIA/k8s-nim-operator/internal/conditions"
51-
platform "github.com/NVIDIA/k8s-nim-operator/internal/controller/platform"
5251
"github.com/NVIDIA/k8s-nim-operator/internal/k8sutil"
5352
"github.com/NVIDIA/k8s-nim-operator/internal/nimparser"
5453
nimparserutils "github.com/NVIDIA/k8s-nim-operator/internal/nimparser/utils"
@@ -86,7 +85,6 @@ type NIMCacheReconciler struct {
8685
client.Client
8786
scheme *runtime.Scheme
8887
log logr.Logger
89-
Platform platform.Platform
9088
orchestratorType k8sutil.OrchestratorType
9189
updater conditions.Updater
9290
recorder record.EventRecorder
@@ -96,12 +94,11 @@ type NIMCacheReconciler struct {
9694
var _ shared.Reconciler = &NIMCacheReconciler{}
9795

9896
// NewNIMCacheReconciler creates a new reconciler for NIMCache with the given platform.
99-
func NewNIMCacheReconciler(client client.Client, scheme *runtime.Scheme, log logr.Logger, platform platform.Platform) *NIMCacheReconciler {
97+
func NewNIMCacheReconciler(client client.Client, scheme *runtime.Scheme, log logr.Logger) *NIMCacheReconciler {
10098
return &NIMCacheReconciler{
101-
Client: client,
102-
scheme: scheme,
103-
log: log,
104-
Platform: platform,
99+
Client: client,
100+
scheme: scheme,
101+
log: log,
105102
}
106103
}
107104

0 commit comments

Comments
 (0)