Skip to content

Commit 631ca4f

Browse files
karthikvetrivelkarthikvetrivel
committed
Add GPUCluster Helm install with ClusterPolicy/NVIDIADriver coexistence
Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>
1 parent 1d1d53f commit 631ca4f

13 files changed

Lines changed: 391 additions & 14 deletions

‎controllers/nvidiadriver_controller.go‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -121,15 +121,30 @@ func (r *NVIDIADriverReconciler) Reconcile(ctx context.Context, req ctrl.Request
121121
}
122122
clusterPolicyInstance := clusterPolicyList.Items[0]
123123

124-
// Ensure that ClusterPolicy is configured to use NVIDIADriver CRD
124+
// Ensure the NVIDIADriver CR has a consumer: either the ClusterPolicy delegates its
125+
// driver to the NVIDIADriver CRD, or a GPUCluster exists. GPUCluster does
126+
// not manage the driver itself — it is either preinstalled on the host (no NVIDIADriver
127+
// CR) or installed via NVIDIADriver CRs, so any CR that exists alongside one is in use.
125128
if !clusterPolicyInstance.Spec.Driver.UseNvidiaDriverCRDType() {
126-
msg := "useNvidiaDriverCRD is not enabled in ClusterPolicy"
127-
logger.V(consts.LogLevelWarning).Info("NVIDIADriver reconciliation skipped", "reason", msg)
128-
instance.Status.State = nvidiav1alpha1.Disabled
129-
if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.Reconciled, msg); condErr != nil {
130-
logger.Error(condErr, "failed to set condition")
129+
gpuClusters := &nvidiav1alpha1.GPUClusterList{}
130+
if err := r.List(ctx, gpuClusters); err != nil {
131+
wrappedErr := fmt.Errorf("error getting GPUCluster list: %w", err)
132+
logger.Error(err, "error getting GPUCluster list")
133+
instance.Status.State = nvidiav1alpha1.NotReady
134+
if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.ReconcileFailed, err.Error()); condErr != nil {
135+
logger.Error(condErr, "failed to set condition")
136+
}
137+
return reconcile.Result{}, wrappedErr
138+
}
139+
if len(gpuClusters.Items) == 0 {
140+
msg := "useNvidiaDriverCRD is not enabled in ClusterPolicy and no GPUCluster exists"
141+
logger.V(consts.LogLevelWarning).Info("NVIDIADriver reconciliation skipped", "reason", msg)
142+
instance.Status.State = nvidiav1alpha1.Disabled
143+
if condErr := r.conditionUpdater.SetConditionsError(ctx, instance, conditions.Reconciled, msg); condErr != nil {
144+
logger.Error(condErr, "failed to set condition")
145+
}
146+
return reconcile.Result{}, nil
131147
}
132-
return reconcile.Result{}, nil
133148
}
134149

135150
// Create a new InfoCatalog which is a generic interface for passing information to state managers

‎controllers/nvidiadriver_controller_test.go‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -111,13 +111,14 @@ func TestReconcile(t *testing.T) {
111111
require.NoError(t, corev1.AddToScheme(scheme))
112112

113113
tests := []struct {
114-
name string
115-
useCRD *bool
116-
driverEnabled *bool
117-
spec nvidiav1alpha1.NVIDIADriverSpec
118-
validator validator.Validator
119-
error error
120-
expectedLog string
114+
name string
115+
useCRD *bool
116+
driverEnabled *bool
117+
gpuClusterExists bool
118+
spec nvidiav1alpha1.NVIDIADriverSpec
119+
validator validator.Validator
120+
error error
121+
expectedLog string
121122
}{
122123
{
123124
name: "ClusterPolicy has driver CRD false → reconciliation skips driver",
@@ -128,6 +129,16 @@ func TestReconcile(t *testing.T) {
128129
error: nil,
129130
expectedLog: "useNvidiaDriverCRD is not enabled in ClusterPolicy",
130131
},
132+
{
133+
name: "driver CRD false but GPUCluster exists → reconciliation proceeds",
134+
useCRD: ptr.To(false),
135+
gpuClusterExists: true,
136+
validator: &FakeNodeSelectorValidator{
137+
CustomError: errors.New("fake list error"),
138+
},
139+
error: nil,
140+
expectedLog: "nodeSelector validation failed",
141+
},
131142
{
132143
name: "ClusterPolicy has driver CRD true but validator errors",
133144
useCRD: ptr.To(true),
@@ -188,6 +199,10 @@ func TestReconcile(t *testing.T) {
188199

189200
// Initialize fake client with ClusterPolicy (driver optional)
190201
clientBuilder := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cp, driver)
202+
if tc.gpuClusterExists {
203+
gc := &nvidiav1alpha1.GPUCluster{ObjectMeta: metav1.ObjectMeta{Name: "config"}}
204+
clientBuilder = clientBuilder.WithObjects(gc)
205+
}
191206
client := clientBuilder.Build()
192207

193208
updater := &FakeConditionUpdater{}

‎deployments/gpu-operator/templates/_helpers.tpl‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,3 +78,10 @@ Full image name with tag
7878
{{- define "driver-manager.fullimage" -}}
7979
{{- .Values.driver.manager.repository -}}/{{- .Values.driver.manager.image -}}:{{- .Values.driver.manager.version -}}
8080
{{- end }}
81+
82+
{{/*
83+
Full image name with tag
84+
*/}}
85+
{{- define "validator.fullimage" -}}
86+
{{- .Values.validator.repository -}}/{{- .Values.validator.image -}}:{{- .Values.validator.version | default .Chart.AppVersion -}}
87+
{{- end }}

‎deployments/gpu-operator/templates/cleanup_crd.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ spec:
4040
- delete
4141
- --filepath=/opt/gpu-operator/nvidia.com_clusterpolicies.yaml
4242
- --filepath=/opt/gpu-operator/nvidia.com_nvidiadrivers.yaml
43+
- --filepath=/opt/gpu-operator/nvidia.com_gpuclusters.yaml
4344
{{- if .Values.nfd.enabled }}
4445
- --filepath=/opt/gpu-operator/nfd-api-crds.yaml
4546
{{- end }}

‎deployments/gpu-operator/templates/clusterpolicy.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
{{- /* ClusterPolicy (device-plugin stack) and GPUCluster (DRA stack) may coexist; per-node
2+
ownership is decided by the nvidia.com/gpu-operator.resource-allocation.mode label. */ -}}
13
apiVersion: nvidia.com/v1
24
kind: ClusterPolicy
35
metadata:

‎deployments/gpu-operator/templates/clusterrole.yaml‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,9 @@ rules:
107107
- clusterpolicies
108108
- clusterpolicies/finalizers
109109
- clusterpolicies/status
110+
- gpuclusters
111+
- gpuclusters/finalizers
112+
- gpuclusters/status
110113
- nvidiadrivers
111114
- nvidiadrivers/finalizers
112115
- nvidiadrivers/status
@@ -153,3 +156,123 @@ rules:
153156
{{- if .Values.operator.cleanupCRD }}
154157
- delete
155158
{{- end }}
159+
- apiGroups:
160+
- resource.nvidia.com
161+
resources:
162+
- computedomains
163+
verbs:
164+
- get
165+
- list
166+
- watch
167+
- create
168+
- update
169+
- patch
170+
- delete
171+
- apiGroups:
172+
- resource.nvidia.com
173+
resources:
174+
- computedomains/status
175+
verbs:
176+
- get
177+
- list
178+
- watch
179+
- create
180+
- update
181+
- patch
182+
- delete
183+
- apiGroups:
184+
- resource.nvidia.com
185+
resources:
186+
- computedomaincliques
187+
verbs:
188+
- get
189+
- list
190+
- watch
191+
- create
192+
- update
193+
- patch
194+
- delete
195+
- apiGroups:
196+
- coordination.k8s.io
197+
resources:
198+
- leases
199+
verbs:
200+
- get
201+
- create
202+
- update
203+
- apiGroups:
204+
- resource.k8s.io
205+
resources:
206+
- resourceclaims
207+
verbs:
208+
- get
209+
- list
210+
- watch
211+
- create
212+
- update
213+
- patch
214+
- delete
215+
- apiGroups:
216+
- resource.k8s.io
217+
resources:
218+
- resourceclaimtemplates
219+
verbs:
220+
- get
221+
- list
222+
- watch
223+
- create
224+
- update
225+
- patch
226+
- delete
227+
- apiGroups:
228+
- resource.k8s.io
229+
resources:
230+
- deviceclasses
231+
verbs:
232+
- get
233+
- list
234+
- watch
235+
- create
236+
- update
237+
- patch
238+
- delete
239+
- apiGroups:
240+
- resource.k8s.io
241+
resources:
242+
- resourceslices
243+
verbs:
244+
- get
245+
- list
246+
- watch
247+
- create
248+
- update
249+
- patch
250+
- delete
251+
- apiGroups:
252+
- resource.k8s.io
253+
resources:
254+
- resourceclaims/status
255+
verbs:
256+
- update
257+
- apiGroups:
258+
- ""
259+
resources:
260+
- pods
261+
verbs:
262+
- get
263+
- list
264+
- watch
265+
- patch
266+
- apiGroups:
267+
- admissionregistration.k8s.io
268+
resources:
269+
- validatingadmissionpolicies
270+
- validatingadmissionpolicybindings
271+
verbs:
272+
- get
273+
- list
274+
- watch
275+
- create
276+
- update
277+
- patch
278+
- delete

‎deployments/gpu-operator/templates/gpucluster.yaml‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
{{- if .Values.gpuCluster.enabled }}
2+
apiVersion: nvidia.com/v1alpha1
3+
kind: GPUCluster
4+
metadata:
5+
name: gpu-cluster
6+
labels:
7+
{{- include "gpu-operator.labels" . | nindent 4 }}
8+
app.kubernetes.io/component: "gpu-operator"
9+
{{- if .Values.operator.cleanupCRD }}
10+
# CR cleanup is handled during pre-delete hook
11+
# Add below annotation so that helm doesn't attempt to cleanup CR twice
12+
annotations:
13+
"helm.sh/resource-policy": keep
14+
{{- end }}
15+
spec:
16+
draDriver:
17+
{{- if .Values.gpuCluster.draDriver.repository }}
18+
repository: {{ .Values.gpuCluster.draDriver.repository }}
19+
{{- end }}
20+
{{- if .Values.gpuCluster.draDriver.image }}
21+
image: {{ .Values.gpuCluster.draDriver.image }}
22+
{{- end }}
23+
{{- if .Values.gpuCluster.draDriver.version }}
24+
version: {{ .Values.gpuCluster.draDriver.version | quote }}
25+
{{- end }}
26+
{{- if .Values.gpuCluster.draDriver.imagePullPolicy }}
27+
imagePullPolicy: {{ .Values.gpuCluster.draDriver.imagePullPolicy }}
28+
{{- end }}
29+
{{- if .Values.gpuCluster.draDriver.imagePullSecrets }}
30+
imagePullSecrets: {{ toYaml .Values.gpuCluster.draDriver.imagePullSecrets | nindent 6 }}
31+
{{- end }}
32+
{{- if .Values.gpuCluster.draDriver.featureGates }}
33+
featureGates: {{ toYaml .Values.gpuCluster.draDriver.featureGates | nindent 6 }}
34+
{{- end }}
35+
{{- if .Values.gpuCluster.draDriver.gpus.kubeletPlugin }}
36+
gpus:
37+
kubeletPlugin: {{ toYaml .Values.gpuCluster.draDriver.gpus.kubeletPlugin | nindent 8 }}
38+
{{- end }}
39+
computeDomains:
40+
enabled: {{ .Values.gpuCluster.draDriver.computeDomains.enabled }}
41+
{{- if .Values.gpuCluster.draDriver.computeDomains.controller }}
42+
controller: {{ toYaml .Values.gpuCluster.draDriver.computeDomains.controller | nindent 8 }}
43+
{{- end }}
44+
{{- if .Values.gpuCluster.draDriver.computeDomains.kubeletPlugin }}
45+
kubeletPlugin: {{ toYaml .Values.gpuCluster.draDriver.computeDomains.kubeletPlugin | nindent 8 }}
46+
{{- end }}
47+
dcgm:
48+
enabled: {{ .Values.dcgm.enabled }}
49+
{{- if .Values.dcgm.repository }}
50+
repository: {{ .Values.dcgm.repository }}
51+
{{- end }}
52+
{{- if .Values.dcgm.image }}
53+
image: {{ .Values.dcgm.image }}
54+
{{- end }}
55+
{{- if .Values.dcgm.version }}
56+
version: {{ .Values.dcgm.version | quote }}
57+
{{- end }}
58+
{{- if .Values.dcgm.imagePullPolicy }}
59+
imagePullPolicy: {{ .Values.dcgm.imagePullPolicy }}
60+
{{- end }}
61+
{{- if .Values.dcgm.imagePullSecrets }}
62+
imagePullSecrets: {{ toYaml .Values.dcgm.imagePullSecrets | nindent 6 }}
63+
{{- end }}
64+
{{- if .Values.dcgm.args }}
65+
args: {{ toYaml .Values.dcgm.args | nindent 6 }}
66+
{{- end }}
67+
{{- if .Values.dcgm.env }}
68+
env: {{ toYaml .Values.dcgm.env | nindent 6 }}
69+
{{- end }}
70+
{{- if .Values.dcgm.resources }}
71+
resources: {{ toYaml .Values.dcgm.resources | nindent 6 }}
72+
{{- end }}
73+
{{- if .Values.dcgm.hostNetwork }}
74+
hostNetwork: {{ .Values.dcgm.hostNetwork }}
75+
{{- end }}
76+
dcgmExporter:
77+
enabled: {{ .Values.dcgmExporter.enabled }}
78+
{{- if .Values.dcgmExporter.repository }}
79+
repository: {{ .Values.dcgmExporter.repository }}
80+
{{- end }}
81+
{{- if .Values.dcgmExporter.image }}
82+
image: {{ .Values.dcgmExporter.image }}
83+
{{- end }}
84+
{{- if .Values.dcgmExporter.version }}
85+
version: {{ .Values.dcgmExporter.version | quote }}
86+
{{- end }}
87+
{{- if .Values.dcgmExporter.imagePullPolicy }}
88+
imagePullPolicy: {{ .Values.dcgmExporter.imagePullPolicy }}
89+
{{- end }}
90+
{{- if .Values.dcgmExporter.imagePullSecrets }}
91+
imagePullSecrets: {{ toYaml .Values.dcgmExporter.imagePullSecrets | nindent 6 }}
92+
{{- end }}
93+
{{- if .Values.dcgmExporter.annotations }}
94+
annotations: {{ toYaml .Values.dcgmExporter.annotations | nindent 6 }}
95+
{{- end }}
96+
{{- if .Values.dcgmExporter.args }}
97+
args: {{ toYaml .Values.dcgmExporter.args | nindent 6 }}
98+
{{- end }}
99+
{{- if .Values.dcgmExporter.env }}
100+
env: {{ toYaml .Values.dcgmExporter.env | nindent 6 }}
101+
{{- end }}
102+
{{- if .Values.dcgmExporter.resources }}
103+
resources: {{ toYaml .Values.dcgmExporter.resources | nindent 6 }}
104+
{{- end }}
105+
{{- if .Values.dcgmExporter.hostPID }}
106+
hostPID: {{ .Values.dcgmExporter.hostPID }}
107+
{{- end }}
108+
{{- if .Values.dcgmExporter.hostNetwork }}
109+
hostNetwork: {{ .Values.dcgmExporter.hostNetwork }}
110+
{{- end }}
111+
{{- if .Values.dcgmExporter.hpcJobMapping }}
112+
hpcJobMapping: {{ toYaml .Values.dcgmExporter.hpcJobMapping | nindent 6 }}
113+
{{- end }}
114+
{{- if .Values.dcgmExporter.enablePodLabels }}
115+
enablePodLabels: {{ .Values.dcgmExporter.enablePodLabels }}
116+
{{- end }}
117+
{{- if .Values.dcgmExporter.enablePodUID }}
118+
enablePodUID: {{ .Values.dcgmExporter.enablePodUID }}
119+
{{- end }}
120+
{{- if .Values.dcgmExporter.podLabelAllowlistRegex }}
121+
podLabelAllowlistRegex: {{ toYaml .Values.dcgmExporter.podLabelAllowlistRegex | nindent 6 }}
122+
{{- end }}
123+
{{- if .Values.dcgmExporter.service }}
124+
service: {{ toYaml .Values.dcgmExporter.service | nindent 6 }}
125+
{{- end }}
126+
{{- if .Values.dcgmExporter.serviceMonitor }}
127+
serviceMonitor: {{ toYaml .Values.dcgmExporter.serviceMonitor | nindent 6 }}
128+
{{- end }}
129+
{{- if and (.Values.dcgmExporter.config) (.Values.dcgmExporter.config.name) }}
130+
config:
131+
name: {{ .Values.dcgmExporter.config.name }}
132+
{{- end }}
133+
hostPaths:
134+
rootFS: {{ .Values.hostPaths.rootFS }}
135+
driverInstallDir: {{ .Values.hostPaths.driverInstallDir }}
136+
{{- if .Values.hostPaths.kubeletRootDir }}
137+
kubeletRootDir: {{ .Values.hostPaths.kubeletRootDir }}
138+
{{- end }}
139+
daemonsets: {{ toYaml .Values.daemonsets | nindent 4 }}
140+
{{- end }}

‎deployments/gpu-operator/templates/operator.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ spec:
5959
fieldPath: metadata.namespace
6060
- name: "DRIVER_MANAGER_IMAGE"
6161
value: "{{ include "driver-manager.fullimage" . }}"
62+
- name: "VALIDATOR_IMAGE"
63+
value: "{{ include "validator.fullimage" . }}"
64+
{{- if .Values.operator.env }}
65+
{{- toYaml .Values.operator.env | nindent 8 }}
66+
{{- end }}
6267
livenessProbe:
6368
httpGet:
6469
path: /healthz

0 commit comments

Comments
 (0)