Skip to content

Commit 19c9f62

Browse files
karthikvetrivelkarthikvetrivel
committed
Add GPUCluster controller with singleton status and GPU node labeling
Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>
1 parent 0bfd54d commit 19c9f62

11 files changed

Lines changed: 1193 additions & 29 deletions

‎bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,30 @@ metadata:
154154
}
155155
}
156156
},
157+
{
158+
"apiVersion": "nvidia.com/v1alpha1",
159+
"kind": "GPUCluster",
160+
"metadata": {
161+
"name": "gpu-cluster"
162+
},
163+
"spec": {
164+
"draDriver": {
165+
"repository": "registry.k8s.io/dra-driver-nvidia",
166+
"image": "dra-driver-nvidia-gpu",
167+
"version": "v0.4.0",
168+
"imagePullPolicy": "IfNotPresent",
169+
"computeDomains": {
170+
"enabled": true
171+
}
172+
},
173+
"dcgm": {
174+
"enabled": false
175+
},
176+
"dcgmExporter": {
177+
"enabled": true
178+
}
179+
}
180+
},
157181
{
158182
"apiVersion": "nvidia.com/v1alpha1",
159183
"kind": "NVIDIADriver",
@@ -224,6 +248,8 @@ spec:
224248
relatedImages:
225249
- name: gpu-operator-image
226250
image: ghcr.io/nvidia/gpu-operator:main-latest
251+
- name: nvidia-dra-driver-image
252+
image: registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.0@sha256:dd17d49170f62071d13ef5de9b8f0ff26c66793f9d802c34532c0cb4f8a1d1f3
227253
- name: dcgm-exporter-image
228254
image: nvcr.io/nvidia/k8s/dcgm-exporter:4.5.3-4.8.2-distroless@sha256:60d3b00ac80b4ae77f94dae2f943685605585ad9e92fdccda3154d009ae317cc
229255
- name: dcgm-image
@@ -337,6 +363,49 @@ spec:
337363
path: state
338364
x-descriptors:
339365
- 'urn:alm:descriptor:text'
366+
- name: gpuclusters.nvidia.com
367+
kind: GPUCluster
368+
version: v1alpha1
369+
group: nvidia.com
370+
displayName: GPUCluster
371+
description: GPUCluster allows you to configure the DRA-based GPU software stack
372+
resources:
373+
- kind: ServiceAccount
374+
name: ''
375+
version: v1
376+
- kind: DaemonSet
377+
name: ''
378+
version: apps/v1
379+
- kind: Deployment
380+
name: ''
381+
version: apps/v1
382+
- kind: ConfigMap
383+
name: ''
384+
version: v1
385+
- kind: Pod
386+
name: ''
387+
version: v1
388+
- kind: status
389+
name: ''
390+
version: v1
391+
statusDescriptors:
392+
- description: The current state of the GPUCluster.
393+
displayName: State
394+
path: state
395+
x-descriptors:
396+
- 'urn:alm:descriptor:text'
397+
- name: computedomains.resource.nvidia.com
398+
kind: ComputeDomain
399+
version: v1beta1
400+
group: resource.nvidia.com
401+
displayName: ComputeDomain
402+
description: ComputeDomain prepares a set of nodes to run a multi-node workload in.
403+
- name: computedomaincliques.resource.nvidia.com
404+
kind: ComputeDomainClique
405+
version: v1beta1
406+
group: resource.nvidia.com
407+
displayName: ComputeDomainClique
408+
description: ComputeDomainClique describes a clique within a ComputeDomain.
340409
- name: clusterpolicies.nvidia.com
341410
kind: ClusterPolicy
342411
version: v1
@@ -652,6 +721,9 @@ spec:
652721
- clusterpolicies
653722
- clusterpolicies/finalizers
654723
- clusterpolicies/status
724+
- gpuclusters
725+
- gpuclusters/finalizers
726+
- gpuclusters/status
655727
- nvidiadrivers
656728
- nvidiadrivers/finalizers
657729
- nvidiadrivers/status
@@ -755,6 +827,7 @@ spec:
755827
- get
756828
- list
757829
- watch
830+
- patch
758831
- apiGroups:
759832
- ""
760833
resources:
@@ -792,6 +865,117 @@ spec:
792865
- update
793866
- patch
794867
- delete
868+
- apiGroups:
869+
- resource.nvidia.com
870+
resources:
871+
- computedomains
872+
verbs:
873+
- get
874+
- list
875+
- watch
876+
- create
877+
- update
878+
- patch
879+
- delete
880+
- apiGroups:
881+
- resource.nvidia.com
882+
resources:
883+
- computedomains/status
884+
verbs:
885+
- get
886+
- list
887+
- watch
888+
- create
889+
- update
890+
- patch
891+
- delete
892+
- apiGroups:
893+
- resource.nvidia.com
894+
resources:
895+
- computedomaincliques
896+
verbs:
897+
- get
898+
- list
899+
- watch
900+
- create
901+
- update
902+
- patch
903+
- delete
904+
- apiGroups:
905+
- coordination.k8s.io
906+
resources:
907+
- leases
908+
verbs:
909+
- get
910+
- create
911+
- update
912+
- apiGroups:
913+
- resource.k8s.io
914+
resources:
915+
- resourceclaims
916+
verbs:
917+
- get
918+
- list
919+
- watch
920+
- create
921+
- update
922+
- patch
923+
- delete
924+
- apiGroups:
925+
- resource.k8s.io
926+
resources:
927+
- resourceclaimtemplates
928+
verbs:
929+
- get
930+
- list
931+
- watch
932+
- create
933+
- update
934+
- patch
935+
- delete
936+
- apiGroups:
937+
- resource.k8s.io
938+
resources:
939+
- deviceclasses
940+
verbs:
941+
- get
942+
- list
943+
- watch
944+
- create
945+
- update
946+
- patch
947+
- delete
948+
- apiGroups:
949+
- resource.k8s.io
950+
resources:
951+
- resourceslices
952+
verbs:
953+
- get
954+
- list
955+
- watch
956+
- create
957+
- update
958+
- patch
959+
- delete
960+
- apiGroups:
961+
- resource.k8s.io
962+
resources:
963+
- resourceclaims/status
964+
verbs:
965+
- update
966+
- apiGroups:
967+
- admissionregistration.k8s.io
968+
resources:
969+
- validatingadmissionpolicies
970+
- validatingadmissionpolicybindings
971+
verbs:
972+
- get
973+
- list
974+
- watch
975+
- create
976+
- update
977+
- patch
978+
- delete
795979
permissions:
796980
- serviceAccountName: gpu-operator
797981
rules:
@@ -828,6 +1012,18 @@ spec:
8281012
- update
8291013
- patch
8301014
- delete
1015+
- apiGroups:
1016+
- apps
1017+
resources:
1018+
- deployments
1019+
verbs:
1020+
- create
1021+
- get
1022+
- list
1023+
- watch
1024+
- update
1025+
- patch
1026+
- delete
8311027
- apiGroups:
8321028
- ""
8331029
resources:
@@ -938,6 +1134,8 @@ spec:
9381134
fieldPath: metadata.namespace
9391135
- name: "VALIDATOR_IMAGE"
9401136
value: "ghcr.io/nvidia/gpu-operator:main-latest"
1137+
- name: "DRA_DRIVER_IMAGE"
1138+
value: "registry.k8s.io/dra-driver-nvidia/dra-driver-nvidia-gpu:v0.4.0@sha256:dd17d49170f62071d13ef5de9b8f0ff26c66793f9d802c34532c0cb4f8a1d1f3"
9411139
- name: "GFD_IMAGE"
9421140
value: "nvcr.io/nvidia/k8s-device-plugin:v0.19.3@sha256:25cc340fe6fd53c101e16fc452f503e7a92c219c64a80ed5381784b522dbbf77"
9431141
- name: "CONTAINER_TOOLKIT_IMAGE"

‎cmd/gpu-operator/main.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,16 @@ func main() {
230230
setupLog.Error(err, "unable to create controller", "controller", "NodeLabeling")
231231
os.Exit(1)
232232
}
233+
234+
if err = (&controllers.GPUClusterReconciler{
235+
Namespace: operatorNamespace,
236+
Client: mgr.GetClient(),
237+
Scheme: mgr.GetScheme(),
238+
ClusterInfo: clusterInfo,
239+
}).SetupWithManager(ctx, mgr); err != nil {
240+
setupLog.Error(err, "unable to create controller", "controller", "GPUCluster")
241+
os.Exit(1)
242+
}
233243
// +kubebuilder:scaffold:builder
234244
if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
235245
setupLog.Error(err, "unable to set up health check")

‎config/rbac/role.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ rules:
132132
- nvidia.com
133133
resources:
134134
- '*'
135+
- gpuclusters
135136
- nvidiadrivers
136137
verbs:
137138
- create
@@ -144,12 +145,22 @@ rules:
144145
- apiGroups:
145146
- nvidia.com
146147
resources:
148+
- clusterpolicies
149+
verbs:
150+
- get
151+
- list
152+
- watch
153+
- apiGroups:
154+
- nvidia.com
155+
resources:
156+
- gpuclusters/finalizers
147157
- nvidiadrivers/finalizers
148158
verbs:
149159
- update
150160
- apiGroups:
151161
- nvidia.com
152162
resources:
163+
- gpuclusters/status
153164
- nvidiadrivers/status
154165
verbs:
155166
- get
@@ -164,6 +175,17 @@ rules:
164175
- roles
165176
verbs:
166177
- '*'
178+
- apiGroups:
179+
- resource.k8s.io
180+
resources:
181+
- resourceclaimtemplates
182+
verbs:
183+
- create
184+
- delete
185+
- get
186+
- list
187+
- update
188+
- watch
167189
- apiGroups:
168190
- route.openshift.io
169191
resources:

‎controllers/active_config.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/**
2+
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
**/
16+
17+
package controllers
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
"sigs.k8s.io/controller-runtime/pkg/client"
24+
25+
gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
26+
nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
27+
)
28+
29+
// resolveActiveConfig determines which cluster-wide configuration drives the operands: a
30+
// ClusterPolicy takes precedence, otherwise the DRA-based GPUCluster. The two CRs are
31+
// mutually exclusive, so at most one of the returned values is non-nil; both nil means
32+
// neither CR exists.
33+
//
34+
// TODO: with multiple CRs of a kind, the tie-breaker is list order, which is not
35+
// guaranteed. Resolve the singleton the reconcilers actually selected instead.
36+
func resolveActiveConfig(ctx context.Context, c client.Client) (*gpuv1.ClusterPolicy, *nvidiav1alpha1.GPUCluster, error) {
37+
clusterPolicies := &gpuv1.ClusterPolicyList{}
38+
if err := c.List(ctx, clusterPolicies); err != nil {
39+
return nil, nil, fmt.Errorf("failed to list ClusterPolicy: %w", err)
40+
}
41+
if len(clusterPolicies.Items) > 0 {
42+
return &clusterPolicies.Items[0], nil, nil
43+
}
44+
45+
gpuClusters := &nvidiav1alpha1.GPUClusterList{}
46+
if err := c.List(ctx, gpuClusters); err != nil {
47+
return nil, nil, fmt.Errorf("failed to list GPUCluster: %w", err)
48+
}
49+
if len(gpuClusters.Items) > 0 {
50+
return nil, &gpuClusters.Items[0], nil
51+
}
52+
53+
return nil, nil, nil
54+
}

0 commit comments

Comments
 (0)