Skip to content

Commit b101993

Browse files
karthikvetrivelkarthikvetrivel
committed
Add golden render tests for GPUCluster operand manifests
Signed-off-by: Karthik Vetrivel <kvetrivel@nvidia.com>
1 parent 631ca4f commit b101993

7 files changed

Lines changed: 1801 additions & 0 deletions

‎internal/state/gpucluster_render_test.go‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/**
2+
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
**/
16+
17+
package state
18+
19+
import (
20+
"context"
21+
"os"
22+
"path/filepath"
23+
"testing"
24+
25+
"github.com/stretchr/testify/require"
26+
corev1 "k8s.io/api/core/v1"
27+
"k8s.io/apimachinery/pkg/api/resource"
28+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
29+
"k8s.io/utils/ptr"
30+
31+
nvidiav1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
32+
nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
33+
)
34+
35+
// fullSpecGPUCluster returns a CR exercising the optional DRA driver knobs: the
36+
// computeDomains capability with controller/kubelet-plugin overrides and per-container
37+
// env/resources on the gpus kubelet plugin.
38+
func fullSpecGPUCluster() *nvidiav1alpha1.GPUCluster {
39+
cr := sampleGPUCluster()
40+
cr.Spec.DRADriver.GPUs.KubeletPlugin = nvidiav1alpha1.DRADriverKubeletPluginSpec{
41+
Env: []nvidiav1.EnvVar{{Name: "GPUS_EXTRA", Value: "1"}},
42+
Resources: &nvidiav1.ResourceRequirements{
43+
Requests: corev1.ResourceList{
44+
corev1.ResourceCPU: resource.MustParse("100m"),
45+
corev1.ResourceMemory: resource.MustParse("128Mi"),
46+
},
47+
},
48+
HealthcheckPort: ptr.To(int32(52000)),
49+
}
50+
cr.Spec.DRADriver.ComputeDomains = nvidiav1alpha1.DRADriverComputeDomainsSpec{
51+
Enabled: ptr.To(true),
52+
Controller: nvidiav1alpha1.DRADriverControllerSpec{
53+
Env: []nvidiav1.EnvVar{{Name: "CD_CONTROLLER_EXTRA", Value: "1"}},
54+
},
55+
KubeletPlugin: nvidiav1alpha1.DRADriverKubeletPluginSpec{
56+
Env: []nvidiav1.EnvVar{{Name: "CD_PLUGIN_EXTRA", Value: "1"}},
57+
},
58+
}
59+
return cr
60+
}
61+
62+
// TestGPUClusterRenderGolden renders each GPUCluster operand's manifests end to end and
63+
// byte-compares the full YAML stream against fixtures in testdata/golden, so unintended
64+
// template changes surface as diffs (mirroring the NVIDIADriver renderer tests).
65+
func TestGPUClusterRenderGolden(t *testing.T) {
66+
cases := []struct {
67+
name string
68+
render func(t *testing.T) []*unstructured.Unstructured
69+
}{
70+
{
71+
// gpus capability only: computeDomains defaults to disabled in-code.
72+
name: "gpucluster-dra-driver-gpus-only",
73+
render: func(t *testing.T) []*unstructured.Unstructured {
74+
s := newTestDRAState(t)
75+
objs, err := s.getManifestObjects(context.Background(), sampleGPUCluster(), draSupportedCatalog())
76+
require.NoError(t, err)
77+
return objs
78+
},
79+
},
80+
{
81+
name: "gpucluster-dra-driver-full-spec",
82+
render: func(t *testing.T) []*unstructured.Unstructured {
83+
s := newTestDRAState(t)
84+
objs, err := s.getManifestObjects(context.Background(), fullSpecGPUCluster(), draSupportedCatalog())
85+
require.NoError(t, err)
86+
return objs
87+
},
88+
},
89+
{
90+
name: "gpucluster-dra-validation",
91+
render: func(t *testing.T) []*unstructured.Unstructured {
92+
s := newTestDRAValidationState(t)
93+
objs, err := s.getManifestObjects(context.Background(), sampleGPUCluster(), draSupportedCatalog())
94+
require.NoError(t, err)
95+
return objs
96+
},
97+
},
98+
{
99+
name: "gpucluster-dcgm",
100+
render: func(t *testing.T) []*unstructured.Unstructured {
101+
s := newTestDCGMState(t)
102+
cr := sampleGPUCluster()
103+
cr.Spec.DCGM = &nvidiav1.DCGMSpec{Enabled: ptr.To(true)}
104+
objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog())
105+
require.NoError(t, err)
106+
return objs
107+
},
108+
},
109+
{
110+
// dcgm disabled: the exporter runs its embedded nv-hostengine.
111+
name: "gpucluster-dcgm-exporter-embedded",
112+
render: func(t *testing.T) []*unstructured.Unstructured {
113+
s := newTestDCGMExporterState(t, false)
114+
objs, err := s.getManifestObjects(context.Background(), exporterCR(&nvidiav1.DCGMExporterSpec{}), draSupportedCatalog())
115+
require.NoError(t, err)
116+
return objs
117+
},
118+
},
119+
{
120+
// dcgm enabled: the exporter wires DCGM_REMOTE_HOSTENGINE_INFO to the
121+
// standalone hostengine Service.
122+
name: "gpucluster-dcgm-exporter-remote-engine",
123+
render: func(t *testing.T) []*unstructured.Unstructured {
124+
s := newTestDCGMExporterState(t, false)
125+
cr := exporterCR(&nvidiav1.DCGMExporterSpec{})
126+
cr.Spec.DCGM = &nvidiav1.DCGMSpec{Enabled: ptr.To(true)}
127+
objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog())
128+
require.NoError(t, err)
129+
return objs
130+
},
131+
},
132+
}
133+
134+
for _, tc := range cases {
135+
t.Run(tc.name, func(t *testing.T) {
136+
objs := tc.render(t)
137+
require.NotEmpty(t, objs)
138+
139+
actual, err := getYAMLString(objs)
140+
require.NoError(t, err)
141+
142+
expected, err := os.ReadFile(filepath.Join(manifestResultDir, tc.name+".yaml"))
143+
require.NoError(t, err)
144+
require.Equal(t, string(expected), actual)
145+
})
146+
}
147+
}

‎internal/state/testdata/golden/gpucluster-dcgm-exporter-embedded.yaml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: nvidia-dcgm-exporter-dra
5+
namespace: test-operator
6+
---
7+
apiVersion: rbac.authorization.k8s.io/v1
8+
kind: Role
9+
metadata:
10+
name: nvidia-dcgm-exporter-dra
11+
namespace: test-operator
12+
rules:
13+
- apiGroups:
14+
- resource.k8s.io
15+
resources:
16+
- resourceclaims
17+
verbs:
18+
- get
19+
- list
20+
- watch
21+
---
22+
apiVersion: rbac.authorization.k8s.io/v1
23+
kind: RoleBinding
24+
metadata:
25+
name: nvidia-dcgm-exporter-dra
26+
namespace: test-operator
27+
roleRef:
28+
apiGroup: rbac.authorization.k8s.io
29+
kind: Role
30+
name: nvidia-dcgm-exporter-dra
31+
subjects:
32+
- kind: ServiceAccount
33+
name: nvidia-dcgm-exporter-dra
34+
namespace: test-operator
35+
---
36+
apiVersion: resource.k8s.io/v1
37+
kind: ResourceClaimTemplate
38+
metadata:
39+
name: nvidia-dcgm-exporter-admin
40+
namespace: test-operator
41+
spec:
42+
spec:
43+
devices:
44+
requests:
45+
- exactly:
46+
adminAccess: true
47+
allocationMode: All
48+
deviceClassName: gpu.nvidia.com
49+
name: admin-gpus
50+
---
51+
apiVersion: v1
52+
kind: Service
53+
metadata:
54+
annotations:
55+
prometheus.io/scrape: "true"
56+
labels:
57+
app: nvidia-dcgm-exporter-dra
58+
name: nvidia-dcgm-exporter-dra
59+
namespace: test-operator
60+
spec:
61+
ports:
62+
- name: gpu-metrics
63+
port: 9400
64+
protocol: TCP
65+
targetPort: 9400
66+
selector:
67+
app: nvidia-dcgm-exporter-dra
68+
type: ClusterIP
69+
---
70+
apiVersion: apps/v1
71+
kind: DaemonSet
72+
metadata:
73+
labels:
74+
app: nvidia-dcgm-exporter-dra
75+
name: nvidia-dcgm-exporter-dra
76+
namespace: test-operator
77+
spec:
78+
selector:
79+
matchLabels:
80+
app: nvidia-dcgm-exporter-dra
81+
template:
82+
metadata:
83+
labels:
84+
app: nvidia-dcgm-exporter-dra
85+
spec:
86+
automountServiceAccountToken: false
87+
containers:
88+
- env:
89+
- name: DCGM_EXPORTER_LISTEN
90+
value: :9400
91+
- name: DCGM_EXPORTER_KUBERNETES
92+
value: "true"
93+
- name: DCGM_EXPORTER_COLLECTORS
94+
value: /etc/dcgm-exporter/dcp-metrics-included.csv
95+
- name: NODE_NAME
96+
valueFrom:
97+
fieldRef:
98+
fieldPath: spec.nodeName
99+
image: nvcr.io/nvidia/k8s/dcgm-exporter:test
100+
livenessProbe:
101+
httpGet:
102+
path: /health
103+
port: 9400
104+
initialDelaySeconds: 45
105+
periodSeconds: 5
106+
name: nvidia-dcgm-exporter-dra
107+
ports:
108+
- containerPort: 9400
109+
name: metrics
110+
readinessProbe:
111+
httpGet:
112+
path: /health
113+
port: 9400
114+
initialDelaySeconds: 45
115+
resources:
116+
claims:
117+
- name: admin-gpus
118+
securityContext:
119+
privileged: true
120+
volumeMounts:
121+
- mountPath: /var/lib/kubelet/pod-resources
122+
name: pod-gpu-resources
123+
readOnly: true
124+
nodeSelector:
125+
nvidia.com/gpu-operator.resource-allocation.mode: dra
126+
nvidia.com/gpu.deploy.dcgm-exporter: "true"
127+
priorityClassName: system-node-critical
128+
resourceClaims:
129+
- name: admin-gpus
130+
resourceClaimTemplateName: nvidia-dcgm-exporter-admin
131+
serviceAccountName: nvidia-dcgm-exporter-dra
132+
tolerations:
133+
- effect: NoSchedule
134+
key: nvidia.com/gpu
135+
operator: Exists
136+
volumes:
137+
- hostPath:
138+
path: /var/lib/kubelet/pod-resources
139+
name: pod-gpu-resources
140+
updateStrategy:
141+
type: RollingUpdate
142+
---

0 commit comments

Comments
 (0)