fix: add NVIDIA env vars to on-demand orc jobs as well

Author: mo-getter

helm/fiftyone-teams-app/templates/delegated-operator-job-configmap.yaml

@@ -44,6 +44,17 @@ data:
   {{- $mergedVolumeMounts := $jobConfig.volumeMounts | default $baseTpl.volumeMounts | default list }}
   {{- $mergedVolumes := $jobConfig.volumes | default $baseTpl.volumes | default list }}
 
+  {{- /* When the executor requests a GPU, expose that same GPU to its telemetry
+         sidecar for read-only metrics. This sets env vars only — the sidecar never
+         requests its own nvidia.com/gpu, so it consumes no GPU quota and the
+         executor keeps the allocated GPU. Honored where the NVIDIA runtime accepts
+         NVIDIA_VISIBLE_DEVICES (most setups; GKE blocks env-based exposure). */}}
+  {{- $gpuRequested := or (not (empty (dig "limits" "nvidia.com/gpu" "" $mergedResources))) (not (empty (dig "requests" "nvidia.com/gpu" "" $mergedResources))) }}
+  {{- $sidecarEnv := dict }}
+  {{- if $gpuRequested }}
+  {{- $sidecarEnv = dict "NVIDIA_VISIBLE_DEVICES" "all" "NVIDIA_DRIVER_CAPABILITIES" "compute,utility" }}
+  {{- end }}
+
   {{- if $.Values.telemetry.enabled }}
     {{- $hasSocketVolume := false }}
     {{- range $vol := $mergedVolumes }}
@@ -143,7 +154,7 @@ data:
               {{- end }}
           {{- if $.Values.telemetry.enabled }}
           initContainers:
-            {{- include "telemetry.native-sidecar" (dict "ctx" $ "serviceType" "delegated-operator" "targetName" "fiftyone delegated" "executor" true "targetContainer" "executor") | nindent 12 }}
+            {{- include "telemetry.native-sidecar" (dict "ctx" $ "serviceType" "delegated-operator" "targetName" "fiftyone delegated" "executor" true "targetContainer" "executor" "sidecarEnv" $sidecarEnv) | nindent 12 }}
           {{- end }}
           {{- with $mergedNodeSelector }}
           nodeSelector:

helm/fiftyone-teams-app/values.yaml

@@ -898,8 +898,6 @@ delegatedOperatorDeployments:
     #       cpu: 8
     #       ephemeral-storage: 1Gi
     #       memory: 16Gi
-    #   # GPU metrics appear automatically — the telemetry sidecar reads this
-    #   # operator's GPU without requesting its own allocation.
     #   volumes:
     #     - name: shm-vol
     #       emptyDir:

tests/unit/helm/delegated-operator-job-configmap_test.go

@@ -754,6 +754,112 @@ func (s *doK8sConfigMapTemplateTest) TestTelemetrySocketInjection() {
 	}
 }
 
+// TestTelemetrySidecarGpuEnv verifies that when a delegated-operator job's
+// executor requests a GPU (via resources.limits or resources.requests), its
+// telemetry native-sidecar is given the NVIDIA_* env vars needed to read GPU
+// metrics, without the sidecar requesting its own nvidia.com/gpu allocation.
+// When no GPU is requested, the sidecar receives no NVIDIA_* env vars.
+func (s *doK8sConfigMapTemplateTest) TestTelemetrySidecarGpuEnv() {
+	const gpuResource = "nvidia.com/gpu"
+	jobKey := "cpuDefault.yaml"
+
+	findContainer := func(containers []corev1.Container, name string) *corev1.Container {
+		for i, c := range containers {
+			if c.Name == name {
+				return &containers[i]
+			}
+		}
+		return nil
+	}
+	envValue := func(env []corev1.EnvVar, name string) (string, bool) {
+		for _, e := range env {
+			if e.Name == name {
+				return e.Value, true
+			}
+		}
+		return "", false
+	}
+
+	// gpuKey escapes the dot in nvidia.com/gpu so helm --set treats the whole
+	// string as a single map key rather than a nested path.
+	gpuKey := func(section string) string {
+		return "delegatedOperatorJobTemplates.jobs.cpuDefault.resources." +
+			section + ".nvidia\\.com/gpu"
+	}
+
+	testCases := []struct {
+		name      string
+		values    map[string]string
+		expectGpu bool
+	}{
+		{
+			name: "gpuInLimitsExposesEnvToSidecar",
+			values: map[string]string{
+				"telemetry.enabled": "true",
+				"delegatedOperatorJobTemplates.jobs.cpuDefault.unused": "nil",
+				gpuKey("limits"): "1",
+			},
+			expectGpu: true,
+		},
+		{
+			name: "gpuInRequestsExposesEnvToSidecar",
+			values: map[string]string{
+				"telemetry.enabled": "true",
+				"delegatedOperatorJobTemplates.jobs.cpuDefault.unused": "nil",
+				gpuKey("requests"): "1",
+			},
+			expectGpu: true,
+		},
+		{
+			name: "noGpuOmitsEnvFromSidecar",
+			values: map[string]string{
+				"telemetry.enabled": "true",
+				"delegatedOperatorJobTemplates.jobs.cpuDefault.unused": "nil",
+			},
+			expectGpu: false,
+		},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		s.Run(testCase.name, func() {
+			subT := s.T()
+			subT.Parallel()
+
+			options := &helm.Options{SetValues: testCase.values}
+			output := helm.RenderTemplate(subT, options, s.chartPath, s.releaseName, s.templates)
+
+			var configMap corev1.ConfigMap
+			helm.UnmarshalK8SYaml(subT, output, &configMap)
+
+			job := s.renderJob(configMap.Data, jobKey)
+
+			// The native-sidecar runs as an initContainer (restartPolicy: Always).
+			sidecar := findContainer(job.Spec.Template.Spec.InitContainers, "telemetry-sidecar")
+			s.Require().NotNil(sidecar, "telemetry-sidecar initContainer not found")
+
+			visibleDevices, hasVisibleDevices := envValue(sidecar.Env, "NVIDIA_VISIBLE_DEVICES")
+			driverCaps, hasDriverCaps := envValue(sidecar.Env, "NVIDIA_DRIVER_CAPABILITIES")
+
+			if testCase.expectGpu {
+				s.True(hasVisibleDevices, "sidecar should have NVIDIA_VISIBLE_DEVICES")
+				s.Equal("all", visibleDevices, "NVIDIA_VISIBLE_DEVICES value mismatch")
+				s.True(hasDriverCaps, "sidecar should have NVIDIA_DRIVER_CAPABILITIES")
+				s.Equal("compute,utility", driverCaps, "NVIDIA_DRIVER_CAPABILITIES value mismatch")
+
+				// The sidecar reads the executor's GPU; it must not request its own.
+				_, limitsHasGpu := sidecar.Resources.Limits[corev1.ResourceName(gpuResource)]
+				_, requestsHasGpu := sidecar.Resources.Requests[corev1.ResourceName(gpuResource)]
+				s.False(limitsHasGpu, "sidecar must not request nvidia.com/gpu in limits")
+				s.False(requestsHasGpu, "sidecar must not request nvidia.com/gpu in requests")
+			} else {
+				s.False(hasVisibleDevices, "sidecar should not have NVIDIA_VISIBLE_DEVICES when no GPU requested")
+				s.False(hasDriverCaps, "sidecar should not have NVIDIA_DRIVER_CAPABILITIES when no GPU requested")
+			}
+		})
+	}
+}
+
 func (s *doK8sConfigMapTemplateTest) loadTestFile(filename, chartVersion string) string {
 	content, err := os.ReadFile(filename)
 	s.NoError(err, "Failed to read test file: %s", filename)