Skip to content

Commit 5dcbd30

Browse files
[EBPF] gpu: Support ARM64 NVML library discovery (#54720)
<!-- dd-meta {"pullId":"df13230b-a434-451b-972e-ac9007c02168","source":"chat","resourceId":"86800824-17f2-4a85-9551-be5b7bf8a830","workflowId":"2ce581a4-a03e-4c0a-a1a1-dda5b27f5a0c","codeChangeId":"2ce581a4-a03e-4c0a-a1a1-dda5b27f5a0c","sourceType":"bits_ai_sre"} --> ### What does this PR do? Bits AI SRE Investigation • [View in Bits AI SRE Investigation](https://ddstaging.datadoghq.com/bits-ai/investigations/8418dc69-efd2-44d3-ad70-91e14223f69d) Add standard ARM64 (`aarch64-linux-gnu`) NVML library paths for host and NVIDIA GPU Operator installations. ### Motivation GPU checks on ARM64 GPU nodes fail because NVML discovery searches only x86_64 library directories. This causes all GPU metrics to fail on affected ARM64 nodes and can trigger incorrect health responses. ### Describe how you validated your changes Unit tests added ### Additional Notes --- PR by Bits - [View session in Datadog](https://ddstaging.datadoghq.com/code/86800824-17f2-4a85-9551-be5b7bf8a830) Comment @DataDog to request changes Co-authored-by: datadog-bits <263423550+datadog-bits@users.noreply.github.com> Co-authored-by: guillermo.julian <guillermo.julian@datadoghq.com>
1 parent 9525b84 commit 5dcbd30

4 files changed

Lines changed: 35 additions & 4 deletions

File tree

pkg/config/env/environment_containers.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -407,8 +407,10 @@ func getDefaultNvmlPaths() []string {
407407
}
408408

409409
systemPaths := []string{
410-
"/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1", // default system install
411-
"/run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1", // nvidia-gpu-operator install
410+
"/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1", // default system install
411+
"/run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1", // nvidia-gpu-operator install
412+
"/usr/lib/aarch64-linux-gnu/libnvidia-ml.so.1", // default system install on ARM64
413+
"/run/nvidia/driver/usr/lib/aarch64-linux-gnu/libnvidia-ml.so.1", // nvidia-gpu-operator install on ARM64
412414
}
413415

414416
hostRoot := os.Getenv("HOST_ROOT")

pkg/config/env/environment_containers_test.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ package env
1010
import (
1111
"os"
1212
"path/filepath"
13+
"runtime"
1314
"testing"
1415

1516
"github.com/stretchr/testify/assert"
@@ -67,6 +68,21 @@ func Test_merge(t *testing.T) {
6768
}
6869
}
6970

71+
func TestGetDefaultNvmlPathsIncludesSupportedLinuxArchitectures(t *testing.T) {
72+
if runtime.GOOS != "linux" {
73+
t.Skip("NVML default paths are only defined on Linux")
74+
}
75+
76+
t.Setenv("HOST_ROOT", "/host")
77+
78+
assert.Equal(t, []string{
79+
"/host/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1",
80+
"/host/run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1",
81+
"/host/usr/lib/aarch64-linux-gnu/libnvidia-ml.so.1",
82+
"/host/run/nvidia/driver/usr/lib/aarch64-linux-gnu/libnvidia-ml.so.1",
83+
}, getDefaultNvmlPaths())
84+
}
85+
7086
func TestDetectPodmanInHomeDir(t *testing.T) {
7187
tmp := t.TempDir()
7288

pkg/gpu/safenvml/lib.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -436,8 +436,10 @@ func GetSafeNvmlLib() (SafeNVML, error) {
436436
// is imported by nearly every binary in the repo.
437437
func generateDefaultNvmlPaths() []string {
438438
systemPaths := []string{
439-
"/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1", // default system install
440-
"/run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1", // nvidia-gpu-operator install
439+
"/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1", // default system install
440+
"/run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1", // nvidia-gpu-operator install
441+
"/usr/lib/aarch64-linux-gnu/libnvidia-ml.so.1", // default system install on ARM64
442+
"/run/nvidia/driver/usr/lib/aarch64-linux-gnu/libnvidia-ml.so.1", // nvidia-gpu-operator install on ARM64
441443
}
442444

443445
hostRoot := os.Getenv("HOST_ROOT")

pkg/gpu/safenvml/lib_test.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,17 @@ func TestPopulateCapabilities(t *testing.T) {
104104
}
105105
}
106106

107+
func TestGenerateDefaultNvmlPathsIncludesSupportedLinuxArchitectures(t *testing.T) {
108+
t.Setenv("HOST_ROOT", "/host")
109+
110+
require.Equal(t, []string{
111+
"/host/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1",
112+
"/host/run/nvidia/driver/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1",
113+
"/host/usr/lib/aarch64-linux-gnu/libnvidia-ml.so.1",
114+
"/host/run/nvidia/driver/usr/lib/aarch64-linux-gnu/libnvidia-ml.so.1",
115+
}, generateDefaultNvmlPaths())
116+
}
117+
107118
// Tests for the NvmlAPIError type
108119
func TestNvmlAPIError(t *testing.T) {
109120
// Define test cases with different error codes and API names

0 commit comments

Comments
 (0)