Merge pull request #482 from arnaldo2792/migrate-ecs-cdi

Migrate ECS to CDI

Author: arnaldo2792

packages/docker-engine/daemon-nvidia-json

@@ -25,5 +25,9 @@ std = { version = "v1", helpers = ["join_array"] }
   {{/if}}
   {{/each}}
   {{/if}}
-  "selinux-enabled": true
+  "selinux-enabled": true,
+  "features": {
+    "cdi": true
+  },
+  "cdi-spec-dirs": ["/etc/cdi/"]
 }

packages/nvidia-container-toolkit/0001-ldcache-fix-parsing-for-aarch64.patch

@@ -0,0 +1,46 @@
+From 018a8150cab0969c63b82a1e93047599e56de167 Mon Sep 17 00:00:00 2001
+From: Arnaldo Garcia Rincon <[email protected]>
+Date: Wed, 23 Apr 2025 01:09:31 +0000
+Subject: [PATCH] ldcache: fix parsing for aarch64
+
+The architecture flag for aarch64 is currently missing from the
+supported architecture flags list. This omission causes the getEntries
+function to exclude all libraries found on aarch64 hosts. As a result
+helper programs like nvidia-ctk are unable to generate CDI
+specifications for the aarch64 architecture.
+
+This fix adds the missing aarch64 architecture flag, using the same
+value as defined in libnvidia-container[1], which maintains a more
+comprehensive list of supported architectures.
+
+[1]: https://github.com/NVIDIA/libnvidia-container/blob/a198166e1c1166f4847598438115ea97dacc7a92/src/ldcache.h#L21
+
+Signed-off-by: Arnaldo Garcia Rincon <[email protected]>
+---
+ internal/ldcache/ldcache.go | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/internal/ldcache/ldcache.go b/internal/ldcache/ldcache.go
+index 4daf95b..455048c 100644
+--- a/internal/ldcache/ldcache.go
++++ b/internal/ldcache/ldcache.go
+@@ -47,6 +47,7 @@ const (
+ 	flagArchX8664   = 0x0300
+ 	flagArchX32     = 0x0800
+ 	flagArchPpc64le = 0x0500
++	flagArchAarch64 = 0x0a00
+ )
+ 
+ var errInvalidCache = errors.New("invalid ld.so.cache file")
+@@ -195,6 +196,8 @@ func (c *ldcache) getEntries() []entry {
+ 		switch e.Flags & flagArchMask {
+ 		case flagArchX8664:
+ 			fallthrough
++		case flagArchAarch64:
++			fallthrough
+ 		case flagArchPpc64le:
+ 			bits = 64
+ 		case flagArchX32:
+-- 
+2.49.0
+

packages/nvidia-container-toolkit/generate-cdi-specs.service

@@ -0,0 +1,34 @@
+[Unit]
+Description=Generate CDI specifications
+# This has to be executed after the kernel modules are loaded
+# otherwise the userspace component of the driver will fail to
+# query the /dev devices
+After=load-tesla-kernel-modules.service load-open-gpu-kernel-modules.service
+# Running this unit after nvidia persistenced ensures that
+# the /dev devices are created and the hardware set to
+# persistence mode.
+Requires=nvidia-persistenced.service
+After=nvidia-persistenced.service
+# Block manual interactions with this service. It doesn't
+# make sense to regenerate CDI specifications as features
+# that might change the GPU (e.g. MIG) are not supported in
+# ECS
+RefuseManualStart=true
+RefuseManualStop=true
+
+[Service]
+Type=oneshot
+# Explanation of the options:
+# --format json: to be consistent across Bottlerocket's variants
+# --mode nvml: the default mode ("auto") resolves to this already, make it explicit
+# --device-name-strategy uuid: the ECS agent only supports device UUIDs
+# --output /etc/cdi/nvidia.json: store the CDI specifications at this location
+ExecStart=/usr/bin/nvidia-ctk cdi generate --format json \
+  --mode nvml \
+  --device-name-strategy uuid \
+  --output /etc/cdi/nvidia.json
+RemainAfterExit=true
+StandardError=journal+console
+
+[Install]
+RequiredBy=preconfigured.target

packages/nvidia-container-toolkit/nvidia-container-toolkit-config-ecs.toml

@@ -3,3 +3,6 @@ root = "/"
 path = "/usr/bin/nvidia-container-cli"
 environment = []
 ldconfig = "@/sbin/ldconfig"
+
+[nvidia-container-runtime]
+mode = "cdi"

packages/nvidia-container-toolkit/nvidia-container-toolkit.spec

@@ -22,6 +22,8 @@ Source3: nvidia-gpu-devices.rules
 Source4: nvidia-container-toolkit-tmpfiles-ecs.conf
 Source5: nvidia-container-toolkit-tmpfiles-k8s.conf
 Source6: nvidia-container-toolkit-config-k8s
+Source7: generate-cdi-specs.service
+Patch0001: 0001-ldcache-fix-parsing-for-aarch64.patch
 
 BuildRequires: %{_cross_os}glibc-devel
 Requires: %{_cross_os}libnvidia-container
@@ -34,6 +36,7 @@ Requires: (%{name}-k8s if %{_cross_os}variant-family(aws-k8s))
 %package ecs
 Summary: Files specific for the ECS variants
 Requires: %{name}
+Requires: %{name}-cdi-specs
 Conflicts: %{name}-k8s
 
 %description ecs
@@ -47,6 +50,13 @@ Conflicts: %{name}-ecs
 %description k8s
 %{summary}.
 
+%package cdi-specs
+Summary: Tools to generate CDI specifications
+Requires: %{name}
+
+%description cdi-specs
+%{summary}.
+
 %prep
 %autosetup -n %{gorepo}-%{gover} -p1
 %cross_go_setup %{gorepo}-%{gover} %{goproject} %{goimport}
@@ -69,6 +79,7 @@ install -d %{buildroot}%{_cross_bindir}
 install -d %{buildroot}%{_cross_tmpfilesdir}
 install -d %{buildroot}%{_cross_templatedir}
 install -d %{buildroot}%{_cross_udevrulesdir}
+install -d %{buildroot}%{_cross_unitdir}
 install -d %{buildroot}%{_cross_datadir}/nvidia-container-toolkit
 install -d %{buildroot}%{_cross_factorydir}/nvidia-container-runtime
 install -d %{buildroot}%{_cross_templatedir}/nvidia-container-runtime
@@ -82,6 +93,7 @@ install -p -m 0644 %{S:3} %{buildroot}%{_cross_udevrulesdir}/90-nvidia-gpu-devic
 install -m 0644 %{S:4} %{buildroot}%{_cross_tmpfilesdir}/nvidia-container-toolkit-ecs.conf
 install -m 0644 %{S:5} %{buildroot}%{_cross_tmpfilesdir}/nvidia-container-toolkit-k8s.conf
 install -m 0644 %{S:6} %{buildroot}%{_cross_templatedir}/nvidia-container-runtime/
+install -m 0644 %{S:7} %{buildroot}%{_cross_unitdir}/
 
 %files
 %license LICENSE
@@ -100,3 +112,6 @@ install -m 0644 %{S:6} %{buildroot}%{_cross_templatedir}/nvidia-container-runtim
 %{_cross_factorydir}/nvidia-container-runtime/nvidia-container-toolkit-config-k8s.toml
 %{_cross_templatedir}/nvidia-container-runtime/nvidia-container-toolkit-config-k8s
 %{_cross_tmpfilesdir}/nvidia-container-toolkit-k8s.conf
+
+%files cdi-specs
+%{_cross_unitdir}/generate-cdi-specs.service