Separate the snap by cuda version on v4

.github/workflows/check.yaml

@@ -46,7 +46,7 @@ jobs:
 
       - name: Lint yaml files
         run: |
-          yamllint .yamllint snap/snapcraft.yaml
+          yamllint .yamllint snap/snapcraft.yaml.in
 
   unit:
     name: Unit
@@ -106,11 +106,16 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [[ubuntu-24.04], [self-hosted, jammy, ARM64]]
+        cuda-version: [11, 12, 13]
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0  # Complete git history is required to generate the version from git tags.
 
+      - name: Build snapcraft.yaml files
+        run: |
+          CUDA_VERSION=${{ matrix.cuda-version }} envsubst '$CUDA_VERSION'  < snap/snapcraft.yaml.in > snap/snapcraft.yaml
+
       - name: Verify snap builds successfully
         id: build
         uses: canonical/action-build@v1
@@ -121,7 +126,7 @@ jobs:
       - name: Upload the built snap
         uses: actions/upload-artifact@v4
         with:
-          name: snap_${{ env.SYSTEM_ARCH }}
+          name: snap_${{ matrix.cuda-version }}_${{ env.SYSTEM_ARCH }}
           path: ${{ steps.build.outputs.snap }}
 
   func:
@@ -132,6 +137,7 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [[ubuntu-24.04], [self-hosted, jammy, ARM64]]
+        cuda-version: [11, 12, 13]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -143,7 +149,7 @@ jobs:
       - name: Download snap file artifact
         uses: actions/download-artifact@v4
         with:
-          name: snap_${{ env.SYSTEM_ARCH }}
+          name: snap_${{ matrix.cuda-version }}_${{ env.SYSTEM_ARCH }}
 
       - name: Set up Python 3.10
         uses: actions/setup-python@v5
@@ -165,7 +171,7 @@ jobs:
         run: |
           BASE_VERSION_SANITIZED=${{ matrix.runs-on }}
           BASE_VERSION_SANITIZED=${BASE_VERSION_SANITIZED//./-}
-          echo "ARTIFACT_SUFFIX=$BASE_VERSION_SANITIZED-${{ env.SYSTEM_ARCH }}" >> $GITHUB_ENV
+          echo "ARTIFACT_SUFFIX=$BASE_VERSION_SANITIZED-${{ matrix.cuda-version }}-${{ env.SYSTEM_ARCH }}" >> $GITHUB_ENV
 
       - name: Rename Functional Test Coverage Artifact
         run: |

.github/workflows/promote.yaml

@@ -12,7 +12,15 @@ on:
         description: 'Channel Promotion, e.g. latest/edge -> latest/candidate'
         required: true
         type: choice
-        options: ['latest/edge -> latest/candidate', 'latest/candidate -> latest/stable', 'core24/edge -> core24/candidate', 'core24/candidate -> core24/stable']
+        options:
+          [
+            'v4-cuda11/edge -> v4-cuda11/candidate',
+            'v4-cuda11/candidate -> v4-cuda11/stable',
+            'v4-cuda12/edge -> v4-cuda12/candidate',
+            'v4-cuda12/candidate -> v4-cuda12/stable',
+            'v4-cuda13/edge -> v4-cuda13/candidate',
+            'v4-cuda13/candidate -> v4-cuda13/stable'
+          ]
 
 jobs:
   promote-snap:

.github/workflows/release.yaml

@@ -23,6 +23,7 @@ jobs:
       fail-fast: false
       matrix:
         runs-on: [[ubuntu-24.04], [self-hosted, jammy, ARM64]]
+        cuda-version: [11, 12, 13]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -34,7 +35,7 @@ jobs:
       - name: Download the built snap from check workflow
         uses: actions/download-artifact@v4
         with:
-          name: snap_${{ env.SYSTEM_ARCH }}
+          name: snap_${{ matrix.cuda-version }}_${{ env.SYSTEM_ARCH }}
 
       - name: Find the downloaded snap file
         run: echo "SNAP_FILE=$(find . -name "*.snap")" >> $GITHUB_ENV
@@ -45,7 +46,7 @@ jobs:
         with:
           snap: ${{ env.SNAP_FILE }}
           # Comma-separated list of channels to release the snap to.
-          release: core24/edge,latest/edge
+          release: v4-cuda${{ matrix.cuda-version }}/edge
 
   notify-on-release-failure:
     runs-on: ubuntu-latest

.gitignore

@@ -42,5 +42,8 @@ dist/
 *~
 *.bak
 
+# snapcraft file is dynamically generated
+snap/snapcraft.yaml
+
 # Note: for editor-specific files, please don't add them here, as they are specific to your environment, not the project.
 # Instead, consider using a global gitignore on your workstation.

README.md

@@ -5,10 +5,24 @@ The snap consists of [dcgm](https://developer.nvidia.com/dcgm) and [dcgm-exporte
 
 ## Build the snap
 
+From v4, the `snapcraft.yaml` file is dynamically generated using the `snapcraft.yaml.in` file
+as template. To create the file run:
+
+```shell
+# generate v4 using cuda11 packages
+CUDA_VERSION=11 envsubst '$CUDA_VERSION'  < snap/snapcraft.yaml.in > snap/snapcraft.yaml
+
+# generate v4 using cuda12 packages
+CUDA_VERSION=12 envsubst '$CUDA_VERSION'  < snap/snapcraft.yaml.in > snap/snapcraft.yaml
+
+# generate v4 using cuda13 packages
+CUDA_VERSION=13 envsubst '$CUDA_VERSION'  < snap/snapcraft.yaml.in > snap/snapcraft.yaml
+```
+
 You can build the snap locally by using the command:
 
 ```shell
-snapcraft --use-lxd
+snapcraft pack -v
 ```
 
 ## Test

snap/local/files/run_dcgm_exporter.sh

@@ -2,7 +2,9 @@
 set -euo pipefail
 
 # Build the argument list for the dcgm-exporter command
-args=()
+# startup validation dynamic generates and check libs that are not possible in snap confinement
+# See https://github.com/NVIDIA/dcgm-exporter/issues/553
+args=("--disable-startup-validate")
 
 nv_hostengine_port="$(snapctl get nv-hostengine-port)"
 dcgm_exporter_address="$(snapctl get dcgm-exporter-address)"
@@ -25,4 +27,4 @@ else
     echo "Error: DCGM exporter metrics file not found or empty: $dcgm_exporter_metrics_file_path, using default"
 fi
 
-exec "$SNAP/bin/dcgm-exporter" "${args[@]}"
+exec "dcgm-exporter" "${args[@]}"

snap/snapcraft.yaml → snap/snapcraft.yaml.in

@@ -19,9 +19,27 @@ description: |
   **How-To**
   ---
 
+  **How to select a channel:**
+
+  You need to have NVIDIA drivers installed on your system to use this snap.\
+  You can check the version of your NVIDIA driver with the command:
+
+     cat /proc/driver/nvidia/version
+
+  With that information, you can check the compatible CUDA version here:\
+  https://docs.nvidia.com/deploy/cuda-compatibility/minor-version-compatibility.html
+
+  Keep in mind that:
+  DCGM v4 has different releases for different CUDA versions. E.g. if your system has the Driver\
+  version >= 580 it uses CUDA 13, so you should install the snap from the channel `v4-cuda13`
+
+     sudo snap install dcgm --channel=v4-cuda13/stable
+
+  DCGM v3 ships a single package with binaries that works from CUDA 10 up to CUDA 12.
+
   **How to install the snap:**
 
-     sudo snap install dcgm
+     sudo snap install dcgm --channel=<CHANNEL>
 
   **How to enable metrics collection:**
 
@@ -129,20 +147,8 @@ apps:
     restart-condition: on-abort
     environment:
       DCGM_HOME_DIR: "${SNAP_COMMON}"
-  dcgmproftester10:
-    command: bin/run_dcgmproftester10.sh
-    plugs:
-      - network-bind
-      - opengl
-      - hardware-observe
-  dcgmproftester11:
-    command: usr/bin/dcgmproftester11
-    plugs:
-      - network-bind
-      - opengl
-      - hardware-observe
-  dcgmproftester12:
-    command: usr/bin/dcgmproftester12
+  dcgmproftester:
+    command: usr/bin/dcgmproftester${CUDA_VERSION}
     plugs:
       - network-bind
       - opengl
@@ -165,33 +171,38 @@ parts:
     after:
       - cuda-sources
     plugin: nil
-    stage-packages: [datacenter-gpu-manager=1:3.3.8]
+
+    build-packages:
+      - git
+
+    override-pull: |
+      craftctl default
+      apt download datacenter-gpu-manager-4-cuda${CUDA_VERSION}=1:4.4.1-1
+      apt download $(apt-cache depends datacenter-gpu-manager-4-cuda${CUDA_VERSION} | awk '/Depends:/ {print $2}')
+
+    override-build: |
+      craftctl default
+      dpkg-deb -x datacenter-gpu-manager-4-cuda${CUDA_VERSION}_*.deb $SNAPCRAFT_PART_INSTALL/
+      dpkg-deb -x datacenter-gpu-manager-4-core*.deb $SNAPCRAFT_PART_INSTALL/
+
     # override prime to set version
     override-prime: |
       craftctl default
       # Locate dcgm .deb file
-      DEB_FILE=$(ls $HOME/parts/dcgm/stage_packages/datacenter-gpu-manager_*.deb)
+      DEB_FILE=$(ls $HOME/parts/dcgm/src/datacenter-gpu-manager-4-cuda*.deb)
 
       # Extract the version from the .deb file
       DCGM_VERSION=$(dpkg-deb -f "$DEB_FILE" Version)
-      GIT_VERSION=$(git -C $CRAFT_PROJECT_DIR describe --always --dirty --abbrev=10)
+      GIT_VERSION=$(git -C $CRAFT_PROJECT_DIR describe --always)
 
       # Set the Snap version to the same as dcgm deb file
-      craftctl set version="${DCGM_VERSION#1:}+snap-${GIT_VERSION}"
+      craftctl set version="${CUDA_VERSION}-${DCGM_VERSION#1:}-${GIT_VERSION}"
 
   # This is the DCGM exporter
   dcgm-exporter:
-    plugin: go
-    build-snaps:
-      - go
-    source: https://github.com/NVIDIA/dcgm-exporter.git
-    source-type: git
-    source-tag: 3.3.8-3.6.0
-    # override build to get the default csv files from the upstream
-    override-build: |
-      craftctl default
-      mkdir -p $SNAPCRAFT_PART_INSTALL/etc/dcgm-exporter
-      cp etc/default-counters.csv etc/dcp-metrics-included.csv $SNAPCRAFT_PART_INSTALL/etc/dcgm-exporter/
+    plugin: nil
+    stage-packages:
+      - datacenter-gpu-manager-exporter=4.5.2-1
 
   # wrappers supporting snap options
   wrapper:
@@ -200,8 +211,11 @@ parts:
     organize:
       run_nv_hostengine.sh: bin/
       run_dcgm_exporter.sh: bin/
-      run_dcgmproftester10.sh: bin/
 
 layout:
   /etc/dcgm-exporter:
     symlink: $SNAP/etc/dcgm-exporter
+  /usr/lib/x86_64-linux-gnu/libdcgm.so.4:
+    bind-file: $SNAP/usr/lib/x86_64-linux-gnu/libdcgm.so.4.4.1
+  /usr/libexec/datacenter-gpu-manager-4:
+    bind: $SNAP/usr/libexec/datacenter-gpu-manager-4

tests/functional/conftest.py

@@ -15,6 +15,7 @@ def install_dcgm_snap():
 
     # Manually connect the 'hardware-observe' interface as auto-connect is not allowed
     subprocess.check_call("sudo snap connect dcgm:hardware-observe".split())
+    subprocess.check_call("sudo snap connect dcgm:opengl".split())
 
     subprocess.check_call("sudo snap start dcgm.dcgm-exporter".split())