fix: preserve hybrid graphics and publish stability (#1239)

Author: ahmedadan

.github/workflows/build-aarch64.yml

@@ -78,6 +78,7 @@ jobs:
         with:
           storage-backend: btrfs
           update-podman: true
+          native-overlay: true
           install-tools: '["just"]'
 
       - name: Capture build timestamp

.github/workflows/build.yml

@@ -95,6 +95,7 @@ jobs:
         with:
           storage-backend: btrfs
           update-podman: true
+          native-overlay: true
           install-tools: '["just"]'
 
       - name: Generate BuildStream CI config

.github/workflows/execute-release.yml

@@ -143,7 +143,8 @@ jobs:
         [
           {"image":"dakota","source_tag":"${{ needs.freshness-check.outputs.build_sha }}","target_tag":"stable"},
           {"image":"dakota-nvidia","source_tag":"${{ needs.freshness-check.outputs.build_sha }}","target_tag":"stable"},
-          {"image":"dakota-gaming","source_tag":"${{ needs.freshness-check.outputs.build_sha }}","target_tag":"stable"}
+          {"image":"dakota-gaming","source_tag":"${{ needs.freshness-check.outputs.build_sha }}","target_tag":"stable"},
+          {"image":"dakota-nvidia-gaming","source_tag":"${{ needs.freshness-check.outputs.build_sha }}","target_tag":"stable"}
         ]
       cosign_identity_regexp: ^https://github\.com/projectbluefin/dakota/\.github/workflows/publish\.yml@refs/heads/(testing|gh-readonly-queue/testing/.+)$
       # fast_forward_branch omitted: handled by update-main-bookmark job below,
@@ -234,7 +235,7 @@ jobs:
         id: digests
         run: |
           set -euo pipefail
-          for image in dakota dakota-nvidia dakota-gaming; do
+          for image in dakota dakota-nvidia dakota-gaming dakota-nvidia-gaming; do
             digest=$(skopeo inspect --no-tags "docker://ghcr.io/projectbluefin/${image}:stable" \
               | jq -r '.Digest' | cut -c1-19)
             echo "${image}=${digest}" >> "$GITHUB_OUTPUT"
@@ -245,6 +246,7 @@ jobs:
           DAKOTA_DIGEST: ${{ steps.digests.outputs.dakota }}
           NVIDIA_DIGEST: ${{ steps.digests.outputs.dakota-nvidia }}
           GAMING_DIGEST: ${{ steps.digests.outputs.dakota-gaming }}
+          NVIDIA_GAMING_DIGEST: ${{ steps.digests.outputs.dakota-nvidia-gaming }}
         run: |
           set -euo pipefail
           TAG=$(gh release list --repo "${{ github.repository }}" --limit 1 --json tagName --jq '.[0].tagName')
@@ -258,6 +260,7 @@ jobs:
             "| \`dakota\` | \`:stable\` | \`${DAKOTA_DIGEST}\` |" \
             "| \`dakota-nvidia\` | \`:stable\` | \`${NVIDIA_DIGEST}\` |" \
             "| \`dakota-gaming\` | \`:stable\` | \`${GAMING_DIGEST}\` |" \
+            "| \`dakota-nvidia-gaming\` | \`:stable\` | \`${NVIDIA_GAMING_DIGEST}\` |" \
             '' \
             '---' \
             '')
@@ -398,7 +401,7 @@ jobs:
         run: |
           set -euo pipefail
           sudo apt-get update -qq && sudo apt-get install -y -qq skopeo
-          for image in dakota dakota-nvidia dakota-gaming; do
+          for image in dakota dakota-nvidia dakota-gaming dakota-nvidia-gaming; do
             promoted_digest=$(skopeo inspect --no-tags \
               "docker://ghcr.io/projectbluefin/${image}:${BUILD_SHA}" | jq -r .Digest)
             stable_digest=$(skopeo inspect --no-tags \
@@ -459,6 +462,15 @@ jobs:
           delete-only-untagged-versions: 'true'
         continue-on-error: true
 
+      - name: Prune untagged GHCR versions for dakota-nvidia-gaming
+        uses: actions/delete-package-versions@e5bc658cc4c965c472efe991f8beea3981499c55 # v5.0.0
+        with:
+          package-name: dakota-nvidia-gaming
+          package-type: container
+          min-versions-to-keep: 100
+          delete-only-untagged-versions: 'true'
+        continue-on-error: true
+
       - name: Summary — ready for next cycle
         run: |
           {

.github/workflows/publish.yml

@@ -141,14 +141,14 @@ jobs:
         id: timestamp
         run: echo "created=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"
 
-      # update-podman: true — newer podman from Ubuntu resolute for bst2 + push.
-      # Note: setup-runner btrfs does not expose loopback-free; watch for disk
-      # pressure and upstream the option if needed.
+      # Use the runner's Podman stack with native rootful overlay storage.
+      # FUSE-backed storage below chunkify's kernel overlay can return ESTALE.
       - name: Setup runner
         uses: projectbluefin/actions/bootc-build/setup-runner@v1
         with:
           storage-backend: btrfs
           update-podman: true
+          native-overlay: true
           install-tools: '["just"]'
 
       # Fetch-only BST config — pull artifact from remote CAS, no build/push
@@ -315,6 +315,7 @@ jobs:
         with:
           storage-backend: btrfs
           update-podman: true
+          native-overlay: true
           install-tools: '["just"]'
 
       # BST config for remote CAS — buildstream-sbom calls bst show --deps all

docs/skills/ci.md

@@ -2038,6 +2038,43 @@ unacceptably slow even when the overlay is on the correct BTRFS volume. Dakota's
 uses that recipe and then transfers the rootful result back to the runner user's
 podman store before lint and push.
 
+### GitHub runner Podman must use native rootful overlay storage (2026-07-29)
+
+**Symptom:** `just chunkify` fails immediately after fakecap restoration while
+walking the mounted rootfs:
+
+```
+Error: scanning /chunkah for files
+Caused by:
+  0: failed to walk rootfs
+  1: Stale file handle (os error 116)
+```
+
+**Root cause:** the `ubuntu-24.04` runner image update from `20260720.247.2`
+to `20260726.254.1` added a static Podman 5.8 stack under `/usr/local` and an
+`/etc/containers/storage.conf` that selects `fuse-overlayfs`. Installing Podman
+5.7 from Ubuntu Resolute does not remove those files: `/usr/local/bin/podman`
+still wins path resolution while the host contains a mixture of runner and
+Resolute packages. Dakota then creates a kernel overlay from `podman image
+mount` and asks chunkah to scan that overlay through a Podman bind mount. The
+FUSE-backed lower store can return `ESTALE` under that nested access pattern.
+
+**Rule:** use the shared `setup-runner` action's explicit native-overlay mode at
+every Dakota call site:
+
+```yaml
+with:
+  storage-backend: btrfs
+  update-podman: false
+  native-overlay: true
+```
+
+The shared action must keep this mode opt-in until other consumers validate it.
+It must reject `native-overlay: true` together with `update-podman: true`, verify
+the runner Podman version, and fail unless rootful `podman info` reports native
+overlay without a `fuse-overlayfs` mount program. Do not remove or replace
+runner binaries ad hoc in Dakota workflows.
+
 ### actions/cache does not create the cache directory on a cold miss — podman bind-mounts fail (2026-06-13)
 
 `actions/cache` only *restores* an existing archive; on a cache miss it does

docs/skills/oci-layers.md

@@ -193,16 +193,23 @@ In a bootc/OSTree-based immutable OS like Dakota, `/etc` is mutable, user-owned,
 2. **Inject Bootc Kernel Arguments**: Install kargs TOML files directly under `/usr/lib/bootc/kargs.d/` (e.g. `/usr/lib/bootc/kargs.d/20-zswap.toml`). Bootc automatically reads these on switch/upgrade and applies them.
 3. **Vendor Sysctl parameters**: Store system defaults under `/usr/lib/sysctl.d/*.conf` (e.g. `60-swappiness.conf`) instead of `/etc/sysctl.conf`.
 
-### Never install a real directory at a GL/ extension path from a layer (2026-07-18)
-
-In the composed image, several paths under `%{libdir}/GL/` are symlinks into
-the Mesa GL extension tree (e.g. `GL/glvnd/egl_vendor.d ->
-../default/glvnd/egl_vendor.d`). A layer that installs a real directory at
-one of those paths shadows the symlink at OCI merge time and evicts the
-files behind it — installing an EGL vendor ICD to
-`%{libdir}/GL/glvnd/egl_vendor.d/` removed Mesa's `50_mesa.json` from
-GLVND's view and with it the llvmpipe fallback. Vendor ICDs belong in
-`/etc/glvnd/egl_vendor.d` (fdsdk's libglvnd searches only `/etc/glvnd` and
-the GL extension dir — never `/usr/share/glvnd`). The same shadowing hazard
-applies to any `GL/` path: check with `ls -ld` on a composed image before
-choosing an install location under `GL/`.
+### Preserve FDSDK GL merge symlinks when adding vendor backends (2026-07-18; updated 2026-07-29)
+
+In the composed image, paths under `%{libdir}/GL/` are symlinks into the Mesa
+GL extension tree. A layer that installs a real directory at one of those
+paths shadows the symlink at OCI merge time and evicts the Mesa files behind
+it.
+
+Two load-bearing examples:
+
+- `GL/glvnd/egl_vendor.d -> ../default/glvnd/egl_vendor.d`: install vendor
+  ICDs in `/etc/glvnd/egl_vendor.d`; fdsdk's libglvnd searches `/etc/glvnd`
+  and the GL extension directory, not `/usr/share/glvnd`.
+- `GL/lib/gbm -> ../default/lib/gbm`: install an NVIDIA GBM backend in
+  `GL/default/lib/gbm`, not `GL/lib/gbm`. Replacing this symlink hides Mesa's
+  `dri_gbm.so`; Intel/AMD iGPU + NVIDIA dGPU laptops then fall back to the
+  connector-less NVIDIA GPU and show a black GDM screen.
+
+Final NVIDIA OCI assembly must assert that `GL/lib/gbm` remains a symlink and
+that both `dri_gbm.so` and `nvidia-drm_gbm.so` resolve through it. Check the
+composed path with `ls -ld` before choosing any install location under `GL/`.

docs/skills/release-promotion.md

@@ -24,6 +24,11 @@ testing (trunk) → build.yml → publish.yml → :testing tag
 
 `main` is a release bookmark only. It is fast-forwarded by `execute-release.yml` after each successful promotion. Do not open PRs against `main`.
 
+Stable promotion advances all four x86 variants together: `dakota`,
+`dakota-nvidia`, `dakota-gaming`, and `dakota-nvidia-gaming`. A missing or
+unverified SHA-pinned variant must fail promotion instead of leaving the stable
+variant set partially updated.
+
 Do not conflate "publish is healthy" with "stable promotion is healthy".
 
 ## When to Use
@@ -348,10 +353,12 @@ flow (issue 1073). The key differences:
 
 ### Keep stable variant lists aligned
 
-When adding an image variant to stable promotion, update the reusable release
-matrix, release-note digest collection, post-release digest verification, and
-untagged package cleanup together. The release workflow can otherwise promote
-only part of the variant set or report success without verifying the new image.
+The stable x86 set is `dakota`, `dakota-nvidia`, `dakota-gaming`, and
+`dakota-nvidia-gaming`. When adding or removing an image variant, update the
+reusable release matrix, release-note digest collection and table, post-release
+digest verification, and untagged package cleanup together. The release
+workflow can otherwise promote only part of the variant set or report success
+without verifying the new image.
 
 ## Rollback
 

elements/bluefin-nvidia/nvidia-drivers.bst

@@ -175,11 +175,12 @@ config:
             ln -sf "$target" "$LIBDIR/${subdir}${name}"
             ;;
         G)
-            # Relocated to the backend dir freedesktop-sdk's Mesa
-            # compiles into libgbm; the link IS the backend (its
-            # entrypoints live in the target library).
-            install -d "$LIBDIR/GL/lib/gbm"
-            ln -sf "../../../$target" "$LIBDIR/GL/lib/gbm/$name"
+            # FDSDK exposes GL/lib/gbm as a symlink to the Mesa extension's
+            # GL/default/lib/gbm. Install into that target: creating a real
+            # GL/lib/gbm here shadows Mesa's dri_gbm.so on hybrid systems.
+            GBM_DIR="$LIBDIR/GL/default/lib/gbm"
+            install -d "$GBM_DIR"
+            ln -srf "$LIBDIR/$target" "$GBM_DIR/$name"
             ;;
         esac
     done

elements/oci/bluefin-nvidia.bst

@@ -56,6 +56,21 @@ config:
     - |
       dconf update /layer/etc/dconf/db
 
+    - |
+      # FDSDK's Mesa merge path must stay a symlink so libgbm sees both
+      # Mesa's DRI backend and NVIDIA's backend on hybrid systems.
+      GBM_DIR="/layer%{libdir}/GL/lib/gbm"
+      if [ ! -L "$GBM_DIR" ]; then
+        echo "FATAL: $GBM_DIR is not the FDSDK Mesa merge symlink" >&2
+        exit 1
+      fi
+      for backend in dri_gbm.so nvidia-drm_gbm.so; do
+        if [ ! -e "$GBM_DIR/$backend" ]; then
+          echo "FATAL: GBM backend $GBM_DIR/$backend is missing" >&2
+          exit 1
+        fi
+      done
+
     - |
       cd "%{install-root}"
       build-oci <<EOF