ci(build): route x86 variants through BuildBox remote execution (#1227)

* ci(build): route x86 variants through BuildBox remote execution with fail-closed verification

* refactor(ci): slim the config generator since the checker owns shape assertions

Author: ahmedadan

.github/actions/generate-bst-ci-config/action.yml

@@ -1,12 +1,12 @@
 name: Generate BuildStream CI config
-description: Write BuildStream CI config for the runner-based Dakota workflow.
+description: Write BuildStream CI config for remote builds and local artifact consumers.
 
 inputs:
   enable-remote-execution:
-    description: Retained for compatibility; Dakota uses the GitHub runner for builds and cache.projectbluefin.io for artifact/source caches.
+    description: Route build actions and CAS storage through cache.projectbluefin.io.
     required: true
   enable-push:
-    description: Push artifacts and sources to the remote cache. Set false for build jobs.
+    description: Publish built artifacts and fetched sources during the BuildStream build.
     required: false
     default: 'true'
 
@@ -23,13 +23,40 @@ runs:
         WORKSPACE_ROOT="${GITHUB_WORKSPACE:-$PWD}"
         mkdir -p "$WORKSPACE_ROOT/logs"
 
-        if [[ "${ENABLE_REMOTE_EXECUTION:-false}" == "true" ]]; then
-          echo "::error::Dakota CI uses the GitHub runner for the BuildStream build; remote execution is intentionally disabled."
+        if [[ "${ENABLE_REMOTE_EXECUTION:-false}" == "true" ]] &&
+           { [[ -z "${CASD_CLIENT_CERT:-}" ]] || [[ -z "${CASD_CLIENT_KEY:-}" ]]; }; then
+          echo "::error::remote execution was requested but CASD client credentials are missing"
           exit 1
         fi
 
         printf '%s\n' "${CASD_CLIENT_CERT:-}" > "$WORKSPACE_ROOT/client.crt"
         printf '%s\n' "${CASD_CLIENT_KEY:-}" > "$WORKSPACE_ROOT/client.key"
+        chmod 0600 "$WORKSPACE_ROOT/client.crt" "$WORKSPACE_ROOT/client.key"
+
+        # Every remote service entry shares one endpoint tuning and mTLS
+        # identity; the stanzas expand inside the unquoted heredocs below.
+        CONNECTION_CONFIG=$(cat <<'EOF'
+            connection-config:
+              keepalive-time: 180
+              retry-limit: 5
+              retry-delay: 1000
+              request-timeout: 180
+        EOF
+        )
+        CLIENT_AUTH=$(cat <<'EOF'
+            auth:
+              client-key: /src/client.key
+              client-cert: /src/client.crt
+        EOF
+        )
+
+        if [[ "${ENABLE_REMOTE_EXECUTION:-false}" == "true" ]]; then
+          # Four BuildBox action slots × four jobs per action is the initial
+          # conservative ceiling on the 32-thread, 128 GiB executor.
+          MAX_JOBS=4
+        else
+          MAX_JOBS=1
+        fi
 
         cat > "$WORKSPACE_ROOT/buildstream-ci.conf" <<'BSTCONF'
         scheduler:
@@ -46,10 +73,10 @@ runs:
         logdir: /src/logs
         BSTCONF
 
-        cat >> "$WORKSPACE_ROOT/buildstream-ci.conf" <<'BSTCONFBUILD'
+        cat >> "$WORKSPACE_ROOT/buildstream-ci.conf" <<BSTCONFBUILD
         build:
           retry-failed: True
-          max-jobs: 1
+          max-jobs: ${MAX_JOBS}
         BSTCONFBUILD
 
         if [[ -n "${CASD_CLIENT_CERT:-}" ]] && [[ -n "${CASD_CLIENT_KEY:-}" ]]; then
@@ -60,55 +87,27 @@ runs:
           servers:
           - url: https://cache.projectbluefin.io:11002
             push: ${PUSH_FLAG}
-            connection-config:
-              keepalive-time: 180
-              retry-limit: 5
-              retry-delay: 1000
-              request-timeout: 180
-            auth:
-              client-key: /src/client.key
-              client-cert: /src/client.crt
+        ${CONNECTION_CONFIG}
+        ${CLIENT_AUTH}
           - url: https://gbm.gnome.org:11003
             push: false
-            connection-config:
-              keepalive-time: 180
-              retry-limit: 5
-              retry-delay: 1000
-              request-timeout: 180
+        ${CONNECTION_CONFIG}
           - url: https://cache.freedesktop-sdk.io:11001
             push: false
-            connection-config:
-              keepalive-time: 180
-              retry-limit: 5
-              retry-delay: 1000
-              request-timeout: 180
+        ${CONNECTION_CONFIG}
 
         source-caches:
           servers:
           - url: https://cache.projectbluefin.io:11002
             push: ${PUSH_FLAG}
-            connection-config:
-              keepalive-time: 180
-              retry-limit: 5
-              retry-delay: 1000
-              request-timeout: 180
-            auth:
-              client-key: /src/client.key
-              client-cert: /src/client.crt
+        ${CONNECTION_CONFIG}
+        ${CLIENT_AUTH}
           - url: https://gbm.gnome.org:11003
             push: false
-            connection-config:
-              keepalive-time: 180
-              retry-limit: 5
-              retry-delay: 1000
-              request-timeout: 180
+        ${CONNECTION_CONFIG}
           - url: https://cache.freedesktop-sdk.io:11001
             push: false
-            connection-config:
-              keepalive-time: 180
-              retry-limit: 5
-              retry-delay: 1000
-              request-timeout: 180
+        ${CONNECTION_CONFIG}
         BSTCONFCACHE
         fi
 
@@ -118,67 +117,37 @@ runs:
           cache-buildtrees: never
         BSTCONFCACHEBLOCK
 
-        if ! grep -Eq '^[[:space:]]*max-jobs:[[:space:]]*1[[:space:]]*$' "$WORKSPACE_ROOT/buildstream-ci.conf"; then
-          echo "::error::buildstream-ci.conf is missing build.max-jobs: 1; the runner-based Dakota build requires this cap."
-          exit 1
+        if [[ "${ENABLE_REMOTE_EXECUTION:-false}" == "true" ]]; then
+          # The top-level storage service makes the runner's local casd a
+          # remote-backed cache, so artifact payloads stay on the BuildBox host.
+          # Keep this out of fetch-only publish configs: export must materialize
+          # the final artifact on the runner.
+          cat >> "$WORKSPACE_ROOT/buildstream-ci.conf" <<BSTCONFREMOTE
+          storage-service:
+            url: https://cache.projectbluefin.io:11002
+        ${CONNECTION_CONFIG}
+        ${CLIENT_AUTH}
+
+        remote-execution:
+          execution-service:
+            url: https://cache.projectbluefin.io:11002
+        ${CONNECTION_CONFIG}
+        ${CLIENT_AUTH}
+          storage-service:
+            url: https://cache.projectbluefin.io:11002
+        ${CONNECTION_CONFIG}
+        ${CLIENT_AUTH}
+          action-cache-service:
+            url: https://cache.projectbluefin.io:11002
+            push: true
+        ${CONNECTION_CONFIG}
+        ${CLIENT_AUTH}
+        BSTCONFREMOTE
+          echo "Remote execution enabled: yes"
+        else
+          echo "Remote execution enabled: no"
         fi
 
         cp "$WORKSPACE_ROOT/buildstream-ci.conf" "$WORKSPACE_ROOT/logs/buildstream-ci.conf"
-        echo "Remote execution enabled: no"
         echo "=== BuildStream CI config ==="
         cat "$WORKSPACE_ROOT/buildstream-ci.conf"
-
-        if [[ -n "${CASD_CLIENT_CERT:-}" ]] && [[ -n "${CASD_CLIENT_KEY:-}" ]]; then
-          cat > "$WORKSPACE_ROOT/buildstream-push.conf" <<'BSTCONFPUSH'
-        scheduler:
-          on-error: continue
-          fetchers: 16
-          builders: 2
-          network-retries: 3
-
-        logging:
-          message-format: '[%{wallclock}][%{elapsed}][%{key}][%{element}] %{action} %{message}'
-          error-lines: 80
-
-        cachedir: /root/.cache/buildstream
-        logdir: /src/logs
-
-        build:
-          retry-failed: True
-          max-jobs: 1
-
-        cache:
-          cache-buildtrees: never
-
-        artifacts:
-          servers:
-          - url: https://cache.projectbluefin.io:11002
-            push: true
-            connection-config:
-              keepalive-time: 180
-              retry-limit: 5
-              retry-delay: 1000
-              request-timeout: 180
-            auth:
-              client-key: /src/client.key
-              client-cert: /src/client.crt
-          - url: https://gbm.gnome.org:11003
-            push: false
-            connection-config:
-              keepalive-time: 180
-              retry-limit: 5
-              retry-delay: 1000
-              request-timeout: 180
-          - url: https://cache.freedesktop-sdk.io:11001
-            push: false
-            connection-config:
-              keepalive-time: 180
-              retry-limit: 5
-              retry-delay: 1000
-              request-timeout: 180
-
-        BSTCONFPUSH
-          cp "$WORKSPACE_ROOT/buildstream-push.conf" "$WORKSPACE_ROOT/logs/buildstream-push.conf"
-          echo "=== BuildStream push config ==="
-          cat "$WORKSPACE_ROOT/buildstream-push.conf"
-        fi

.github/copilot-instructions.md

@@ -170,12 +170,13 @@ re-run the pre-flight until it is clean. Only then proceed.
 - "The stale build is for a different branch, it won't interfere" → **It uses the same runners and CAS. Cancel it.**
 - "I already cancelled one build, that's enough" → **Cancel ALL of them. Run the pre-flight again.**
 
-Concurrent BST builds share the same `ubuntu-24.04` runner pool and the same remote CAS
-write bandwidth at `cache.projectbluefin.io:11002`. Two concurrent builds do not halve
-wall time — they more than double it and risk 6-hour timeouts with
-`Cached elements after warm: 0`.
+Independent BST **workflow runs** share the same remote executor and CAS. Never
+start a second build workflow while one is active. The four x86_64 matrix jobs inside
+one `build.yml` run are the intentional exception: they start together and are bounded
+by the BuildBox backend's four global action slots. Do not cancel matrix siblings or
+serialize them; their CAS payloads stay remote and BuildBox enforces capacity.
 
-**One build. Field clear first. No exceptions.**
+**One build workflow run, with its four coordinated variants. Field clear first.**
 
 **Pre-flight is atomic:** cancel → verify clear → push/dispatch must complete in one
 uninterrupted sequence. Cancelling the active build and then not pushing the
@@ -191,8 +192,8 @@ lesson (2026-07-09).
 ## CI overview
 
 - **Schedule:** nightly at 13:00 UTC (after gnome-build-meta nightly ~08:00 UTC finish)
-- **Publish triggers:** `merge_group`, `schedule`, `workflow_dispatch` (not `pull_request`)
-- **Remote cache:** `cache.projectbluefin.io:11002` (mTLS — `CASD_CLIENT_CERT` + `CASD_CLIENT_KEY`)
+- **Build triggers:** BST-affecting `push: testing`, daily schedule, or `workflow_dispatch` (not PR/merge-group)
+- **Remote build:** BuildBox execution + remote CAS at `cache.projectbluefin.io:11002` (mTLS — `CASD_CLIENT_CERT` + `CASD_CLIENT_KEY`)
 - **Image:** `ghcr.io/projectbluefin/dakota:{testing,stable,next,btw}` and `:<sha>` (`:latest` is never published)
 
 ## Key architecture

.github/workflows/build.yml

@@ -48,9 +48,11 @@ jobs:
     if: (!cancelled()) && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'push')
     runs-on: ubuntu-24.04
     timeout-minutes: 350
+    # The BuildBox backend has four action slots. Start every image variant
+    # together; the backend remains the global execution-capacity limit.
     strategy:
       fail-fast: false
-      max-parallel: 1
+      max-parallel: 4
       matrix:
         include:
           - variant: default
@@ -101,17 +103,11 @@ jobs:
           CASD_CLIENT_KEY: ${{ secrets.CASD_CLIENT_KEY }}
         uses: ./.github/actions/generate-bst-ci-config
         with:
-          enable-remote-execution: "false"
-          enable-push: "false"
-
-      # Warm-up only; the build step pulls missing artifacts on demand.
-      - name: Pull prebuilt artifacts from remote CAS
-        continue-on-error: true
-        env:
-          BST_FLAGS: -o x86_64_v3 false --no-interactive --config /src/buildstream-ci.conf
-        run: |
-          just bst artifact pull --deps all ${{ matrix.element }}
-        timeout-minutes: 60
+          enable-remote-execution: "true"
+          # BuildStream adds artifact/source push queues to the build when the
+          # configured remotes are writable. RE and the caches share one CAS,
+          # so payloads remain on the BuildBox host and are deduplicated there.
+          enable-push: "true"
 
       - name: Count elements for progress tracking
         id: count
@@ -139,16 +135,19 @@ jobs:
           ELEMENT_TOTAL: ${{ steps.count.outputs.total }}
         run: |
           set -o pipefail
+          CONSOLE_LOG="logs/bst-console-${{ matrix.variant }}.log"
           just bst build --deps none ${{ matrix.element }} 2>&1 \
+            | tee "$CONSOLE_LOG" \
             | python3 files/scripts/bst-progress.py
-        timeout-minutes: 330
 
-      - name: Push OCI artifact to remote CAS
-        env:
-          BST_FLAGS: -o x86_64_v3 false --no-interactive --config /src/buildstream-push.conf
-        run: |
-          just bst artifact push --deps run ${{ matrix.element }}
-        timeout-minutes: 120
+          # Fail closed: a green cache hit is not proof that BuildStream loaded
+          # the remote executor. The startup banner is emitted whenever the
+          # remote-execution block is active, even if no action needs rebuilding.
+          if ! grep -Fq "Remote Execution Configuration" "$CONSOLE_LOG"; then
+            echo "::error::BuildStream did not load the remote execution configuration"
+            exit 1
+          fi
+        timeout-minutes: 330
 
       - name: Upload build logs
         if: always()

Justfile

@@ -78,6 +78,7 @@ bst *ARGS:
 [group('dev')]
 check-publish-workflow:
     python3 scripts/check_publish_workflow.py
+    python3 -m unittest scripts.test_check_publish_workflow
 
 [group('dev')]
 monitor-pipeline BUILD_RUN_ID="":

docs/ci.md

@@ -6,7 +6,7 @@
 |---|---|---|
 | `validate` | `pull_request` | `bst show` — graph + patch check (~15 min) |
 | `e2e` | `pull_request` when `elements/`, `files/`, `patches/`, `Justfile`, or `project.conf` changed | Smoke test in QEMU via projectbluefin/testsuite |
-| `build` | `push: testing/next` (paths-ignore: `.github/workflows/**`, `docs/**`, `**.md`, `AGENTS.md`), `merge_group`, `workflow_dispatch`, `schedule: daily 13:00 UTC` — skips on `pull_request` | Full OCI build (~60–90 min) |
+| `build` | `push: testing` (BST-affecting paths), `workflow_dispatch`, `schedule: daily 13:00 UTC` — skips on `pull_request`/`merge_group` | Four x86 variants concurrently through remote BuildBox execution; artifacts land in the remote CAS |
 | `build-aarch64` | `push: testing/main` (BST-affecting paths only), `workflow_run` from `publish.yml` on `testing`, `workflow_dispatch` | ARM64 — fully decoupled, never blocks release |
 
 ## Publish pipeline (publish.yml)
@@ -51,11 +51,11 @@ push to testing (BST-affecting) or daily 13:00 UTC schedule
 
 ## Schedule
 
-Build fires daily at 13:00 UTC (`schedule:` in `build.yml`), plus on every BST-affecting push to `testing` or `next`, `merge_group`, and `workflow_dispatch`.
+Build fires daily at 13:00 UTC, on BST-affecting pushes to `testing`, and through `workflow_dispatch`. The nightly-next dispatcher invokes the same workflow explicitly on `next`; PR and merge-queue events do not run it.
 
-## Remote cache
+## Remote execution and cache
 
-`cache.projectbluefin.io:11002` — mTLS via `CASD_CLIENT_CERT` + `CASD_CLIENT_KEY`.
+`cache.projectbluefin.io:11002` — BuildBox 1.4.11 execution, remote CAS, artifact/source caches, and action cache behind mTLS via `CASD_CLIENT_CERT` + `CASD_CLIENT_KEY`. Build jobs fail closed; publish uses a fetch-only configuration so it can materialize images locally.
 
 ## Published images
 

docs/skills/buildstream.md

@@ -138,24 +138,25 @@ Changing a `kind: stack` dependency does not always invalidate downstream `compo
 
 Dakota's `patches/` directory is still carrying a small downstream patch queue for FSDK and GNOME Build Meta behavior that has not landed in the pinned upstream release yet. Every patch should carry an `Upstream-Status` header plus an `Exit` line so the next engineer can tell whether the workaround is still pending upstream or should be dropped. This is especially important when the patch is only needed because the junction is still older than the stock GNOME OS baseline.
 
-### Warm-cache builds still take 90-120 min — this is normal (2026-06-23)
+### Legacy runner-local warm-cache timing (2026-06-23)
 
-Even with a fully warm remote CAS, a full build takes 90-120 min. Common misconception: "cache is hot = fast build." Actual breakdown:
+> Superseded for x86 CI by remote-backed CAS and BuildBox execution on
+> 2026-07-28. The 90–150 minute range described the old explicit pre-pull and
+> runner-local assembly path; do not use it to justify reintroducing those steps.
 
-- **Pull volume:** ~1,400 elements × a few seconds each / 32 parallel fetchers = 15-30 min just for network pulls
-- **Two parallel jobs:** `default` and `nvidia` both run simultaneously, each hitting the same CAS endpoint, halving effective bandwidth per job
-- **OCI assembly is sequential:** After all elements pull/build, `oci/bluefin.bst` runs chunkify + image assembly — single-threaded, typically 20-40 min on its own
-- **Cold elements:** Any junction ref bump (Renovate PRs for distrobox, gnome-build-meta, etc.) invalidates those subtrees → full recompile from source adds 30-90 min
+Cold junction changes can still take time because actions genuinely need to run,
+but they run on the remote executor. Measure the new path before tuning.
 
-Do not cancel a build under 120 min just because it "seems slow." Historical range for successful builds: 90-150 min.
+### Fetcher count needs measurement under remote-backed CAS (2026-06-23)
 
-### 32 fetchers is the right setting for cache.projectbluefin.io (2026-06-23)
-
-`buildstream-ci.conf` uses `fetchers: 32` (BST default is 10). With default + nvidia running simultaneously = 64 concurrent gRPC streams. The CAS server is a Hetzner AX102-U (1 Gbit/s uplink, NVMe Gen4) and can serve 64 streams comfortably. The bottleneck is network bandwidth (~125 MB/s total), not server capacity. Do not reduce fetchers without evidence of server-side saturation.
+`buildstream-ci.conf` retains `fetchers: 32` for the initial RE rollout. Four
+variant jobs can create more metadata requests than the old two-job model, while
+top-level remote storage avoids transferring file payloads through the runners.
+Change this value only from observed endpoint saturation or latency evidence.
 
 ### Avoid /dev/stdin redirection in remote sandboxes (2026-07-25)
 
-BuildStream elements that write inline configuration files using `install -Dm644 /dev/stdin ... <<'EOF'` fail in remote execution sandboxes (like BuildBarn) where `/dev/stdin` is not available as a standard character device. Write inline files using a two-step pattern: create the destination file with `install -Dm644 /dev/null target`, then populate it with `cat > target <<'EOF'`.
+BuildStream elements that write inline configuration files using `install -Dm644 /dev/stdin ... <<'EOF'` fail in remote execution sandboxes (such as BuildBox) where `/dev/stdin` is not available as a standard character device. Write inline files using a two-step pattern: create the destination file with `install -Dm644 /dev/null target`, then populate it with `cat > target <<'EOF'`.
 
 ### overlap-whitelist required for base system file replacement