Stop the manifest merge from booting BuildKit it never uses (#1175)

The merge job runs only `docker buildx imagetools create`, a registry-only operation needing no builder, but setup-buildx-action's default container driver booted moby/buildkit from Docker Hub — whose timeouts killed three dev builds and stranded :dev. Pin driver: docker on both merge jobs; build jobs keep the container driver and get a fail-soft pre-pull retry. Red/green test as Wall 6 in test_workflow_safety_invariants.py.

Author: new-usemame

.github/workflows/docker-image-build-dev.yml

@@ -122,6 +122,24 @@ jobs:
           password: ${{ secrets.GH_PAT || secrets.GITHUB_TOKEN }}
 
       # --- 3. Build & Push (Digest Only) ---
+      # Unlike the merge job, this one genuinely needs BuildKit, so it can't
+      # drop the container driver. Pre-pull with backoff instead: the Docker
+      # Hub timeouts that red these builds are transient, and
+      # setup-buildx-action boots from the local image when one is present.
+      # Fail-soft — if the pull never lands, or the action's default image tag
+      # moves in a future buildx, fall through and let it pull as before.
+      - name: Pre-pull BuildKit image (Docker Hub flake guard)
+        run: |
+          set -u
+          for attempt in 1 2 3; do
+            if docker pull moby/buildkit:buildx-stable-1; then
+              exit 0
+            fi
+            echo "buildkit pull attempt ${attempt} failed; retrying in $((attempt * 15))s"
+            sleep "$((attempt * 15))"
+          done
+          echo "::warning::Could not pre-pull moby/buildkit after 3 attempts; letting setup-buildx-action try."
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
         with:
@@ -243,6 +261,24 @@ jobs:
           password: ${{ secrets.GH_PAT || secrets.GITHUB_TOKEN }}
 
       # --- 3. Build & Push (Digest Only) ---
+      # Unlike the merge job, this one genuinely needs BuildKit, so it can't
+      # drop the container driver. Pre-pull with backoff instead: the Docker
+      # Hub timeouts that red these builds are transient, and
+      # setup-buildx-action boots from the local image when one is present.
+      # Fail-soft — if the pull never lands, or the action's default image tag
+      # moves in a future buildx, fall through and let it pull as before.
+      - name: Pre-pull BuildKit image (Docker Hub flake guard)
+        run: |
+          set -u
+          for attempt in 1 2 3; do
+            if docker pull moby/buildkit:buildx-stable-1; then
+              exit 0
+            fi
+            echo "buildkit pull attempt ${attempt} failed; retrying in $((attempt * 15))s"
+            sleep "$((attempt * 15))"
+          done
+          echo "::warning::Could not pre-pull moby/buildkit after 3 attempts; letting setup-buildx-action try."
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
         with:
@@ -388,8 +424,17 @@ jobs:
           pattern: digests-*
           merge-multiple: true
 
-      - name: Set up Docker Buildx
+      # `imagetools create` below is a registry-only operation — it reads the
+      # per-arch manifests and writes a manifest list. It needs no BuildKit.
+      # The default docker-container driver would boot moby/buildkit pulled
+      # from Docker Hub, whose egress from GitHub runners intermittently times
+      # out; on 2026-07-27 that killed three consecutive dev builds at "booting
+      # buildkit" *after* both arch builds had already pushed, stranding :dev.
+      # `driver: docker` reuses the local dockerd, so this job touches only GHCR.
+      - name: Set up Docker Buildx (local driver — no Docker Hub pull)
         uses: docker/setup-buildx-action@v4
+        with:
+          driver: docker
 
       - name: Create manifest list and push
         working-directory: /tmp/digests

.github/workflows/docker-image-build-release.yml

@@ -95,6 +95,21 @@ jobs:
           # pbs-cache mirror; falls back to GITHUB_TOKEN for push-only contexts.
           password: ${{ secrets.GH_PAT || secrets.GITHUB_TOKEN }}
 
+      # See the dev workflow for the full note. This job builds, so it needs
+      # BuildKit; pre-pull with backoff to survive a transient Docker Hub
+      # timeout. Fail-soft — falls through to the action's own pull.
+      - name: Pre-pull BuildKit image (Docker Hub flake guard)
+        run: |
+          set -u
+          for attempt in 1 2 3; do
+            if docker pull moby/buildkit:buildx-stable-1; then
+              exit 0
+            fi
+            echo "buildkit pull attempt ${attempt} failed; retrying in $((attempt * 15))s"
+            sleep "$((attempt * 15))"
+          done
+          echo "::warning::Could not pre-pull moby/buildkit after 3 attempts; letting setup-buildx-action try."
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd  # v4.0.0
 
@@ -191,8 +206,16 @@ jobs:
           pattern: digests-*
           merge-multiple: true
 
-      - name: Set up Docker Buildx
+      # Registry-only manifest merge — see the dev workflow for the full note.
+      # The default container driver boots moby/buildkit from Docker Hub, which
+      # intermittently times out from GitHub runners. Here that is worse than on
+      # dev: the tag is already published, so a failure leaves
+      # `docker pull ...:vX.Y.Z` 404ing for every user (the v4.0.169 failure
+      # mode). `driver: docker` keeps this job on GHCR only.
+      - name: Set up Docker Buildx (local driver — no Docker Hub pull)
         uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd  # v4.0.0
+        with:
+          driver: docker
 
       - name: Compute tag list
         id: tags

tests/unit/test_workflow_safety_invariants.py

@@ -495,6 +495,77 @@ def test_integration_tests_job_authenticates_to_ghcr():
     )
 
 
+# ─── Wall 6: manifest-merge jobs don't boot BuildKit ───────────────────
+#
+# `docker buildx imagetools create` is a registry-only operation: it
+# reads the per-arch manifests and writes a manifest list. It needs no
+# builder. But `docker/setup-buildx-action` defaults to the
+# docker-container driver, which boots moby/buildkit pulled from Docker
+# Hub — so a job that only merges manifests took a hard dependency on a
+# third-party registry it never otherwise touches.
+#
+# Docker Hub egress from GitHub-hosted runners intermittently times out.
+# On 2026-07-27 three consecutive dev builds died at "booting buildkit"
+# with `Get "https://registry-1.docker.io/v2/": context deadline
+# exceeded` — after both arch builds had already succeeded and pushed.
+# The :dev tag went stale and the household canary stopped receiving
+# merges. The same shape sits in the release workflow's merge job, where
+# it is worse: the tag publishes, the manifest never lands, and
+# `docker pull ...:vX.Y.Z` 404s for everyone (the v4.0.169 failure mode).
+#
+# So: a job that merges manifests and does not itself build an image
+# must not set up a container-driver buildx.
+
+
+def _job_text(job: dict) -> str:
+    """Flatten every step's `run` + `uses` into one searchable string."""
+    parts = []
+    for step in _steps(job):
+        parts.append(str(step.get("run", "")))
+        parts.append(str(step.get("uses", "")))
+    return "\n".join(parts)
+
+
+def test_manifest_merge_jobs_do_not_boot_container_buildkit():
+    """Registry-only manifest merges must not pull moby/buildkit.
+
+    Applies to any job that runs `imagetools create` but never builds an
+    image. Such a job may either omit setup-buildx-action entirely (the
+    buildx CLI plugin is preinstalled on GitHub runners) or pin
+    `driver: docker`, which reuses the local dockerd. What it may not do
+    is take the default container driver.
+    """
+    offenders = []
+    for path in all_workflows():
+        wf = _load(path)
+        for job_name, job in (wf.get("jobs") or {}).items():
+            if not isinstance(job, dict):
+                continue
+            text = _job_text(job)
+            if "imagetools create" not in text:
+                continue
+            # A job that genuinely builds an image needs a real builder.
+            builds = "docker/build-push-action" in text or re.search(
+                r"docker\s+buildx\s+build\b", text
+            )
+            if builds:
+                continue
+            for step in _steps(job):
+                if not str(step.get("uses", "")).startswith(
+                    "docker/setup-buildx-action"
+                ):
+                    continue
+                driver = (step.get("with") or {}).get("driver")
+                if driver != "docker":
+                    offenders.append(f"{path.name}:{job_name}")
+    assert not offenders, (
+        "Manifest-merge job(s) boot a container-driver BuildKit for a "
+        "registry-only `imagetools create`, taking a needless Docker Hub "
+        "dependency that has already broken the dev channel: "
+        f"{offenders}. Drop the setup-buildx step or pin `driver: docker`."
+    )
+
+
 def test_all_workflows_have_minimum_permissions_block():
     """Every workflow that does any mutating action (commenting,
     labeling, merging) must have an explicit top-level OR job-level