x64: switch CI to docker-run + docker-commit; fix TARGET_ARCH env collision

jgillis · claude · jgillis · commit b5904c5b6202 · 2026-05-26T09:11:34.000+02:00
The multi-stage Dockerfile build was tripping GNU make's built-in
implicit-rule variable TARGET_ARCH. Make's default `%.o: %.c` recipe is
`$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c`, so an inherited
`TARGET_ARCH=x86_64` env var splices `x86_64` as a positional gcc
argument inside any sub-make that relies on the built-in pattern rule
(e.g. Julia's bundled libwhich). Failure surfaces as
`gcc: error: x86_64: linker input file not found` and looks like a
parallel-make race.

Fix: rename to JULIA_TARGET_ARCH everywhere + defensively
`unset TARGET_ARCH` at the top of build-julia.sh.

Also switch the workflow from docker/build-push-action to plain
`docker run` + `docker commit` per user request — same env contract as
the proven local build, no BuildKit-induced surprises. Local build of
the renamed script: 77m53s, multi-target sysimage, no libunwind, smoketest green.

Co-Authored-By: Claude Opus 4.7 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
@@ -4,7 +4,6 @@ on:
   push:
     branches: [docker-manylinux_2_28-x64]
     paths:
-      - Dockerfile
       - scripts/**
       - .github/workflows/build-image.yml
   workflow_dispatch:
@@ -31,23 +30,34 @@ concurrency:
 env:
   IMAGE: ghcr.io/${{ github.repository_owner }}/manylinux_2_28-x64-polyglot
   BASE_IMAGE: ghcr.io/jgillis/manylinux_2_28-x64:production
-  TARGET_ARCH: x86_64
-  # BUILD_JOBS=1 to bypass the BuildKit `docker build` + parallel-make race
-  # in Julia's deps phase (`gcc: error: x86_64: linker input file not found`
-  # from configure scripts under -j>1). ~25 min deps phase vs ~12 with -j4.
-  BUILD_JOBS: 1
+  JULIA_TARGET_ARCH: x86_64
+  # Local proven build runs at -j6 in ~78 min. GitHub-hosted ubuntu-latest is
+  # 4-core/16 GB; -j4 keeps LLVM linking under the OOM line and matches the
+  # build-julia.sh default. Going lower trades wall time for nothing.
+  BUILD_JOBS: 4
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    timeout-minutes: 180
     steps:
       - uses: actions/checkout@v4
+
       - uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - uses: docker/setup-buildx-action@v3
+
+      - name: Free up disk space
+        # ubuntu-latest ships ~14 GB free; LLVM build + container layers easily
+        # push past that. Strip the largest known-unused payloads up front.
+        run: |
+          set -eux
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+                      /opt/hostedtoolcache/CodeQL
+          sudo docker image prune -af
+          df -h /
 
       - name: Resolve tags
         id: tags
@@ -61,21 +71,72 @@ jobs:
             echo "tag_jl=${{ env.IMAGE }}:julia${JULIA_VERSION}"
           } >> "$GITHUB_OUTPUT"
 
-      - name: Build (and conditionally push) image
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: Dockerfile
-          build-args: |
-            BASE_IMAGE=${{ env.BASE_IMAGE }}
-            TARGET_ARCH=${{ env.TARGET_ARCH }}
-            JULIA_VERSION=${{ steps.tags.outputs.julia_version }}
-            BUILD_JOBS=${{ env.BUILD_JOBS }}
-          push: ${{ github.event.inputs.push != 'false' }}
-          tags: |
-            ${{ steps.tags.outputs.tag_prod }}
-            ${{ steps.tags.outputs.tag_sha }}
-            ${{ steps.tags.outputs.tag_jl }}
-          cache-from: type=gha,scope=manylinux_2_28-x64-polyglot
-          cache-to: type=gha,scope=manylinux_2_28-x64-polyglot,mode=max
-          provenance: false
+      - name: Pull base image
+        run: docker pull "$BASE_IMAGE"
+
+      - name: Build polyglot inside container (docker run)
+        # docker run + docker commit (NOT docker build). The multi-stage
+        # Dockerfile approach + BuildKit was fighting the same GNU make
+        # TARGET_ARCH name-collision bug that hit local builds, and gave
+        # confusing failures because BuildKit obscures the env. docker run
+        # is plain: same env contract as the proven local build script.
+        run: |
+          set -eux
+          docker run \
+            --name polyglot-build \
+            -v "$PWD/scripts:/work/scripts:ro" \
+            -e JULIA_VERSION="${{ steps.tags.outputs.julia_version }}" \
+            -e JULIA_TARGET_ARCH="${{ env.JULIA_TARGET_ARCH }}" \
+            -e BUILD_JOBS="${{ env.BUILD_JOBS }}" \
+            "$BASE_IMAGE" \
+            bash /work/scripts/run-in-container.sh
+
+      - name: Commit container into polyglot image
+        # Bake metadata (ENV, WORKDIR, LABEL) via --change since docker commit
+        # otherwise just snapshots the filesystem. Keep the base image's PATH
+        # and prepend /opt/julia/bin + /opt/rustup/cargo/bin.
+        run: |
+          set -eux
+          docker commit \
+            --change 'ENV JULIA_HOME=/opt/julia' \
+            --change 'ENV JULIA_PATH=/opt/julia' \
+            --change 'ENV JULIA_VERSION=${{ steps.tags.outputs.julia_version }}' \
+            --change 'ENV CARGO_HOME=/opt/rustup/cargo' \
+            --change 'ENV RUSTUP_HOME=/opt/rustup' \
+            --change 'ENV PATH=/opt/julia/bin:/opt/rustup/cargo/bin:/opt/rh/gcc-toolset-14/root/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' \
+            --change 'ENV CARGO_BUILD_TARGET=${{ env.JULIA_TARGET_ARCH }}-unknown-linux-gnu' \
+            --change 'WORKDIR /work' \
+            --change 'LABEL org.opencontainers.image.title=manylinux_2_28 polyglot (Julia + Rust + C/C++)' \
+            --change 'LABEL org.opencontainers.image.source=https://github.com/${{ github.repository }}' \
+            --change 'LABEL casadi.polyglot.julia.version=${{ steps.tags.outputs.julia_version }}' \
+            --change 'LABEL casadi.polyglot.arch=${{ env.JULIA_TARGET_ARCH }}' \
+            polyglot-build \
+            "${{ steps.tags.outputs.tag_prod }}"
+          docker tag "${{ steps.tags.outputs.tag_prod }}" "${{ steps.tags.outputs.tag_sha }}"
+          docker tag "${{ steps.tags.outputs.tag_prod }}" "${{ steps.tags.outputs.tag_jl }}"
+
+      - name: Smoketest committed image
+        run: |
+          set -eux
+          docker run --rm "${{ steps.tags.outputs.tag_prod }}" bash -lc '
+            set -e
+            julia --version
+            julia -e "println(\"Julia \", VERSION, \" — LLVM \", Base.libllvm_version, \" — CPU \", Sys.CPU_NAME)"
+            rustc --version
+            cargo --version
+            cbindgen --version
+            test ! -e /opt/julia/lib/libunwind.so
+            echo "polyglot post-commit OK"
+          '
+
+      - name: Push (conditional)
+        if: github.event_name == 'push' || github.event.inputs.push != 'false'
+        run: |
+          set -eux
+          docker push "${{ steps.tags.outputs.tag_prod }}"
+          docker push "${{ steps.tags.outputs.tag_sha }}"
+          docker push "${{ steps.tags.outputs.tag_jl }}"
+
+      - name: Cleanup builder container
+        if: always()
+        run: docker rm -f polyglot-build 2>/dev/null || true
diff --git a/Dockerfile b/Dockerfile
diff --git a/scripts/build-julia.sh b/scripts/build-julia.sh
@@ -7,18 +7,25 @@
 #                                    into src/{signals-unix.c,stackwalk.c}
 #
 # Args: $1 = path to extracted Julia source tree (containing Make.inc)
-# Env:  TARGET_ARCH  (x86_64 | aarch64)   — defaults to native
+# Env:  JULIA_TARGET_ARCH  (x86_64 | aarch64)   — defaults to native
 #       BUILD_JOBS                         — make parallelism (default 4)
 set -euo pipefail
 SRC="${1:?Julia source tree path required}"
-TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}"
+JULIA_TARGET_ARCH="${JULIA_TARGET_ARCH:-$(uname -m)}"
 BUILD_JOBS="${BUILD_JOBS:-4}"
 
+# GNU make's built-in implicit recipe for `%.o: %.c` is
+#   $(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c $<
+# so an inherited TARGET_ARCH env var (e.g. "x86_64") leaks in as a positional
+# gcc arg via Julia's sub-makes (libwhich, OpenBLAS, ...), failing with
+# `gcc: error: x86_64: linker input file not found`. Strip it defensively.
+unset TARGET_ARCH
+
 cd "$SRC"
 
 # JULIA_CPU_TARGET — multi-target string matching JuliaCI's official binary
 # builds (utilities/build_envs.sh).
-case "$TARGET_ARCH" in
+case "$JULIA_TARGET_ARCH" in
   x86_64)
     JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1)"
     ;;
@@ -27,7 +34,7 @@ case "$TARGET_ARCH" in
     JULIA_CPU_TARGET="generic"
     ;;
   *)
-    echo "Unsupported TARGET_ARCH=$TARGET_ARCH" >&2; exit 1
+    echo "Unsupported JULIA_TARGET_ARCH=$JULIA_TARGET_ARCH" >&2; exit 1
     ;;
 esac
 
@@ -36,18 +43,6 @@ USE_BINARYBUILDER=0
 JULIA_CPU_TARGET=${JULIA_CPU_TARGET}
 DISABLE_LIBUNWIND:=1
 EOF
-# HOSTCC must be a single token. Make.inc line 445 unconditionally does
-# `HOSTCC = \$(CC)` which then gets `-m\$(BINARY)` appended → HOSTCC becomes
-# the two-word "gcc -m64". libwhich.mk's recipe `\$(MAKE) -C scratch/...
-# CC="\$(HOSTCC)" libwhich` then expands to `make ... CC="gcc -m64" libwhich`.
-# Under BuildKit's docker build with -j>1, the sub-make's MAKEFLAGS-driven
-# re-tokenization splits this and a stray ARCH token ("x86_64" or "aarch64")
-# leaks in as a positional arg to gcc — `gcc: error: x86_64: linker input
-# file not found`. Setting HOSTCC in Make.user does NOT work because line
-# 445 overwrites it. The fix is to pass HOSTCC/HOSTCXX on the make command
-# line so they win over makefile assignments.
-MAKE_VARS=(HOSTCC=gcc HOSTCXX=g++)
-
 echo "=== Make.user ==="
 cat Make.user
 echo
@@ -137,7 +132,7 @@ echo
 
 echo "=== Build ==="
 date
-time make -j"$BUILD_JOBS" "${MAKE_VARS[@]}"
+time make -j"$BUILD_JOBS"
 date
 echo
 
diff --git a/scripts/run-in-container.sh b/scripts/run-in-container.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+# Orchestrate the full polyglot image build inside the base manylinux container.
+# Invoked by the CI workflow under `docker run --name polyglot-build ...`;
+# the workflow then `docker commit`s the resulting container into the image.
+#
+# Required env vars:
+#   JULIA_VERSION       e.g. 1.12.6
+#   JULIA_TARGET_ARCH   x86_64 | aarch64   (NOT named TARGET_ARCH — that name
+#                                           collides with GNU make's built-in
+#                                           implicit-rule variable and poisons
+#                                           the `%.o: %.c` recipe — see skill
+#                                           gnu-make-target-arch-env-collision)
+#   BUILD_JOBS          make parallelism (CI defaults to 4)
+set -euo pipefail
+
+: "${JULIA_VERSION:?JULIA_VERSION required}"
+: "${JULIA_TARGET_ARCH:?JULIA_TARGET_ARCH required}"
+: "${BUILD_JOBS:=4}"
+
+# Defensive: this var must NEVER reach `make`. See skill referenced above.
+unset TARGET_ARCH
+
+echo "=== Install build-time packages ==="
+# flex     — not pre-installed in manylinux_2_28 / pypa bases
+# perl-core — OpenSSL Configure needs Data::Dumper + IPC::Cmd
+( dnf install -y flex perl-core 2>&1 | tail -5 ) \
+  || ( yum install -y flex perl-core 2>&1 | tail -5 )
+( dnf clean all 2>/dev/null || yum clean all ) >/dev/null 2>&1 || true
+rm -rf /var/cache/dnf /var/cache/yum 2>/dev/null || true
+
+echo
+echo "=== Fetch Julia ${JULIA_VERSION} source ==="
+mkdir -p /build
+cd /build
+curl -fsSL -o "julia-${JULIA_VERSION}-full.tar.gz" \
+  "https://github.com/JuliaLang/julia/releases/download/v${JULIA_VERSION}/julia-${JULIA_VERSION}-full.tar.gz"
+tar xzf "julia-${JULIA_VERSION}-full.tar.gz"
+rm "julia-${JULIA_VERSION}-full.tar.gz"
+
+echo
+echo "=== Build Julia (delegates to scripts/build-julia.sh) ==="
+JULIA_TARGET_ARCH="$JULIA_TARGET_ARCH" BUILD_JOBS="$BUILD_JOBS" \
+  /work/scripts/build-julia.sh "/build/julia-${JULIA_VERSION}"
+
+echo
+echo "=== Move julia install into /opt/julia ==="
+mv "/build/julia-${JULIA_VERSION}/usr" /opt/julia
+# Strip everything else — sources, srccache, scratch, intermediate artifacts.
+# Leaves /opt/julia as the only thing committed into the final image.
+rm -rf "/build/julia-${JULIA_VERSION}" /build/*.tar.gz
+rmdir /build 2>/dev/null || true
+
+echo
+echo "=== Install Rust + cbindgen (idempotent: skips if already present) ==="
+/work/scripts/install-rust.sh
+
+# Add the per-target Rust triple.
+case "${JULIA_TARGET_ARCH}" in
+  x86_64)  RUSTUP_HOME=/opt/rustup CARGO_HOME=/opt/rustup/cargo rustup target add x86_64-unknown-linux-gnu ;;
+  aarch64) RUSTUP_HOME=/opt/rustup CARGO_HOME=/opt/rustup/cargo rustup target add aarch64-unknown-linux-gnu ;;
+esac
+
+echo
+echo "=== Drop cargo profile shim into /etc/profile.d ==="
+UP=$(echo "${JULIA_TARGET_ARCH}_unknown_linux_gnu" | tr '[:lower:]' '[:upper:]')
+GCC=$( command -v "${CC:-gcc}" 2>/dev/null || command -v gcc )
+printf 'export CARGO_TARGET_%s_LINKER=%s\nexport CARGO_TARGET_%s_RUSTFLAGS=-C prefer-dynamic\n' \
+    "$UP" "$GCC" "$UP" > /etc/profile.d/polyglot-cargo.sh
+
+echo
+echo "=== In-container smoketest ==="
+export PATH=/opt/julia/bin:/opt/rustup/cargo/bin:$PATH
+julia --version
+julia -e 'println("Julia ", VERSION, " — LLVM ", Base.libllvm_version, " — CPU ", Sys.CPU_NAME)'
+rustc --version
+cargo --version
+cbindgen --version
+test ! -e /opt/julia/lib/libunwind.so
+echo "polyglot pre-commit OK"
