build: completely refactor static build to support cross-compilation

Author: mhx

.docker/Dockerfile.alpine

@@ -1,91 +1,109 @@
 # syntax=docker/dockerfile:1
 ARG ARCH=amd64
 FROM $ARCH/alpine:latest
+
+ARG ARCH
+ARG TARGET_ARCHS
+ARG TOOLCHAIN_OPTIMIZE=2
+
 RUN apk update
 RUN apk upgrade
-RUN apk add --no-cache \
+RUN qemu_pkgs=""; \
+    for arch in $(echo $TARGET_ARCHS | tr ',' ' '); do \
+        qemu_pkgs="$qemu_pkgs qemu-$arch"; \
+    done; \
+    apk add --no-cache \
         bash-completion \
         build-base \
         wget \
         curl \
         vim \
         tar \
         ripgrep \
-        py3-pip \
         less \
         gcc \
         g++ \
         clang18 \
         clang19 \
+        clang20 \
+        lld \
         git \
+        xz \
+        gzip \
+        bzip2 \
+        zstd \
         ccache \
-        samurai \
+        ninja-build \
         cmake \
         make \
         bison \
         flex \
         ronn \
+        perf \
+        py3-pip \
         fuse \
-        fuse-dev \
-        fuse-static \
         fuse3 \
-        fuse3-dev \
-        fuse3-static \
         pkgconf \
-        binutils-dev \
-        libevent-dev \
-        libevent-static \
-        linux-headers \
-        date-dev \
-        range-v3-dev \
-        zlib-static \
-        libucontext-dev \
-        libdwarf-dev \
-        elfutils-dev \
-        utfcpp \
-        nlohmann-json \
         meson \
         autoconf \
         strace \
-        gdb
-
-# Install UPX
-ARG ARCH
-# UPX isn't supported on RISC-V yet, so skip this
-RUN if [ "$ARCH" != "riscv64" ]; then \
-        wget -O - https://github.com/upx/upx/releases/download/v5.0.0/upx-5.0.0-$(bash -c "echo \${0//v8/}" $ARCH)_linux.tar.xz | tar -xJf - -C /usr/local/bin --strip-components=1 --wildcards "*/upx"; \
-    else \
-        echo "Skipping UPX installation for RISC-V architecture while it is not supported."; \
-        echo "See https://github.com/upx/upx/discussions/793 for details."; \
-    fi
-
-# Install mistletoe
-RUN pip3 install --break-system-packages --root-user-action ignore mistletoe
+        gdb \
+        gmp-dev \
+        zlib-static \
+        zstd-static \
+        zlib-dev \
+        zstd-dev \
+        mpfr-dev \
+        mpc1-dev \
+        isl-dev \
+        texinfo \
+        linux-headers \
+        gdb-multiarch \
+        rsync \
+        screen \
+        $qemu_pkgs
 
-# Install size-optimized and static-link-optimized libstdc++
-RUN apk add --no-cache gmp-dev mpfr-dev mpc1-dev isl-dev
-COPY install-libstdcxx.sh /usr/local/bin/install-libstdcxx.sh
-RUN bash /usr/local/bin/install-libstdcxx.sh
+COPY fetch.sh /usr/local/bin/fetch.sh
 
 # Install mold
 COPY install-mold.sh /usr/local/bin/install-mold.sh
-RUN bash /usr/local/bin/install-mold.sh
+RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \
+    bash /usr/local/bin/install-mold.sh
 
 # Install bloaty
 COPY install-bloaty.sh /usr/local/bin/install-bloaty.sh
-RUN if [ "$ARCH" != "riscv64" ]; then \
+RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \
+    if [ "$ARCH" == "amd64" ]; then \
         bash /usr/local/bin/install-bloaty.sh; \
     else \
-        echo "Skipping bloaty installation for RISC-V."; \
+        echo "Skipping bloaty installation for $ARCH."; \
     fi
 
+RUN apk del linux-headers
+
+# Install size-optimized and static-link-optimized libstdc++
+COPY install-toolchain.sh /usr/local/bin/install-toolchain.sh
+RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \
+    bash /usr/local/bin/install-toolchain.sh $TOOLCHAIN_OPTIMIZE $TARGET_ARCHS
+
+RUN apk del zlib-dev zstd-dev
+
 # Install all static libraries
 COPY install-static-libs.sh /usr/local/bin/install-static-libs.sh
-RUN bash /usr/local/bin/install-static-libs.sh gcc clang-19 :alpine
 
-# Install the *real* ninja...
-# TODO: move this up if we change anything
-RUN apk add --no-cache ninja-build
+RUN --mount=type=cache,id=ccache,target=/root/.ccache --mount=type=cache,id=pkgcache,target=/root/.pkgcache \
+    bash /usr/local/bin/install-static-libs.sh :all $TARGET_ARCHS
+
+# Install UPX
+# UPX isn't supported on RISC-V/S390 (yet), so skip this
+RUN if [ "$ARCH" == "amd64" ] || [ "$ARCH" == "arm64v8" ]; then \
+        wget -O - https://github.com/upx/upx/releases/download/v5.0.2/upx-5.0.2-$(bash -c "echo \${0//v8/}" $ARCH)_linux.tar.xz | tar -xJf - -C /usr/local/bin --strip-components=1 --wildcards "*/upx"; \
+    else \
+        echo "Skipping UPX installation for $ARCH architecture while it is not supported."; \
+        echo "See https://github.com/upx/upx/discussions/793 for details."; \
+    fi
+
+RUN pip3 install --break-system-packages --root-user-action ignore mistletoe
 
 # Set up git & user
 RUN git config --global --add safe.directory /workspace

.docker/Dockerfile.ubuntu

@@ -72,13 +72,6 @@ RUN apt install -y \
         libflac++-dev \
         nlohmann-json3-dev \
         python3-mistletoe
-### XXX: no more static libs for Ubuntu
-# COPY install-static-libs.sh /usr/local/bin/install-static-libs.sh
-# # TODO: gcc-14 cannot build the current version of libunwind
-# RUN bash /usr/local/bin/install-static-libs.sh gcc-13 clang-18 :ubuntu
-ARG ARCH
-# upx-4.2.2 is broken for arm64 (https://github.com/upx/upx/issues/758)
-RUN wget -O - https://github.com/upx/upx/releases/download/v5.0.0/upx-5.0.0-$(bash -c "echo \${0//v8/}" $ARCH)_linux.tar.xz | tar -xJf - -C /usr/local/bin --strip-components=1 --wildcards "*/upx"
 # current ronn version is horribly broken
 RUN gem install ronn-ng
 # RUN useradd -g users -u 1000 -m mhx

.docker/Makefile

@@ -1,7 +1,7 @@
 IMAGE_UBUNTU="dwarfs-buildenv"
 IMAGE_DEBIAN="dwarfs-buildenv-debian"
 IMAGE_ALPINE="dwarfs-buildenv-alpine"
-IMAGE_ALPINE_RISCV="dwarfs-buildenv-alpine-riscv64"
+IMAGE_ALPINE_NEWARCH="dwarfs-buildenv-alpine-newarch"
 IMAGE_UBUNTU2204="dwarfs-buildenv-ubuntu2204"
 IMAGE_FEDORA="dwarfs-buildenv-fedora"
 IMAGE_ARCH="dwarfs-buildenv-arch"
@@ -34,16 +34,16 @@ run_debian:
 	@docker run $(COMMON_RUN_OPTS) $(IMAGE_DEBIAN)
 
 build_alpine:
-	docker build -f Dockerfile.alpine -t $(IMAGE_ALPINE) .
+	docker buildx build -f Dockerfile.alpine --build-arg TARGET_ARCHS=x86_64,aarch64,riscv64,ppc64le,s390x,arm,i386 -t $(IMAGE_ALPINE) .
 
 run_alpine:
 	@docker run $(COMMON_RUN_OPTS) $(IMAGE_ALPINE)
 
-build_alpine_riscv:
-	docker build -f Dockerfile.alpine -t $(IMAGE_ALPINE_RISCV) --build-arg ARCH=riscv64 .
+build_alpine_newarch:
+	docker buildx build -f Dockerfile.alpine --build-arg TARGET_ARCHS=s390x -t $(IMAGE_ALPINE_NEWARCH) .
 
-run_alpine_riscv:
-	@docker run $(COMMON_RUN_OPTS) $(IMAGE_ALPINE_RISCV)
+run_alpine_newarch:
+	@docker run $(COMMON_RUN_OPTS) $(IMAGE_ALPINE_NEWARCH)
 
 build_ubuntu2204:
 	docker build -f Dockerfile.ubuntu-2204 -t $(IMAGE_UBUNTU2204) .

.docker/build-linux.sh

@@ -9,8 +9,8 @@ mkdir -p "$LOCAL_REPO_PATH"
 LAST_UPDATE_FILE="$LOCAL_REPO_PATH/last-update"
 
 WORKFLOW_LOG_DIR="/artifacts/workflow-logs/${GITHUB_RUN_ID}"
-NINJA_LOG_FILE="${WORKFLOW_LOG_DIR}/ninja-${BUILD_ARCH},${BUILD_DIST},${BUILD_TYPE}.log"
-BUILD_LOG_FILE="${WORKFLOW_LOG_DIR}/build-${BUILD_ARCH},${BUILD_DIST},${BUILD_TYPE}.log"
+NINJA_LOG_FILE="${WORKFLOW_LOG_DIR}/ninja-${BUILD_ARCH},${CROSS_ARCH},${BUILD_DIST},${BUILD_TYPE}.log"
+BUILD_LOG_FILE="${WORKFLOW_LOG_DIR}/build-${BUILD_ARCH},${CROSS_ARCH},${BUILD_DIST},${BUILD_TYPE}.log"
 mkdir -p "$WORKFLOW_LOG_DIR"
 
 log() {
@@ -77,7 +77,7 @@ cd build
 # Stick to clang-18, clang-19 has a regression for nilsimsa performance
 if [[ "$BUILD_DIST" == "alpine" ]]; then
   GCC_VERSION=
-  CLANG_VERSION=-19
+  CLANG_VERSION=-20
 elif [[ "$BUILD_DIST" == "ubuntu-2204" ]]; then
   GCC_VERSION=-12
   CLANG_VERSION=-15
@@ -114,6 +114,9 @@ case "-$BUILD_TYPE-" in
       *-ubuntu-*|*-debian-*)
         export CC=gcc$GCC_VERSION CXX=g++$GCC_VERSION
         ;;
+      *)
+        export CC=gcc CXX=g++
+        ;;
     esac
     export COMPILER=gcc
     ;;
@@ -182,7 +185,16 @@ esac
 
 case "-$BUILD_TYPE-" in
   *-static-*)
-    export LDFLAGS="${LDFLAGS} -fuse-ld=mold"
+    case "$CROSS_ARCH" in
+      ppc64le)
+         # https://github.com/rui314/mold/issues/1490
+         CMAKE_ARGS="${CMAKE_ARGS} -DDISABLE_MOLD=1"
+         export LDFLAGS="${LDFLAGS} -fuse-ld=lld"
+         ;;
+      *)
+         export LDFLAGS="${LDFLAGS} -fuse-ld=mold"
+         ;;
+    esac
     ;;
 esac
 
@@ -287,39 +299,49 @@ if [[ "-$BUILD_TYPE-" == *-shared-* ]]; then
 fi
 
 if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
-  if [[ "-$BUILD_TYPE-" == *-relsize-* ]]; then
-    _LIBSTDCXXDIR="/opt/static-libs/libstdc++-Os/lib"
-    if [[ "$ARCH" == "aarch64" ]]; then
-      # Similar to the issue with *not* linking against `gcc_eh` in the CMakeLists.txt,
-      # if we link against the `gcc_eh` from the `-Os` build, we run into exactly the
-      # same issue. So we temporarily copy the size-optimized `libgcc.a` to a directory
-      # we then use for linking.
-      _GCCLIBDIR="/tmp/gcclib"
-      mkdir -p "$_GCCLIBDIR"
-      cp -a "$_LIBSTDCXXDIR"/gcc/*/*/libgcc.a "$_GCCLIBDIR"
-    else
-      _GCCLIBDIR=$(ls -d1 $_LIBSTDCXXDIR/gcc/*/*)
-    fi
-    LDFLAGS="${LDFLAGS} -L$_GCCLIBDIR -L$_LIBSTDCXXDIR"
+  # A size-optimized libgcc/libstdc++/musl will only save about 10k for the universal
+  # and fuse-extract binaries, so we'll just stick to using the -O2 toolchain.
+  _SYSROOT="/opt/cross/O2"
+  _MARCH="${CROSS_ARCH}"
+  if [[ -z "$CROSS_ARCH" ]]; then
+    _MARCH="${ARCH}"
   fi
-  export LDFLAGS="${LDFLAGS} -L/opt/static-libs/$COMPILER/lib"
-  if [[ "$ARCH" == "aarch64" ]]; then
-    # For some reason, this dependency of libunwind is not resolved on aarch64
-    export LDFLAGS="${LDFLAGS} -lz"
+  if [[ "$_MARCH" == "arm" ]]; then
+    _TARGET="${_MARCH}-linux-musleabihf"
+  else
+    _TARGET="${_MARCH}-alpine-linux-musl"
   fi
-  CMAKE_ARGS="${CMAKE_ARGS} -DSTATIC_BUILD_DO_NOT_USE=1 -DWITH_UNIVERSAL_BINARY=1 -DWITH_FUSE_EXTRACT_BINARY=1"
+  export CC="${_TARGET}-${CC}"
+  export CXX="${_TARGET}-${CXX}"
+  export STRIP_TOOL="${_TARGET}-strip"
+  export PATH="$_SYSROOT/usr/bin:$PATH"
+
+  _staticprefix="/opt/static-libs/$COMPILER/$_TARGET"
   if [[ "$BUILD_TYPE" == *-minimal-* ]]; then
-    _jemallocprefix="/opt/static-libs/$COMPILER-jemalloc-minimal"
+    _jemallocprefix="/opt/static-libs/$COMPILER-jemalloc-minimal/$_TARGET"
   else
     CMAKE_ARGS="${CMAKE_ARGS} -DWITH_PXATTR=1"
-    _jemallocprefix="/opt/static-libs/$COMPILER-jemalloc-full"
+    _jemallocprefix="/opt/static-libs/$COMPILER-jemalloc-full/$_TARGET"
   fi
   if [[ "$BUILD_TYPE" == *-libressl-* ]]; then
-    _sslprefix="/opt/static-libs/$COMPILER-libressl"
+    _sslprefix="/opt/static-libs/$COMPILER-libressl/$_TARGET"
   else
-    _sslprefix="/opt/static-libs/$COMPILER-openssl"
+    _sslprefix="/opt/static-libs/$COMPILER-openssl/$_TARGET"
+  fi
+
+  export LDFLAGS="${LDFLAGS} -static-libgcc -L$_staticprefix/lib -L$_sslprefix/lib"
+  export CFLAGS="${CFLAGS} -isystem $_staticprefix/include"
+  export CXXFLAGS="${CXXFLAGS} -isystem $_staticprefix/include"
+
+  if [[ "$_MARCH" == "i386" ]]; then
+    export LDFLAGS="${LDFLAGS} -lucontext"
+  fi
+
+  CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_SYSROOT=$_SYSROOT -DCMAKE_FIND_ROOT_PATH=$_staticprefix;$_sslprefix;$_jemallocprefix -DCMAKE_PREFIX_PATH=$_staticprefix;$_sslprefix;$_jemallocprefix -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY -DSTATIC_BUILD_DO_NOT_USE=1 -DWITH_UNIVERSAL_BINARY=1 -DWITH_FUSE_EXTRACT_BINARY=1"
+
+  if [[ -n "$CROSS_ARCH" ]]; then
+    CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_SYSTEM_NAME=Linux -DCMAKE_SYSTEM_PROCESSOR=$_MARCH -DCMAKE_CROSSCOMPILING_EMULATOR=/usr/bin/qemu-$_MARCH -DFOLLY_HAVE_UNALIGNED_ACCESS=OFF -DFOLLY_HAVE_WEAK_SYMBOLS=ON -DFOLLY_HAVE_LINUX_VDSO=OFF -DFOLLY_HAVE_WCHAR_SUPPORT=OFF -DHAVE_VSNPRINTF_ERRORS=OFF"
   fi
-  CMAKE_ARGS="${CMAKE_ARGS} -DSTATIC_BUILD_EXTRA_PREFIX=/opt/static-libs/$COMPILER;$_sslprefix;$_jemallocprefix"
 fi
 
 INSTALLDIR="$HOME/install"

.docker/check_lib_versions.py

@@ -23,6 +23,13 @@
     # "microsoft/mimalloc",
     "jemalloc/jemalloc",
     "tukaani-project/xz",
+    "davea42/libdwarf-code",
+    "libevent/libevent",
+    "nlohmann/json",
+    "HowardHinnant/date",
+    "nemtrif/utfcpp",
+    "ericniebler/range-v3",
+    "greg7mdp/parallel-hashmap",
 ]
 
 

.docker/fetch.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -ex
+
+URL=$1
+FILE=${2:-$(basename "$URL")}
+SHA512SUM="${3}"
+
+URLHASH=$(echo -n "$URL" | sha256sum | awk '{print $1}')
+CACHEDIR="$HOME/.pkgcache"
+CACHEFILE="$CACHEDIR/$URLHASH"
+
+if [ ! -f "$CACHEFILE" ]; then
+    rm -f "$CACHEFILE.tmp"
+    curl --retry 5 -L "$URL" > "$CACHEFILE.tmp"
+    if [ -n "$SHA512SUM" ]; then
+        echo "$SHA512SUM  $CACHEFILE.tmp" | sha512sum -c -
+        if [ $? -ne 0 ]; then
+            echo "Checksum verification failed for $URL"
+            rm -f "$CACHEFILE.tmp"
+            exit 1
+        fi
+    fi
+    mv -f "$CACHEFILE.tmp" "$CACHEFILE"
+fi
+
+if [ "$FILE" == "-" ]; then
+    cat "$CACHEFILE"
+else
+    cp "$CACHEFILE" "$FILE"
+fi

.docker/install-bloaty.sh

@@ -6,13 +6,16 @@ cd "$HOME"
 mkdir pkgs
 cd pkgs
 
-git clone --recurse-submodules https://github.com/google/bloaty
+git clone --depth=1 --recurse-submodules --shallow-submodules https://github.com/google/bloaty
 cd bloaty
 mkdir build
 cd build
 
+export PATH="/usr/lib/ccache/bin:$PATH"
+
 cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local
 ninja
+ccache -s
 ninja install
 
 cd "$HOME"