fix: portable Linux binaries (glibc 2.28 baseline) — Raspberry Pi & Amazon Linux 2023 out of the box

.github/workflows/ci.yml

@@ -261,55 +261,72 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v6
 
-      - name: Install build requirements
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y \
-            build-essential \
-            ninja-build \
-            automake \
-            cmake \
-            gcc \
-            git \
-            libboost-all-dev \
-            libcurl4-openssl-dev \
-            libgflags-dev \
-            libssl-dev \
-            zlib1g-dev
-          sudo apt-get clean
-          sudo rm -rf /var/lib/apt/lists/*
-
+      # Build deps (compilers, Boost, OpenSSL, ...) live inside the manylinux
+      # build container — the host only needs docker, python, and zip (all
+      # preinstalled on the runner image).
       - name: Restore Arrow cache
         id: arrow-cache
         uses: actions/cache/restore@v5
         with:
-          path: build/third_party
-          key: arrow-linux-${{ matrix.platform }}-${{ matrix.duckdb_channel }}-apache-arrow-23.0.1-${{ hashFiles('third_party/Arrow_CMakeLists.txt.in', 'third_party/patch_arrow.cmake') }}
+          # portable-deps holds the static OpenSSL + Boost prefixes built by
+          # scripts/build_portable_linux.sh (stamp-file guarded).
+          path: |
+            build/third_party
+            build/portable-deps
+          key: arrow-linux-portable228-v2-${{ matrix.platform }}-${{ matrix.duckdb_channel }}-apache-arrow-23.0.1-${{ hashFiles('third_party/Arrow_CMakeLists.txt.in', 'third_party/patch_arrow.cmake', 'scripts/build_portable_linux.sh') }}
           restore-keys: |
-            arrow-linux-${{ matrix.platform }}-${{ matrix.duckdb_channel }}-apache-arrow-23.0.1-
+            arrow-linux-portable228-v2-${{ matrix.platform }}-${{ matrix.duckdb_channel }}-apache-arrow-23.0.1-
 
       - name: Setup sccache
         uses: mozilla-actions/sccache-action@v0.0.10
 
-      - name: Configure Project
-        uses: threeal/cmake-action@v2.1.0
-        with:
-          generator: Ninja
-          run-build: true
-          options: |
-            CMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }}
-            GIZMOSQL_ENTERPRISE=ON
-            WITH_OPENTELEMETRY=ON
-            GIZMOSQL_DUCKDB_CHANNEL=${{ matrix.duckdb_channel }}
-            CMAKE_C_COMPILER_LAUNCHER=sccache
-            CMAKE_CXX_COMPILER_LAUNCHER=sccache
+      - name: Build (manylinux_2_28 container — glibc 2.28 baseline)
+        # Compile inside AlmaLinux 8 (glibc 2.28) with gcc-toolset-14 so the
+        # released binaries run out of the box on Raspberry Pi OS (bullseye+),
+        # Amazon Linux 2023, Ubuntu 20.04+, Debian 11+, etc. Same gcc 14
+        # codegen as the Ubuntu runner — only the glibc *baseline* changes;
+        # glibc's optimized routines are chosen at runtime on the target via
+        # IFUNC, so there is no performance cost. The script self-checks the
+        # produced binaries' max GLIBC/GLIBCXX symbol versions and fails the
+        # build on any baseline regression. sccache (a static musl binary,
+        # arch-matched) is mounted in and the GHA cache env forwarded so
+        # compile caching keeps working inside the container.
+        # The workspace is mounted at the SAME path as on the host so that
+        # absolute paths CMake bakes into build artifacts (e.g. the
+        # $<TARGET_FILE:gizmosql_client> path compiled into the integration
+        # test binary) remain valid when the tests later run on the host.
+        run: |
+          docker run --rm \
+            -v "$PWD":"$PWD" \
+            -v "$SCCACHE_PATH":/usr/local/bin/sccache:ro \
+            -e REPO_ROOT="$PWD" \
+            -e CMAKE_BUILD_TYPE=${{ env.CMAKE_BUILD_TYPE }} \
+            -e SCCACHE_GHA_ENABLED \
+            -e ACTIONS_CACHE_URL \
+            -e ACTIONS_RESULTS_URL \
+            -e ACTIONS_RUNTIME_TOKEN \
+            "quay.io/pypa/manylinux_2_28_$(uname -m)" \
+            bash "$PWD/scripts/build_portable_linux.sh" ${{ matrix.duckdb_channel }} build
+          sudo chown -R "$(id -u):$(id -g)" build
+
+      - name: Smoke test out-of-the-box startup (Raspberry Pi OS userland + Amazon Linux 2023)
+        # debian:bookworm-slim is the same userland as Raspberry Pi OS
+        # (bookworm). Both targets were broken before the manylinux build
+        # (binaries needed GLIBC_2.38 / GLIBCXX_3.4.32).
+        run: |
+          docker run --rm -v "$PWD/build:/bin-under-test:ro" debian:bookworm-slim bash -c \
+            "apt-get update -qq >/dev/null && apt-get install -y -qq libcurl4 >/dev/null && /bin-under-test/gizmosql_server${{ env.bin_suffix }} --version"
+          docker run --rm -v "$PWD/build:/bin-under-test:ro" amazonlinux:2023 bash -c \
+            "/bin-under-test/gizmosql_server${{ env.bin_suffix }} --version"
 
       - name: Save Arrow cache
         if: steps.arrow-cache.outputs.cache-hit != 'true'
         uses: actions/cache/save@v5
         with:
-          path: build/third_party
-          key: arrow-linux-${{ matrix.platform }}-${{ matrix.duckdb_channel }}-apache-arrow-23.0.1-${{ hashFiles('third_party/Arrow_CMakeLists.txt.in', 'third_party/patch_arrow.cmake') }}
+          path: |
+            build/third_party
+            build/portable-deps
+          key: arrow-linux-portable228-v2-${{ matrix.platform }}-${{ matrix.duckdb_channel }}-apache-arrow-23.0.1-${{ hashFiles('third_party/Arrow_CMakeLists.txt.in', 'third_party/patch_arrow.cmake', 'scripts/build_portable_linux.sh') }}
 
       - name: Start MinIO and setup bucket
         run: |

CHANGELOG.md

@@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - **Windows ARM64 builds** — native `arm64` Windows binaries (for Snapdragon X-class and other Windows-on-Arm machines) now ship alongside the existing `amd64` ones, in both the stable and LTS channels: signed CLI zips (`gizmosql_cli_windows_arm64.zip`, `gizmosql_cli_windows_arm64_lts.zip`) and signed MSI installers (`GizmoSQL-arm64.msi`, `GizmoSQL-arm64-lts.msi`), all with build-provenance attestations. Built natively on GitHub's `windows-11-arm` hosted runners (no cross-compilation), with the integration test suite running on ARM64 in CI. The MSI now packages the VC++ runtime DLLs captured from the build runner's VC redist (matching the binary architecture) instead of the MSI runner's x64 `System32` copies.
 
+### Fixed
+
+- **Linux release binaries now run out of the box on Raspberry Pi OS, Amazon Linux 2023, Ubuntu 20.04+, and Debian 11+.** The Linux CLI binaries (both arches, both channels) are now compiled in a `manylinux_2_28` container (AlmaLinux 8, glibc 2.28 baseline, gcc-toolset-14) instead of directly on the Ubuntu 24.04 runners — previously they required `GLIBC_2.38` / `GLIBCXX_3.4.32`, which made them fail on any distro older than ~mid-2024 (e.g. Raspberry Pi OS bookworm, Amazon Linux 2023) with "GLIBC_2.38 not found". No performance impact: the same gcc 14 code generation is used, and glibc selects its optimized routines at runtime on the target machine (IFUNC). CI now enforces the glibc 2.28 ceiling on every build (symbol-version check in `scripts/build_portable_linux.sh`) and smoke-tests the produced binaries in `debian:bookworm` (the Raspberry Pi OS userland) and `amazonlinux:2023` containers, so a portability regression can never ship silently.
+
 ### Changed
 
 - **Quick Start version auto-sync now happens at docs-publish time instead of via a commit to `main`.** The `deploy-docs` workflow runs `scripts/update_docs_version.sh` against the latest release tag right before publishing GitHub Pages, and is triggered after a successful release via `workflow_run`. This replaces the `sync-docs-version` CI job, which tried to push the version bump to `main` and was rejected by the branch-protection ruleset (`github-actions[bot]` is not exempt). The live docs always reflect the newest release with no secrets and no push to the protected branch.

CMakeLists.txt

@@ -213,6 +213,20 @@ if(WIN32 AND VCPKG_TARGET_TRIPLET)
     endif()
 endif()
 
+# Forward OpenSSL hints into the OpenSSL-consuming ExternalProjects (Arrow,
+# jwt-cpp; cpp-httplib has its own forwarding below). On standard Linux/macOS
+# hosts they find the system OpenSSL on their own; in the portable manylinux
+# build (scripts/build_portable_linux.sh) the static OpenSSL 3 lives in a
+# custom prefix, so pass it down when the caller set it.
+set(GIZMOSQL_EP_OPENSSL_ROOT_ARG "")
+set(GIZMOSQL_EP_OPENSSL_STATIC_ARG "")
+if(OPENSSL_ROOT_DIR)
+    set(GIZMOSQL_EP_OPENSSL_ROOT_ARG "-DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR}")
+endif()
+if(DEFINED OPENSSL_USE_STATIC_LIBS)
+    set(GIZMOSQL_EP_OPENSSL_STATIC_ARG "-DOPENSSL_USE_STATIC_LIBS=${OPENSSL_USE_STATIC_LIBS}")
+endif()
+
 # Forward iOS cross-compilation settings to Arrow ExternalProject
 if(CMAKE_SYSTEM_NAME STREQUAL "iOS")
     set(ARROW_SIMD_LEVEL "NONE")

Dockerfile.ci

@@ -11,11 +11,11 @@
 #     entrypoint actually needs. No C/C++ build toolchain — the server and
 #     client binaries are prebuilt in CI and COPYed in.
 
-# NOTE on the base image: it MUST be Debian trixie (13) or newer, NOT bookworm
-# (12). The gizmosql_server / gizmosql_client binaries are built on the CI
-# Ubuntu 24.04 runners (glibc 2.39, libstdc++ GLIBCXX_3.4.32) and require a
-# runtime with at least that glibc/libstdc++. bookworm (glibc 2.36) is too old
-# and fails at runtime with "GLIBC_2.38 not found". trixie ships glibc 2.41.
+# NOTE on the base image: the gizmosql_server / gizmosql_client binaries are
+# built in a manylinux_2_28 container (scripts/build_portable_linux.sh) with a
+# glibc 2.28 baseline, so any Debian 11+/Ubuntu 20.04+ base works — including
+# bookworm. trixie is used simply because it is the current Debian stable for
+# python:3.12-slim.
 
 # ──────────────────────────────────────────────────────────────────────────
 # Builder stage: fetch relocatable CLI tooling

scripts/build_portable_linux.sh

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+# Build portable Linux gizmosql binaries with a glibc 2.28 baseline.
+#
+# Runs INSIDE a quay.io/pypa/manylinux_2_28_{x86_64,aarch64} container
+# (AlmaLinux 8, glibc 2.28, gcc-toolset-14 on PATH). Binaries produced here
+# run out of the box on any distro with glibc >= 2.28: Raspberry Pi OS
+# (bullseye/bookworm), Amazon Linux 2023, Ubuntu 20.04+, Debian 11+, etc.
+# gcc-toolset statically links the newer-than-EL8 libstdc++/libgcc pieces,
+# so the binary's dynamic GLIBCXX requirement stays at the EL8 baseline.
+#
+# Usage (from the repo root; mount the workspace at the SAME path as the
+# host and pass it via REPO_ROOT, so absolute paths baked into artifacts —
+# e.g. the client path compiled into the integration test binary — stay
+# valid when tests run on the host afterwards):
+#   docker run --rm -v "$PWD":"$PWD" -e REPO_ROOT="$PWD" \
+#     quay.io/pypa/manylinux_2_28_$(uname -m) \
+#     bash "$PWD/scripts/build_portable_linux.sh" <duckdb_channel> [build_dir]
+#
+#   duckdb_channel : stable | lts
+#   build_dir      : CMake build dir relative to the repo root
+#                    (default: build)
+#
+# Dependency prefixes (static OpenSSL 3 + static Boost program_options) are
+# built into <build_dir>/portable-deps with stamp files, so caching the build
+# dir across CI runs skips the ~8 min dependency build.
+set -euxo pipefail
+
+DUCKDB_CHANNEL="${1:-stable}"
+BUILD_DIR="${2:-build}"
+REPO_ROOT="${REPO_ROOT:-/work}"
+DEPS_PREFIX="${REPO_ROOT}/${BUILD_DIR}/portable-deps"
+NPROC=$(nproc)
+
+OPENSSL_VERSION=3.5.1
+BOOST_VERSION=1.89.0
+
+# The workspace (and any cache-restored ExternalProject clones) are owned by
+# the host runner user, while this container runs as root. Without this, git
+# fails with "detected dubious ownership" — which silently breaks GizmoSQL's
+# own `git describe` version detection AND DuckDB's (DuckDB then falls back
+# to v0.0.1, making every runtime extension install 404).
+git config --global --add safe.directory '*'
+
+cd "${REPO_ROOT}"
+
+# ---------------------------------------------------------------------------
+# System packages (headers only / build tools — nothing here ends up as a
+# runtime dependency except libcurl.so.4, which every target distro ships).
+# ---------------------------------------------------------------------------
+dnf install -y --setopt=install_weak_deps=False \
+    ninja-build \
+    flex \
+    libcurl-devel \
+    zlib-devel \
+    perl-IPC-Cmd \
+    perl-Pod-Html
+
+# ---------------------------------------------------------------------------
+# Static OpenSSL 3 (cpp-httplib v0.16+ and jwt-cpp need >= 3.0; EL8 ships
+# 1.1.1). no-shared so the final binaries carry no libssl/libcrypto runtime
+# dependency.
+# ---------------------------------------------------------------------------
+if [ ! -f "${DEPS_PREFIX}/.openssl-${OPENSSL_VERSION}.stamp" ]; then
+    rm -rf /tmp/openssl-src
+    mkdir -p /tmp/openssl-src
+    curl -fsSL "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz" \
+        | tar -xz -C /tmp/openssl-src --strip-components=1
+    pushd /tmp/openssl-src
+    ./config no-shared no-tests no-docs --prefix="${DEPS_PREFIX}" --libdir=lib
+    make -j"${NPROC}" build_libs
+    make install_dev
+    popd
+    rm -rf /tmp/openssl-src
+    touch "${DEPS_PREFIX}/.openssl-${OPENSSL_VERSION}.stamp"
+fi
+
+# ---------------------------------------------------------------------------
+# Boost: headers + static program_options (the only compiled Boost lib we
+# link; uuid/algorithm are header-only). Built from source because EL8's
+# boost 1.66 predates BoostConfig.cmake, which our CMake (>= 4, no FindBoost
+# module) requires.
+# ---------------------------------------------------------------------------
+if [ ! -f "${DEPS_PREFIX}/.boost-${BOOST_VERSION}.stamp" ]; then
+    BOOST_UNDER=${BOOST_VERSION//./_}
+    rm -rf /tmp/boost-src
+    mkdir -p /tmp/boost-src
+    curl -fsSL "https://archives.boost.io/release/${BOOST_VERSION}/source/boost_${BOOST_UNDER}.tar.gz" \
+        | tar -xz -C /tmp/boost-src --strip-components=1
+    pushd /tmp/boost-src
+    ./bootstrap.sh --with-libraries=program_options --prefix="${DEPS_PREFIX}"
+    ./b2 -j"${NPROC}" link=static variant=release install
+    popd
+    rm -rf /tmp/boost-src
+    touch "${DEPS_PREFIX}/.boost-${BOOST_VERSION}.stamp"
+fi
+
+# ---------------------------------------------------------------------------
+# Configure + build. gcc-toolset-14 is already first on PATH in the
+# manylinux image; CMAKE_PREFIX_PATH points the build at the static deps.
+# ---------------------------------------------------------------------------
+SCCACHE_ARGS=()
+if command -v sccache >/dev/null 2>&1; then
+    SCCACHE_ARGS=(
+        -DCMAKE_C_COMPILER_LAUNCHER=sccache
+        -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+    )
+fi
+
+cmake -B "${BUILD_DIR}" -G Ninja \
+    -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" \
+    -DCMAKE_PREFIX_PATH="${DEPS_PREFIX}" \
+    -DOPENSSL_ROOT_DIR="${DEPS_PREFIX}" \
+    -DOPENSSL_USE_STATIC_LIBS=TRUE \
+    -DCMAKE_EXE_LINKER_FLAGS="-L${DEPS_PREFIX}/lib" \
+    -DGIZMOSQL_ENTERPRISE=ON \
+    -DWITH_OPENTELEMETRY=ON \
+    -DGIZMOSQL_DUCKDB_CHANNEL="${DUCKDB_CHANNEL}" \
+    "${SCCACHE_ARGS[@]}"
+
+cmake --build "${BUILD_DIR}"
+
+# ---------------------------------------------------------------------------
+# Portability self-check: fail the build if the produced binaries require
+# glibc newer than 2.28 or any GLIBCXX newer than EL8's system libstdc++
+# (gcc 8 = GLIBCXX_3.4.25, CXXABI_1.3.11) — i.e. if the baseline ever
+# silently regresses.
+# ---------------------------------------------------------------------------
+check_baseline() {
+    local bin="$1"
+    local bad
+    bad=$( (objdump -T "$bin"; objdump -p "$bin") | grep -oE 'GLIBC_2\.[0-9]+(\.[0-9]+)?' | sort -Vu | awk -F'GLIBC_' '$2+0 > 0 {split($2,v,"."); if (v[1] > 2 || (v[1] == 2 && v[2] > 28)) print "GLIBC_" $2}' || true)
+    local badxx
+    badxx=$( (objdump -T "$bin"; objdump -p "$bin") | grep -oE 'GLIBCXX_3\.4\.[0-9]+' | sort -Vu | awk -F'GLIBCXX_3.4.' '$2+0 > 25 {print "GLIBCXX_3.4." $2}' || true)
+    if [ -n "${bad}${badxx}" ]; then
+        echo "ERROR: $bin requires symbols above the glibc 2.28 / EL8 baseline:" >&2
+        echo "${bad}" "${badxx}" >&2
+        exit 1
+    fi
+    echo "OK: $bin is glibc 2.28 baseline clean"
+}
+
+for bin in "${BUILD_DIR}"/gizmosql_server* "${BUILD_DIR}"/gizmosql_client*; do
+    [ -f "$bin" ] && [ -x "$bin" ] && check_baseline "$bin"
+done

tests/integration/test_max_metadata_size.cpp

@@ -284,7 +284,7 @@ TEST(MaxMetadataEnvVar, EnvVarRaisesLimit) {
   // after the `Serve()` call returns ready; we don't have that signal here,
   // so probe via repeated connect attempts up to a timeout.
   bool reachable = false;
-  auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(15);
+  auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30);
   while (std::chrono::steady_clock::now() < deadline) {
     arrow::flight::FlightClientOptions opts;
     auto loc = arrow::flight::Location::ForGrpcTcp("localhost", kPort);
@@ -300,6 +300,15 @@ TEST(MaxMetadataEnvVar, EnvVarRaisesLimit) {
     }
     std::this_thread::sleep_for(std::chrono::milliseconds(200));
   }
+  if (!reachable) {
+    // Tear down BEFORE asserting: ASSERT_TRUE returns from the test body,
+    // and destroying the still-joinable server_thread would call
+    // std::terminate and abort the entire test binary ("terminate called
+    // without an active exception").
+    ShutdownFlightServer();
+    if (server_thread.joinable()) server_thread.join();
+    gizmosql::CleanupServerResources();
+  }
   ASSERT_TRUE(reachable) << "env-var server failed to come up";
 
   auto status = TrySelectWithBigHeader(kPort, kUser, kPass,

third_party/Arrow_CMakeLists.txt.in

@@ -56,5 +56,7 @@ ExternalProject_Add(
       ${ARROW_VCPKG_MANIFEST_ARG}
       ${ARROW_GRPC_ZLIB_ARG}
       ${ARROW_BOOST_SOURCE_ARG}
+      ${GIZMOSQL_EP_OPENSSL_ROOT_ARG}
+      ${GIZMOSQL_EP_OPENSSL_STATIC_ARG}
       ${ARROW_IOS_ARGS}
 )

third_party/DuckDB_CMakeLists.txt.in

@@ -13,6 +13,17 @@ ExternalProject_Add(
     GIT_SHALLOW TRUE
     CMAKE_ARGS
           -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/third_party/duckdb
+          # Pin the libdir: Red Hat-family hosts (e.g. the manylinux_2_28
+          # portable-build container) default CMAKE_INSTALL_LIBDIR to lib64,
+          # but the root CMakeLists hard-codes third_party/duckdb/lib paths.
+          -DCMAKE_INSTALL_LIBDIR=lib
+          # Pin DuckDB's version to the tag we check out instead of relying
+          # on `git describe` inside DuckDB's configure. If git fails there
+          # (e.g. "dubious ownership" when a cache-restored clone is read by
+          # the container root user), DuckDB silently builds as v0.0.1 and
+          # every runtime extension INSTALL 404s against
+          # extensions.duckdb.org/v0.0.1/.
+          -DOVERRIDE_GIT_DESCRIBE=@DUCKDB_GIT_TAG@
           -DCMAKE_C_COMPILER_LAUNCHER=${CMAKE_C_COMPILER_LAUNCHER}
           -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}
           "-DDUCKDB_EXTENSION_CONFIGS=${DUCKDB_EXTENSION_CONFIGS}"

third_party/JWTCPP_CMakeLists.txt.in

@@ -14,5 +14,7 @@ ExternalProject_Add(
     CMAKE_ARGS
       -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/third_party/jwt-cpp
       -DJWT_BUILD_EXAMPLES=OFF
+      ${GIZMOSQL_EP_OPENSSL_ROOT_ARG}
+      ${GIZMOSQL_EP_OPENSSL_STATIC_ARG}
     BUILD_COMMAND ""  # This is a header only library
 )