Add wheel-builder stage to pre-build Python dependencies

Introduce a multi-stage build approach where Python packages are
compiled into wheels in a dedicated wheel-builder stage, then
installed in the final image without requiring build dependencies.

Benefits:
- Build dependencies (gcc, python-devel, etc.) are no longer
  installed/removed in the final image stage
- Improved build caching - wheel-builder layer is reused if
  package list doesn't change
- Cleaner separation between compilation and runtime

Signed-off-by: Riccardo Pittau <elfosardo@gmail.com>

Author: elfosardo

Dockerfile

@@ -27,6 +27,42 @@ RUN git clone https://github.com/ipxe/ipxe.git && \
 COPY prepare-efi.sh /bin/
 RUN prepare-efi.sh centos
 
+## Build Python wheels for all dependencies
+FROM $BASE_IMAGE AS wheel-builder
+
+# build arguments for source build customization
+ARG UPPER_CONSTRAINTS_FILE=upper-constraints.txt
+ARG IRONIC_SOURCE=7fe20fe31f0e184c68b7029e04acdb5d286fc4b2 # main
+ARG SUSHY_SOURCE
+
+# Set environment variables for the build script
+ENV IRONIC_SOURCE=${IRONIC_SOURCE}
+ENV SUSHY_SOURCE=${SUSHY_SOURCE}
+ENV UPPER_CONSTRAINTS_FILE=${UPPER_CONSTRAINTS_FILE}
+
+# Install build dependencies for compiling Python packages
+RUN --mount=type=cache,target=/var/cache/dnf \
+    echo "install_weak_deps=False" >> /etc/dnf/dnf.conf && \
+    echo "tsflags=nodocs" >> /etc/dnf/dnf.conf && \
+    echo "keepcache=1" >> /etc/dnf/dnf.conf && \
+    dnf install -y \
+        gcc \
+        git-core \
+        python3.12-devel \
+        python3.12-pip \
+        python3.12-setuptools
+
+# Install wheel and pin pip/setuptools versions
+RUN python3.12 -m pip install --no-cache-dir pip==24.1 setuptools==74.1.2 wheel
+
+# Copy sources and package lists
+COPY sources /sources/
+COPY ${UPPER_CONSTRAINTS_FILE} ironic-packages-list /tmp/
+COPY build-wheels.sh /bin/
+
+# Build all wheels
+RUN /bin/build-wheels.sh
+
 # build actual image
 FROM $BASE_IMAGE
 
@@ -43,21 +79,16 @@ ARG PKGS_LIST=main-packages-list.txt
 ARG EXTRA_PKGS_LIST
 ARG PATCH_LIST
 
-# build arguments for source build customization
-ARG UPPER_CONSTRAINTS_FILE=upper-constraints.txt
-ARG IRONIC_SOURCE=7fe20fe31f0e184c68b7029e04acdb5d286fc4b2 # master
-ARG SUSHY_SOURCE
-
-COPY sources /sources/
-COPY ${UPPER_CONSTRAINTS_FILE} ironic-packages-list ${PKGS_LIST} \
-     ${EXTRA_PKGS_LIST:-$PKGS_LIST} ${PATCH_LIST:-$PKGS_LIST} \
-     /tmp/
+COPY ${PKGS_LIST} ${EXTRA_PKGS_LIST:-$PKGS_LIST} ${PATCH_LIST:-$PKGS_LIST} /tmp/
 COPY ironic-config/inspector.ipxe.j2 ironic-config/httpd-ironic-api.conf.j2 \
      ironic-config/ipxe_config.template ironic-config/dnsmasq.conf.j2 \
      /templates/
 COPY prepare-image.sh patch-image.sh configure-nonroot.sh /bin/
 COPY scripts/ /bin/
 
+# Copy pre-built wheels from wheel-builder stage
+COPY --from=wheel-builder /wheels /wheels
+
 RUN --mount=type=cache,target=/var/cache/dnf \
     prepare-image.sh && \
      rm -f /bin/prepare-image.sh

build-wheels.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/bash
+
+set -euxo pipefail
+
+# This script builds Python wheels for all Ironic dependencies.
+# It runs in the wheel-builder stage of the Docker build.
+
+UPPER_CONSTRAINTS_PATH="/tmp/${UPPER_CONSTRAINTS_FILE:-}"
+
+# If the content of the upper-constraints file is empty,
+# we assume we're on the master branch
+if [[ ! -s "${UPPER_CONSTRAINTS_PATH}" ]]; then
+    UPPER_CONSTRAINTS_PATH="/tmp/upper-constraints.txt"
+    curl -L https://releases.openstack.org/constraints/upper/master -o "${UPPER_CONSTRAINTS_PATH}"
+fi
+
+# Install jinja2 to render the package list template
+python3.12 -m pip install --no-cache-dir jinja2
+
+IRONIC_PKG_LIST="/tmp/ironic-packages-list"
+IRONIC_PKG_LIST_FINAL="/tmp/ironic-packages-list-final"
+
+# Render the Jinja2 template for the package list
+python3.12 -c 'import os; import sys; import jinja2; sys.stdout.write(jinja2.Template(sys.stdin.read()).render(env=os.environ, path=os.path))' < "${IRONIC_PKG_LIST}" > "${IRONIC_PKG_LIST_FINAL}"
+
+# Remove sushy constraint if building from source
+if [[ -n ${SUSHY_SOURCE:-} ]]; then
+    sed -i '/^sushy===/d' "${UPPER_CONSTRAINTS_PATH}"
+fi
+
+# Build wheels for all packages
+# Note: some packages may not produce wheels (pure Python), but pip wheel handles this
+python3.12 -m pip wheel \
+    --wheel-dir=/wheels \
+    --no-cache-dir \
+    -r "${IRONIC_PKG_LIST_FINAL}" \
+    -c "${UPPER_CONSTRAINTS_PATH}"
+
+# Also build wheels for runtime dependencies not in the main list
+python3.12 -m pip wheel \
+    --wheel-dir=/wheels \
+    --no-cache-dir \
+    jinja2 watchdog \
+    -c "${UPPER_CONSTRAINTS_PATH}"
+
+echo "Wheels built successfully in /wheels"
+ls -la /wheels/

prepare-image.sh

@@ -12,55 +12,35 @@ EOF
 IRONIC_UID=997
 IRONIC_GID=994
 
-declare -a BUILD_DEPS=(
-    gcc
-    git-core
-    python3.12-devel
-    python3.12-setuptools
-)
-
 dnf upgrade -y
 
 # NOTE(dtantsur): pip is a requirement of python3 in CentOS
-dnf install -y python3.12-pip "${BUILD_DEPS[@]}"
+dnf install -y python3.12-pip
 
 # NOTE(elfosardo): pinning pip and setuptools version to avoid
 # incompatibilities and errors during packages installation;
 # versions should be updated regularly, for example
 # after cutting a release branch.
 python3.12 -m pip install --no-cache-dir pip==24.1 setuptools==74.1.2
 
-UPPER_CONSTRAINTS_PATH="/tmp/${UPPER_CONSTRAINTS_FILE:-}"
-
-# NOTE(elfosardo): if the content of the upper-constraints file is empty,
-# we give as assumed that we're on the master branch
-if [[ ! -s "${UPPER_CONSTRAINTS_PATH}" ]]; then
-    UPPER_CONSTRAINTS_PATH="/tmp/upper-constraints.txt"
-    curl -L https://releases.openstack.org/constraints/upper/master -o "${UPPER_CONSTRAINTS_PATH}"
-fi
-
-# NOTE(elfosardo): install dependencies constrained
-python3.12 -m pip install jinja2 watchdog -c "${UPPER_CONSTRAINTS_PATH}"
-
-IRONIC_PKG_LIST="/tmp/ironic-packages-list"
-IRONIC_PKG_LIST_FINAL="/tmp/ironic-packages-list-final"
+# Install from pre-built wheels (built in wheel-builder stage)
+# No compilation needed here - wheels are already built
+python3.12 -m pip install \
+    --no-cache-dir \
+    --no-index \
+    --find-links=/wheels \
+    --ignore-installed \
+    --prefix /usr \
+    /wheels/*.whl
 
-python3.12 -c 'import os; import sys; import jinja2; sys.stdout.write(jinja2.Template(sys.stdin.read()).render(env=os.environ, path=os.path))' < "${IRONIC_PKG_LIST}" > "${IRONIC_PKG_LIST_FINAL}"
-
-if [[ -n ${SUSHY_SOURCE:-} ]]; then
-    sed -i '/^sushy===/d' "${UPPER_CONSTRAINTS_PATH}"
-fi
-
-python3.12 -m pip install --no-cache-dir --ignore-installed --prefix /usr -r "${IRONIC_PKG_LIST_FINAL}" -c "${UPPER_CONSTRAINTS_PATH}"
+# Clean up wheels after installation
+rm -rf /wheels
 
 # ironic system configuration
 mkdir -p /var/log/ironic /var/lib/ironic
 getent group ironic > /dev/null || groupadd -r ironic -g "${IRONIC_GID}"
 getent passwd ironic > /dev/null || useradd -r -g ironic -u "${IRONIC_UID}" -s /sbin/nologin ironic -d /var/lib/ironic
 
-# clean installed build dependencies
-dnf remove -y "${BUILD_DEPS[@]}"
-
 xargs -rtd'\n' dnf install -y < /tmp/"${PKGS_LIST}"
 
 if [[ -n "${EXTRA_PKGS_LIST:-}" ]]; then