diff --git a/.github/workflows/build_package_sources.yml b/.github/workflows/build_package_sources.yml new file mode 100644 index 00000000000..d622b9fef47 --- /dev/null +++ b/.github/workflows/build_package_sources.yml @@ -0,0 +1,193 @@ +# ============================================================================ # +# Copyright (c) 2022 - 2026 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # +# +# Workflow: diff apt and pip packages between a base and target image, then +# build a source image containing source code for all added packages. +# +# Inputs: +# push_to_NGC: boolean, default false +# Push the built source image to NGC, otherwise push to GHCR. +# environment: string, no default +# The environment to build the source image for. + +on: + push: + workflow_dispatch: + inputs: + push_to_NGC: + required: false + type: boolean + default: false + description: 'Push the built source image to NGC, otherwise push to GHCR.' + environment: + required: false + type: string + +name: Build package sources + +jobs: + diff-packages: + name: Diff apt/pip packages (CUDA ${{ matrix.cuda }}) + runs-on: ubuntu-latest + outputs: + base_image: ${{ steps.images.outputs.base_image }} + strategy: + fail-fast: false + matrix: + cuda: ['12.6', '13.0'] + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Set image names + id: images + run: | + echo "base_image=nvcr.io/nvidia/cuda:${{ matrix.cuda }}.0-runtime-ubuntu24.04" >> $GITHUB_OUTPUT + cuda_major=$(echo "${{ matrix.cuda }}" | cut -d. -f1) + echo "target_image=nvcr.io/nvidia/nightly/cuda-quantum:cu${cuda_major}-latest" >> $GITHUB_OUTPUT + + - name: Log in to container registries + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin 2>/dev/null || true + echo "${{ secrets.DOCKERHUB_READONLY_TOKEN }}" | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin 2>/dev/null || true + + - name: Pull images + run: | + docker pull "${{ steps.images.outputs.base_image }}" + docker pull "${{ steps.images.outputs.target_image }}" + + - name: Diff package lists + run: | + chmod +x scripts/diff_image_packages.sh + ./scripts/diff_image_packages.sh \ + "${{ steps.images.outputs.base_image }}" \ + "${{ steps.images.outputs.target_image }}" \ + package-source-diff + + - name: Upload package lists + uses: actions/upload-artifact@v4 + with: + name: package-source-diff-cu${{ matrix.cuda }}.zip + path: package-source-diff/ + retention-days: 1 + + build-source-image: + name: Build source image (CUDA ${{ matrix.cuda }}) + needs: diff-packages + runs-on: linux-amd64-cpu32 + strategy: + fail-fast: false + matrix: + cuda: ['12.6', '13.0'] + permissions: + contents: read + packages: write + + environment: + name: ${{ inputs.environment || 'default' }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive + + - name: Download package lists + uses: actions/download-artifact@v4 + with: + name: package-source-diff-cu${{ matrix.cuda }}.zip + + - name: Restore package-source-diff directory layout + run: | + # Ensure package-source-diff/ exists with list files for Dockerfile COPY + if [ ! -f package-source-diff/apt_packages.txt ]; then + mkdir -p package-source-diff + mv apt_packages.txt pip_packages.txt package-source-diff/ 2>/dev/null || true + fi + touch package-source-diff/apt_packages.txt package-source-diff/pip_packages.txt + ls -la package-source-diff/ + + - name: Generate tpls lock file + run: | + chmod +x scripts/generate_tpls_lock.sh + ./scripts/generate_tpls_lock.sh tpls_commits.lock + cat tpls_commits.lock + wc -l tpls_commits.lock || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build source image + id: build + uses: docker/build-push-action@v5 + with: + context: . + file: docker/build/package_sources.Dockerfile + build-args: | + base_image=${{ needs.diff-packages.outputs.base_image }} + load: true + tags: package-sources:latest + outputs: type=docker,dest=/tmp/package-sources.tar + push: false + + - name: Log in to GitHub CR + if: github.event.inputs.push_to_NGC != 'true' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Log in to NGC + if: github.event.inputs.push_to_NGC == 'true' + uses: docker/login-action@v3 + with: + registry: nvcr.io + username: '$oauthtoken' + password: ${{ secrets.NGC_CREDENTIALS }} + + - name: Tag and push to GHCR or NGC + id: push + run: | + docker load --input /tmp/package-sources.tar + cuda_major=$(echo "${{ matrix.cuda }}" | cut -d. -f1) + target_image="nvcr.io/nvidia/nightly/cuda-quantum:cu${cuda_major}-latest" + tag_suffix="${target_image##*:}" + if [ "${{ github.event.inputs.push_to_NGC }}" = "true" ]; then + # Derive push destination from target_image: .../cuda-quantum:tag -> .../cuda-quantum-src:tag + repo_part="${target_image%:*}" + tag="${repo_part}-src:${tag_suffix}" + else + owner_lower=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') + tag="ghcr.io/${owner_lower}/cuda-quantum-src:${tag_suffix}" + fi + docker tag package-sources:latest "$tag" + docker push "$tag" + echo "image_tag=$tag" | tee -a $GITHUB_OUTPUT + + - name: Summary + run: | + base_image="${{ needs.diff-packages.outputs.base_image }}" + cuda_major=$(echo "${{ matrix.cuda }}" | cut -d. -f1); target_image="nvcr.io/nvidia/nightly/cuda-quantum:cu${cuda_major}-latest" + echo "## Package source diff (CUDA ${{ matrix.cuda }})" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Base image:** \`${base_image}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Target image:** \`${target_image}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Apt packages added:** $(wc -l < package-source-diff/apt_packages.txt 2>/dev/null || echo 0)" >> $GITHUB_STEP_SUMMARY + echo "- **Pip packages added:** $(wc -l < package-source-diff/pip_packages.txt 2>/dev/null || echo 0)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Built image \`package-sources:latest\` (ubuntu:24.04 with sources under \`/sources/apt\`, \`/sources/pip\`, and \`/sources/tpls\`)." >> $GITHUB_STEP_SUMMARY + if [ -n "${{ steps.push.outputs.image_tag }}" ]; then + echo "- **Pushed:** \`${{ steps.push.outputs.image_tag }}\`" >> $GITHUB_STEP_SUMMARY + fi diff --git a/docker/build/package_sources.Dockerfile b/docker/build/package_sources.Dockerfile new file mode 100644 index 00000000000..a63b3e9dc0f --- /dev/null +++ b/docker/build/package_sources.Dockerfile @@ -0,0 +1,154 @@ +# ============================================================================ # +# Copyright (c) 2022 - 2026 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # +# +# Builds an Ubuntu 24.04 image containing source code for a set of apt and pip +# packages plus the repo's tpls/ (third-party library) source. Tpls are cloned +# at build time using .gitmodules and a lock file (commit + path per line) via +# git clone --no-checkout --filter=tree:0 + fetch + checkout. +# +# Build from repo root with package-source-diff/ and tpls_commits.lock (or generate with scripts/generate_tpls_lock.sh): +# docker build -t package-sources:latest -f docker/build/package_sources.Dockerfile . +# +# base_image is the base image to use for the build. +# +# Expects in build context: +# package-source-diff/apt_packages.txt - one apt package name per line +# package-source-diff/pip_packages.txt - one pip package==version per line +# tpls_commits.lock - " " per submodule (same as install_prerequisites.sh -l) +# .gitmodules - submodule paths and URLs +# scripts/clone_tpls_from_lock.sh - clone script +# NOTICE, LICENSE - attribution + +ARG base_image=ubuntu:24.04 +FROM ${base_image} + +SHELL ["/bin/bash", "-c"] +ARG DEBIAN_FRONTEND=noninteractive + +# Install deps for fetching apt source, pip sdists, and cloning tpls +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + build-essential \ + curl \ + dpkg-dev \ + git \ + jq \ + python3 \ + python3-pip \ + unzip \ + && python3 -m pip install --upgrade unearth --break-system-packages \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Install necessary repository for librdmac1 +RUN apt-get update && apt-get install -y --no-install-recommends gnupg wget \ + && wget -qO - "https://www.mellanox.com/downloads/ofed/RPM-GPG-KEY-Mellanox" | apt-key add - \ + && mkdir -p /etc/apt/sources.list.d && wget -q -nc --no-check-certificate -P /etc/apt/sources.list.d "https://linux.mellanox.com/public/repo/mlnx_ofed/5.3-1.0.0.1/ubuntu20.04/mellanox_mlnx_ofed.list" \ + && echo 'deb-src http://linux.mellanox.com/public/repo/mlnx_ofed/5.3-1.0.0.1/ubuntu20.04/$(ARCH) ./' >> /etc/apt/sources.list.d/mellanox_mlnx_ofed.list \ + && apt-get update -y + +# Enable source repositories (Ubuntu 24.04 DEB822 format) +RUN if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then \ + sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources; \ + else \ + sed -i '/^# deb-src/s/^# //' /etc/apt/sources.list 2>/dev/null || true; \ + fi +RUN apt-get update + +ENV SOURCES_ROOT=/sources +RUN mkdir -p "${SOURCES_ROOT}/apt" "${SOURCES_ROOT}/pip" "${SOURCES_ROOT}/tpls" "${SOURCES_ROOT}/scripts" + +ENV SCRIPTS_DIR=${SOURCES_ROOT}/scripts + +# Copy .gitmodules, tpls lock file, clone script, package lists, and pip sdist fetcher +COPY .gitmodules "${SCRIPTS_DIR}"/.gitmodules +COPY tpls_commits.lock "${SCRIPTS_DIR}"/tpls_commits.lock +COPY scripts/clone_tpls_from_lock.sh "${SCRIPTS_DIR}"/clone_tpls_from_lock.sh +COPY package-source-diff/apt_packages.txt "${SCRIPTS_DIR}"/apt_packages.txt +COPY package-source-diff/pip_packages.txt "${SCRIPTS_DIR}"/pip_packages.txt + +# Copy attribution +COPY NOTICE LICENSE "${SOURCES_ROOT}/" + +# Fetch apt source, pip sdists, and clone tpls in parallel (prefix lines so logs stay readable) +RUN apt-get update && set -o pipefail && \ + ( set -o pipefail; cd "${SOURCES_ROOT}/apt" && \ + chmod 777 . && \ + : > "${SOURCES_ROOT}/apt/apt_omitted_packages.txt" && \ + while IFS= read -r pkg || [ -n "$pkg" ]; do \ + [ -z "$pkg" ] && continue; \ + apt-get source -y "$pkg" || echo "$pkg" >> "${SOURCES_ROOT}/apt/apt_omitted_packages.txt"; \ + done < "${SCRIPTS_DIR}"/apt_packages.txt; \ + ) 2>&1 | sed 's/^/[apt] /' & \ + ( set -o pipefail; : > "${SOURCES_ROOT}/pip/pip_omitted_packages.txt" && \ + cd "${SOURCES_ROOT}/pip" && \ + while IFS= read -r package || [ -n "$package" ]; do \ + [ -z "$package" ] && continue; \ + url=$(unearth --no-binary "$package" 2>/dev/null | jq -r '.link.url'); \ + if [ -n "$url" ] && [ "$url" != "null" ]; then \ + curl -fsSL -O "$url" || echo "$package" >> pip_omitted_packages.txt; \ + else \ + echo "$package" >> pip_omitted_packages.txt; \ + fi; \ + done < "${SCRIPTS_DIR}"/pip_packages.txt; \ + ) 2>&1 | sed 's/^/[pip] /' & \ + ( set -o pipefail; SOURCES_ROOT="${SOURCES_ROOT}" GITMODULES="${SCRIPTS_DIR}"/.gitmodules lock_file="${SCRIPTS_DIR}"/tpls_commits.lock \ + bash "${SCRIPTS_DIR}"/clone_tpls_from_lock.sh ) 2>&1 | sed 's/^/[tpls] /' & \ + wait + +RUN echo -e "apt_omitted_packages.txt:\n$(cat ${SOURCES_ROOT}/apt/apt_omitted_packages.txt)" +RUN echo -e "pip_omitted_packages.txt:\n$(cat ${SOURCES_ROOT}/pip/pip_omitted_packages.txt)" + +# For omitted apt packages (no source available), extract license/copyright/EULA from the .deb +RUN echo "Retrieving EULA/copyright for omitted apt packages..." && \ + mkdir -p "${SOURCES_ROOT}/apt/licenses" /tmp/deb_extract && \ + while IFS= read -r pkg || [ -n "$pkg" ]; do \ + [ -z "$pkg" ] && continue; \ + ( cd /tmp/deb_extract && apt-get download "$pkg" 2>/dev/null ) || true; \ + deb=$(ls /tmp/deb_extract/*.deb 2>/dev/null | head -1); \ + if [ -n "$deb" ]; then \ + dpkg-deb -R "$deb" "/tmp/deb_extract/${pkg}_pkg" 2>/dev/null || true; \ + dest="${SOURCES_ROOT}/apt/licenses/${pkg}"; \ + mkdir -p "$dest"; \ + find "/tmp/deb_extract/${pkg}_pkg" \( -iname "*license*" -o -iname "*eula*" -o -iname "*copyright*" \) -exec cp -a {} "$dest/" \; 2>/dev/null || true; \ + rm -rf "/tmp/deb_extract/${pkg}_pkg" /tmp/deb_extract/*.deb; \ + fi; \ + done < "${SOURCES_ROOT}/apt/apt_omitted_packages.txt"; \ + rm -rf /tmp/deb_extract + +# For omitted pip packages (no sdist), get EULA/license from the wheel: fetch wheel from PyPI, extract, copy license/EULA/copyright files +RUN echo "Retrieving EULA/license for omitted pip packages..." && \ + mkdir -p "${SOURCES_ROOT}/pip/licenses" /tmp/wheel_extract && \ + while IFS= read -r package || [ -n "$package" ]; do \ + [ -z "$package" ] && continue; \ + name="${package%%==*}"; \ + version="${package#*==}"; \ + [ -z "$name" ] || [ -z "$version" ] || [ "$version" = "$package" ] && continue; \ + url=$(curl -sS "https://pypi.org/pypi/${name}/${version}/json" 2>/dev/null | jq -r '.urls[] | select(.packagetype=="bdist_wheel") | select(.filename | test("manylinux.*x86_64|manylinux_2.*x86_64")) | .url' 2>/dev/null | head -1); \ + if [ -n "$url" ] && [ "$url" != "null" ]; then \ + if curl -fsSL -o /tmp/pip_wheel.whl "$url" 2>/dev/null; then \ + (cd /tmp/wheel_extract && unzip -o -q /tmp/pip_wheel.whl 2>/dev/null) || true; \ + dest="${SOURCES_ROOT}/pip/licenses/${name}"; \ + mkdir -p "$dest"; \ + find /tmp/wheel_extract -type f \( -iname "*license*" -o -iname "*eula*" -o -iname "*copyright*" \) -exec cp -an {} "$dest/" \; 2>/dev/null || true; \ + if [ -z "$(ls -A "$dest" 2>/dev/null)" ]; then \ + license_text=$(curl -sS "https://pypi.org/pypi/${name}/${version}/json" 2>/dev/null | jq -r '.info.license // .info.license_expression // empty'); \ + [ -n "$license_text" ] && [ "$license_text" != "null" ] && echo "$license_text" > "$dest/LICENSE_from_PyPI.txt"; \ + fi; \ + find /tmp/wheel_extract -mindepth 1 -delete 2>/dev/null || rm -rf /tmp/wheel_extract/*; \ + fi; \ + rm -f /tmp/pip_wheel.whl; \ + fi; \ + done < "${SOURCES_ROOT}/pip/pip_omitted_packages.txt"; \ + rm -rf /tmp/wheel_extract + +# Summary +RUN echo "apt: $(find ${SOURCES_ROOT}/apt -maxdepth 1 -type d 2>/dev/null | wc -l) dirs" && \ + echo "pip: $(find ${SOURCES_ROOT}/pip -maxdepth 1 -type f \( -name '*.tar.gz' -o -name '*.zip' \) 2>/dev/null | wc -l) sdists" && \ + echo "tpls: $(find ${SOURCES_ROOT}/tpls -maxdepth 1 -mindepth 1 -type d 2>/dev/null | wc -l) libraries" + +WORKDIR ${SOURCES_ROOT} diff --git a/scripts/clone_tpls_from_lock.sh b/scripts/clone_tpls_from_lock.sh new file mode 100644 index 00000000000..ac663a91995 --- /dev/null +++ b/scripts/clone_tpls_from_lock.sh @@ -0,0 +1,45 @@ +# ============================================================================ # +# Copyright (c) 2022 - 2026 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # +# +# Clones each tpl submodule at the commit given in the lock file into +# SOURCES_ROOT/tpls/. Uses git clone --no-checkout --filter=tree:0 +# then fetch + checkout for minimal history. +# +# Usage: SOURCES_ROOT=/sources .gitmodules=/path/.gitmodules lock_file=/path/tpls_lock.txt \ +# ./scripts/clone_tpls_from_lock.sh + +set -euo pipefail + +: "${SOURCES_ROOT:?SOURCES_ROOT not set}" +: "${GITMODULES:?GITMODULES not set}" +: "${lock_file:?lock_file not set}" + +[ -f "$lock_file" ] || { echo "Lock file not found: $lock_file" >&2; exit 1; } + +mkdir -p "${SOURCES_ROOT}/tpls" +cd "${SOURCES_ROOT}/tpls" + +while IFS= read -r line || [ -n "$line" ]; do + [ -z "$line" ] && continue + commit="${line%% *}" + path="${line#* }" + [ -z "$commit" ] || [ -z "$path" ] && continue + + repo="$(git config --file "$GITMODULES" --get "submodule.${path}.url")" || { + echo "WARN: no url for $path" >&2 + continue + } + lib="$(basename "$path")" + dest="${SOURCES_ROOT}/tpls/${lib}" + + echo "Cloning $lib @ $commit from $repo ..." + git clone --no-checkout --filter=tree:0 "$repo" "$dest" \ + && git -C "$dest" fetch --depth 1 origin "$commit" \ + && git -C "$dest" checkout --detach FETCH_HEAD \ + || { echo "Failed to clone $lib" >&2; rm -rf "$dest" 2>/dev/null; true; } +done < "$lock_file" diff --git a/scripts/diff_image_packages.sh b/scripts/diff_image_packages.sh new file mode 100644 index 00000000000..b7f47cda463 --- /dev/null +++ b/scripts/diff_image_packages.sh @@ -0,0 +1,74 @@ +# ============================================================================ # +# Copyright (c) 2022 - 2026 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # +# +# Diffs apt and pip packages between a base image and a target image. +# The target image is assumed to be built FROM the base image. +# Outputs: +# - apt_added: apt package names installed in target but not in base +# - pip_added: pip package==version lines installed in target but not in base +# +# Usage: +# ./scripts/diff_image_packages.sh +# Writes /apt_packages.txt and /pip_packages.txt + +set -euo pipefail + +BASE_IMAGE="${1:?Usage: $0 }" +TARGET_IMAGE="${2:?Usage: $0 }" +OUTPUT_DIR="${3:?Usage: $0 }" + +mkdir -p "$OUTPUT_DIR" + +# Get apt list: manually installed package names (works in minimal images) +# Override ENTRYPOINT so the image runs exactly "sh -c '...'" (NVIDIA images set ENTRYPOINT). +get_apt_list() { + docker run --rm --entrypoint sh "$1" -c \ + 'apt-mark showmanual 2>/dev/null | sort -u' || true +} + +# Get pip list: package==version (freeze format) for reproducible source download +get_pip_list() { + docker run --rm --entrypoint sh "$1" -c \ + 'pip list --format=freeze 2>/dev/null || python3 -m pip list --format=freeze 2>/dev/null || true' | \ + grep -v '^#' | sort -u || true +} + +BASE_APT=$(mktemp) +TARGET_APT=$(mktemp) +BASE_PIP=$(mktemp) +TARGET_PIP=$(mktemp) +trap "rm -f $BASE_APT $TARGET_APT $BASE_PIP $TARGET_PIP" EXIT + +echo "Collecting apt list from base image: $BASE_IMAGE" +get_apt_list "$BASE_IMAGE" > "$BASE_APT" +echo "Collecting apt list from target image: $TARGET_IMAGE" +get_apt_list "$TARGET_IMAGE" > "$TARGET_APT" + +echo "Collecting pip list from base image: $BASE_IMAGE" +get_pip_list "$BASE_IMAGE" > "$BASE_PIP" +echo "Collecting pip list from target image: $TARGET_IMAGE" +get_pip_list "$TARGET_IMAGE" > "$TARGET_PIP" + +# Apt diff: package names in target but not in base +comm -13 <(sort -u "$BASE_APT") <(sort -u "$TARGET_APT") | grep -v '^$' > "$OUTPUT_DIR/apt_packages.txt" || true + +# Pip diff: package==version in target but not in base (by package name) +# We compare by package name so we get "added" packages; for versions we use target's version +BASE_PIP_NAMES=$(mktemp) +trap "rm -f $BASE_APT $TARGET_APT $BASE_PIP $TARGET_PIP $BASE_PIP_NAMES" EXIT +awk -F'==' '{print $1}' "$BASE_PIP" | sort -u > "$BASE_PIP_NAMES" +# Lines in TARGET_PIP whose package name is not in BASE_PIP +while IFS= read -r line; do + pkg="${line%%==*}" + if ! grep -qxF "$pkg" "$BASE_PIP_NAMES" 2>/dev/null; then + echo "$line" + fi +done < "$TARGET_PIP" > "$OUTPUT_DIR/pip_packages.txt" || true + +echo "Wrote $OUTPUT_DIR/apt_packages.txt ($(wc -l < "$OUTPUT_DIR/apt_packages.txt" || echo 0) packages)" +echo "Wrote $OUTPUT_DIR/pip_packages.txt ($(wc -l < "$OUTPUT_DIR/pip_packages.txt" || echo 0) packages)" diff --git a/scripts/generate_tpls_lock.sh b/scripts/generate_tpls_lock.sh new file mode 100644 index 00000000000..a493082e235 --- /dev/null +++ b/scripts/generate_tpls_lock.sh @@ -0,0 +1,49 @@ +# ============================================================================ # +# Copyright (c) 2022 - 2026 NVIDIA Corporation & Affiliates. # +# All rights reserved. # +# # +# This source code and the accompanying materials are made available under # +# the terms of the Apache License 2.0 which accompanies this distribution. # +# ============================================================================ # +# +# Writes a lock file with one line per submodule: " ". +# Same format as: git config --file .gitmodules --get-regexp '^submodule\..*\.path$' \ +# | awk '{print $2}' | while read p; do printf "%s %s\n" "$(git rev-parse HEAD:$p)" "$p"; done +# Used so the package_sources image (or install_prerequisites.sh -l) can clone each tpl at a pinned commit. +# Must be run from repo root with submodules initialized. +# +# Usage: ./scripts/generate_tpls_lock.sh [output_file] +# Default output: tpls_commits.lock (repo root) +# +# This will produce a file that looks like: +# $ cat tpls_commits.lock +# fc8d07cfe54ba9f5019453dfdb112491246ee017 tpls/fmt +# f8d7d77c06936315286eb55f8de22cd23c188571 tpls/googletest-src +# 7cbf1a2591520c2491aa35339f227775f4d3adf6 tpls/llvm +# 81fe2d424f05e5596772caeaa0e2b7e6518da92c tpls/eigen +# 102a354e0ceded132bb1f38b5d0be90806a3070b tpls/armadillo +# b702f6db50b5f92f4e9a8aeb4b5f985bcbba38f4 tpls/ensmallen +# 287333ee00555aaece5a5cf6acc9040563c6f642 tpls/spdlog +# 67b6156299d22f70e38db3a68fe7ec2a00022739 tpls/xtl +# cccd75769b332f7cc726d702555180d74bd78953 tpls/xtensor +# 8b48ff878c168b51fe5ef7b8c728815b9e1a9857 tpls/pybind11 +# 304800611697e8abd8fd424bafa72a45f644f9ed tpls/qpp +# d202b82fbccf897604a18e035c09e1330dffd082 tpls/cpr +# 7609450f71434bdc9fbd9491a9505b423c2a8496 tpls/asio +# 94a011b9f7c0a991e5382927a2dbe5a7d9a056b8 tpls/Crow +# 42e0b9e099180e8570407c33f87b4683cac00d81 tpls/Stim + +set -euo pipefail + +OUTPUT="${1:-tpls_commits.lock}" + +repo_root="$(git rev-parse --show-toplevel)" +cd "$repo_root" + +OUTPUT="${OUTPUT:-tpls_commits.lock}" +: > "$OUTPUT" +while read -r p; do + [ -z "$p" ] && continue + sha="$(git rev-parse "HEAD:$p" 2>/dev/null)" || continue + printf "%s %s\n" "$sha" "$p" >> "${OUTPUT:-tpls_commits.lock}" +done < <(git config -f .gitmodules --get-regexp '^submodule\..*\.path$' | awk '{print $2}')