Build package sources #40
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # ============================================================================ # | |
| # Copyright (c) 2022 - 2026 NVIDIA Corporation & Affiliates. # | |
| # All rights reserved. # | |
| # # | |
| # This source code and the accompanying materials are made available under # | |
| # the terms of the Apache License 2.0 which accompanies this distribution. # | |
| # ============================================================================ # | |
| # | |
| # Workflow: diff apt and pip packages between a base and target image, then | |
| # build a source image containing source code for all added packages. | |
| # Optionally diff pip packages added by the macOS wheel (provide artifact_url). | |
| # | |
| # Inputs: | |
| # push_to_NGC: boolean, default false | |
| # Push the built source image to NGC, otherwise push to GHCR. | |
| # environment: string, no default | |
| # The environment to build the source image for. | |
| # artifact_url: string, no default | |
| # If set, diff pip packages added by the macOS wheel from this artifact URL | |
| # (e.g. .../actions/runs/RUN_ID/artifacts/ARTIFACT_ID for pycudaq-3.13-darwin-arm64). | |
| # python_version: string, default 3.13 | |
| # Python version for the macOS diff (must match the wheel). | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| artifact_url: | |
| required: false | |
| type: string | |
| description: 'Optional: URL of macOS wheel artifact to diff pip packages (e.g. .../runs/RUN_ID/artifacts/ARTIFACT_ID)' | |
| python_version: | |
| required: false | |
| type: string | |
| default: '3.13' | |
| description: 'Python version for macOS diff (must match the wheel)' | |
| name: Build package sources | |
| jobs: | |
| diff-packages-cudaq: | |
| name: Diff apt/pip packages ${{ matrix.release }} (CUDA ${{ matrix.cuda }}) | |
| runs-on: ubuntu-latest | |
| outputs: | |
| base_image: ${{ steps.images.outputs.base_image }} | |
| target_image: ${{ steps.images.outputs.target_image }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cuda: ['12.6', '13.0'] | |
| release: ['cudaq', 'cudaqx'] | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - name: Set image names | |
| id: images | |
| run: | | |
| cuda_major=$(echo "${{ matrix.cuda }}" | cut -d. -f1) | |
| if [ "${{ matrix.release }}" = "cudaq" ]; then | |
| echo "base_image=nvcr.io/nvidia/cuda:${{ matrix.cuda }}.0-runtime-ubuntu24.04" | tee -a $GITHUB_OUTPUT | |
| echo "target_image=nvcr.io/nvidia/nightly/cuda-quantum:cu${cuda_major}-latest" | tee -a $GITHUB_OUTPUT | |
| else | |
| # cudaqx: devcontainer base, cudaqx target | |
| cu_tag=$(echo "${{ matrix.cuda }}" | tr -d .) | |
| echo "base_image=ghcr.io/nvidia/cuda-quantum-devcontainer:amd64-cu${{ matrix.cuda }}-gcc11-main" | tee -a $GITHUB_OUTPUT | |
| echo "target_image=ghcr.io/nvidia/cudaqx:cu${cuda_major}-latest" | tee -a $GITHUB_OUTPUT | |
| fi | |
| - name: Log in to container registries | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin 2>/dev/null || true | |
| echo "${{ secrets.DOCKERHUB_READONLY_TOKEN }}" | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin 2>/dev/null || true | |
| - name: Pull images | |
| run: | | |
| docker pull "${{ steps.images.outputs.base_image }}" | |
| docker pull "${{ steps.images.outputs.target_image }}" | |
| - name: Diff package lists | |
| run: | | |
| chmod +x scripts/diff_image_packages.sh | |
| ./scripts/diff_image_packages.sh \ | |
| "${{ steps.images.outputs.base_image }}" \ | |
| "${{ steps.images.outputs.target_image }}" \ | |
| package-source-diff | |
| - name: Package diff step summary | |
| run: | | |
| echo "## Diffed packages (${{ steps.images.outputs.base_image }} vs ${{ steps.images.outputs.target_image }})" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Apt packages added" >> $GITHUB_STEP_SUMMARY | |
| if [ -s package-source-diff/apt_packages.txt ]; then | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| cat package-source-diff/apt_packages.txt >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "_None_" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Pip packages added" >> $GITHUB_STEP_SUMMARY | |
| if [ -s package-source-diff/pip_packages.txt ]; then | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| cat package-source-diff/pip_packages.txt >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "_None_" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload package lists | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: package-source-diff-cu${{ matrix.cuda }}-${{ matrix.release }}.zip | |
| path: package-source-diff/ | |
| retention-days: 1 | |
| diff-macos-package: | |
| name: Diff pip packages (macOS arm64) | |
| if: inputs.artifact_url != '' | |
| runs-on: macos-26 | |
| timeout-minutes: 20 | |
| permissions: | |
| contents: read | |
| actions: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Parse artifact URL | |
| id: parse | |
| run: | | |
| url="${{ inputs.artifact_url }}" | |
| run_id=$(echo "$url" | sed -n 's|.*/runs/\([0-9]*\)/.*|\1|p') | |
| artifact_id=$(echo "$url" | sed -n 's|.*/artifacts/\([0-9]*\)$|\1|p') | |
| if [ -z "$run_id" ] || [ -z "$artifact_id" ]; then | |
| echo "Could not parse run_id and artifact_id from: $url" | |
| exit 1 | |
| fi | |
| echo "run_id=$run_id" >> $GITHUB_OUTPUT | |
| echo "artifact_id=$artifact_id" >> $GITHUB_OUTPUT | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ inputs.python_version }} | |
| - name: Capture pip packages (before) | |
| run: | | |
| pip list --format=freeze | grep -v '^#' | sort -u > pip_before.txt | |
| echo "Packages before: $(wc -l < pip_before.txt)" | |
| - name: Download wheel artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| run-id: ${{ steps.parse.outputs.run_id }} | |
| artifact-ids: ${{ steps.parse.outputs.artifact_id }} | |
| path: wheel/ | |
| github-token: ${{ github.token }} | |
| - name: Install wheel | |
| run: | | |
| whl=$(find wheel -name '*.whl' 2>/dev/null | head -1) | |
| if [ -z "$whl" ]; then | |
| echo "No .whl file found under wheel/" | |
| find wheel -type f | |
| exit 1 | |
| fi | |
| echo "Installing $whl" | |
| pip install "$whl" | |
| # Capture package name (PEP 427: name-version-...); normalize to pip form (dashes) | |
| installed_pkg=$(basename "$whl" .whl | sed -E 's/-[0-9]+\.[0-9]+(\.[0-9]+)?.*//' | tr '_' '-') | |
| echo "$installed_pkg" > installed_wheel_package.txt | |
| - name: Capture pip packages (after) | |
| run: | | |
| pip list --format=freeze | grep -v '^#' | sort -u > pip_after.txt | |
| echo "Packages after: $(wc -l < pip_after.txt)" | |
| - name: Diff added pip packages | |
| id: diff | |
| run: | | |
| exclude_pkg='' | |
| [ -f installed_wheel_package.txt ] && exclude_pkg=$(cat installed_wheel_package.txt) | |
| awk -F'==' '{print $1}' pip_before.txt | sort -u > pip_before_names.txt | |
| while IFS= read -r line; do | |
| pkg="${line%%==*}" | |
| if [ -n "$exclude_pkg" ] && [ "$pkg" = "$exclude_pkg" ]; then | |
| continue | |
| fi | |
| if ! grep -qxF "$pkg" pip_before_names.txt 2>/dev/null; then | |
| echo "$line" | |
| fi | |
| done < pip_after.txt > pip_added.txt || true | |
| count=$(wc -l < pip_added.txt | tr -d ' ') | |
| echo "count=$count" >> $GITHUB_OUTPUT | |
| echo "## Pip packages added by macOS wheel" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Count: **$count**" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| cat pip_added.txt >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| - name: Upload macOS pip diff artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: macos-pip-diff.zip | |
| path: | | |
| pip_added.txt | |
| pip_before.txt | |
| pip_after.txt | |
| retention-days: 7 | |
| build-source-image: | |
| name: Build source image CUDA ${{ matrix.cuda }} | |
| needs: [diff-packages-cudaq, diff-macos-package] | |
| if: always() && needs.diff-packages-cudaq.result == 'success' && (needs.diff-macos-package.result == 'success' || needs.diff-macos-package.result == 'skipped') | |
| runs-on: linux-amd64-cpu32 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cuda: ['12.6', '13.0'] | |
| permissions: | |
| contents: read | |
| packages: write | |
| environment: | |
| name: ${{ inputs.environment || 'default' }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| submodules: recursive | |
| - name: Set base image | |
| id: base-image | |
| run: echo "base_image=nvcr.io/nvidia/cuda:${{ matrix.cuda }}.0-runtime-ubuntu24.04" >> $GITHUB_OUTPUT | |
| - name: Download package lists (cudaq) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: package-source-diff-cu${{ matrix.cuda }}-cudaq.zip | |
| path: package-source-diff-cudaq | |
| - name: Download package lists (cudaqx) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: package-source-diff-cu${{ matrix.cuda }}-cudaqx.zip | |
| path: package-source-diff-cudaqx | |
| - name: Restore package-source-diff directory layout | |
| run: | | |
| mkdir -p package-source-diff | |
| for variant in cudaq cudaqx; do | |
| dir="package-source-diff-${variant}" | |
| apt_src="$dir/apt_packages.txt"; [ -f "$dir/package-source-diff/apt_packages.txt" ] && apt_src="$dir/package-source-diff/apt_packages.txt" | |
| pip_src="$dir/pip_packages.txt"; [ -f "$dir/package-source-diff/pip_packages.txt" ] && pip_src="$dir/package-source-diff/pip_packages.txt" | |
| [ -f "$apt_src" ] && cp "$apt_src" "package-source-diff/apt_packages_${variant}.txt" || touch "package-source-diff/apt_packages_${variant}.txt" | |
| [ -f "$pip_src" ] && cp "$pip_src" "package-source-diff/pip_packages_${variant}.txt" || touch "package-source-diff/pip_packages_${variant}.txt" | |
| done | |
| touch package-source-diff/pip_packages_macos.txt | |
| ls -la package-source-diff/ | |
| - name: Download macOS pip diff | |
| if: github.event.inputs.artifact_url != '' | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: macos-pip-diff.zip | |
| path: macos-pip-diff | |
| - name: Apply macOS pip diff to package list | |
| if: github.event.inputs.artifact_url != '' | |
| run: cp macos-pip-diff/pip_added.txt package-source-diff/pip_packages_macos.txt | |
| - name: Generate trimmed package lists (cudaqx minus cudaq, macos minus cudaq/cudaqx) | |
| run: | | |
| d=package-source-diff | |
| # apt: cudaqx trimmed = packages in cudaqx not in cudaq | |
| if [ -f "$d/apt_packages_cudaqx.txt" ]; then | |
| if [ -f "$d/apt_packages_cudaq.txt" ]; then | |
| comm -23 <(sort -u "$d/apt_packages_cudaqx.txt") <(sort -u "$d/apt_packages_cudaq.txt") | grep -v '^$' > "$d/apt_packages_cudaqx_trimmed.txt" || true | |
| else | |
| cp "$d/apt_packages_cudaqx.txt" "$d/apt_packages_cudaqx_trimmed.txt" | |
| fi | |
| else | |
| : > "$d/apt_packages_cudaqx_trimmed.txt" | |
| fi | |
| # pip: cudaqx trimmed = packages in cudaqx (by name) not in cudaq | |
| awk -F'==' '{print $1}' "$d/pip_packages_cudaq.txt" 2>/dev/null | sort -u > /tmp/pip_cudaq_names.txt || true | |
| : > "$d/pip_packages_cudaqx_trimmed.txt" | |
| if [ -f "$d/pip_packages_cudaqx.txt" ]; then | |
| while IFS= read -r line; do | |
| [ -z "$line" ] && continue | |
| pkg="${line%%==*}" | |
| grep -qxF "$pkg" /tmp/pip_cudaq_names.txt 2>/dev/null || echo "$line" >> "$d/pip_packages_cudaqx_trimmed.txt" | |
| done < "$d/pip_packages_cudaqx.txt" | |
| fi | |
| # macOS pip: trimmed = packages in macos not in cudaq and not in cudaqx | |
| { awk -F'==' '{print $1}' "$d/pip_packages_cudaq.txt" 2>/dev/null; awk -F'==' '{print $1}' "$d/pip_packages_cudaqx.txt" 2>/dev/null; } | sort -u > /tmp/pip_union_names.txt | |
| : > "$d/pip_packages_macos_trimmed.txt" | |
| if [ -f "$d/pip_packages_macos.txt" ] && [ -s "$d/pip_packages_macos.txt" ]; then | |
| while IFS= read -r line; do | |
| [ -z "$line" ] && continue | |
| pkg="${line%%==*}" | |
| grep -qxF "$pkg" /tmp/pip_union_names.txt 2>/dev/null || echo "$line" >> "$d/pip_packages_macos_trimmed.txt" | |
| done < "$d/pip_packages_macos.txt" | |
| fi | |
| ls -la "$d"/*trimmed* 2>/dev/null || true | |
| - name: Generate pip attribution (NOTICE_PIP per variant + CUDA version) | |
| run: | | |
| cuda_major=$(echo "${{ matrix.cuda }}" | cut -d. -f1) | |
| suffix="_cu${cuda_major}" | |
| python3 scripts/generate_pip_attribution.py \ | |
| package-source-diff/pip_packages_cudaq.txt \ | |
| -o "NOTICE_PIP_cudaq${suffix}" | |
| python3 scripts/generate_pip_attribution.py \ | |
| package-source-diff/pip_packages_cudaqx.txt \ | |
| -o "NOTICE_PIP_cudaqx${suffix}" | |
| python3 scripts/generate_pip_attribution.py \ | |
| package-source-diff/pip_packages_macos.txt \ | |
| -o "NOTICE_PIP_macos${suffix}" | |
| ls -la NOTICE_PIP_* | |
| head -50 "NOTICE_PIP_cudaq${suffix}" | |
| - name: Generate apt attribution (NOTICE_APT per variant + CUDA version) | |
| run: | | |
| cuda_major=$(echo "${{ matrix.cuda }}" | cut -d. -f1) | |
| suffix="_cu${cuda_major}" | |
| python3 scripts/generate_apt_attribution.py \ | |
| package-source-diff/apt_packages_cudaq.txt \ | |
| -o "NOTICE_APT_cudaq${suffix}" | |
| python3 scripts/generate_apt_attribution.py \ | |
| package-source-diff/apt_packages_cudaqx.txt \ | |
| -o "NOTICE_APT_cudaqx${suffix}" | |
| ls -la NOTICE_APT_* | |
| head -50 "NOTICE_APT_cudaq${suffix}" | |
| - name: Generate tpls lock file | |
| run: | | |
| chmod +x scripts/generate_tpls_lock.sh | |
| ./scripts/generate_tpls_lock.sh tpls_commits.lock | |
| cat tpls_commits.lock | |
| wc -l tpls_commits.lock || true | |
| - name: Download CUDA-Q prerequisite sources | |
| run: | | |
| set -euo pipefail | |
| # Generate cudaq_prereqs.lock for the current configuration | |
| bash scripts/install_prerequisites.sh -l | |
| cat cudaq_prereqs.lock | |
| mkdir -p prereqs | |
| while IFS= read -r line; do | |
| # Skip comments and empty lines | |
| [ -z "$line" ] && continue | |
| case "$line" in | |
| \#*) continue ;; | |
| esac | |
| name="" | |
| type="" | |
| url="" | |
| ref="" | |
| for kv in $line; do | |
| key="${kv%%=*}" | |
| val="${kv#*=}" | |
| case "$key" in | |
| name) name="$val" ;; | |
| type) type="$val" ;; | |
| url) url="$val" ;; | |
| ref) ref="$val" ;; | |
| esac | |
| done | |
| if [ -z "$name" ] || [ -z "$type" ]; then | |
| echo "Skipping malformed lock entry: $line" | |
| continue | |
| fi | |
| # Ensure a dedicated subdirectory for each prerequisite | |
| mkdir -p "prereqs/${name}" | |
| case "$type" in | |
| tar) | |
| fname="$(basename "$url")" | |
| dest="prereqs/${name}/${fname}" | |
| echo "Downloading tar archive for $name from $url -> $dest" | |
| curl -L "$url" -o "$dest" | |
| ;; | |
| sh) | |
| fname="$(basename "$url")" | |
| dest="prereqs/${name}/${fname}" | |
| echo "Downloading shell installer for $name from $url -> $dest" | |
| curl -L "$url" -o "$dest" | |
| chmod +x "$dest" | |
| ;; | |
| pem) | |
| dest="prereqs/${name}/${name}.pem" | |
| echo "Downloading PEM bundle for $name from $url -> $dest" | |
| curl -L "$url" -o "$dest" | |
| ;; | |
| git) | |
| dest_dir="prereqs/${name}" | |
| echo "Cloning git repo for $name from $url (ref=${ref:-HEAD}) -> $dest_dir" | |
| git clone "$url" "$dest_dir" | |
| if [ -n "${ref:-}" ]; then | |
| git -C "$dest_dir" checkout "$ref" | |
| fi | |
| ;; | |
| script) | |
| # No direct URL; record the script/target so downstream tooling | |
| # can resolve the concrete sources if desired. | |
| echo "$line" > "prereqs/${name}/lockentry" | |
| ;; | |
| *) | |
| echo "Unknown type '$type' in lock entry: $line" | |
| ;; | |
| esac | |
| done < cudaq_prereqs.lock | |
| echo "Contents of prereqs/:" | |
| ls -R prereqs | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build source image | |
| id: build | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: docker/build/package_sources.Dockerfile | |
| build-args: | | |
| base_image=${{ steps.base-image.outputs.base_image }} | |
| load: true | |
| tags: package-sources:latest | |
| outputs: type=docker,dest=/tmp/package-sources.tar | |
| push: false | |
| - name: Log in to GitHub CR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ github.token }} | |
| - name: Tag and push amd64 image | |
| id: push | |
| run: | | |
| docker load --input /tmp/package-sources.tar | |
| cuda_major=$(echo "${{ matrix.cuda }}" | cut -d. -f1) | |
| target_image="nvcr.io/nvidia/nightly/cuda-quantum:cu${cuda_major}-latest" | |
| tag_suffix="${target_image##*:}" | |
| owner_lower=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') | |
| tag="ghcr.io/${owner_lower}/cuda-quantum-src:${tag_suffix}" | |
| tag_amd64="${tag}-amd64" | |
| tag_arm64="${tag}-arm64" | |
| docker tag package-sources:latest "$tag_amd64" | |
| docker push "$tag_amd64" | |
| echo "image_tag=$tag" | tee -a $GITHUB_OUTPUT | |
| echo "tag_amd64=$tag_amd64" | tee -a $GITHUB_OUTPUT | |
| echo "tag_arm64=$tag_arm64" | tee -a $GITHUB_OUTPUT | |
| - name: Create arm64 image (copy /sources from amd64) | |
| run: | | |
| tag_amd64="${{ steps.push.outputs.tag_amd64 }}" | |
| tag_arm64="${{ steps.push.outputs.tag_arm64 }}" | |
| base_image="${{ steps.base-image.outputs.base_image }}" | |
| # Extract /sources from amd64 image | |
| cid=$(docker create "$tag_amd64") | |
| docker cp "$cid:/sources" ./sources-from-amd64 | |
| docker rm "$cid" | |
| # Build arm64 image: same base, add /sources | |
| cat > Dockerfile.arm64 << EOF | |
| FROM ${base_image} | |
| COPY sources-from-amd64 /sources | |
| EOF | |
| docker buildx build --platform linux/arm64 -f Dockerfile.arm64 -t "$tag_arm64" --load . | |
| docker push "$tag_arm64" | |
| - name: Create and push multi-arch manifest | |
| run: | | |
| tag="${{ steps.push.outputs.image_tag }}" | |
| tag_amd64="${{ steps.push.outputs.tag_amd64 }}" | |
| tag_arm64="${{ steps.push.outputs.tag_arm64 }}" | |
| docker buildx imagetools create --tag "$tag" "$tag_amd64" "$tag_arm64" | |
| - name: Summary | |
| run: | | |
| base_image="${{ steps.base-image.outputs.base_image }}" | |
| echo "## Package source diff (CUDA ${{ matrix.cuda }})" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Base image:** \`${base_image}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Built image ${{ steps.push.outputs.image_tag }}" >> $GITHUB_STEP_SUMMARY |