Deployments #5633
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| # We update the build caches during the pull_request_target event, | |
| # and deploy the images via workflow dispatch on branches with deployment permissions. | |
| # A dispatch event is automatically triggered when the pull_request_target event | |
| # triggers the workflow. | |
| workflow_dispatch: | |
| inputs: | |
| update_registry_cache: | |
| type: boolean | |
| description: Create or update the build caches on the container registry. The build terminates after that and no images are pushed. | |
| required: false | |
| default: false | |
| cache_base: | |
| required: false | |
| type: string | |
| description: 'The name of the branch to use as cache base.' | |
| default: main | |
| platforms: | |
| type: string | |
| description: The platforms to build the Docker images for. A semicolon separated list creates a separate list for each platform, whereas a comma separate list creates a multi-platform build. | |
| required: false | |
| default: default | |
| create_release: | |
| type: string | |
| description: Create a GitHub release with the given version number (e.g. 0.3.0). A GitHub release with that version must not exist already. | |
| required: false | |
| default: '' | |
| workflow_run: | |
| workflows: | |
| - PR merged | |
| types: | |
| - completed | |
| # PR merged workflow runs after merge into certain branches an on pull_request_target; | |
| # there should hence be no need for additional branch filtering here. | |
| push: | |
| branches: | |
| - 'releases/*' | |
| - 'staging/*' | |
| - 'experimental/*' | |
| - 'features/*' | |
| name: Deployments # do not change name without updating workflow_run triggers | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.ref_name }} | |
| cancel-in-progress: false | |
| jobs: | |
| metadata: | |
| name: Prepare build | |
| if: github.event_name != 'push' || github.event.created | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| actions: read | |
| outputs: | |
| pull_request_number: ${{ steps.pr_info.outputs.pr_number }} | |
| pull_request_base: ${{ steps.pr_info.outputs.pr_base }} | |
| pull_request_commit: ${{ steps.pr_info.outputs.merge_commit }} | |
| llvm_commit: ${{ steps.build_config.outputs.llvm_commit }} | |
| pybind11_commit: ${{ steps.build_config.outputs.pybind11_commit }} | |
| cache_base: ${{ steps.build_info.outputs.cache_base }} | |
| cache_target: ${{ steps.build_info.outputs.cache_target }} | |
| multi_platform: ${{ steps.build_info.outputs.multi_platform }} | |
| platforms: ${{ steps.build_info.outputs.platforms }} | |
| build_dependencies: ${{ steps.build_config.outputs.build_dependencies }} | |
| create_packages: ${{ steps.build_config.outputs.create_packages }} | |
| environment: ${{ steps.build_info.outputs.environment }} | |
| steps: | |
| - name: Retrieve PR info | |
| if: github.event_name == 'workflow_run' | |
| id: pr_info | |
| run: | | |
| artifacts_url=${{ github.event.workflow_run.artifacts_url }} | |
| artifacts=$(gh api $artifacts_url -q '.artifacts[] | {name: .name, url: .archive_download_url}') | |
| for artifact in `echo "$artifacts"`; do | |
| name=`echo $artifact | jq -r '.name'` | |
| if [ "$name" == "metadata_pr" ]; then | |
| url=`echo $artifact | jq -r '.url'` | |
| gh api $url > metadata.zip | |
| unzip -d metadata metadata.zip | |
| for file in `find metadata/ -type f`; do | |
| cat "$file" >> metadata.txt | |
| done | |
| pr_number=`cat metadata.txt | grep -o 'pr-number: \S*' | cut -d ' ' -f 2` | |
| pr_base=`cat metadata.txt | grep -o 'pr-base: \S*' | cut -d ' ' -f 2` | |
| merge_commit=`cat metadata.txt | grep -o 'source-sha: \S*' | cut -d ' ' -f 2` | |
| rm -rf metadata.zip metadata.txt metadata | |
| echo "pr_number=$pr_number" >> $GITHUB_OUTPUT | |
| echo "pr_base=$pr_base" >> $GITHUB_OUTPUT | |
| echo "merge_commit=$merge_commit" >> $GITHUB_OUTPUT | |
| fi | |
| done | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Determine build info | |
| id: build_info | |
| run: | | |
| if ${{ github.event_name == 'workflow_dispatch' }}; then | |
| defaults="${{ (inputs.update_registry_cache && 'linux/arm64 linux/amd64') || 'linux/amd64,linux/arm64' }}" | |
| requested_platforms=`echo "${{ inputs.platforms }}" | tr + , | sed "s#default#$defaults#g"` | |
| else | |
| requested_platforms="linux/arm64 linux/amd64" | |
| fi | |
| if [ -n "$(echo "$requested_platforms" | egrep ',')" ]; then | |
| multi_platform="{\"ids\":[]}" | |
| platform_flag=`echo "$requested_platforms" | tr -d ' '` | |
| multi_platform=`echo $multi_platform | jq ".ids |= . + [\"multiplat\"]"` | |
| platform={\"multiplat\":{\"docker_flag\":\"$platform_flag\"}} | |
| multi_platform=`echo $multi_platform | jq ". |= . + $platform"` | |
| fi | |
| platforms="{\"ids\":[]}" | |
| for input in `echo "$requested_platforms" | tr , ' ' | tr ';' ' '`; do | |
| platform_id=`echo $input | sed 's/linux\///g' | tr -d ' '` | |
| minimal_base_image=`([ "$platform_id" == "amd64" ] && echo amd64/almalinux:8) || ([ "$platform_id" == "arm64" ] && echo arm64v8/almalinux:8) || echo` | |
| platforms=`echo $platforms | jq ".ids |= . + [\"$platform_id\"]"` | |
| platform={\"$platform_id\":{\"docker_flag\":\"$input\",\"minimal_base_image\":\"$minimal_base_image\"}} | |
| platforms=`echo $platforms | jq ". |= . + $platform"` | |
| done | |
| cache_base=${{ (github.event_name == 'push' && 'main') || inputs.cache_base || steps.pr_info.outputs.pr_base }} | |
| cache_target=${{ (github.event_name != 'workflow_dispatch' && (steps.pr_info.outputs.pr_base || github.ref_name)) || (inputs.update_registry_cache && github.ref_name) || '' }} | |
| environment=${{ (github.event_name != 'workflow_run' && 'ghcr-deployment') || '' }} | |
| # Store deployment info | |
| release_title="${{ inputs.create_release }}" | |
| release_version=`echo "$release_title" | (egrep -o "([0-9]{1,}\.)+[0-9]{1,}([A-Za-z0-9_\-\.]*)" || true)` | |
| echo "source-sha: ${{ github.sha }}" >> deployment_info.txt | |
| echo "release-title: $release_title" >> deployment_info.txt | |
| echo "release-version: $release_version" >> deployment_info.txt | |
| echo "cache_base=$cache_base" >> $GITHUB_OUTPUT | |
| echo "cache_target=$cache_target" >> $GITHUB_OUTPUT | |
| echo "multi_platform=$(echo $multi_platform)" >> $GITHUB_OUTPUT | |
| echo "platforms=$(echo $platforms)" >> $GITHUB_OUTPUT | |
| echo "environment=$environment" >> $GITHUB_OUTPUT | |
| - name: Upload deployment info | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: deployment_info # changing the artifact name requires updating other workflows | |
| path: deployment_info.txt | |
| retention-days: 30 | |
| if-no-files-found: error | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: "${{ steps.pr_info.outputs.merge_commit }}" | |
| - name: Configure build | |
| id: build_config | |
| run: | | |
| echo "llvm_commit=$(git rev-parse @:./tpls/llvm)" >> $GITHUB_OUTPUT | |
| echo "pybind11_commit=$(git rev-parse @:./tpls/pybind11)" >> $GITHUB_OUTPUT | |
| if ${{ github.event_name != 'workflow_run' || steps.pr_info.outputs.pr_number != '' }}; then | |
| echo "build_dependencies=true" >> $GITHUB_OUTPUT | |
| fi | |
| if ${{ github.event_name == 'workflow_dispatch' && ! inputs.update_registry_cache }}; then | |
| echo "create_packages=true" >> $GITHUB_OUTPUT | |
| fi | |
| devdeps: | |
| name: Build dev dependencies | |
| needs: metadata | |
| if: needs.metadata.outputs.build_dependencies == 'true' | |
| strategy: | |
| matrix: | |
| platform: ${{ fromJson(needs.metadata.outputs.platforms).ids }} | |
| toolchain: [clang16, gcc11, gcc12] | |
| fail-fast: false | |
| uses: ./.github/workflows/dev_environment.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platforms: ${{ fromJson(needs.metadata.outputs.platforms)[format('{0}', matrix.platform)].docker_flag }} | |
| dockerfile: build/devdeps.Dockerfile | |
| build_config_id: ${{ matrix.toolchain }} | |
| build_args: | | |
| toolchain=${{ matrix.toolchain }} | |
| registry_cache_from: ${{ needs.metadata.outputs.cache_base }} | |
| update_registry_cache: ${{ needs.metadata.outputs.cache_target }} | |
| pull_request_number: ${{ needs.metadata.outputs.pull_request_number }} | |
| pull_request_commit: ${{ needs.metadata.outputs.pull_request_commit }} | |
| checkout_submodules: true | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| # needed only for the cloudposse GitHub action | |
| matrix_key: ${{ matrix.platform }}-${{ matrix.toolchain }} | |
| wheeldeps: | |
| name: Build wheel dependencies | |
| needs: metadata | |
| if: needs.metadata.outputs.build_dependencies == 'true' | |
| strategy: | |
| matrix: | |
| # There are currently no multi-platform manylinux images available. | |
| # See https://github.com/pypa/manylinux/issues/1306. | |
| platform: ${{ fromJson(needs.metadata.outputs.platforms).ids }} | |
| cuda_version: ["12.6", "13.0"] | |
| fail-fast: false | |
| uses: ./.github/workflows/dev_environment.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platforms: ${{ fromJson(needs.metadata.outputs.platforms)[format('{0}', matrix.platform)].docker_flag }} | |
| dockerfile: build/devdeps.manylinux.Dockerfile | |
| build_config_id: cu${{ matrix.cuda_version }}-gcc11 | |
| build_args: | | |
| base_image=ghcr.io/nvidia/pypa/manylinux_2_28${{ (matrix.platform == 'arm64' && '_aarch64') || (matrix.platform == 'amd64' && '_x86_64') || '' }}:latest | |
| cuda_version=${{ matrix.cuda_version }} | |
| toolchain=gcc11 | |
| distro=rhel8 | |
| llvm_commit=${{ needs.metadata.outputs.llvm_commit }} | |
| pybind11_commit=${{ needs.metadata.outputs.pybind11_commit }} | |
| registry_cache_from: ${{ needs.metadata.outputs.cache_base }} | |
| update_registry_cache: ${{ needs.metadata.outputs.cache_target }} | |
| pull_request_number: ${{ needs.metadata.outputs.pull_request_number }} | |
| pull_request_commit: ${{ needs.metadata.outputs.pull_request_commit }} | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| # needed only for the cloudposse GitHub action | |
| matrix_key: ${{ matrix.platform }}-cu${{ matrix.cuda_version }}-python | |
| source_build: | |
| name: Build cross-platform dependencies | |
| needs: metadata | |
| if: needs.metadata.outputs.build_dependencies == 'true' | |
| strategy: | |
| matrix: | |
| platform: ${{ fromJson(needs.metadata.outputs.platforms).ids }} | |
| cuda_version: ["12.6", "13.0"] | |
| fail-fast: false | |
| uses: ./.github/workflows/dev_environment.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platforms: ${{ fromJson(needs.metadata.outputs.platforms)[format('{0}', matrix.platform)].docker_flag }} | |
| dockerfile: build/assets.Dockerfile | |
| build_config_id: cu${{ matrix.cuda_version }}-llvm | |
| build_target: prereqs | |
| build_args: | | |
| base_image=${{ fromJson(needs.metadata.outputs.platforms)[format('{0}', matrix.platform)].minimal_base_image }} | |
| cuda_version=${{ matrix.cuda_version }} | |
| toolchain=llvm | |
| registry_cache_from: ${{ needs.metadata.outputs.cache_base }} | |
| update_registry_cache: ${{ needs.metadata.outputs.cache_target }} | |
| pull_request_number: ${{ needs.metadata.outputs.pull_request_number }} | |
| pull_request_commit: ${{ needs.metadata.outputs.pull_request_commit }} | |
| checkout_submodules: true | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| # needed only for the cloudposse GitHub action | |
| matrix_key: ${{ matrix.platform }}-cu${{ matrix.cuda_version }}-installer | |
| openmpi: | |
| name: Build Open MPI | |
| needs: metadata | |
| if: needs.metadata.outputs.build_dependencies == 'true' | |
| strategy: | |
| matrix: | |
| platform: ${{ fromJson(needs.metadata.outputs.platforms).ids }} | |
| cuda_version: ["12.6", "13.0"] | |
| fail-fast: false | |
| uses: ./.github/workflows/dev_environment.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platforms: ${{ fromJson(needs.metadata.outputs.platforms)[format('{0}', matrix.platform)].docker_flag }} | |
| dockerfile: build/devdeps.ompi.Dockerfile | |
| build_config_id: cu${{ matrix.cuda_version }} | |
| build_args: | | |
| cuda_version=${{ matrix.cuda_version }} | |
| registry_cache_from: ${{ needs.metadata.outputs.cache_base }} | |
| update_registry_cache: ${{ needs.metadata.outputs.cache_target }} | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| # needed only for the cloudposse GitHub action | |
| matrix_key: ${{ matrix.platform }}-cu${{ matrix.cuda_version }}-ompi | |
| dispatch: | |
| name: Dispatch deployments | |
| needs: [metadata, devdeps, wheeldeps, openmpi] | |
| if: github.event_name != 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/github-script@v7 | |
| with: | |
| # We want this dispatch to trigger additional workflows. | |
| github-token: ${{ secrets.REPO_BOT_ACCESS_TOKEN }} | |
| script: | | |
| github.rest.actions.createWorkflowDispatch({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| workflow_id: 'deployments.yml', | |
| ref: '${{ needs.metadata.outputs.pull_request_base || github.ref_name }}', | |
| inputs: { | |
| platforms: 'linux/arm64 linux/amd64', | |
| cache_base: '${{ needs.metadata.outputs.cache_base }}', | |
| }, | |
| }) | |
| # This job is needed only when using the cloudposse GitHub action to read | |
| # the output of a matrix job. This is a workaround due to current GitHub | |
| # limitations that may not be needed if the work started here concludes: | |
| # https://github.com/actions/runner/pull/2477 | |
| config: | |
| name: Configure build | |
| needs: [metadata, devdeps, wheeldeps, source_build, openmpi] | |
| if: needs.metadata.outputs.create_packages == 'true' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| json: "${{ steps.read_json.outputs.result }}" | |
| steps: | |
| - uses: cloudposse/[email protected] | |
| id: read_json | |
| with: | |
| matrix-step-name: dev_environment | |
| - name: Print config.outputs.json | |
| run: | | |
| echo '${{ steps.read_json.outputs.result }}' | |
| coverage: | |
| name: Update code coverage | |
| needs: [metadata, config] | |
| strategy: | |
| matrix: | |
| platform: [amd64] | |
| toolchain: [clang16] | |
| fail-fast: false | |
| uses: ./.github/workflows/generate_cc.yml | |
| secrets: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| with: | |
| platform: linux/${{ matrix.platform }} | |
| devdeps_image: ${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-{1}', matrix.platform, matrix.toolchain)] }} | |
| export_environment: false | |
| extdevdeps: | |
| name: Create dev environment | |
| needs: [metadata, config, openmpi] | |
| strategy: | |
| matrix: | |
| platform: ${{ fromJson(needs.metadata.outputs.platforms).ids }} | |
| cuda_version: ["12.6", "13.0"] | |
| fail-fast: false | |
| uses: ./.github/workflows/dev_environment.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platforms: ${{ fromJson(needs.metadata.outputs.platforms)[format('{0}', matrix.platform)].docker_flag }} | |
| dockerfile: build/devdeps.ext.Dockerfile | |
| build_config_id: cu${{ matrix.cuda_version }}-gcc11 | |
| build_args: | | |
| cuda_version=${{ matrix.cuda_version }} | |
| base_image=${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-gcc11', matrix.platform)] }} | |
| ompidev_image=${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-cu{1}-ompi', matrix.platform, matrix.cuda_version)] }} | |
| registry_cache_from: ${{ needs.metadata.outputs.cache_base }} | |
| update_registry_cache: ${{ needs.metadata.outputs.cache_target }} | |
| environment: ${{ needs.metadata.outputs.environment }} | |
| # needed only for the cloudposse GitHub action | |
| matrix_key: ${{ matrix.platform }}-cu${{ matrix.cuda_version }}-ext | |
| # This job is needed only when using the cloudposse GitHub action to read | |
| # the output of a matrix job. This is a workaround due to current GitHub | |
| # limitations that may not be needed if the work started here concludes: | |
| # https://github.com/actions/runner/pull/2477 | |
| ext_config: | |
| name: Configure build | |
| needs: extdevdeps | |
| runs-on: ubuntu-latest | |
| outputs: | |
| json: "${{ steps.read_json.outputs.result }}" | |
| steps: | |
| - uses: cloudposse/[email protected] | |
| id: read_json | |
| with: | |
| matrix-step-name: dev_environment | |
| - name: Print ext_config.outputs.json | |
| run: | | |
| echo '${{ steps.read_json.outputs.result }}' | |
| docker_images: | |
| name: Create Docker images | |
| needs: [metadata, config, ext_config] | |
| strategy: | |
| matrix: | |
| platform: ${{ fromJson(needs.metadata.outputs.platforms).ids }} | |
| cuda_version: ["12.6", "13.0"] | |
| fail-fast: false | |
| uses: ./.github/workflows/docker_images.yml | |
| secrets: | |
| NGC_CREDENTIALS: ${{ secrets.NGC_CREDENTIALS }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platforms: ${{ fromJson(needs.metadata.outputs.platforms)[format('{0}', matrix.platform)].docker_flag }} | |
| cuda_version: ${{ matrix.cuda_version }} | |
| ompidev_image: ${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-cu{1}-ompi', matrix.platform, matrix.cuda_version)] }} | |
| devdeps_image: ${{ fromJson(needs.ext_config.outputs.json).image_hash[format('{0}-cu{1}-ext', matrix.platform, matrix.cuda_version)] }} | |
| build_docs: ${{ matrix.cuda_version == '12.6' && matrix.platform == fromJson(needs.metadata.outputs.platforms).ids[0] }} | |
| environment: ghcr-deployment | |
| digest_config: | |
| name: Aggregate digests | |
| needs: docker_images | |
| runs-on: ubuntu-latest | |
| outputs: | |
| json: "${{ steps.read.outputs.result }}" | |
| steps: | |
| - name: Read matrix outputs | |
| id: read | |
| uses: cloudposse/[email protected] | |
| with: | |
| matrix-step-name: docker_images | |
| - name: Print full digest config JSON | |
| run: | | |
| echo "Received matrix outputs:" | |
| echo '${{ steps.read.outputs.result }}' | jq . | |
| stitch: | |
| name: Stitch CUDA ${{ matrix.cuda }} images | |
| needs: [digest_config, ext_config] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| cuda: ["12.6", "13.0"] | |
| environment: ghcr-deployment | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up environment | |
| run: | | |
| docker context create builder_context || echo "Context exists" | |
| docker context use builder_context | |
| - name: Setup buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| endpoint: builder_context | |
| - name: Login to GitHub CR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ github.token }} | |
| - name: Log in to NGC | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: nvcr.io | |
| username: '$oauthtoken' | |
| password: ${{ secrets.NGC_CREDENTIALS }} | |
| - name: metadata for jobs | |
| id: metadata | |
| run: | | |
| set -euo pipefail | |
| DIGESTS_JSON='${{ needs.digest_config.outputs.json }}' | |
| EXT_JSON='${{ needs.ext_config.outputs.json }}' | |
| echo "digests_json<<EOF" | tee -a "$GITHUB_OUTPUT" | |
| echo "$DIGESTS_JSON" | tee -a "$GITHUB_OUTPUT" | |
| echo "EOF" | tee -a "$GITHUB_OUTPUT" | |
| echo "ext_json<<EOF" | tee -a "$GITHUB_OUTPUT" | |
| echo "$EXT_JSON" | tee -a "$GITHUB_OUTPUT" | |
| echo "EOF" | tee -a "$GITHUB_OUTPUT" | |
| - name: Stitch devdeps image | |
| id: stitch_devdeps_image | |
| run: | | |
| cuda="${{ matrix.cuda }}" | |
| cuda_major="${cuda%%.*}" | |
| EXT_JSON='${{ steps.metadata.outputs.ext_json }}' | |
| DIGESTS_JSON='${{ steps.metadata.outputs.digests_json }}' | |
| refs="" | |
| for arch in arm64 amd64; do | |
| key="${arch}-cu${cuda}-ext" | |
| image=$(echo "$EXT_JSON" | jq -r ".image_hash[\"$key\"]") | |
| refs="$refs $image" | |
| done | |
| registry=$(echo "$image" | cut -d'@' -f1) | |
| # Retrieve image tag from DIGEST_JSON, just look at amd64 and filter it out | |
| key="amd64-cu${cuda}-dev-image" | |
| dev_image=$(echo "$DIGESTS_JSON" | jq -r --arg k "$key" '.image_tag[$k]') | |
| # Remove 'amd64-' from the whole string | |
| stitched_tag="${dev_image/amd64-/}" | |
| # push to cuda-quantum-devdeps instead of cuda-quantum-dev | |
| stitched_tag="${stitched_tag/cuda-quantum-dev:/cuda-quantum-devdeps:}" | |
| echo "Stitching devdeps image with tag: $stitched_tag" | |
| echo "References: $refs" | |
| docker buildx imagetools create --tag "$stitched_tag" $refs | |
| digest=$(docker buildx imagetools inspect "$stitched_tag" | awk '/^Digest:/ { print $2 }') | |
| stitched_output=$(cat stitched_digests.json 2>/dev/null || echo '{}') | |
| stitched_output=$(echo "$stitched_output" | jq --arg key "devdeps-cu${cuda_major}" --arg val "$registry@$digest" '. + {($key): $val}') | |
| echo "$stitched_output" | tee stitched_digests.json | |
| - name: Stitch debug image | |
| id: stitch_debug_image | |
| run: | | |
| set -euo pipefail | |
| DIGESTS_JSON='${{ steps.metadata.outputs.digests_json }}' | |
| cuda="${{ matrix.cuda }}" | |
| cuda_major="${cuda%%.*}" | |
| refs="" | |
| for arch in amd64 arm64; do | |
| key="${arch}-cu${cuda}-dev-image" | |
| digest=$(echo "$DIGESTS_JSON" | jq -r ".digest[\"$key\"]") | |
| arch_reference=$(echo "$DIGESTS_JSON" | jq -r ".image_tag[\"$key\"]") | |
| registry=$(echo "$arch_reference" | cut -d':' -f1) | |
| refs="$refs $registry@$digest" | |
| done | |
| # Retrieve image tag from DIGEST_JSON, just look at amd64 and filter it out | |
| key="amd64-cu${cuda}-dev-image" | |
| dev_image=$(echo "$DIGESTS_JSON" | jq -r --arg k "$key" '.image_tag[$k]') | |
| # Remove 'amd64-' from the whole string | |
| stitched_tag="${dev_image/amd64-/}" | |
| echo "Stitching debug image with tag: $stitched_tag" | |
| echo "References: $refs" | |
| docker buildx imagetools create --tag "$stitched_tag" $refs | |
| digest=$(docker buildx imagetools inspect "$stitched_tag" | awk '/^Digest:/ { print $2 }') | |
| stitched_output=$(cat stitched_digests.json 2>/dev/null || echo '{}') | |
| stitched_output=$(echo "$stitched_output" | jq --arg key "dev-cu${cuda_major}" --arg val "$registry@$digest" '. + {($key): $val}') | |
| echo "$stitched_output" | tee stitched_digests.json | |
| - name: Stitch release image | |
| id: stitch_release_image | |
| run: | | |
| set -euo pipefail | |
| DIGESTS_JSON='${{ steps.metadata.outputs.digests_json }}' | |
| cuda="${{ matrix.cuda }}" | |
| cuda_major="${cuda%%.*}" | |
| image="image" | |
| refs="" | |
| for arch in amd64 arm64; do | |
| key="${arch}-cu${cuda}-${image}" | |
| digest=$(echo "$DIGESTS_JSON" | jq -r ".digest[\"$key\"]") | |
| arch_reference=$(echo "$DIGESTS_JSON" | jq -r ".image_tag[\"$key\"]") | |
| registry=$(echo "$arch_reference" | cut -d':' -f1) | |
| refs="$refs $registry@$digest" | |
| done | |
| is_versioned=${{ github.ref_type == 'tag' || startsWith(github.ref_name, 'releases/') || startsWith(github.ref_name, 'staging/') }} | |
| has_continuous_deployment=${{ startsWith(github.ref_name, 'experimental/') || github.ref_name == 'main' }} | |
| push_to_ngc=`($is_versioned || $has_continuous_deployment) && echo true || echo` | |
| echo "push to NGC? $push_to_ngc" | |
| if [[ "$push_to_ngc" == "true" ]]; then | |
| registry=${registry/ghcr.io\/nvidia/nvcr.io\/nvidia\/nightly} | |
| fi | |
| image_tag="cu${cuda_major}-" | |
| if ${{ github.event.pull_request.number != '' }} || [ -n "$(echo ${{ github.ref_name }} | grep pull-request/)" ]; then | |
| pr_number=`echo ${{ github.ref_name }} | grep -o [0-9]*` | |
| image_tag+=pr-${pr_number:-${{ github.event.pull_request.number }}} | |
| elif ${{ github.ref_type == 'branch' && github.ref_name == 'main' }}; then | |
| image_tag+="latest" | |
| elif "$is_versioned" == "true"; then | |
| image_tag+=`echo ${{ github.ref_name }} | egrep -o "([0-9]{1,}\.)+[0-9]{1,}"` | |
| else | |
| image_tag+=`echo ${{ github.ref_name }} | tr '/' '-'` | |
| fi | |
| suffix="-base" | |
| image_tag="${image_tag}${suffix}" | |
| stitched_tag="${registry}:${image_tag}" | |
| echo "Stitching release image with tag: $stitched_tag" | |
| echo "References: $refs" | |
| docker buildx imagetools create --tag "$stitched_tag" $refs | |
| digest=$(docker buildx imagetools inspect "$stitched_tag" | awk '/^Digest:/ { print $2 }') | |
| stitched_output=$(cat stitched_digests.json 2>/dev/null || echo '{}') | |
| stitched_output=$(echo "$stitched_output" | jq --arg key "base-cu${cuda_major}" --arg val "$registry@$digest" '. + {($key): $val}') | |
| echo "$stitched_output" | tee stitched_digests.json | |
| - name: Read stitched_digests.json | |
| id: stitched_json | |
| run: | | |
| echo "json=$(jq -c . stitched_digests.json)" | tee -a $GITHUB_OUTPUT | |
| - name: Write matrix outputs | |
| uses: cloudposse/[email protected] | |
| with: | |
| matrix-step-name: stitched_images | |
| matrix-key: cu${{ matrix.cuda }} | |
| outputs: | | |
| json: ${{ steps.stitched_json.outputs.json }} | |
| aggregate_stitched_images: | |
| name: Aggregate Stitched Images | |
| needs: stitch | |
| runs-on: ubuntu-latest | |
| outputs: | |
| json: "${{ steps.read.outputs.result }}" | |
| steps: | |
| - name: Read matrix outputs | |
| id: read | |
| uses: cloudposse/[email protected] | |
| with: | |
| matrix-step-name: stitched_images | |
| - name: Print full digest config JSON | |
| run: | | |
| echo "Received matrix outputs:" | |
| echo '${{ steps.read.outputs.result }}' | jq . | |
| summary: | |
| name: Write deployment summary | |
| needs: [metadata, aggregate_stitched_images] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Create image summary | |
| id: summary | |
| run: | | |
| STITCHED_JSON='${{ needs.aggregate_stitched_images.outputs.json }}' | |
| echo "$STITCHED_JSON" | jq . | |
| for cuda in 12.6 13.0; do | |
| cuda_major="${cuda%%.*}" | |
| artifact_name="image_cu${cuda_major}_publishing" | |
| info_file="$artifact_name.txt" | |
| key="base-cu${cuda_major}" | |
| cudaq_reference=$(echo "$STITCHED_JSON" | jq -r --arg cuda "cu${cuda}" --arg key "$key" '.json[$cuda][$key]') | |
| key="dev-cu${cuda_major}" | |
| dev_reference=$(echo "$STITCHED_JSON" | jq -r --arg cuda "cu${cuda}" --arg key "$key" '.json[$cuda][$key]') | |
| key="devdeps-cu${cuda_major}" | |
| devdeps_reference=$(echo "$STITCHED_JSON" | jq -r --arg cuda "cu${cuda}" --arg key "$key" '.json[$cuda][$key]') | |
| # Parse JSON and join docker_flag values (e.g., "linux/amd64,linux/arm64") | |
| platforms=$(echo '${{ needs.metadata.outputs.platforms }}' \ | |
| | jq -r '[.ids[] as $id | .[$id].docker_flag] | join(",")') | |
| { | |
| echo "source-sha: ${{ github.sha }}" | |
| echo "cuda-quantum-image: ${cudaq_reference}" | |
| echo "cuda-quantum-dev-image: ${dev_reference}" | |
| echo "cuda-quantum-devdeps-image: ${devdeps_reference}" | |
| echo "platforms: ${platforms}" | |
| echo "cuda-version: ${cuda}" | |
| cat .github/workflows/config/gitlab_commits.txt | |
| } | tee "$info_file" | |
| echo "Created $info_file" | |
| done | |
| - name: Upload image_cu12_publishing | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: image_cu12_publishing | |
| path: image_cu12_publishing.txt | |
| retention-days: 30 | |
| if-no-files-found: error | |
| - name: Upload image_cu13_publishing | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: image_cu13_publishing | |
| path: image_cu13_publishing.txt | |
| retention-days: 30 | |
| if-no-files-found: error | |
| python_wheels: | |
| name: Create Python wheels | |
| needs: [metadata, config] | |
| strategy: | |
| matrix: | |
| platform: ${{ fromJson(needs.metadata.outputs.platforms).ids }} | |
| python_version: ['3.11', '3.12', '3.13'] | |
| cuda_version: ["12.6", "13.0"] | |
| fail-fast: false | |
| uses: ./.github/workflows/python_wheels.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platform: ${{ fromJson(needs.metadata.outputs.platforms)[format('{0}', matrix.platform)].docker_flag }} | |
| python_version: ${{ matrix.python_version }} | |
| cuda_version: ${{ matrix.cuda_version }} | |
| devdeps_image: ${{ fromJson(needs.config.outputs.json).image_hash[format('{0}-cu{1}-python', matrix.platform, matrix.cuda_version)] }} | |
| create_staging_info: ${{ matrix.python_version == '3.11' }} # staging info is the same for all python versions | |
| binaries: | |
| name: Create CUDA Quantum installer | |
| needs: [metadata, config] | |
| strategy: | |
| matrix: | |
| platform: ${{ fromJson(needs.metadata.outputs.platforms).ids }} | |
| cuda_version: ["12.6", "13.0"] | |
| fail-fast: false | |
| uses: ./.github/workflows/prebuilt_binaries.yml | |
| secrets: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_READONLY_TOKEN: ${{ secrets.DOCKERHUB_READONLY_TOKEN }} | |
| with: | |
| platform: ${{ fromJson(needs.metadata.outputs.platforms)[format('{0}', matrix.platform)].docker_flag }} | |
| platform_base_image: ${{ fromJson(needs.metadata.outputs.platforms)[format('{0}', matrix.platform)].minimal_base_image }} | |
| build_config_id: cu${{ matrix.cuda_version }}-llvm | |
| cuda_version: ${{ matrix.cuda_version }} | |
| build_cache: ${{ fromJson(needs.config.outputs.json).build_cache[format('{0}-cu{1}-installer', matrix.platform, matrix.cuda_version)] }} | |
| environment: ghcr-deployment |