Release Windows packages #100
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release Windows packages | |
| on: | |
| # Trigger from another workflow (typically to build dev packages and then test them) | |
| workflow_call: | |
| inputs: | |
| release_type: | |
| description: The type of build version to produce ("nightly", or "dev") | |
| type: string | |
| default: "dev" | |
| package_suffix: | |
| type: string | |
| s3_subdir: | |
| description: "Subdirectory to push the Python packages" | |
| type: string | |
| default: "v2" | |
| s3_staging_subdir: | |
| description: "Staging subdirectory to push the packages" | |
| type: string | |
| default: "v2-staging" | |
| repository: | |
| description: "Repository to checkout. Otherwise, defaults to `github.repository`." | |
| type: string | |
| ref: | |
| description: "Branch, tag or SHA to checkout. Defaults to the reference or SHA that triggered the workflow." | |
| type: string | |
| # Trigger manually (typically to test the workflow or manually build a release [candidate]) | |
| workflow_dispatch: | |
| inputs: | |
| release_type: | |
| description: The type of build version to produce ("nightly", or "dev") | |
| type: string | |
| default: "dev" | |
| package_suffix: | |
| type: string | |
| s3_subdir: | |
| description: "Subdirectory to push the Python packages" | |
| type: string | |
| default: "v2" | |
| s3_staging_subdir: | |
| description: "Staging subdirectory to push the packages" | |
| type: string | |
| default: "v2-staging" | |
| families: | |
| description: "A comma separated list of AMD GPU families, e.g. `gfx94X,gfx103x`, or empty for the default list" | |
| type: string | |
| extra_cmake_options: | |
| description: "Extra options to pass to the CMake configure command" | |
| type: string | |
| # Trigger on a schedule to build nightly release candidates. | |
| schedule: | |
| # Runs at 04:00 AM UTC, which is 8:00 PM PST (UTC-8) | |
| - cron: '0 04 * * *' | |
| permissions: | |
| contents: read | |
| jobs: | |
| setup_metadata: | |
| if: ${{ github.repository_owner == 'ROCm' || github.event_name != 'schedule' }} | |
| runs-on: ubuntu-24.04 | |
| env: | |
| S3_SUBDIR: ${{ inputs.s3_subdir || 'v2' }} | |
| S3_STAGING_SUBDIR: ${{ inputs.s3_staging_subdir || 'v2-staging' }} | |
| release_type: ${{ inputs.release_type || 'nightly' }} | |
| outputs: | |
| version: ${{ steps.release_information.outputs.version }} | |
| release_type: ${{ env.release_type }} | |
| package_targets: ${{ steps.configure.outputs.package_targets }} | |
| cloudfront_url: ${{ steps.release_information.outputs.cloudfront_url }} | |
| cloudfront_staging_url: ${{ steps.release_information.outputs.cloudfront_staging_url }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: ${{ inputs.repository || github.repository }} | |
| ref: ${{ inputs.ref || '' }} | |
| - name: Setup Python | |
| uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 | |
| with: | |
| python-version: 3.12 | |
| # Compute version suffix based on inputs (default to 'rc') | |
| - name: Set variables for nightly release | |
| if: ${{ env.release_type == 'nightly' }} | |
| run: | | |
| version_suffix="$(printf 'rc%(%Y%m%d)T')" | |
| echo "version_suffix=${version_suffix}" >> $GITHUB_ENV | |
| echo "cloudfront_base_url=https://rocm.nightlies.amd.com" >> $GITHUB_ENV | |
| - name: Set variables for development release | |
| if: ${{ env.release_type == 'dev' }} | |
| run: | | |
| version_suffix=".dev0+${{ github.sha }}" | |
| echo "version_suffix=${version_suffix}" >> $GITHUB_ENV | |
| echo "cloudfront_base_url=https://d25kgig7rdsyks.cloudfront.net" >> $GITHUB_ENV | |
| - name: Generate release information | |
| id: release_information | |
| run: | | |
| base_version=$(jq -r '.["rocm-version"]' version.json) | |
| echo "version=${base_version}${version_suffix}" >> $GITHUB_OUTPUT | |
| echo "cloudfront_url=${cloudfront_base_url}/${{ env.S3_SUBDIR }}" >> $GITHUB_OUTPUT | |
| echo "cloudfront_staging_url=${cloudfront_base_url}/${{ env.S3_STAGING_SUBDIR }}" >> $GITHUB_OUTPUT | |
| - name: Generating package target matrix | |
| id: configure | |
| env: | |
| AMDGPU_FAMILIES: ${{ inputs.families }} | |
| THEROCK_PACKAGE_PLATFORM: "windows" | |
| run: python ./build_tools/github_actions/fetch_package_targets.py | |
| windows_packages: | |
| name: ${{ matrix.target_bundle.amdgpu_family }}::Build Windows | |
| runs-on: ${{ github.repository_owner == 'ROCm' && 'azure-windows-scale-rocm' || 'windows-2022' }} | |
| needs: [setup_metadata] | |
| permissions: | |
| contents: write | |
| actions: write # Added permission to trigger workflows | |
| id-token: write # Added permission for AWS S3 upload | |
| defaults: | |
| run: | |
| shell: bash | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target_bundle: ${{ fromJSON(needs.setup_metadata.outputs.package_targets) }} | |
| env: | |
| TEATIME_LABEL_GH_GROUP: 1 | |
| BUILD_DIR: B:\build | |
| CACHE_DIR: "${{github.workspace}}/.cache" | |
| CCACHE_DIR: "${{github.workspace}}/.cache/ccache" | |
| CCACHE_MAXSIZE: "4000M" | |
| DIST_ARCHIVE: "B:/build/artifacts/therock-dist-windows-${{ matrix.target_bundle.amdgpu_family }}${{ inputs.package_suffix }}-${{ needs.setup_metadata.outputs.version }}.tar.gz" | |
| RELEASE_TYPE: "${{ needs.setup_metadata.outputs.release_type }}" | |
| S3_BUCKET_TAR: "therock-${{ needs.setup_metadata.outputs.release_type }}-tarball" | |
| S3_BUCKET_PY: "therock-${{ needs.setup_metadata.outputs.release_type }}-python" | |
| S3_SUBDIR: ${{ inputs.s3_subdir || 'v2' }} | |
| S3_STAGING_SUBDIR: ${{ inputs.s3_staging_subdir || 'v2-staging' }} | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: ${{ inputs.repository || github.repository }} | |
| ref: ${{ inputs.ref || '' }} | |
| - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 | |
| with: | |
| python-version: 3.12 | |
| - name: Install python deps | |
| run: | | |
| pip install -r requirements.txt | |
| # TODO(amd-justchen): share with build_windows_packages.yml. Include in VM image? Dockerfile? | |
| - name: Install requirements | |
| run: | | |
| choco install --no-progress -y ccache | |
| # ninja pinned due to a bug in the 1.13.0 release: | |
| # https://github.com/ninja-build/ninja/issues/2616 | |
| choco install --no-progress -y ninja --version 1.12.1 | |
| choco install --no-progress -y strawberryperl | |
| echo "$PATH;C:\Strawberry\c\bin" >> $GITHUB_PATH | |
| choco install --no-progress -y awscli | |
| echo "$PATH;C:\Program Files\Amazon\AWSCLIV2" >> $GITHUB_PATH | |
| - uses: iterative/setup-dvc@4bdfd2b0f6f1ad7e08afadb03b1a895c352a5239 # v2.0.0 | |
| with: | |
| version: '3.62.0' | |
| # After other installs, so MSVC get priority in the PATH. | |
| - name: Configure MSVC | |
| uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 | |
| # TODO(#762): Move to script and share with build_windows_packages.yml. | |
| - name: Runner health status | |
| run: | | |
| ccache --zero-stats | |
| python ./build_tools/health_status.py | |
| # TODO: We shouldn't be using a cache on actual release branches, but it | |
| # really helps for iteration time. | |
| - name: Enable cache | |
| uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 | |
| with: | |
| path: ${{ env.CACHE_DIR }} | |
| key: windows-package-matrix-v1-${{ matrix.target_bundle.amdgpu_family }}-${{ github.sha }} | |
| restore-keys: | | |
| windows-package-matrix-v1-${{ matrix.target_bundle.amdgpu_family }}- | |
| - name: Fetch sources | |
| timeout-minutes: 30 | |
| run: | | |
| git config fetch.parallel 10 | |
| git config --global core.symlinks true | |
| git config --global core.longpaths true | |
| python ./build_tools/fetch_sources.py --jobs 12 | |
| - name: Configure Projects | |
| env: | |
| amdgpu_families: ${{ matrix.target_bundle.amdgpu_family }} | |
| package_version: "ADHOCBUILD" | |
| extra_cmake_options: ${{ inputs.extra_cmake_options }} | |
| run: | | |
| # clear cache before build and after download | |
| ccache -z | |
| python3 build_tools/github_actions/build_configure.py | |
| - name: Build therock-dist | |
| run: cmake --build "${{ env.BUILD_DIR }}" --target therock-dist | |
| - name: Build therock-archives | |
| run: cmake --build "${{ env.BUILD_DIR }}" --target therock-archives | |
| - name: Compress dist folder | |
| run: | | |
| cd ${{ env.BUILD_DIR }}/dist/rocm | |
| echo "Compressing ${{ env.DIST_ARCHIVE }}" | |
| tar cfz "${{ env.DIST_ARCHIVE }}" --force-local . | |
| - name: Build Python Packages | |
| run: | | |
| python ./build_tools/build_python_packages.py \ | |
| --artifact-dir=${{ env.BUILD_DIR }}/artifacts \ | |
| --dest-dir=${{ env.BUILD_DIR }}/packages \ | |
| --version=${{ needs.setup_metadata.outputs.version }} | |
| - name: Build report | |
| if: ${{ !cancelled() }} | |
| shell: bash | |
| run: | | |
| if [ -d "${{ env.BUILD_DIR }}" ]; then | |
| echo "Build dir:" | |
| echo "------------" | |
| ls -lh "${{ env.BUILD_DIR }}" | |
| echo "CCache Stats:" | |
| echo "-------------" | |
| ccache -s | |
| else | |
| echo "[ERROR] Build directory ${{ env.BUILD_DIR }} does not exist. Skipping report!" | |
| echo " This should only happen if the CI is cancelled before the build step." | |
| exit 1 # Stop the CI as build did not happen | |
| fi | |
| - name: Configure AWS Credentials | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| aws-region: us-east-2 | |
| role-to-assume: arn:aws:iam::692859939525:role/therock-${{ env.RELEASE_TYPE }}-releases | |
| - name: Upload Releases to staging S3 | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| run: | | |
| aws s3 cp ${{ env.DIST_ARCHIVE }} s3://${{ env.S3_BUCKET_TAR }} | |
| aws s3 cp ${{ env.BUILD_DIR }}/packages/dist/ s3://${{ env.S3_BUCKET_PY }}/${{ env.S3_STAGING_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }}/ \ | |
| --recursive --no-follow-symlinks \ | |
| --exclude "*" \ | |
| --include "*.whl" \ | |
| --include "*.tar.gz" | |
| - name: (Re-)Generate Python package release index for staging | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| run: | | |
| pip install boto3 packaging | |
| python ./build_tools/third_party/s3_management/manage.py ${{ env.S3_STAGING_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }} | |
| ## TODO: Restrict uploading to the non-staging S3 directory until sanity checks and all validation tests have successfully passed. | |
| - name: Upload Releases to S3 | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| run: | | |
| aws s3 cp ${{ env.DIST_ARCHIVE }} s3://${{ env.S3_BUCKET_TAR }} | |
| aws s3 cp ${{ env.BUILD_DIR }}/packages/dist/ s3://${{ env.S3_BUCKET_PY }}/${{ env.S3_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }}/ \ | |
| --recursive --no-follow-symlinks \ | |
| --exclude "*" \ | |
| --include "*.whl" \ | |
| --include "*.tar.gz" | |
| # TODO(marbre): guard against race conditions where multiple workflows update the index at the same time? | |
| # Moving the index computation server-side could help | |
| - name: (Re-)Generate Python package release index | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| run: | | |
| pip install boto3 packaging | |
| python ./build_tools/third_party/s3_management/manage.py ${{ env.S3_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }} | |
| - name: Trigger building PyTorch wheels | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| uses: benc-uk/workflow-dispatch@e2e5e9a103e331dad343f381a29e654aea3cf8fc # v1.2.4 | |
| with: | |
| workflow: release_windows_pytorch_wheels.yml | |
| inputs: | | |
| { "amdgpu_family": "${{ matrix.target_bundle.amdgpu_family }}", | |
| "release_type": "${{ env.RELEASE_TYPE }}", | |
| "s3_subdir": "${{ env.S3_SUBDIR }}", | |
| "s3_staging_subdir": "${{ env.S3_STAGING_SUBDIR }}", | |
| "cloudfront_url": "${{ needs.setup_metadata.outputs.cloudfront_url }}", | |
| "cloudfront_staging_url": "${{ needs.setup_metadata.outputs.cloudfront_staging_url }}", | |
| "rocm_version": "${{ needs.setup_metadata.outputs.version }}", | |
| "ref": "${{ inputs.ref || '' }}" | |
| } | |
| - name: Save cache | |
| uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 | |
| if: ${{ !cancelled() }} | |
| with: | |
| path: ${{ env.CACHE_DIR }} | |
| key: windows-package-matrix-v1-${{ matrix.target_bundle.amdgpu_family }}-${{ github.sha }} |