Release portable Linux packages #97
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release portable Linux packages | |
| on: | |
| # Trigger from another workflow (typically to build dev packages and then test them) | |
| workflow_call: | |
| inputs: | |
| release_type: | |
| description: The type of build version to produce ("nightly", or "dev") | |
| type: string | |
| default: "dev" | |
| package_suffix: | |
| type: string | |
| s3_subdir: | |
| description: "Subdirectory to push the packages" | |
| type: string | |
| default: "v2" | |
| s3_staging_subdir: | |
| description: "Staging subdirectory to push the packages" | |
| type: string | |
| default: "v2-staging" | |
| repository: | |
| description: "Repository to checkout. Otherwise, defaults to `github.repository`." | |
| type: string | |
| ref: | |
| description: "Branch, tag or SHA to checkout. Defaults to the reference or SHA that triggered the workflow." | |
| type: string | |
| # Trigger manually (typically to test the workflow or manually build a release [candidate]) | |
| workflow_dispatch: | |
| inputs: | |
| release_type: | |
| description: The type of build version to produce ("nightly", or "dev") | |
| type: string | |
| default: "dev" | |
| package_suffix: | |
| type: string | |
| s3_subdir: | |
| description: "Subdirectory to push the packages" | |
| type: string | |
| default: "v2" | |
| s3_staging_subdir: | |
| description: "Staging subdirectory to push the packages" | |
| type: string | |
| default: "v2-staging" | |
| families: | |
| description: "Comma separated list of AMD GPU families, e.g. `gfx94X,gfx103x`" | |
| type: string | |
| # Trigger on a schedule to build nightly release candidates. | |
| schedule: | |
| # Runs at 04:00 AM UTC, which is 8:00 PM PST (UTC-8) | |
| - cron: '0 04 * * *' | |
| permissions: | |
| contents: read | |
| jobs: | |
| setup_metadata: | |
| if: ${{ github.repository_owner == 'ROCm' || github.event_name != 'schedule' }} | |
| runs-on: ubuntu-24.04 | |
| env: | |
| S3_SUBDIR: ${{ inputs.s3_subdir || 'v2' }} | |
| S3_STAGING_SUBDIR: ${{ inputs.s3_staging_subdir || 'v2-staging' }} | |
| release_type: ${{ inputs.release_type || 'nightly' }} | |
| outputs: | |
| version: ${{ steps.release_information.outputs.version }} | |
| release_type: ${{ env.release_type }} | |
| package_targets: ${{ steps.configure.outputs.package_targets }} | |
| cloudfront_url: ${{ steps.release_information.outputs.cloudfront_url }} | |
| cloudfront_staging_url: ${{ steps.release_information.outputs.cloudfront_staging_url }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: ${{ inputs.repository || github.repository }} | |
| ref: ${{ inputs.ref || '' }} | |
| - name: Setup Python | |
| uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 | |
| with: | |
| python-version: 3.12 | |
| # Compute version suffix based on inputs (default to 'rc') | |
| - name: Set variables for nightly release | |
| if: ${{ env.release_type == 'nightly' }} | |
| run: | | |
| version_suffix="$(printf 'rc%(%Y%m%d)T')" | |
| echo "version_suffix=${version_suffix}" >> $GITHUB_ENV | |
| echo "cloudfront_base_url=https://rocm.nightlies.amd.com" >> $GITHUB_ENV | |
| - name: Set variables for development release | |
| if: ${{ env.release_type == 'dev' }} | |
| run: | | |
| version_suffix=".dev0+${{ github.sha }}" | |
| echo "version_suffix=${version_suffix}" >> $GITHUB_ENV | |
| echo "cloudfront_base_url=https://d25kgig7rdsyks.cloudfront.net" >> $GITHUB_ENV | |
| - name: Generate release information | |
| id: release_information | |
| run: | | |
| base_version=$(jq -r '.["rocm-version"]' version.json) | |
| echo "version=${base_version}${version_suffix}" >> $GITHUB_OUTPUT | |
| echo "cloudfront_url=${cloudfront_base_url}/${{ env.S3_SUBDIR }}" >> $GITHUB_OUTPUT | |
| echo "cloudfront_staging_url=${cloudfront_base_url}/${{ env.S3_STAGING_SUBDIR }}" >> $GITHUB_OUTPUT | |
| - name: Generating package target matrix | |
| id: configure | |
| env: | |
| AMDGPU_FAMILIES: ${{ inputs.families }} | |
| THEROCK_PACKAGE_PLATFORM: "linux" | |
| run: python ./build_tools/github_actions/fetch_package_targets.py | |
| portable_linux_packages: | |
| name: ${{ matrix.target_bundle.amdgpu_family }}::Build Portable Linux | |
| runs-on: ${{ github.repository_owner == 'ROCm' && 'azure-linux-scale-rocm' || 'ubuntu-24.04' }} | |
| needs: [setup_metadata] | |
| permissions: | |
| contents: write | |
| actions: write # Added permission to trigger workflows | |
| id-token: write # Added permission for AWS S3 upload | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target_bundle: ${{ fromJSON(needs.setup_metadata.outputs.package_targets) }} | |
| env: | |
| TEATIME_LABEL_GH_GROUP: 1 | |
| OUTPUT_DIR: ${{ github.workspace }}/output | |
| BUILD_IMAGE: ghcr.io/rocm/therock_build_manylinux_x86_64@sha256:3ac188c17f88f08ce522297b616d8308361b8e9a9b31bcc3c3bbb9429b1efa6c | |
| DIST_ARCHIVE: "${{ github.workspace }}/output/therock-dist-linux-${{ matrix.target_bundle.amdgpu_family }}${{ inputs.package_suffix }}-${{ needs.setup_metadata.outputs.version }}.tar.gz" | |
| FILE_NAME: "therock-dist-linux-${{ matrix.target_bundle.amdgpu_family }}${{ inputs.package_suffix }}-${{ needs.setup_metadata.outputs.version }}.tar.gz" | |
| RELEASE_TYPE: "${{ needs.setup_metadata.outputs.release_type }}" | |
| S3_BUCKET_TAR: "therock-${{ needs.setup_metadata.outputs.release_type }}-tarball" | |
| S3_BUCKET_PY: "therock-${{ needs.setup_metadata.outputs.release_type }}-python" | |
| S3_SUBDIR: ${{ inputs.s3_subdir || 'v2' }} | |
| S3_STAGING_SUBDIR: ${{ inputs.s3_staging_subdir || 'v2-staging' }} | |
| steps: | |
| - name: "Checking out repository" | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: ${{ inputs.repository || github.repository }} | |
| ref: ${{ inputs.ref || '' }} | |
| - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 | |
| with: | |
| python-version: 3.12 | |
| # TODO: We shouldn't be using a cache on actual release branches, but it | |
| # really helps for iteration time. | |
| - name: Enable cache | |
| uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 | |
| with: | |
| path: ${{ env.OUTPUT_DIR }}/caches | |
| key: portable-linux-package-matrix-v1-${{ matrix.target_bundle.amdgpu_family }}-${{ github.sha }} | |
| restore-keys: | | |
| portable-linux-package-matrix-v1-${{ matrix.target_bundle.amdgpu_family }}- | |
| - name: Install the AWS tool | |
| run: ./dockerfiles/install_awscli.sh | |
| - name: Fetch sources | |
| timeout-minutes: 30 | |
| run: | | |
| # Prefetch docker container in background. | |
| docker pull ${{ env.BUILD_IMAGE }} & | |
| ./build_tools/fetch_sources.py --jobs 10 | |
| wait | |
| - name: Build Projects | |
| run: | | |
| ./build_tools/linux_portable_build.py \ | |
| --image=${{ env.BUILD_IMAGE }} \ | |
| --output-dir=${{ env.OUTPUT_DIR }} \ | |
| -- \ | |
| "-DTHEROCK_AMDGPU_FAMILIES=${{ matrix.target_bundle.amdgpu_family }}" | |
| cd ${{ env.OUTPUT_DIR }}/build/dist/rocm | |
| echo "Building ${{ env.DIST_ARCHIVE }}" | |
| tar cfz "${{ env.DIST_ARCHIVE }}" . | |
| - name: Setup Python | |
| uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 | |
| with: | |
| python-version: 3.12 | |
| - name: Build Python Packages | |
| run: | | |
| ./build_tools/linux_portable_build.py \ | |
| --image=${{ env.BUILD_IMAGE }} \ | |
| --output-dir=${{ env.OUTPUT_DIR }}/packages \ | |
| --build-python-only \ | |
| --artifact-dir=${{ env.OUTPUT_DIR }}/build/artifacts \ | |
| -- \ | |
| "--version=${{ needs.setup_metadata.outputs.version }}" | |
| - name: Build Report | |
| if: ${{ !cancelled() }} | |
| run: | | |
| echo "Full SDK du:" | |
| echo "------------" | |
| du -h -d 1 ${{ env.OUTPUT_DIR }}/build/dist/rocm | |
| - name: Configure AWS Credentials | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 | |
| with: | |
| aws-region: us-east-2 | |
| role-to-assume: arn:aws:iam::692859939525:role/therock-${{ env.RELEASE_TYPE }}-releases | |
| - name: Upload Releases to staging S3 | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| run: | | |
| aws s3 cp ${{ env.OUTPUT_DIR }}/packages/dist/ s3://${{ env.S3_BUCKET_PY }}/${{ env.S3_STAGING_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }}/ \ | |
| --recursive --no-follow-symlinks \ | |
| --exclude "*" \ | |
| --include "*.whl" \ | |
| --include "*.tar.gz" | |
| - name: (Re-)Generate Python package release index for staging | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| run: | | |
| pip install boto3 packaging | |
| python ./build_tools/third_party/s3_management/manage.py ${{ env.S3_STAGING_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }} | |
| ## TODO: Restrict uploading to the non-staging S3 directory until ROCm sanity checks and all validation tests have successfully passed. | |
| - name: Upload Releases to S3 | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| run: | | |
| aws s3 cp ${{ env.DIST_ARCHIVE }} s3://${{ env.S3_BUCKET_TAR }} | |
| aws s3 cp ${{ env.OUTPUT_DIR }}/packages/dist/ s3://${{ env.S3_BUCKET_PY }}/${{ env.S3_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }}/ \ | |
| --recursive --no-follow-symlinks \ | |
| --exclude "*" \ | |
| --include "*.whl" \ | |
| --include "*.tar.gz" | |
| - name: (Re-)Generate Python package release index | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| run: | | |
| pip install boto3 packaging | |
| python ./build_tools/third_party/s3_management/manage.py ${{ env.S3_SUBDIR }}/${{ matrix.target_bundle.amdgpu_family }} | |
| - name: Trigger building PyTorch wheels | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| uses: benc-uk/workflow-dispatch@e2e5e9a103e331dad343f381a29e654aea3cf8fc # v1.2.4 | |
| with: | |
| workflow: release_portable_linux_pytorch_wheels.yml | |
| inputs: | | |
| { "amdgpu_family": "${{ matrix.target_bundle.amdgpu_family }}", | |
| "release_type": "${{ env.RELEASE_TYPE }}", | |
| "s3_subdir": "${{ env.S3_SUBDIR }}", | |
| "s3_staging_subdir": "${{ env.S3_STAGING_SUBDIR }}", | |
| "cloudfront_url": "${{ needs.setup_metadata.outputs.cloudfront_url }}", | |
| "cloudfront_staging_url": "${{ needs.setup_metadata.outputs.cloudfront_staging_url }}", | |
| "rocm_version": "${{ needs.setup_metadata.outputs.version }}", | |
| "ref": "${{ inputs.ref || '' }}" | |
| } | |
| - name: URL-encode .tar URL | |
| id: url-encode-tar | |
| run: python -c "from urllib.parse import quote; print('tar_url=https://therock-${{ env.RELEASE_TYPE }}-tarball.s3.amazonaws.com/' + quote('therock-dist-linux-${{ matrix.target_bundle.amdgpu_family }}${{ inputs.package_suffix }}-${{ needs.setup_metadata.outputs.version }}.tar.gz'))" >> ${GITHUB_OUTPUT} | |
| - name: Trigger build JAX wheels | |
| if: ${{ github.repository_owner == 'ROCm' }} | |
| uses: benc-uk/workflow-dispatch@e2e5e9a103e331dad343f381a29e654aea3cf8fc # v1.2.4 | |
| with: | |
| workflow: build_linux_jax_wheels.yml | |
| inputs: | | |
| { "amdgpu_family": "${{ matrix.target_bundle.amdgpu_family }}", | |
| "python_versions": "3.12", | |
| "release_type": "${{ env.RELEASE_TYPE }}", | |
| "s3_subdir": "${{ env.S3_STAGING_SUBDIR }}", | |
| "rocm_version": "${{ needs.setup_metadata.outputs.version }}", | |
| "tar_url": "${{ steps.url-encode-tar.outputs.tar_url }}" | |
| } | |
| - name: Save cache | |
| uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 | |
| if: ${{ !cancelled() }} | |
| with: | |
| path: ${{ env.OUTPUT_DIR }}/caches | |
| key: portable-linux-package-matrix-v1-${{ matrix.target_bundle.amdgpu_family }}-${{ github.sha }} |