Update S3 HTML dependencies for download.pytorch.org nightly and test #21
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update S3 HTML dependencies for download.pytorch.org nightly and test | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| mode: | |
| description: 'Operation mode' | |
| required: true | |
| type: choice | |
| default: update-packages | |
| options: | |
| - update-packages | |
| - create-target | |
| dryrun: | |
| description: 'Enable dry run mode' | |
| required: true | |
| type: choice | |
| default: enabled | |
| options: | |
| - enabled | |
| - disabled | |
| package: | |
| description: 'Package to update (only for update-packages mode)' | |
| required: false | |
| type: choice | |
| default: torch | |
| options: | |
| - all | |
| - torch | |
| - triton | |
| - torchtune | |
| - torch_xpu | |
| - vllm | |
| target: | |
| description: 'Target to create (e.g., rocm7.2, cu130) - only for create-target mode' | |
| required: false | |
| type: string | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| update: | |
| runs-on: ubuntu-22.04 | |
| environment: pytorchbot-env | |
| container: | |
| image: continuumio/miniconda3:23.10.0-1 | |
| steps: | |
| - name: Validate inputs for create-target mode | |
| if: ${{ github.event.inputs.mode == 'create-target' }} | |
| shell: bash | |
| env: | |
| TARGET: ${{ github.event.inputs.target }} | |
| run: | | |
| set -e | |
| if [[ -z "${TARGET}" ]]; then | |
| echo "ERROR: target is required for create-target mode" | |
| exit 1 | |
| fi | |
| # Validate target matches expected patterns | |
| if [[ ! "${TARGET}" =~ ^(cu[0-9]+|rocm[0-9]+\.[0-9]+)$ ]]; then | |
| echo "ERROR: Invalid target format '${TARGET}'" | |
| echo "Valid formats:" | |
| echo " - CUDA: cu118, cu121, cu126, cu128, cu129, cu130, etc." | |
| echo " - ROCm: rocm5.7, rocm6.0, rocm6.4, rocm7.1, rocm7.2, etc." | |
| exit 1 | |
| fi | |
| echo "Target '${TARGET}' is valid" | |
| - name: Configure AWS credentials | |
| id: aws_creds | |
| uses: aws-actions/configure-aws-credentials@50ac8dd1e1b10d09dac7b8727528b91bed831ac0 # v3.0.2 | |
| with: | |
| role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_s3_update | |
| aws-region: us-east-1 | |
| - name: Checkout repository test-infra | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: pytorch/test-infra | |
| ref: ${{ github.ref }} | |
| - name: Install dependencies | |
| run: | | |
| pip install -r s3_management/requirements.txt | |
| - name: Update packages | |
| if: ${{ github.event.inputs.mode == 'update-packages' }} | |
| shell: bash | |
| env: | |
| DRYRUN: ${{ github.event.inputs.dryrun }} | |
| PACKAGE: ${{ github.event.inputs.package || 'torch' }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_PYTORCH_ACCOUNT_ID }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_PYTORCH_PUBLIC_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_PYTORCH_PUBLIC_SECRET_ACCESS_KEY }} | |
| R2_BUCKET_NAME: "pytorch-downloads" | |
| run: | | |
| set -ex | |
| ARGS="--package ${PACKAGE}" | |
| if [[ "${DRYRUN}" == "enabled" ]]; then | |
| ARGS="${ARGS} --dry-run" | |
| fi | |
| # shellcheck disable=SC2086 | |
| python3 s3_management/update_dependencies.py ${ARGS} | |
| - name: Create new target | |
| if: ${{ github.event.inputs.mode == 'create-target' }} | |
| shell: bash | |
| env: | |
| TARGET: ${{ github.event.inputs.target }} | |
| DRYRUN: ${{ github.event.inputs.dryrun }} | |
| R2_ACCOUNT_ID: ${{ secrets.R2_PYTORCH_ACCOUNT_ID }} | |
| R2_ACCESS_KEY_ID: ${{ secrets.R2_PYTORCH_PUBLIC_ACCESS_KEY_ID }} | |
| R2_SECRET_ACCESS_KEY: ${{ secrets.R2_PYTORCH_PUBLIC_SECRET_ACCESS_KEY }} | |
| R2_BUCKET_NAME: "pytorch-downloads" | |
| run: | | |
| set -ex | |
| # Create the target directory with torch dependencies | |
| ARGS="--create-target --target ${TARGET}" | |
| if [[ "${DRYRUN}" == "enabled" ]]; then | |
| ARGS="${ARGS} --dry-run" | |
| fi | |
| # shellcheck disable=SC2086 | |
| python3 s3_management/update_dependencies.py ${ARGS} | |
| # Run update-packages for torch to ensure all dependencies are current | |
| ARGS="--package torch" | |
| if [[ "${DRYRUN}" == "enabled" ]]; then | |
| ARGS="${ARGS} --dry-run" | |
| fi | |
| # shellcheck disable=SC2086 | |
| python3 s3_management/update_dependencies.py ${ARGS} |