Merge remote-tracking branch 'upstream/release/2.10' into release/2.1… #6
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Retrieval PyTorch Tests for Target Determination | ||
| on: | ||
| workflow_call: | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| jobs: | ||
| get-label-type: | ||
| name: get-label-type | ||
| # Don't run on forked repos | ||
| if: github.repository_owner == 'pytorch' | ||
| <<<<<<< HEAD | ||
| uses: pytorch/pytorch/.github/workflows/_runner-determinator.yml@release/2.11 | ||
| ======= | ||
| uses: pytorch/pytorch/.github/workflows/_runner-determinator.yml@release/2.10 | ||
| >>>>>>> upstream/release/2.10 | ||
| with: | ||
| triggering_actor: ${{ github.triggering_actor }} | ||
| issue_owner: ${{ github.event.pull_request.user.login || github.event.issue.user.login }} | ||
| curr_branch: ${{ github.head_ref || github.ref_name }} | ||
| curr_ref_type: ${{ github.ref_type }} | ||
| llm-retrieval: | ||
| # Don't run on forked repos | ||
| if: github.repository_owner == 'pytorch' | ||
| runs-on: "${{ needs.get-label-type.outputs.label-type }}linux.4xlarge" | ||
| continue-on-error: true | ||
| needs: get-label-type | ||
| steps: | ||
| - name: Clone PyTorch | ||
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | ||
| with: | ||
| repository: pytorch/pytorch | ||
| fetch-depth: 0 | ||
| path: pytorch | ||
| - name: Setup Linux | ||
| uses: ./pytorch/.github/actions/setup-linux | ||
| - name: Clone CodeLlama | ||
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | ||
| with: | ||
| repository: osalpekar/codellama | ||
| ref: main | ||
| path: codellama | ||
| - name: Clone Target Determination Code | ||
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | ||
| with: | ||
| repository: osalpekar/llm-target-determinator | ||
| ref: v0.0.2 | ||
| path: llm-target-determinator | ||
| - name: Install requirements | ||
| shell: bash | ||
| run: | | ||
| set -euxo pipefail | ||
| python3 -m pip install -r llm-target-determinator/requirements.txt | ||
| cd "${GITHUB_WORKSPACE}/codellama" | ||
| python3 -m pip install -e . | ||
| - name: Fetch CodeLlama Checkpoint | ||
| shell: bash | ||
| run: | | ||
| set -euxo pipefail | ||
| cd "${GITHUB_WORKSPACE}/codellama" | ||
| mkdir "CodeLlama-7b-Python" | ||
| aws s3 cp "s3://target-determinator-assets/CodeLlama-7b-Python" "CodeLlama-7b-Python" --recursive --no-progress | ||
| - name: Fetch indexes | ||
| uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0 | ||
| with: | ||
| max_attempts: 3 | ||
| retry_wait_seconds: 10 | ||
| timeout_minutes: 5 | ||
| shell: bash | ||
| command: | | ||
| set -euxo pipefail | ||
| python3 -m pip install awscli==1.29.40 | ||
| cd "${GITHUB_WORKSPACE}"/llm-target-determinator/assets | ||
| aws s3 cp "s3://target-determinator-assets/indexes/latest" . --recursive | ||
| unzip -o indexer-files\*.zip | ||
| rm indexer-files*.zip | ||
| - name: Run Retriever | ||
| id: run_retriever | ||
| continue-on-error: true # ghstack not currently supported due to problems getting git diff | ||
| shell: bash | ||
| run: | | ||
| set -euxo pipefail | ||
| cd "${GITHUB_WORKSPACE}"/llm-target-determinator | ||
| export PATH="$HOME/.local/bin:$PATH" | ||
| torchrun \ | ||
| --standalone \ | ||
| --nnodes=1 \ | ||
| --nproc-per-node=1 \ | ||
| retriever.py \ | ||
| --experiment-name indexer-files \ | ||
| --pr-parse-format GITDIFF | ||
| cd assets | ||
| zip -r mappings.zip mappings | ||
| - name: Upload results to s3 | ||
| uses: seemethere/upload-artifact-s3@baba72d0712b404f646cebe0730933554ebce96a # v5.1.0 | ||
| if: ${{ steps.run_retriever.outcome == 'success' }} | ||
| with: | ||
| name: llm_results | ||
| retention-days: 14 | ||
| if-no-files-found: warn | ||
| path: llm-target-determinator/assets/mappings.zip | ||
| env: | ||
| AWS_ACCESS_KEY_ID: "" | ||
| AWS_SECRET_ACCESS_KEY: "" | ||
| AWS_SESSION_TOKEN: "" | ||
| AWS_DEFAULT_REGION: "" | ||
| AWS_REGION: "" | ||
| - name: Teardown Linux | ||
| <<<<<<< HEAD | ||
| uses: pytorch/test-infra/.github/actions/teardown-linux@release/2.11 | ||
| ======= | ||
| uses: pytorch/test-infra/.github/actions/teardown-linux@release/2.10 | ||
| >>>>>>> upstream/release/2.10 | ||
| if: always() | ||
