Stabilize DeepCompile ZeRO-3 memory scheduling #8446
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: cpu-torch-latest | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| torch_preset: | |
| description: PyTorch CPU preset to install for manual runs | |
| required: false | |
| default: '2.10.0-cpu' | |
| type: choice | |
| options: | |
| - '2.7.1-cpu' | |
| - '2.8.0-cpu' | |
| - '2.9.1-cpu' | |
| - '2.10.0-cpu' | |
| transformers_version: | |
| description: Hugging Face Transformers PyPI package version to install | |
| required: false | |
| default: '4.50.0' | |
| type: string | |
| transformers_source: | |
| description: Hugging Face Transformers source for manual runs | |
| required: false | |
| default: 'git' | |
| type: choice | |
| options: | |
| - 'pypi' | |
| - 'git' | |
| transformers_ref: | |
| description: Hugging Face Transformers git ref to install when source is git | |
| required: false | |
| default: 'main' | |
| type: string | |
| pull_request: | |
| merge_group: | |
| branches: [ master ] | |
| schedule: | |
| - cron: "0 0 * * *" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-paths: | |
| name: cpu-torch-latest / check paths | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| should_run: ${{ steps.non_pr.outputs.should_run || steps.filter.outputs.run_tests }} | |
| steps: | |
| - id: non_pr | |
| if: github.event_name != 'pull_request' | |
| run: echo "should_run=true" >> "$GITHUB_OUTPUT" | |
| - uses: actions/checkout@v4 | |
| if: github.event_name == 'pull_request' | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| if: github.event_name == 'pull_request' | |
| with: | |
| predicate-quantifier: every | |
| filters: | | |
| run_tests: | |
| - '**' | |
| - '!docs/**' | |
| - '!blogs/**' | |
| - '!deepspeed/inference/v2/**' | |
| - '!tests/unit/inference/v2/**' | |
| unit-tests: | |
| name: cpu-torch-latest / unit tests | |
| needs: check-paths | |
| if: ${{ !cancelled() && (needs.check-paths.result != 'success' || needs.check-paths.outputs.should_run == 'true') }} | |
| runs-on: ubuntu-24.04 | |
| env: | |
| DEFAULT_TORCH_PRESET: '2.10.0-cpu' | |
| DEFAULT_TRANSFORMERS_SOURCE: 'git' | |
| DEFAULT_TRANSFORMERS_VERSION: '4.50.0' | |
| DEFAULT_TRANSFORMERS_REF: 'main' | |
| steps: | |
| - name: Fail if path filter failed | |
| if: needs.check-paths.result != 'success' | |
| run: exit 1 | |
| - uses: actions/checkout@v4 | |
| - id: setup-venv | |
| uses: ./.github/workflows/setup-venv | |
| - name: Install system packages | |
| run: | | |
| sudo apt-get install -y numactl pdsh | |
| - name: Resolve dependency inputs | |
| env: | |
| GITHUB_EVENT_NAME: ${{ github.event_name }} | |
| MANUAL_TORCH_PRESET: ${{ github.event.inputs.torch_preset || '' }} | |
| MANUAL_TRANSFORMERS_SOURCE: ${{ github.event.inputs.transformers_source || '' }} | |
| MANUAL_TRANSFORMERS_VERSION: ${{ github.event.inputs.transformers_version || '' }} | |
| MANUAL_TRANSFORMERS_REF: ${{ github.event.inputs.transformers_ref || '' }} | |
| run: | | |
| if [ "$GITHUB_EVENT_NAME" = 'workflow_dispatch' ] && [ -n "$MANUAL_TORCH_PRESET" ]; then | |
| selected_preset="$MANUAL_TORCH_PRESET" | |
| else | |
| selected_preset="$DEFAULT_TORCH_PRESET" | |
| fi | |
| if [ "$GITHUB_EVENT_NAME" = 'workflow_dispatch' ] && [ -n "$MANUAL_TRANSFORMERS_SOURCE" ]; then | |
| transformers_source="$MANUAL_TRANSFORMERS_SOURCE" | |
| else | |
| transformers_source="$DEFAULT_TRANSFORMERS_SOURCE" | |
| fi | |
| if [ "$GITHUB_EVENT_NAME" = 'workflow_dispatch' ] && [ -n "$MANUAL_TRANSFORMERS_VERSION" ]; then | |
| transformers_version="$MANUAL_TRANSFORMERS_VERSION" | |
| else | |
| transformers_version="$DEFAULT_TRANSFORMERS_VERSION" | |
| fi | |
| if [ "$GITHUB_EVENT_NAME" = 'workflow_dispatch' ] && [ -n "$MANUAL_TRANSFORMERS_REF" ]; then | |
| transformers_ref="$MANUAL_TRANSFORMERS_REF" | |
| else | |
| transformers_ref="$DEFAULT_TRANSFORMERS_REF" | |
| fi | |
| if [ "$transformers_source" = 'git' ] && [ -z "$transformers_ref" ]; then | |
| transformers_ref='main' | |
| fi | |
| case "$selected_preset" in | |
| '2.7.1-cpu') | |
| torch_install_version='2.7.1' | |
| torchvision_install_version='0.22.1' | |
| torch_test_version='2.7' | |
| ;; | |
| '2.8.0-cpu') | |
| torch_install_version='2.8.0' | |
| torchvision_install_version='0.23.0' | |
| torch_test_version='2.8' | |
| ;; | |
| '2.9.1-cpu') | |
| torch_install_version='2.9.1' | |
| torchvision_install_version='0.24.1' | |
| torch_test_version='2.9' | |
| ;; | |
| '2.10.0-cpu') | |
| torch_install_version='2.10.0' | |
| torchvision_install_version='0.25.0' | |
| torch_test_version='2.10' | |
| ;; | |
| *) | |
| echo "Unsupported torch_preset: $selected_preset" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| { | |
| echo "SELECTED_TORCH_PRESET=$selected_preset" | |
| echo "TORCH_INSTALL_VERSION=$torch_install_version" | |
| echo "TORCHVISION_INSTALL_VERSION=$torchvision_install_version" | |
| echo "TORCH_TEST_VERSION=$torch_test_version" | |
| echo "PYTORCH_INDEX_URL=https://download.pytorch.org/whl/cpu" | |
| echo "TRANSFORMERS_SOURCE=$transformers_source" | |
| echo "TRANSFORMERS_VERSION=$transformers_version" | |
| echo "TRANSFORMERS_REF=$transformers_ref" | |
| } >> "$GITHUB_ENV" | |
| echo "Selected PyTorch preset: $selected_preset" | |
| echo "Resolved install tuple: torch==$torch_install_version torchvision==$torchvision_install_version" | |
| echo "Resolved test expectation: torch=$torch_test_version" | |
| echo "Resolved Transformers source: $transformers_source" | |
| echo "Resolved Transformers version: $transformers_version" | |
| echo "Resolved Transformers ref: $transformers_ref" | |
| - name: Install PyTorch | |
| run: | | |
| pip install \ | |
| torch=="$TORCH_INSTALL_VERSION" \ | |
| torchvision=="$TORCHVISION_INSTALL_VERSION" \ | |
| --index-url "$PYTORCH_INDEX_URL" | |
| python -c "import torch; print('torch:', torch.__version__, torch)" | |
| python -c "import torch; print('CUDA available:', torch.cuda.is_available())" | |
| - name: Install Transformers | |
| run: | | |
| case "$TRANSFORMERS_SOURCE" in | |
| 'pypi') | |
| pip install "transformers==$TRANSFORMERS_VERSION" | |
| ;; | |
| 'git') | |
| git clone --filter=blob:none https://github.com/huggingface/transformers /tmp/transformers | |
| cd /tmp/transformers | |
| git checkout "$TRANSFORMERS_REF" | |
| resolved_ref="$(git rev-parse HEAD)" | |
| echo "TRANSFORMERS_RESOLVED_REF=$resolved_ref" >> "$GITHUB_ENV" | |
| echo "Resolved Transformers git ref: $resolved_ref" | |
| pip install . | |
| ;; | |
| *) | |
| echo "Unsupported TRANSFORMERS_SOURCE: $TRANSFORMERS_SOURCE" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| python -c "import transformers; print('transformers:', transformers.__version__, transformers)" | |
| - name: Install deepspeed | |
| run: | | |
| pip install .[dev,autotuning] | |
| ds_report | |
| - name: Reinstall selected Transformers | |
| run: | | |
| case "$TRANSFORMERS_SOURCE" in | |
| 'pypi') | |
| pip install --no-deps --force-reinstall "transformers==$TRANSFORMERS_VERSION" | |
| ;; | |
| 'git') | |
| cd /tmp/transformers | |
| pip install --no-deps --force-reinstall . | |
| ;; | |
| *) | |
| echo "Unsupported TRANSFORMERS_SOURCE: $TRANSFORMERS_SOURCE" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| python -c "import transformers; print('transformers:', transformers.__version__, transformers)" | |
| - name: Python environment | |
| run: | | |
| pip list | |
| - name: Unit tests | |
| run: | | |
| unset TORCH_CUDA_ARCH_LIST # only jit compile for current arch | |
| cd tests | |
| HF_HOME=/tmp/hf_home/ pytest $PYTEST_OPTS --forked -n 4 unit/ --torch_ver="$TORCH_TEST_VERSION" | |
| HF_HOME=/tmp/hf_home/ pytest $PYTEST_OPTS --forked -m 'sequential' unit/ --torch_ver="$TORCH_TEST_VERSION" |