Skip to content

Commit 19072dc

Browse files
ko3n1gko3n1g
authored
ci: Build arbitrary wheels (#777)
* ci: Build arbitrary wheels Signed-off-by: oliver könig <okoenig@nvidia.com> * v2.2.5 Signed-off-by: oliver könig <okoenig@nvidia.com> * fix Signed-off-by: oliver könig <okoenig@nvidia.com> * fix Signed-off-by: oliver könig <okoenig@nvidia.com> --------- Signed-off-by: oliver könig <okoenig@nvidia.com>
1 parent da71f21 commit 19072dc

3 files changed

Lines changed: 302 additions & 138 deletions

File tree

‎.github/workflows/_build.yml‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
name: ~Build wheel template
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
runs-on:
7+
description: "The runner to use for the build"
8+
required: true
9+
type: string
10+
python-version:
11+
description: "The Python version to use for the build"
12+
required: true
13+
type: string
14+
cuda-version:
15+
description: "The CUDA version to use for the build"
16+
required: true
17+
type: string
18+
torch-version:
19+
description: "The PyTorch version to use for the build"
20+
required: true
21+
type: string
22+
cxx11_abi:
23+
description: "The C++11 ABI to use for the build"
24+
required: true
25+
type: string
26+
upload-to-release:
27+
description: "Upload wheel to this release"
28+
required: false
29+
type: boolean
30+
default: false
31+
release-version:
32+
description: "Upload wheel to this release"
33+
required: false
34+
type: string
35+
36+
defaults:
37+
run:
38+
shell: bash -x -e -u -o pipefail {0}
39+
40+
jobs:
41+
build-wheel:
42+
runs-on: ${{ inputs.runs-on }}
43+
name: Build wheel (${{ inputs.release-version }}-${{ inputs.python-version }}-${{ inputs.cuda-version }}-${{ inputs.torch-version }}-${{ inputs.cxx11_abi }})
44+
steps:
45+
- name: Checkout
46+
uses: actions/checkout@v4
47+
with:
48+
ref: ${{ inputs.release-version }}
49+
submodules: recursive
50+
51+
- name: Set up Python
52+
uses: actions/setup-python@v5
53+
with:
54+
python-version: ${{ inputs.python-version }}
55+
56+
- name: Set CUDA and PyTorch versions
57+
run: |
58+
echo "MATRIX_CUDA_VERSION=$(echo ${{ inputs.cuda-version }} | awk -F \. {'print $1 $2'})" >> $GITHUB_ENV
59+
echo "MATRIX_TORCH_VERSION=$(echo ${{ inputs.torch-version }} | awk -F \. {'print $1 "." $2'})" >> $GITHUB_ENV
60+
echo "WHEEL_CUDA_VERSION=$(echo ${{ inputs.cuda-version }} | awk -F \. {'print $1'})" >> $GITHUB_ENV
61+
echo "MATRIX_PYTHON_VERSION=$(echo ${{ inputs.python-version }} | awk -F \. {'print $1 $2'})" >> $GITHUB_ENV
62+
63+
- name: Free up disk space
64+
if: ${{ runner.os == 'Linux' }}
65+
# https://github.com/easimon/maximize-build-space/blob/master/action.yml
66+
# https://github.com/easimon/maximize-build-space/tree/test-report
67+
run: |
68+
sudo rm -rf /usr/share/dotnet
69+
sudo rm -rf /opt/ghc
70+
sudo rm -rf /opt/hostedtoolcache/CodeQL
71+
72+
- name: Set up swap space
73+
if: runner.os == 'Linux'
74+
uses: pierotofy/set-swap-space@v1.0
75+
with:
76+
swap-size-gb: 10
77+
78+
- name: Install CUDA ${{ inputs.cuda-version }}
79+
if: ${{ inputs.cuda-version != 'cpu' }}
80+
uses: Jimver/cuda-toolkit@v0.2.26
81+
id: cuda-toolkit
82+
with:
83+
cuda: ${{ inputs.cuda-version }}
84+
linux-local-args: '["--toolkit"]'
85+
# default method is "local", and we're hitting some error with caching for CUDA 11.8 and 12.1
86+
# method: ${{ (inputs.cuda-version == '11.8.0' || inputs.cuda-version == '12.1.0') && 'network' || 'local' }}
87+
method: "network"
88+
sub-packages: '["nvcc"]'
89+
90+
- name: Install PyTorch ${{ inputs.torch-version }}+cu${{ inputs.cuda-version }}
91+
run: |
92+
pip install --upgrade pip
93+
# For some reason torch 2.2.0 on python 3.12 errors saying no setuptools
94+
pip install setuptools==68.0.0
95+
# With python 3.13 and torch 2.5.1, unless we update typing-extensions, we get error
96+
# AttributeError: attribute '__default__' of 'typing.ParamSpec' objects is not writable
97+
pip install typing-extensions==4.12.2
98+
# We want to figure out the CUDA version to download pytorch
99+
# e.g. we can have system CUDA version being 11.7 but if torch==1.12 then we need to download the wheel from cu116
100+
# This code is ugly, maybe there's a better way to do this.
101+
export TORCH_CUDA_VERSION=$(python -c "from os import environ as env; \
102+
minv = {'2.4': 118, '2.5': 118, '2.6': 118, '2.7': 118}[env['MATRIX_TORCH_VERSION']]; \
103+
maxv = {'2.4': 124, '2.5': 124, '2.6': 126, '2.7': 128}[env['MATRIX_TORCH_VERSION']]; \
104+
print(minv if int(env['MATRIX_CUDA_VERSION']) < 120 else maxv)" \
105+
)
106+
if [[ ${{ inputs.torch-version }} == *"dev"* ]]; then
107+
# pip install --no-cache-dir --pre torch==${{ inputs.torch-version }} --index-url https://download.pytorch.org/whl/nightly/cu${TORCH_CUDA_VERSION}
108+
# Can't use --no-deps because we need cudnn etc.
109+
# Hard-coding this version of pytorch-triton for torch 2.6.0.dev20241001
110+
pip install jinja2
111+
pip install https://download.pytorch.org/whl/nightly/pytorch_triton-3.1.0%2Bcf34004b8a-cp${MATRIX_PYTHON_VERSION}-cp${MATRIX_PYTHON_VERSION}-linux_x86_64.whl
112+
pip install --no-cache-dir --pre https://download.pytorch.org/whl/nightly/cu${TORCH_CUDA_VERSION}/torch-${{ inputs.torch-version }}%2Bcu${TORCH_CUDA_VERSION}-cp${MATRIX_PYTHON_VERSION}-cp${MATRIX_PYTHON_VERSION}-linux_x86_64.whl
113+
else
114+
pip install --no-cache-dir torch==${{ inputs.torch-version }} --index-url https://download.pytorch.org/whl/cu${TORCH_CUDA_VERSION}
115+
fi
116+
nvcc --version
117+
python --version
118+
python -c "import torch; print('PyTorch:', torch.__version__)"
119+
python -c "import torch; print('CUDA:', torch.version.cuda)"
120+
python -c "from torch.utils import cpp_extension; print (cpp_extension.CUDA_HOME)"
121+
shell: bash
122+
123+
- name: Restore build cache
124+
uses: actions/cache/restore@v4
125+
with:
126+
path: build.tar
127+
key: build-${{ inputs.release-version }}-${{ inputs.python-version }}-${{ inputs.cuda-version }}-${{ inputs.torch-version }}-${{ inputs.cxx11_abi }}-${{ github.run_number }}-${{ github.run_attempt }}
128+
restore-keys: |
129+
build-${{ inputs.release-version }}-${{ inputs.python-version }}-${{ inputs.cuda-version }}-${{ inputs.torch-version }}-${{ inputs.cxx11_abi }}-
130+
131+
- name: Unpack build cache
132+
run: |
133+
echo ::group::Adjust timestamps
134+
sudo find / -exec touch -t 197001010000 {} + || true
135+
echo ::endgroup::
136+
137+
if [ -f build.tar ]; then
138+
find . -mindepth 1 -maxdepth 1 ! -name 'build.tar' -exec rm -rf {} +
139+
tar -xpvf build.tar -C .
140+
else
141+
echo "No build.tar found, skipping"
142+
fi
143+
144+
ls -al ./
145+
ls -al build/ || true
146+
ls -al csrc/ || true
147+
148+
- name: Build wheel
149+
id: build_wheel
150+
run: |
151+
# We want setuptools >= 49.6.0 otherwise we can't compile the extension if system CUDA version is 11.7 and pytorch cuda version is 11.6
152+
# https://github.com/pytorch/pytorch/blob/664058fa83f1d8eede5d66418abff6e20bd76ca8/torch/utils/cpp_extension.py#L810
153+
# However this still fails so I'm using a newer version of setuptools
154+
pip install setuptools==68.0.0
155+
pip install ninja packaging wheel
156+
export PATH=/usr/local/nvidia/bin:/usr/local/nvidia/lib64:$PATH
157+
export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
158+
159+
# Limit MAX_JOBS otherwise the github runner goes OOM
160+
export MAX_JOBS=2
161+
export MAMBA_FORCE_BUILD="TRUE"
162+
export MAMBA_FORCE_CXX11_ABI=${{ inputs.cxx11_abi}}
163+
164+
# 5h timeout since GH allows max 6h and we want some buffer
165+
EXIT_CODE=0
166+
timeout 5h python setup.py bdist_wheel --dist-dir=dist || EXIT_CODE=$?
167+
168+
if [ $EXIT_CODE -eq 0 ]; then
169+
tmpname=cu${WHEEL_CUDA_VERSION}torch${MATRIX_TORCH_VERSION}cxx11abi${{ inputs.cxx11_abi }}
170+
wheel_name=$(ls dist/*whl | xargs -n 1 basename | sed "s/-/+$tmpname-/2")
171+
ls dist/*whl |xargs -I {} mv {} dist/${wheel_name}
172+
echo "wheel_name=${wheel_name}" >> $GITHUB_ENV
173+
fi
174+
175+
# Store exit code in GitHub env for later steps
176+
echo "build_exit_code=$EXIT_CODE" | tee -a "$GITHUB_OUTPUT"
177+
178+
# Do not fail the job if timeout killed the build
179+
exit $EXIT_CODE
180+
181+
- name: Log build logs after timeout
182+
if: always() && steps.build_wheel.outputs.build_exit_code == 124
183+
run: |
184+
ls -al ./
185+
tar -cvf build.tar . --atime-preserve=replace
186+
187+
- name: Save build cache timeout
188+
if: always() && steps.build_wheel.outputs.build_exit_code == 124
189+
uses: actions/cache/save@v4
190+
with:
191+
key: build-${{ inputs.release-version }}-${{ inputs.python-version }}-${{ inputs.cuda-version }}-${{ inputs.torch-version }}-${{ inputs.cxx11_abi }}-${{ github.run_number }}-${{ github.run_attempt }}
192+
path: build.tar
193+
194+
- name: Log Built Wheels
195+
run: |
196+
ls dist
197+
198+
- name: Get Release with tag
199+
id: get_current_release
200+
uses: joutvhu/get-release@v1
201+
with:
202+
tag_name: ${{ inputs.release-version }}
203+
env:
204+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
205+
206+
- name: Upload Release Asset
207+
id: upload_release_asset
208+
if: inputs.upload-to-release
209+
uses: actions/upload-release-asset@v1
210+
env:
211+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
212+
with:
213+
upload_url: ${{ steps.get_current_release.outputs.upload_url }}
214+
asset_path: ./dist/${{env.wheel_name}}
215+
asset_name: ${{env.wheel_name}}
216+
asset_content_type: application/*

‎.github/workflows/build.yml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
name: Build wheels
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
runs-on:
7+
description: "The runner to use for the build"
8+
required: true
9+
type: string
10+
default: ubuntu-22.04
11+
python-version:
12+
description: "The Python version to use for the build"
13+
required: true
14+
type: string
15+
cuda-version:
16+
description: "The CUDA version to use for the build"
17+
required: true
18+
type: string
19+
torch-version:
20+
description: "The PyTorch version to use for the build"
21+
required: true
22+
type: string
23+
cxx11_abi:
24+
description: "Enable torch flag C++11 ABI (TRUE/FALSE)"
25+
required: true
26+
type: string
27+
upload-to-release:
28+
description: "Upload wheel to this release"
29+
required: false
30+
type: boolean
31+
default: false
32+
release-version:
33+
description: "Upload wheel to this release"
34+
required: false
35+
type: string
36+
push:
37+
38+
jobs:
39+
build-wheels:
40+
uses: ./.github/workflows/_build.yml
41+
with:
42+
runs-on: ${{ inputs.runs-on || 'ubuntu-22.04' }}
43+
python-version: ${{ inputs.python-version || '3.10' }}
44+
cuda-version: ${{ inputs.cuda-version || '12.9.1' }}
45+
torch-version: ${{ inputs.torch-version || '2.7.1' }}
46+
cxx11_abi: ${{ inputs.cxx11_abi || 'FALSE' }}
47+
upload-to-release: ${{ inputs.upload-to-release || false }}
48+
release-version: ${{ inputs.release-version || 'v2.2.5' }}

0 commit comments

Comments
 (0)