Skip to content

Commit b0ebe72

Browse files
atalmanatalman
committed
Add workflow to build numpy for preview Python (3.15/3.15t) and host on download.pytorch.org
numpy does not yet publish cp315 wheels on PyPI, so torch preview-Python wheel builds and smoke tests fail when pip resolves numpy from an sdist. This adds a workflow_dispatch workflow that builds numpy from source inside the manylinux2_28 builder images for cp315/cp315t on x86_64 and aarch64, repairs the wheels with auditwheel, and uploads them to s3://pytorch/whl/<channel>/ so the pytorch index can serve them.
1 parent 64a6965 commit b0ebe72

2 files changed

Lines changed: 217 additions & 0 deletions

File tree

‎.github/scripts/build_numpy_preview.sh‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
#!/usr/bin/env bash
2+
# Build numpy wheels for preview CPython versions (e.g. 3.15 / 3.15t) inside a
3+
# manylinux builder image and upload them to download.pytorch.org.
4+
#
5+
# Why this exists:
6+
# numpy does not yet publish cp315 wheels on PyPI. When torch preview-Python
7+
# wheels are built or smoke-tested, pip tries to resolve numpy and falls back
8+
# to building it from an sdist, which fails in the constrained build/test
9+
# environment. Pre-building numpy here and hosting it on the pytorch index
10+
# lets those installs resolve a real wheel instead.
11+
#
12+
# This must run inside a pytorch manylinux builder image that ships the target
13+
# interpreters under /opt/python (e.g. pytorch/manylinux2_28-builder:cpu).
14+
#
15+
# Required env:
16+
# ARCH x86_64 | aarch64
17+
# Optional env:
18+
# NUMPY_VERSION numpy version to build (default: 2.5.1)
19+
# PYTHON_VERSIONS space separated (default: "3.15 3.15t")
20+
# CHANNELS download.pytorch.org channels (default: "nightly test")
21+
# MANYWHEEL_VERSION manylinux platform tag version (default: 2_28)
22+
# DRY_RUN "true" builds but does not upload (default: true)
23+
24+
set -euo pipefail
25+
26+
NUMPY_VERSION="${NUMPY_VERSION:-2.5.1}"
27+
PYTHON_VERSIONS="${PYTHON_VERSIONS:-3.15 3.15t}"
28+
CHANNELS="${CHANNELS:-nightly test}"
29+
MANYWHEEL_VERSION="${MANYWHEEL_VERSION:-2_28}"
30+
DRY_RUN="${DRY_RUN:-true}"
31+
ARCH="${ARCH:?ARCH must be set (x86_64|aarch64)}"
32+
33+
PLAT="manylinux_${MANYWHEEL_VERSION}_${ARCH}"
34+
BUILD_DIR="/tmp/numpy-preview-build"
35+
WHEELHOUSE="${BUILD_DIR}/wheelhouse"
36+
37+
rm -rf "${BUILD_DIR}"
38+
mkdir -p "${WHEELHOUSE}"
39+
40+
# 3.15 -> cp315 ; 3.15t -> cp315t
41+
cp_tag() {
42+
local ver="$1" suffix=""
43+
if [[ "${ver}" == *t ]]; then
44+
suffix="t"
45+
ver="${ver%t}"
46+
fi
47+
echo "cp${ver//./}${suffix}"
48+
}
49+
50+
# cp315 -> /opt/python/cp315-cp315/bin/python
51+
py_bin() {
52+
local tag
53+
tag="$(cp_tag "$1")"
54+
echo "/opt/python/${tag}-${tag}/bin/python"
55+
}
56+
57+
echo "==> numpy==${NUMPY_VERSION} arch=${ARCH} plat=${PLAT}"
58+
echo "==> python versions: ${PYTHON_VERSIONS}"
59+
echo "==> channels: ${CHANNELS} dry_run=${DRY_RUN}"
60+
61+
for pyver in ${PYTHON_VERSIONS}; do
62+
tag="$(cp_tag "${pyver}")"
63+
py="$(py_bin "${pyver}")"
64+
65+
if [[ ! -x "${py}" ]]; then
66+
echo "::error::Interpreter for ${pyver} not found at ${py}"
67+
exit 1
68+
fi
69+
70+
echo "==> Building numpy==${NUMPY_VERSION} for ${tag} (${py})"
71+
work="${BUILD_DIR}/${tag}"
72+
mkdir -p "${work}"
73+
74+
"${py}" -m pip install --upgrade pip auditwheel
75+
76+
# --no-binary forces a source build against this exact interpreter so the
77+
# produced extension modules target the preview CPython ABI.
78+
"${py}" -m pip wheel --no-deps --no-binary numpy \
79+
--wheel-dir "${work}" "numpy==${NUMPY_VERSION}"
80+
81+
# numpy's source build emits a linux_<arch> tagged wheel that bundles OpenBLAS
82+
# but references libgfortran/libquadmath from the toolchain; auditwheel vendors
83+
# those in and rewrites the platform tag to a compliant manylinux tag.
84+
for whl in "${work}"/numpy-*.whl; do
85+
echo " auditwheel repair ${whl##*/} -> ${PLAT}"
86+
"${py}" -m auditwheel repair \
87+
--plat "${PLAT}" \
88+
--wheel-dir "${WHEELHOUSE}" \
89+
"${whl}"
90+
done
91+
done
92+
93+
echo "==> Built wheels:"
94+
ls -la "${WHEELHOUSE}"
95+
96+
if [[ "${DRY_RUN}" == "true" ]]; then
97+
echo "==> DRY RUN: skipping upload to download.pytorch.org"
98+
exit 0
99+
fi
100+
101+
# Upload to s3://pytorch/whl/<channel>/ with the same public-read ACL and
102+
# checksum metadata the binary upload workflow uses for torch wheels.
103+
for channel in ${CHANNELS}; do
104+
dest="s3://pytorch/whl/${channel}/"
105+
echo "==> Uploading to ${dest}"
106+
for pkg in "${WHEELHOUSE}"/numpy-*.whl; do
107+
shm_id="$(sha256sum "${pkg}" | awk '{print $1}')"
108+
aws s3 cp "${pkg}" "${dest}" \
109+
--acl public-read \
110+
--metadata "checksum-sha256=${shm_id}"
111+
done
112+
done
113+
114+
echo "==> Done. Run the 'Update S3 HTML indices' workflow (or wait for the"
115+
echo " hourly cron) to publish numpy in the whl/<channel> package indices."

‎.github/workflows/build-numpy-preview.yml‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
name: Build numpy for preview Python
2+
3+
# Builds numpy wheels for preview CPython versions (e.g. 3.15 / 3.15t) that do
4+
# not yet have wheels on PyPI, and uploads them to download.pytorch.org so that
5+
# torch preview-Python wheel builds and smoke tests can resolve numpy.
6+
#
7+
# Run via workflow_dispatch. Defaults to a dry run (build only, no upload);
8+
# set dry_run=false to actually publish.
9+
10+
on:
11+
workflow_dispatch:
12+
inputs:
13+
numpy_version:
14+
description: "numpy version to build from sdist"
15+
type: string
16+
default: "2.5.1"
17+
required: false
18+
python_versions:
19+
description: "Space-separated preview Python versions"
20+
type: string
21+
default: "3.15 3.15t"
22+
required: false
23+
channel:
24+
description: "download.pytorch.org channel to upload to"
25+
type: choice
26+
options:
27+
- nightly
28+
- test
29+
default: nightly
30+
required: false
31+
manywheel_version:
32+
description: "Manylinux platform tag version (e.g. '2_28')"
33+
type: string
34+
default: "2_28"
35+
required: false
36+
dry_run:
37+
description: "Build only, do not upload to S3"
38+
type: boolean
39+
default: true
40+
required: false
41+
42+
concurrency:
43+
group: build-numpy-preview-${{ github.ref }}
44+
cancel-in-progress: false
45+
46+
permissions:
47+
id-token: write
48+
contents: read
49+
50+
jobs:
51+
build:
52+
strategy:
53+
fail-fast: false
54+
matrix:
55+
include:
56+
- arch: x86_64
57+
runner: linux.2xlarge
58+
image: pytorch/manylinux2_28-builder:cpu
59+
- arch: aarch64
60+
runner: linux.arm64.m8g.4xlarge
61+
image: pytorch/manylinux2_28_aarch64-builder:cpu-aarch64
62+
runs-on: ${{ matrix.runner }}
63+
timeout-minutes: 120
64+
container:
65+
image: ${{ matrix.image }}
66+
steps:
67+
- name: Checkout test-infra
68+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
69+
70+
- name: Install AWS CLI
71+
if: ${{ !inputs.dry_run }}
72+
run: |
73+
/opt/python/cp312-cp312/bin/pip install awscli==1.38.23
74+
echo "/opt/python/cp312-cp312/bin" >> "${GITHUB_PATH}"
75+
76+
- name: Configure AWS credentials
77+
if: ${{ !inputs.dry_run }}
78+
uses: aws-actions/configure-aws-credentials@50ac8dd1e1b10d09dac7b8727528b91bed831ac0 # v3.0.2
79+
with:
80+
# Same roles the binary upload workflow uses to write to s3://pytorch/whl/<channel>.
81+
# nightly and test channels are guarded by separate IAM roles.
82+
role-to-assume: arn:aws:iam::749337293305:role/${{ inputs.channel == 'test' && 'gha_workflow_test_build_wheels' || 'gha_workflow_nightly_build_wheels' }}
83+
aws-region: us-east-1
84+
role-duration-seconds: 7200
85+
86+
- name: Build and upload numpy
87+
env:
88+
ARCH: ${{ matrix.arch }}
89+
NUMPY_VERSION: ${{ inputs.numpy_version }}
90+
PYTHON_VERSIONS: ${{ inputs.python_versions }}
91+
CHANNELS: ${{ inputs.channel }}
92+
MANYWHEEL_VERSION: ${{ inputs.manywheel_version }}
93+
DRY_RUN: ${{ inputs.dry_run }}
94+
run: bash .github/scripts/build_numpy_preview.sh
95+
96+
- name: Upload wheels as workflow artifact
97+
if: always()
98+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
99+
with:
100+
name: numpy-preview-${{ matrix.arch }}
101+
path: /tmp/numpy-preview-build/wheelhouse/*.whl
102+
if-no-files-found: warn

0 commit comments

Comments
 (0)