Build numpy for preview Python (3.15/3.15t) and host on download.pytorch.org

.github/scripts/build_numpy_preview.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# Build numpy wheels for preview CPython versions (e.g. 3.15 / 3.15t) inside a
+# manylinux builder image.  The resulting wheels are left in a wheelhouse for
+# the workflow to upload to download.pytorch.org (S3 + R2).
+#
+# Why this exists:
+#   numpy does not yet publish cp315 wheels on PyPI.  When torch preview-Python
+#   wheels are built or smoke-tested, pip tries to resolve numpy and falls back
+#   to building it from an sdist, which fails in the constrained build/test
+#   environment.  Pre-building numpy here and hosting it on the pytorch index
+#   lets those installs resolve a real wheel instead.
+#
+# This must run inside a pytorch manylinux builder image that ships the target
+# interpreters under /opt/python (e.g. pytorch/manylinux2_28-builder:cpu).
+#
+# Required env:
+#   ARCH               x86_64 | aarch64
+# Optional env:
+#   NUMPY_VERSION      numpy version to build            (default: 2.5.1)
+#   PYTHON_VERSIONS    space separated                   (default: "3.15 3.15t")
+#   MANYWHEEL_VERSION  manylinux platform tag version    (default: 2_28)
+
+set -euo pipefail
+
+NUMPY_VERSION="${NUMPY_VERSION:-2.5.1}"
+PYTHON_VERSIONS="${PYTHON_VERSIONS:-3.15 3.15t}"
+MANYWHEEL_VERSION="${MANYWHEEL_VERSION:-2_28}"
+ARCH="${ARCH:?ARCH must be set (x86_64|aarch64)}"
+
+PLAT="manylinux_${MANYWHEEL_VERSION}_${ARCH}"
+BUILD_DIR="/tmp/numpy-preview-build"
+WHEELHOUSE="${BUILD_DIR}/wheelhouse"
+
+rm -rf "${BUILD_DIR}"
+mkdir -p "${WHEELHOUSE}"
+
+# 3.15 -> cp315 ; 3.15t -> cp315t
+cp_tag() {
+  local ver="$1" suffix=""
+  if [[ "${ver}" == *t ]]; then
+    suffix="t"
+    ver="${ver%t}"
+  fi
+  echo "cp${ver//./}${suffix}"
+}
+
+# 3.15  -> /opt/python/cp315-cp315/bin/python
+# 3.15t -> /opt/python/cp315-cp315t/bin/python
+# Free-threaded builds only carry the 't' on the ABI tag, not the interpreter tag.
+py_bin() {
+  local ver="$1" suffix=""
+  if [[ "${ver}" == *t ]]; then
+    suffix="t"
+    ver="${ver%t}"
+  fi
+  local digits="${ver//./}"
+  echo "/opt/python/cp${digits}-cp${digits}${suffix}/bin/python"
+}
+
+echo "==> numpy==${NUMPY_VERSION}  arch=${ARCH}  plat=${PLAT}"
+echo "==> python versions: ${PYTHON_VERSIONS}"
+
+for pyver in ${PYTHON_VERSIONS}; do
+  tag="$(cp_tag "${pyver}")"
+  py="$(py_bin "${pyver}")"
+
+  if [[ ! -x "${py}" ]]; then
+    echo "::error::Interpreter for ${pyver} not found at ${py}"
+    exit 1
+  fi
+
+  echo "==> Building numpy==${NUMPY_VERSION} for ${tag} (${py})"
+  work="${BUILD_DIR}/${tag}"
+  mkdir -p "${work}"
+
+  "${py}" -m pip install --upgrade pip auditwheel
+
+  # --no-binary forces a source build against this exact interpreter so the
+  # produced extension modules target the preview CPython ABI.
+  "${py}" -m pip wheel --no-deps --no-binary numpy \
+    --wheel-dir "${work}" "numpy==${NUMPY_VERSION}"
+
+  # numpy's source build emits a linux_<arch> tagged wheel that bundles OpenBLAS
+  # but references libgfortran/libquadmath from the toolchain; auditwheel vendors
+  # those in and rewrites the platform tag to a compliant manylinux tag.
+  for whl in "${work}"/numpy-*.whl; do
+    echo "    auditwheel repair ${whl##*/} -> ${PLAT}"
+    "${py}" -m auditwheel repair \
+      --plat "${PLAT}" \
+      --wheel-dir "${WHEELHOUSE}" \
+      "${whl}"
+  done
+done
+
+echo "==> Built wheels:"
+ls -la "${WHEELHOUSE}"
+
+# Uploading to S3 (s3://pytorch) and R2 (s3://pytorch-downloads) is handled by
+# the workflow, which manages the two distinct credential contexts.

.github/workflows/build-numpy-preview.yml

@@ -0,0 +1,158 @@
+name: Build numpy for preview Python
+
+# Builds numpy wheels for preview CPython versions (e.g. 3.15 / 3.15t) that do
+# not yet have wheels on PyPI, and uploads them to download.pytorch.org so that
+# torch preview-Python wheel builds and smoke tests can resolve numpy. Wheels
+# are uploaded to both S3 (s3://pytorch) and R2 (s3://pytorch-downloads) since
+# download.pytorch.org is R2-backed.
+#
+# Run via workflow_dispatch. Defaults to a dry run (build only, no upload);
+# set dry_run=false to actually publish.
+#
+# Pull requests that touch this workflow or its build script run the build in
+# dry-run mode (build only, no AWS credentials, no upload) so the numpy build
+# itself can be validated in CI.
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/build-numpy-preview.yml
+      - .github/scripts/build_numpy_preview.sh
+  workflow_dispatch:
+    inputs:
+      numpy_version:
+        description: "numpy version to build from sdist"
+        type: string
+        default: "2.5.1"
+        required: false
+      python_versions:
+        description: "Space-separated preview Python versions"
+        type: string
+        default: "3.15 3.15t"
+        required: false
+      channel:
+        description: "download.pytorch.org channel to upload to"
+        type: choice
+        options:
+          - nightly
+          - test
+        default: nightly
+        required: false
+      manywheel_version:
+        description: "Manylinux platform tag version (e.g. '2_28')"
+        type: string
+        default: "2_28"
+        required: false
+      dry_run:
+        description: "Build only, do not upload to S3"
+        type: boolean
+        default: true
+        required: false
+
+concurrency:
+  group: build-numpy-preview-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: x86_64
+            runner: linux.2xlarge
+            image: pytorch/manylinux2_28-builder:cpu
+          - arch: aarch64
+            runner: linux.arm64.m8g.4xlarge
+            image: pytorch/manylinux2_28_aarch64-builder:cpu-aarch64
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 120
+    container:
+      image: ${{ matrix.image }}
+    steps:
+      - name: Checkout test-infra
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Install AWS CLI
+        if: ${{ github.event_name == 'workflow_dispatch' && !inputs.dry_run }}
+        run: |
+          /opt/python/cp312-cp312/bin/pip install awscli==1.38.23
+          echo "/opt/python/cp312-cp312/bin" >> "${GITHUB_PATH}"
+
+      - name: Configure AWS credentials
+        if: ${{ github.event_name == 'workflow_dispatch' && !inputs.dry_run }}
+        uses: aws-actions/configure-aws-credentials@50ac8dd1e1b10d09dac7b8727528b91bed831ac0 # v3.0.2
+        with:
+          # Same roles the binary upload workflow uses to write to s3://pytorch/whl/<channel>.
+          # nightly and test channels are guarded by separate IAM roles.
+          role-to-assume: arn:aws:iam::749337293305:role/${{ inputs.channel == 'test' && 'gha_workflow_test_build_wheels' || 'gha_workflow_nightly_build_wheels' }}
+          aws-region: us-east-1
+          role-duration-seconds: 7200
+
+      - name: Build numpy
+        env:
+          ARCH: ${{ matrix.arch }}
+          NUMPY_VERSION: ${{ inputs.numpy_version || '2.5.1' }}
+          PYTHON_VERSIONS: ${{ inputs.python_versions || '3.15 3.15t' }}
+          MANYWHEEL_VERSION: ${{ inputs.manywheel_version || '2_28' }}
+        run: bash .github/scripts/build_numpy_preview.sh
+
+      # Upload to S3 (s3://pytorch) using the OIDC-assumed role, then to R2
+      # (s3://pytorch-downloads) using the R2 access keys. download.pytorch.org
+      # is R2-backed, so wheels must land in both. Only on workflow_dispatch with
+      # dry_run disabled — never on pull_request.
+      - name: Upload numpy to S3 (download.pytorch.org)
+        if: ${{ github.event_name == 'workflow_dispatch' && !inputs.dry_run }}
+        env:
+          CHANNEL: ${{ inputs.channel }}
+        run: |
+          set -ex
+          dest="s3://pytorch/whl/${CHANNEL}/"
+          for pkg in /tmp/numpy-preview-build/wheelhouse/numpy-*.whl; do
+            shm_id="$(sha256sum "${pkg}" | awk '{print $1}')"
+            aws s3 cp "${pkg}" "${dest}" \
+              --acl public-read \
+              --metadata "checksum-sha256=${shm_id}"
+          done
+
+      - name: Configure R2 credentials
+        if: ${{ github.event_name == 'workflow_dispatch' && !inputs.dry_run }}
+        env:
+          R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
+          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
+          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
+        run: |
+          # Overwrite the OIDC S3 credentials from the previous step with R2 keys.
+          {
+            echo "AWS_ACCESS_KEY_ID=${R2_ACCESS_KEY_ID}"
+            echo "AWS_SECRET_ACCESS_KEY=${R2_SECRET_ACCESS_KEY}"
+            echo "AWS_SESSION_TOKEN="
+            echo "AWS_DEFAULT_REGION=auto"
+            echo "R2_ACCOUNT_ID=${R2_ACCOUNT_ID}"
+          } >> "${GITHUB_ENV}"
+
+      - name: Upload numpy to R2 (download.pytorch.org)
+        if: ${{ github.event_name == 'workflow_dispatch' && !inputs.dry_run }}
+        env:
+          CHANNEL: ${{ inputs.channel }}
+        run: |
+          set -ex
+          dest="s3://pytorch-downloads/whl/${CHANNEL}/"
+          for pkg in /tmp/numpy-preview-build/wheelhouse/numpy-*.whl; do
+            shm_id="$(sha256sum "${pkg}" | awk '{print $1}')"
+            aws s3 cp "${pkg}" "${dest}" \
+              --metadata "checksum-sha256=${shm_id}" \
+              --endpoint-url "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com"
+          done
+
+      - name: Upload wheels as workflow artifact
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: numpy-preview-${{ matrix.arch }}
+          path: /tmp/numpy-preview-build/wheelhouse/*.whl
+          if-no-files-found: warn