build: use cibuildwheels for PyPi compatible packages

Also use abi3 for producing a wheel that works for Python 3.10+, without having to build different wheels for each Python minor version.

Author: g-husam

cloudbuild.yaml

@@ -13,11 +13,9 @@
 # limitations under the License.
 
 steps:
-  # 1. Build the distribution (sdist and wheel).
-  # This uses scikit-build-core as defined in pyproject.toml to compile C++ extensions.
-  # We set BUILD_TESTING=OFF to ignore tests during the artifact build.
+  # 1. Build the source distribution (sdist).
   - name: 'python:3.10'
-    id: 'build'
+    id: 'build-sdist'
     entrypoint: 'bash'
     args:
       - '-c'
@@ -27,27 +25,47 @@ steps:
           exit 1
         fi
         pip install build
-        
-        echo "Building C++ extensions with editable install..."
-        # Note: pip install -e . will also respect the build-system requirements
+        python -m build --sdist
+
+  # 2. Build the manylinux wheels using cibuildwheel.
+  # This step builds for the native architecture of the GCB worker (x86_64 or ARM64).
+  # Multiple parallel builds on different architectures can safely upload to the same AR repository.
+  - name: 'python:3.10'
+    id: 'build-wheels'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        # Install cibuildwheel and the docker client
+        pip install cibuildwheel
+        apt-get update && apt-get install -y docker.io
+
+        echo "Detected architecture: $(uname -m)"
+
+        echo "Building C++ extensions with editable install (local check)..."
         SKBUILD_CMAKE_ARGS="-DBUILD_TESTING=OFF" pip install -e .
-        
-        echo "Building sdist and wheel..."
-        SKBUILD_CMAKE_ARGS="-DBUILD_TESTING=OFF" python -m build
 
-  # 2. Upload to internal Artifact Registry (AR) for OSS Exit Gate.
-  # OSS Exit Gate fetches artifacts from this repository.
+        echo "Running cibuildwheel..."
+        # cibuildwheel will build for the native architecture by default
+        # using the Python version filters defined in pyproject.toml.
+        cibuildwheel --output-dir dist
+    volumes:
+      - name: 'docker-socket'
+        path: '/var/run/docker.sock'
+    waitFor: ['build-sdist']
+
+  # 3. Upload to internal Artifact Registry (AR) for OSS Exit Gate.
   - name: 'python:3.10'
     id: 'upload-to-ar'
     entrypoint: 'bash'
     args:
       - '-c'
       - |
-        pip install -U twine keyring keyrings.google-artifactregistry-auth
+        pip install twine keyring keyrings.google-artifactregistry-auth
         twine upload --repository-url https://us-python.pkg.dev/oss-exit-gate-prod/${_PROJECT_NAME}--pypi dist/*
-    waitFor: ['build']
+    waitFor: ['build-wheels']
 
-  # 3. Create and upload the manifest to GCS to trigger the Exit Gate publication.
+  # 4. Create and upload the manifest to GCS to trigger the Exit Gate publication.
   # The presence of this file in the specific GCS bucket triggers the verification and publishing process.
   - name: 'gcr.io/cloud-builders/gcloud'
     id: 'trigger-exit-gate'

pyproject.toml

@@ -132,6 +132,10 @@ build-backend = "scikit_build_core.build"
 # Tells scikit-build-core to use setuptools-scm to retrieve the version from git.
 metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
 
+# Enable Stable ABI (abi3) for Python 3.10 and later.
+# This produces a single wheel per architecture that works on all future Python versions.
+wheel.py-api = "cp310"
+
 # Specifies the minimum version of CMake that must be present on the system.
 cmake.version = ">=3.18"
 
@@ -193,3 +197,20 @@ fail_under = 90
 [tool.gcovr]
 fail-under-line = "80"
 #fail-under-branch = "85"
+
+# ===================================================================
+# Tool-specific Configuration for cibuildwheel
+# ===================================================================
+[tool.cibuildwheel]
+# Build only once per architecture (using Python 3.10).
+# Because abi3 is enabled, this wheel will work for 3.10, 3.11, 3.12, 3.13, etc.
+build = "cp310-*"
+# Skip 32-bit builds and musllinux
+skip = "*-manylinux_i686 *-musllinux_*"
+
+[tool.cibuildwheel.linux]
+# Pass the flag to skip tests during the build inside the container
+environment = { SKBUILD_CMAKE_ARGS="-DBUILD_TESTING=OFF" }
+# We need to install the build requirements inside the cibuildwheel container
+before-build = "pip install pybind11 scikit-build-core cmake ninja setuptools-scm"
+