Skip to content

Commit e24def4

Browse files
dellaertdellaert
authored
Merge pull request #2029 from yambati03/develop
Add `cibuildwheel` workflow
2 parents 12cebaf + 895fc88 commit e24def4

File tree

7 files changed

+275
-7
lines changed

7 files changed

+275
-7
lines changed

‎.github/workflows/build-cibw.yml‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
# This workflow builds the Python wheels using cibuildwheel and uploads them to TestPyPI.
2+
# It can be triggered on push to the develop branch or manually via Github Actions.
3+
4+
name: Build Wheels (cibuildwheel)
5+
6+
on:
7+
push:
8+
branches:
9+
- develop
10+
workflow_dispatch:
11+
12+
jobs:
13+
# Get the system time and store it in an output. This is used to tag the wheels.
14+
# This needs to be done in a separate job so that each matrix job in build_wheels can
15+
# access the same timestamp.
16+
get_system_time:
17+
name: Get System Time
18+
runs-on: ubuntu-latest
19+
outputs:
20+
timestamp: ${{ steps.get_time.outputs.timestamp }}
21+
steps:
22+
- name: Get system time
23+
id: get_time
24+
run: echo "timestamp=$(date +'%Y%m%d%H%M')" >> "$GITHUB_OUTPUT"
25+
26+
build_wheels:
27+
name: Build Wheels
28+
needs: get_system_time
29+
runs-on: ${{ matrix.os }}
30+
strategy:
31+
fail-fast: false
32+
matrix:
33+
include:
34+
# Linux x86_64
35+
- os: ubuntu-latest
36+
python_version: "3.10"
37+
cibw_python_version: 310
38+
platform_id: manylinux_x86_64
39+
manylinux_image: manylinux2014
40+
- os: ubuntu-latest
41+
python_version: "3.11"
42+
cibw_python_version: 311
43+
platform_id: manylinux_x86_64
44+
manylinux_image: manylinux2014
45+
- os: ubuntu-latest
46+
python_version: "3.12"
47+
cibw_python_version: 312
48+
platform_id: manylinux_x86_64
49+
manylinux_image: manylinux2014
50+
- os: ubuntu-latest
51+
python_version: "3.13"
52+
cibw_python_version: 313
53+
platform_id: manylinux_x86_64
54+
manylinux_image: manylinux2014
55+
56+
# Linux aarch64
57+
- os: ubuntu-24.04-arm
58+
python_version: "3.10"
59+
cibw_python_version: 310
60+
platform_id: manylinux_aarch64
61+
manylinux_image: manylinux2014
62+
- os: ubuntu-24.04-arm
63+
python_version: "3.11"
64+
cibw_python_version: 311
65+
platform_id: manylinux_aarch64
66+
manylinux_image: manylinux2014
67+
- os: ubuntu-24.04-arm
68+
python_version: "3.12"
69+
cibw_python_version: 312
70+
platform_id: manylinux_aarch64
71+
manylinux_image: manylinux2014
72+
- os: ubuntu-24.04-arm
73+
python_version: "3.13"
74+
cibw_python_version: 313
75+
platform_id: manylinux_aarch64
76+
manylinux_image: manylinux2014
77+
78+
# MacOS x86_64
79+
# - os: macos-13
80+
# python_version: "3.10"
81+
# cibw_python_version: 310
82+
# platform_id: macosx_x86_64
83+
# - os: macos-13
84+
# python_version: "3.11"
85+
# cibw_python_version: 311
86+
# platform_id: macosx_x86_64
87+
# - os: macos-13
88+
# python_version: "3.12"
89+
# cibw_python_version: 312
90+
# platform_id: macosx_x86_64
91+
# - os: macos-13
92+
# python_version: "3.13"
93+
# cibw_python_version: 313
94+
# platform_id: macosx_x86_64
95+
96+
steps:
97+
- name: Checkout
98+
uses: actions/checkout@v4
99+
100+
- name: Set up Python ${{ matrix.python_version }}
101+
uses: actions/setup-python@v5
102+
with:
103+
python-version: ${{ matrix.python_version }}
104+
105+
# Set the DEVELOP flag and the TIMESTAMP environment variables. This is used in the
106+
# top-level CMakeLists.txt to generate the GTSAM_VERSION_STRING.
107+
- name: Set Develop Flag
108+
run: |
109+
echo "DEVELOP=1" >> $GITHUB_ENV
110+
echo "TIMESTAMP=${{ needs.get_system_time.outputs.timestamp }}" >> $GITHUB_ENV
111+
112+
- name: Install Dependencies
113+
run: |
114+
python3 -m pip install -r python/dev_requirements.txt
115+
if [ "$RUNNER_OS" == "Linux" ]; then
116+
sudo apt-get install -y wget libicu-dev python3-pip python3-setuptools libboost-all-dev ninja-build
117+
elif [ "$RUNNER_OS" == "macOS" ]; then
118+
brew install wget icu4c boost ninja python-setuptools
119+
else
120+
echo "$RUNNER_OS not supported"
121+
exit 1
122+
fi
123+
124+
# We first build the Python wrapper module on the host machine. This is done because cibuildwheel
125+
# expects a setup.py file to be present in the project directory.
126+
#
127+
# The Python wrapper module is then rebuilt within the cibuildwheel container before building
128+
# the wheels to ensure platform compatibility.
129+
- name: Run CMake
130+
run: |
131+
cmake . -B build -DGTSAM_BUILD_PYTHON=1 -DGTSAM_PYTHON_VERSION=${{ matrix.python_version }}
132+
133+
- name: Build and test wheels
134+
env:
135+
# Generate the platform identifier. See https://cibuildwheel.pypa.io/en/stable/options/#build-skip.
136+
CIBW_BUILD: cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }}
137+
CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
138+
CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.manylinux_image }}
139+
CIBW_ARCHS: all
140+
CIBW_ENVIRONMENT_PASS_LINUX: DEVELOP TIMESTAMP
141+
142+
# Use build instead of pip wheel to build the wheels. This is recommended by PyPA.
143+
# See https://cibuildwheel.pypa.io/en/stable/options/#build-frontend.
144+
CIBW_BUILD_FRONTEND: "build"
145+
CIBW_BEFORE_ALL: bash {project}/build_tools/wheels/cibw_before_all.sh ${{ matrix.python_version }} {project}
146+
147+
CIBW_BUILD_VERBOSITY: 1
148+
149+
run: bash build_tools/wheels/build_wheels.sh
150+
151+
- name: Store artifacts
152+
uses: actions/upload-artifact@v4
153+
with:
154+
name: cibw-wheels-cp${{ matrix.cibw_python_version }}-${{ matrix.platform_id }}
155+
path: wheelhouse/*.whl
156+
157+
upload_all:
158+
name: Upload All
159+
needs: build_wheels
160+
runs-on: ubuntu-latest
161+
permissions:
162+
id-token: write
163+
steps:
164+
- name: Download Artifacts
165+
uses: actions/download-artifact@v4
166+
with:
167+
path: dist/
168+
merge-multiple: true
169+
170+
- name: Publish to PyPI
171+
uses: pypa/gh-action-pypi-publish@release/v1
172+
with:
173+
verbose: true
174+
packages-dir: dist/
175+
# repository-url: https://test.pypi.org/legacy/

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/build*
1+
/build
22
/debug*
33
.idea
44
*.pyc

‎CMakeLists.txt‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,23 @@ set (GTSAM_VERSION_PATCH 0)
1010
set (GTSAM_PRERELEASE_VERSION "a0")
1111
math (EXPR GTSAM_VERSION_NUMERIC "10000 * ${GTSAM_VERSION_MAJOR} + 100 * ${GTSAM_VERSION_MINOR} + ${GTSAM_VERSION_PATCH}")
1212

13-
if ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "")
13+
# Set the version string for the library.
14+
#
15+
# If the environment variable DEVELOP is set, then the version string will be
16+
# "MAJOR.MINORprerelease.devTIMESTAMP". TIMESTAMP is another environment variable that should be set to the current
17+
# datetime. See build-cibw.yaml for example usage.
18+
#
19+
# If the prerelease version is empty, then the version string will be "MAJOR.MINOR.PATCH". Otherwise, the version
20+
# string will be "MAJOR.MINORprerelease".
21+
if (DEFINED ENV{DEVELOP})
22+
set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}.dev$ENV{TIMESTAMP}")
23+
set (SETUP_NAME "gtsam-develop")
24+
elseif ("${GTSAM_PRERELEASE_VERSION}" STREQUAL "")
1425
set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}.${GTSAM_VERSION_PATCH}")
26+
set (SETUP_NAME "gtsam")
1527
else()
1628
set (GTSAM_VERSION_STRING "${GTSAM_VERSION_MAJOR}.${GTSAM_VERSION_MINOR}${GTSAM_PRERELEASE_VERSION}")
29+
set (SETUP_NAME "gtsam")
1730
endif()
1831

1932
project(GTSAM

‎build_tools/wheels/build_wheels.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
# This script calls cibuildwheel to build the wheels for the project. It is used in the build-cibw.yml workflow in .github/workflows.
4+
# Note that the build/python directory contains the wrapper module built for the specified Python version.
5+
6+
set -e
7+
set -x
8+
9+
python -m pip install cibuildwheel
10+
python -m cibuildwheel build/python --output-dir wheelhouse

‎build_tools/wheels/cibw_before_all.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
3+
# This script is invoked prior to building the wheels with cibuildwheel. It is used in the build-cibw.yml workflow in .github/workflows.
4+
# It installs the necessary dependencies and builds the wrapper module for the specified Python version.
5+
6+
set -e
7+
set -x
8+
9+
PYTHON_VERSION="$1"
10+
PROJECT_DIR="$2"
11+
ARCH=$(uname -m)
12+
13+
export PYTHON="python${PYTHON_VERSION}"
14+
15+
if [ "$(uname)" == "Linux" ]; then
16+
# manylinux2014 is based on CentOS 7, so use yum to install dependencies
17+
yum install -y wget
18+
19+
# Install Boost from source
20+
wget https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.gz --quiet
21+
tar -xzf boost_1_87_0.tar.gz
22+
cd boost_1_87_0
23+
./bootstrap.sh --prefix=/opt/boost
24+
./b2 install --prefix=/opt/boost --with=all
25+
cd ..
26+
elif [ "$(uname)" == "Darwin" ]; then
27+
brew install wget cmake boost
28+
fi
29+
30+
$(which $PYTHON) -m pip install -r $PROJECT_DIR/python/dev_requirements.txt
31+
32+
# Remove build/cache files that were generated on host
33+
rm -rf $PROJECT_DIR/build
34+
rm -rf CMakeCache.txt CMakeFiles
35+
36+
# Build the Python wrapper module
37+
cmake $PROJECT_DIR \
38+
-B build \
39+
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
40+
-DGTSAM_BUILD_TESTS=OFF \
41+
-DGTSAM_BUILD_UNSTABLE=${GTSAM_BUILD_UNSTABLE:-ON} \
42+
-DGTSAM_USE_QUATERNIONS=OFF \
43+
-DGTSAM_WITH_TBB=${GTSAM_WITH_TBB:-OFF} \
44+
-DGTSAM_BUILD_EXAMPLES_ALWAYS=OFF \
45+
-DGTSAM_BUILD_WITH_MARCH_NATIVE=OFF \
46+
-DGTSAM_BUILD_PYTHON=ON \
47+
-DGTSAM_UNSTABLE_BUILD_PYTHON=${GTSAM_BUILD_UNSTABLE:-ON} \
48+
-DGTSAM_PYTHON_VERSION=$PYTHON_VERSION \
49+
-DPYTHON_EXECUTABLE:FILEPATH=$(which $PYTHON) \
50+
-DGTSAM_ALLOW_DEPRECATED_SINCE_V43=OFF \
51+
-DCMAKE_INSTALL_PREFIX=$PROJECT_DIR/gtsam_install
52+
53+
cd $PROJECT_DIR/build/python
54+
55+
# Install the Python wrapper module and generate Python stubs
56+
if [ "$(uname)" == "Linux" ]; then
57+
make -j $(nproc) install
58+
make -j $(nproc) python-stubs
59+
elif [ "$(uname)" == "Darwin" ]; then
60+
make -j $(sysctl -n hw.logicalcpu) install
61+
make -j $(sysctl -n hw.logicalcpu) python-stubs
62+
fi
63+

‎python/CMakeLists.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -284,9 +284,9 @@ endif()
284284
add_custom_target(
285285
python-stubs
286286
COMMAND
287-
${CMAKE_COMMAND} -E env
288-
"PYTHONPATH=${GTSAM_PYTHON_BUILD_DIRECTORY}/$ENV{PYTHONPATH}"
289-
pybind11-stubgen -o . --enum-class-locations \"KernelFunctionType|NoiseFormat:gtsam.gtsam\" --enum-class-locations \"OrderingType:gtsam.gtsam.Ordering\" --numpy-array-use-type-var --ignore-all-errors gtsam
287+
${CMAKE_COMMAND} -E env
288+
"PYTHONPATH=${GTSAM_PYTHON_BUILD_DIRECTORY}/$ENV{PYTHONPATH}"
289+
${PYTHON_EXECUTABLE} -m pybind11_stubgen -o . --enum-class-locations \"KernelFunctionType|NoiseFormat:gtsam.gtsam\" --enum-class-locations \"OrderingType:gtsam.gtsam.Ordering\" --numpy-array-use-type-var --ignore-all-errors gtsam
290290
DEPENDS ${GTSAM_PYTHON_DEPENDENCIES} ${GTSAM_PYTHON_TEST_FILES} ${GTSAM_PYTHON_TARGET}
291291
WORKING_DIRECTORY "${GTSAM_PYTHON_BUILD_DIRECTORY}/"
292292
)

‎python/setup.py.in‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
"""Setup file to install the GTSAM package."""
22

3-
from setuptools import setup, find_namespace_packages
3+
from setuptools import setup, find_namespace_packages, Distribution
44

55
packages = find_namespace_packages(
66
where=".",
@@ -20,8 +20,14 @@ package_data = {
2020
# Cleaner to read in the contents rather than copy them over.
2121
readme_contents = open("${GTSAM_SOURCE_DIR}/README.md").read()
2222

23+
# The cibuildwheel tool won't recognize a wheel as platform-dependent unless the ext_modules option is defined in setup.py. This is used to define C/C++ source files that need to be built for the wheel.
24+
# However, we pre-build our C++ files. Thus, we force cibuildwheel to think that there are ext_modules defined by overwriting the has_ext_modules() function.
25+
class BinaryDistribution(Distribution):
26+
def has_ext_modules(foo):
27+
return True
28+
2329
setup(
24-
name='gtsam',
30+
name='${SETUP_NAME}',
2531
description='Georgia Tech Smoothing And Mapping library',
2632
url='https://gtsam.org/',
2733
version='${GTSAM_VERSION_STRING}', # https://www.python.org/dev/peps/pep-0440/
@@ -46,6 +52,7 @@ setup(
4652
packages=packages,
4753
include_package_data=True,
4854
package_data=package_data,
55+
distclass=BinaryDistribution,
4956
test_suite="gtsam.tests",
5057
install_requires=open("${GTSAM_SOURCE_DIR}/python/requirements.txt").readlines(),
5158
zip_safe=False,

0 commit comments

Comments
 (0)