Add python stubs to built wheels

Adds support for IDE type completion and static type analysis

Signed-off-by: Chad Dombrova <[email protected]>

Cleanup

Fix more type errors in the tests

Add docs on how to build the stubs

Remove reference to make pystubs

Document how to generate stubs without docker

Improve documentation

Author: chadrik

.github/workflows/wheel_workflow.yml

@@ -144,14 +144,21 @@ jobs:
       - name: Build wheels
         uses: pypa/[email protected]
         env:
+          # pass GITHUB_ACTIONS through to the build container so that custom
+          # processes can tell they are running in CI.
+          CIBW_ENVIRONMENT_PASS_LINUX: GITHUB_ACTIONS
           CIBW_BUILD: ${{ matrix.python }}
           CIBW_ARCHS: ${{ matrix.arch }}
           CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux }}
 
       - uses: actions/upload-artifact@v4
         with:
           name: cibw-wheels-${{ matrix.python }}-${{ matrix.manylinux }}
-          path: ./wheelhouse/*.whl
+          path: |
+            ./wheelhouse/*.whl
+            ./wheelhouse/PyOpenColorIO/__init__.pyi
+          # if stub validation fails we want to upload the stubs for users to review
+          if: success() || failure()
 
   # ---------------------------------------------------------------------------
   # Linux ARM Wheels
@@ -213,7 +220,11 @@ jobs:
       - uses: actions/upload-artifact@v4
         with:
           name: cibw-wheels-${{ matrix.python }}-${{ matrix.manylinux }}
-          path: ./wheelhouse/*.whl
+          path: |
+            ./wheelhouse/*.whl
+            ./wheelhouse/PyOpenColorIO/__init__.pyi
+          # if stub validation fails we want to upload the stubs for users to review
+          if: success() || failure()
 
   # ---------------------------------------------------------------------------
   # macOS Wheels

docs/quick_start/installation.rst

@@ -536,3 +536,51 @@ Note: For other user facing environment variables, see :ref:`using_env_vars`.
     for Windows, Linux and macOS respectively. For systems that supports it,
     it is also possible to edit the RPATH of the plugin to add the location of
     the shared OpenColorIO lib.
+
+Updating python type stubs
+==========================
+
+The python stubs are built by analyzing the docstrings generated by pybind11
+and are used to drive IDE code completion and static analysis.  Changes to
+the bindings that result in changes to the stubs should be reviewed together
+in a single pull request, so that regressions are not unwittingly introduced.
+
+The workflow looks like this:
+
+- Install `uv <https://docs.astral.sh/uv/getting-started/installation/>`_ and
+  ``docker``
+- Create the temp build directory, cd into it, and run `cmake /path/to/source`
+- Run ``cmake --build . --target pystubs`` to generate updated stubs in
+  ``src/bindings/python/stubs/__init__.pyi``
+- Commit the new stubs and push to Github
+- In CI, the stubs will be included in the wheels built by ``cibuildwheel``, as
+  defined in ``.github/workflows/wheel_workflow.yml``
+- In CI, one of the ``cibuildwheel`` Github actions will rebuild the stubs to a
+  temp location and verify that they match what has been committed to the repo.
+  This step ensures that if changes to the C++ source code and bindings results
+  in a change to the stubs, developers are notified of the need to regenerate
+  the stubs, so that changes can be reviewed and the rules in ``generate_stubs.py``
+  can be updated, if necessary.
+
+If you want to bypass the use of docker, you can use the following steps to
+build the python bindings and run the stub generator directly (requires python
+3.11+):
+
+.. code-block:: bash
+
+    python -m venv .venv
+    . .venv/bin/activate
+    mkdir build
+    cd build
+    cmake -DPython_EXECUTABLE=$(which python) -DCMAKE_INSTALL_PREFIX=../dist ..
+    cmake --build . --target=install
+    # extract the deps from the pyproject.toml and install them into the virtualenv
+    PY_DEPS=$(python -c "import tomllib;print(tomllib.load(open('../pyproject.toml', 'rb'))['tool']['cibuildwheel']['overrides'][0]['test-requires'])")
+    pip install ${PY_DEPS}
+    PY_VER=$(python -c "import sys;print(f'{sys.version_info[0]}.{sys.version_info[1]}')")
+    # generate stubs
+    PYTHONPATH=../dist/lib/python$PY_VER/site-packages python ../src/bindings/python/stubs/generate_stubs.py
+
+If all else fails and you can't build the stubs locally, you can push changes
+to CI and download an artifact containing the wheel and ``__init__.pyi`` file from
+any job that fails the stub validation.

pyproject.toml

@@ -40,3 +40,33 @@ before-build = "share/ci/scripts/macos/install_docs_env.sh"
 # Disable for now as wheels are built with static dependencies.
 environment = { PATH="$GITHUB_WORKSPACE/doxygen;$PATH", OCIO_PYTHON_LOAD_DLLS_FROM_PATH="0" }
 before-build = 'bash -c "share/ci/scripts/windows/install_docs_env.sh $GITHUB_WORKSPACE/doxygen"'
+
+[[tool.cibuildwheel.overrides]]
+# Trigger the build & validation of the python stubs for certain platforms.
+# The test command acts as a kind of "post-build" callback where it's possible
+# for the stub-generator to import the freshly-built wheel.
+# There are two entry-points which are designed to call generate_stubs.py through
+# this test command:
+# - `cmake /path/to/source; cmake --build . --target pystubs` is called during
+#   local development to generate the stubs and copy them into the git repo to
+#   be committed and reviewed.
+# - in CI, the cibuildwheel action is used to validate that the stubs match what
+#   has been committed to the repo.
+test-requires = "mypy~=1.15.0 stubgenlib~=0.2.0"
+# Note: the python version here must be kept in sync with src/bindings/python/CMakeLists.txt
+select = "cp311-manylinux_*64"
+inherit.test-command = "append"
+test-command = [
+    "python {project}/src/bindings/python/stubs/generate_stubs.py --out-path '/output' --validate-path '{project}/src/bindings/python/stubs/PyOpenColorIO/__init__.pyi'",
+]
+
+[tool.mypy]
+files = [
+    "tests/python/",
+    "src/bindings/python/stubs",
+]
+allow_redefinition = true
+check_untyped_defs = true
+disable_error_code = [
+    "func-returns-value",
+]

src/bindings/python/CMakeLists.txt

@@ -259,3 +259,30 @@ install(TARGETS PyOpenColorIO
 )
 
 install(FILES package/__init__.py DESTINATION ${_PyOpenColorIO_SITE_PACKAGE_DIR})
+install(FILES stubs/PyOpenColorIO/__init__.pyi DESTINATION ${_PyOpenColorIO_SITE_PACKAGE_DIR})
+install(FILES stubs/PyOpenColorIO/py.typed DESTINATION ${_PyOpenColorIO_SITE_PACKAGE_DIR})
+
+###############################################################################
+
+# Note: the python version must be kept in sync with `[[tool.cibuildwheel.overrides]]` in pyproject.toml.
+# The stubs are generated within a container so the version of python does not need to match
+# the version of python that OpenColorIO is being built against.
+# Note: the version of cibuildwheel should be kept in sync with .github/workflows/wheel.yml
+add_custom_command (COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/generate_stubs_local.py
+                        --repo-root ${CMAKE_SOURCE_DIR} --python-version="3.11" --cibuildwheel-version="2.22.0"
+                        --output-dir "${CMAKE_BINARY_DIR}/wheelhouse"
+                    OUTPUT "${CMAKE_BINARY_DIR}/wheelhouse/PyOpenColorIO/__init__.pyi"
+                    DEPENDS "${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/generate_stubs.py"
+                    DEPENDS "${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/generate_stubs_local.py"
+                    COMMENT "pystubs: Generating python stubs"
+                    )
+
+add_custom_command (COMMAND ${CMAKE_COMMAND} -E copy
+                        "${CMAKE_BINARY_DIR}/wheelhouse/PyOpenColorIO/__init__.pyi"
+                        "${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/PyOpenColorIO/__init__.pyi"
+                    OUTPUT "${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/PyOpenColorIO/__init__.pyi"
+                    DEPENDS "${CMAKE_BINARY_DIR}/wheelhouse/PyOpenColorIO/__init__.pyi"
+                    COMMENT "pystubs: Copying generated stubs to source"
+                    )
+
+add_custom_target (pystubs DEPENDS "${CMAKE_SOURCE_DIR}/src/bindings/python/stubs/PyOpenColorIO/__init__.pyi")