Pybind11 free threading (#3158)

* pickup latest pybind11 and support free threaded

* Cleanup

Some stuff wasn't actually thread safe:
  - Made the jitter RNG thread_local in tools/python/src/numpy_returns.cpp:97.
  - Serialized concurrent inference on the same face-recognition or CNN detector instance in tools/python/src/face_recognition.cpp:72 and tools/python/src/ cnn_face_detector.cpp:52.

I also pulled the official pybind11 release.  What was here looks like one of their alpha snapshots.  We should use an official pybind11 release
In particular, I replaced the mixed development pybind11 snapshot with exact upstream pybind11 v3.0.4, commit d03662f0984f652b60e7ddce53d3868002275197. Also added provenance/update instructions in dlib/external/pybind11/README.dlib.

Then some other minor related cleanup

* fix errors

---------

Co-authored-by: Katsumi Yokomichi <yokomichikt@scft.co.jp>

Author: davisking

.github/workflows/build_python.yml

@@ -60,6 +60,36 @@ jobs:
     - name: Test
       run: python -m pytest --ignore docs --ignore dlib
 
+  Ubuntu-Free-Threaded:
+    runs-on: 'ubuntu-latest'
+    steps:
+    - uses: actions/checkout@v6
+    - uses: actions/setup-python@v6
+      with:
+        python-version: '3.13t'
+    - name: Install python deps
+      run: pip install pytest numpy
+    - name: Build
+      run: pip install .
+    - name: Verify free-threading
+      run: python -c "import sys; assert not sys._is_gil_enabled()"
+    - name: Test
+      run: python -m pytest --ignore docs --ignore dlib
+
+  Ubuntu-Python-3-8:
+    runs-on: 'ubuntu-latest'
+    steps:
+    - uses: actions/checkout@v6
+    - uses: actions/setup-python@v6
+      with:
+        python-version: '3.8'
+    - name: Install python deps
+      run: pip install pytest numpy
+    - name: Build
+      run: pip install .
+    - name: Test
+      run: python -m pytest --ignore docs --ignore dlib
+
 # Disabled for now since something is going sideways with python packages on github actions
 # MacOS:
 #   runs-on: 'macos-latest'
@@ -73,4 +103,3 @@ jobs:
 #       pip install .
 #   - name: Test
 #     run: python3 -m pytest --ignore docs --ignore dlib
-

dlib/external/pybind11/CMakeLists.txt

@@ -10,26 +10,15 @@ if(NOT CMAKE_VERSION VERSION_LESS "3.27")
   cmake_policy(GET CMP0148 _pybind11_cmp0148)
 endif()
 
-cmake_minimum_required(VERSION 3.5)
-
-# The `cmake_minimum_required(VERSION 3.5...3.27)` syntax does not work with
-# some versions of VS that have a patched CMake 3.11. This forces us to emulate
-# the behavior using the following workaround:
-if(${CMAKE_VERSION} VERSION_LESS 3.27)
-  cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
-else()
-  cmake_policy(VERSION 3.27)
-endif()
+cmake_minimum_required(VERSION 3.15...4.2)
 
 if(_pybind11_cmp0148)
   cmake_policy(SET CMP0148 ${_pybind11_cmp0148})
   unset(_pybind11_cmp0148)
 endif()
 
 # Avoid infinite recursion if tests include this as a subdirectory
-if(DEFINED PYBIND11_MASTER_PROJECT)
-  return()
-endif()
+include_guard(GLOBAL)
 
 # Extract project version from source
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/include/pybind11/detail/common.h"
@@ -74,16 +63,15 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
 
   set(PYBIND11_MASTER_PROJECT ON)
 
-  if(OSX AND CMAKE_VERSION VERSION_LESS 3.7)
-    # Bug in macOS CMake < 3.7 is unable to download catch
-    message(WARNING "CMAKE 3.7+ needed on macOS to download catch, and newer HIGHLY recommended")
-  elseif(WINDOWS AND CMAKE_VERSION VERSION_LESS 3.8)
-    # Only tested with 3.8+ in CI.
-    message(WARNING "CMAKE 3.8+ tested on Windows, previous versions untested")
-  endif()
-
   message(STATUS "CMake ${CMAKE_VERSION}")
 
+  if(DEFINED SKBUILD AND DEFINED ENV{PYBIND11_GLOBAL_SDIST})
+    message(
+      FATAL_ERROR
+        "PYBIND11_GLOBAL_SDIST is not supported, use nox -s build_global or a pybind11-global SDist instead."
+    )
+  endif()
+
   if(CMAKE_CXX_STANDARD)
     set(CMAKE_CXX_EXTENSIONS OFF)
     set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -100,7 +88,7 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
 else()
   set(PYBIND11_MASTER_PROJECT OFF)
   set(pybind11_system SYSTEM)
-  set(_pybind11_findpython_default OFF)
+  set(_pybind11_findpython_default COMPAT)
 endif()
 
 # Options
@@ -111,54 +99,115 @@ option(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION
        "To enforce that a handle_type_name<> specialization exists" OFF)
 option(PYBIND11_SIMPLE_GIL_MANAGEMENT
        "Use simpler GIL management logic that does not support disassociation" OFF)
-option(PYBIND11_NUMPY_1_ONLY
-       "Disable NumPy 2 support to avoid changes to previous pybind11 versions." OFF)
 set(PYBIND11_INTERNALS_VERSION
     ""
     CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.")
+option(PYBIND11_USE_CROSSCOMPILING "Respect CMAKE_CROSSCOMPILING" OFF)
 
 if(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
   add_compile_definitions(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
 endif()
 if(PYBIND11_SIMPLE_GIL_MANAGEMENT)
   add_compile_definitions(PYBIND11_SIMPLE_GIL_MANAGEMENT)
 endif()
-if(PYBIND11_NUMPY_1_ONLY)
-  add_compile_definitions(PYBIND11_NUMPY_1_ONLY)
-endif()
 
 cmake_dependent_option(
   USE_PYTHON_INCLUDE_DIR
   "Install pybind11 headers in Python include directory instead of default installation prefix"
   OFF "PYBIND11_INSTALL" OFF)
 
-cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" ${_pybind11_findpython_default}
-                       "NOT CMAKE_VERSION VERSION_LESS 3.12" OFF)
+set(PYBIND11_FINDPYTHON
+    ${_pybind11_findpython_default}
+    CACHE STRING "Force new FindPython - NEW, OLD, COMPAT")
 
-# Allow PYTHON_EXECUTABLE if in FINDPYTHON mode and building pybind11's tests
-# (makes transition easier while we support both modes).
-if(PYBIND11_MASTER_PROJECT
-   AND PYBIND11_FINDPYTHON
-   AND DEFINED PYTHON_EXECUTABLE
-   AND NOT DEFINED Python_EXECUTABLE)
-  set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}")
+if(PYBIND11_MASTER_PROJECT)
+
+  # Allow PYTHON_EXECUTABLE if in FINDPYTHON mode and building pybind11's tests
+  # (makes transition easier while we support both modes).
+  if(PYBIND11_FINDPYTHON
+     AND DEFINED PYTHON_EXECUTABLE
+     AND NOT DEFINED Python_EXECUTABLE)
+    set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}")
+  endif()
+
+  # This is a shortcut that is primarily for the venv cmake preset,
+  # but can be used to quickly setup tests manually, too
+  set(PYBIND11_CREATE_WITH_UV
+      ""
+      CACHE STRING "Create a virtualenv if it doesn't exist")
+
+  if(NOT PYBIND11_CREATE_WITH_UV STREQUAL "")
+    set(Python_ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/.venv")
+    if(EXISTS "${Python_ROOT_DIR}")
+      if(EXISTS "${CMAKE_BINARY_DIR}/CMakeCache.txt")
+        message(STATUS "Using existing venv at ${Python_ROOT_DIR}, remove or --fresh to recreate")
+      else()
+        # --fresh used to remove the cache
+        file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/.venv")
+      endif()
+    endif()
+    if(NOT EXISTS "${Python_ROOT_DIR}")
+      find_program(UV uv REQUIRED)
+      # CMake 3.19+ would be able to use COMMAND_ERROR_IS_FATAL
+      message(
+        STATUS "Creating venv with ${UV} venv -p ${PYBIND11_CREATE_WITH_UV} '${Python_ROOT_DIR}'")
+      execute_process(COMMAND ${UV} venv -p ${PYBIND11_CREATE_WITH_UV} "${Python_ROOT_DIR}"
+                      RESULT_VARIABLE _venv_result)
+      if(_venv_result AND NOT _venv_result EQUAL 0)
+        message(FATAL_ERROR "uv venv failed with '${_venv_result}'")
+      endif()
+      message(
+        STATUS
+          "Installing deps with ${UV} pip install -p '${Python_ROOT_DIR}' -r tests/requirements.txt"
+      )
+      execute_process(
+        COMMAND ${UV} pip install -p "${Python_ROOT_DIR}" -r
+                "${CMAKE_CURRENT_SOURCE_DIR}/tests/requirements.txt" RESULT_VARIABLE _pip_result)
+      if(_pip_result AND NOT _pip_result EQUAL 0)
+        message(FATAL_ERROR "uv pip install failed with '${_pip_result}'")
+      endif()
+    endif()
+  else()
+    if(NOT DEFINED Python3_EXECUTABLE
+       AND NOT DEFINED Python_EXECUTABLE
+       AND NOT DEFINED Python_ROOT_DIR
+       AND NOT DEFINED ENV{VIRTUALENV}
+       AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.venv")
+      message(STATUS "Autodetecting Python in virtual environment")
+      set(Python_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/.venv")
+    endif()
+  endif()
 endif()
 
-# NB: when adding a header don't forget to also add it to setup.py
 set(PYBIND11_HEADERS
+    include/pybind11/detail/argument_vector.h
     include/pybind11/detail/class.h
     include/pybind11/detail/common.h
+    include/pybind11/detail/cpp_conduit.h
     include/pybind11/detail/descr.h
+    include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h
+    include/pybind11/detail/exception_translation.h
+    include/pybind11/detail/function_record_pyobject.h
+    include/pybind11/detail/holder_caster_foreign_helpers.h
     include/pybind11/detail/init.h
     include/pybind11/detail/internals.h
+    include/pybind11/detail/native_enum_data.h
+    include/pybind11/detail/pybind11_namespace_macros.h
+    include/pybind11/detail/struct_smart_holder.h
     include/pybind11/detail/type_caster_base.h
     include/pybind11/detail/typeid.h
+    include/pybind11/detail/using_smart_holder.h
+    include/pybind11/detail/value_and_holder.h
     include/pybind11/attr.h
     include/pybind11/buffer_info.h
     include/pybind11/cast.h
     include/pybind11/chrono.h
     include/pybind11/common.h
     include/pybind11/complex.h
+    include/pybind11/conduit/pybind11_conduit_v1.h
+    include/pybind11/conduit/pybind11_platform_abi_id.h
+    include/pybind11/conduit/wrap_include_python_h.h
+    include/pybind11/critical_section.h
     include/pybind11/options.h
     include/pybind11/eigen.h
     include/pybind11/eigen/common.h
@@ -168,20 +217,25 @@ set(PYBIND11_HEADERS
     include/pybind11/eval.h
     include/pybind11/gil.h
     include/pybind11/gil_safe_call_once.h
+    include/pybind11/gil_simple.h
     include/pybind11/iostream.h
     include/pybind11/functional.h
+    include/pybind11/native_enum.h
     include/pybind11/numpy.h
     include/pybind11/operators.h
     include/pybind11/pybind11.h
     include/pybind11/pytypes.h
+    include/pybind11/subinterpreter.h
     include/pybind11/stl.h
     include/pybind11/stl_bind.h
     include/pybind11/stl/filesystem.h
+    include/pybind11/trampoline_self_life_support.h
     include/pybind11/type_caster_pyobject_ptr.h
-    include/pybind11/typing.h)
+    include/pybind11/typing.h
+    include/pybind11/warnings.h)
 
 # Compare with grep and warn if mismatched
-if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12)
+if(PYBIND11_MASTER_PROJECT)
   file(
     GLOB_RECURSE _pybind11_header_check
     LIST_DIRECTORIES false
@@ -199,10 +253,7 @@ if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12)
   endif()
 endif()
 
-# CMake 3.12 added list(TRANSFORM <list> PREPEND
-# But we can't use it yet
-string(REPLACE "include/" "${CMAKE_CURRENT_SOURCE_DIR}/include/" PYBIND11_HEADERS
-               "${PYBIND11_HEADERS}")
+list(TRANSFORM PYBIND11_HEADERS PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
 
 # Cache variable so this can be used in parent projects
 set(pybind11_INCLUDE_DIR
@@ -257,6 +308,9 @@ elseif(USE_PYTHON_INCLUDE_DIR AND DEFINED PYTHON_INCLUDE_DIR)
 endif()
 
 if(PYBIND11_INSTALL)
+  if(DEFINED SKBUILD_PROJECT_NAME AND SKBUILD_PROJECT_NAME STREQUAL "pybind11_global")
+    install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION "${SKBUILD_HEADERS_DIR}")
+  endif()
   install(DIRECTORY ${pybind11_INCLUDE_DIR}/pybind11 DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
   set(PYBIND11_CMAKECONFIG_INSTALL_DIR
       "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}"
@@ -272,25 +326,11 @@ if(PYBIND11_INSTALL)
     tools/${PROJECT_NAME}Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
     INSTALL_DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
 
-  if(CMAKE_VERSION VERSION_LESS 3.14)
-    # Remove CMAKE_SIZEOF_VOID_P from ConfigVersion.cmake since the library does
-    # not depend on architecture specific settings or libraries.
-    set(_PYBIND11_CMAKE_SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P})
-    unset(CMAKE_SIZEOF_VOID_P)
-
-    write_basic_package_version_file(
-      ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
-      VERSION ${PROJECT_VERSION}
-      COMPATIBILITY AnyNewerVersion)
-
-    set(CMAKE_SIZEOF_VOID_P ${_PYBIND11_CMAKE_SIZEOF_VOID_P})
-  else()
-    # CMake 3.14+ natively supports header-only libraries
-    write_basic_package_version_file(
-      ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
-      VERSION ${PROJECT_VERSION}
-      COMPATIBILITY AnyNewerVersion ARCH_INDEPENDENT)
-  endif()
+  # CMake natively supports header-only libraries
+  write_basic_package_version_file(
+    ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion ARCH_INDEPENDENT)
 
   install(
     FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake
@@ -299,6 +339,7 @@ if(PYBIND11_INSTALL)
           tools/pybind11Common.cmake
           tools/pybind11Tools.cmake
           tools/pybind11NewTools.cmake
+          tools/pybind11GuessPythonExtSuffix.cmake
     DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
 
   if(NOT PYBIND11_EXPORT_NAME)
@@ -336,6 +377,17 @@ if(PYBIND11_INSTALL)
   install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc"
           DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/")
 
+  # When building a wheel, include __init__.py's for modules
+  # (see https://github.com/pybind/pybind11/pull/5552)
+  if(DEFINED SKBUILD_PROJECT_NAME AND SKBUILD_PROJECT_NAME STREQUAL "pybind11")
+    file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/empty")
+    file(TOUCH "${CMAKE_CURRENT_BINARY_DIR}/empty/__init__.py")
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/empty/__init__.py"
+            DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/")
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/empty/__init__.py"
+            DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/")
+  endif()
+
   # Uninstall target
   if(PYBIND11_MASTER_PROJECT)
     configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake_uninstall.cmake.in"

dlib/external/pybind11/README.dlib

@@ -0,0 +1,13 @@
+This directory vendors pybind11 v3.0.4 from commit
+d03662f0984f652b60e7ddce53d3868002275197.
+
+The following paths are copied unchanged from the upstream release tag:
+
+- CMakeLists.txt
+- LICENSE
+- README.rst
+- include/
+- tools/
+
+When updating pybind11, replace all of those paths from one tagged upstream
+release.  Do not mix headers or CMake helpers from different revisions.