Add pip install support via scikit-build-core + cibuildwheel (facebookresearch#4862)

Summary:

Add official pip wheel packaging for `faiss-cpu`, enabling
`pip install faiss-cpu` from PyPI. Builds binary wheels across Linux
x86_64/aarch64, macOS arm64/x86_64, and Windows x86_64 for Python 3.10-3.14.
Linux and macOS use Python Stable ABI (abi3) — a single cp310-abi3 wheel
per platform works on all Python 3.10+ versions without per-version rebuilds.

**BLAS strategy:**
- Linux (x86_64 and aarch64): Threaded OpenBLAS (`libopenblaso`). The
  CMake build prefers `libopenblaso` (OpenMP-threaded) or `libopenblasp`
  (pthreads-threaded) over the serial default on RHEL/Fedora-based
  manylinux containers.
- macOS: Apple Accelerate (found automatically).
- Windows: Prebuilt OpenBLAS from GitHub releases (v0.3.30).

**SIMD optimization strategy:**
- Linux and macOS: Dynamic Dispatch (`FAISS_OPT_LEVEL=dd`) compiles SIMD-hot
  kernel files at multiple ISA levels (generic + AVX2 + AVX-512 on x86_64,
  NEON + SVE on aarch64) into a single library. Runtime CPUID detection
  selects the optimal code path automatically.
- Windows: MSVC does not support DD, so uses `FAISS_OPT_LEVEL=generic`.
  Shared libraries (`BUILD_SHARED_LIBS=ON`) are used to avoid MSVC's
  single-pass static archive scanning issue.

**New files:**
- `pyproject.toml`: scikit-build-core build backend config with cibuildwheel
  settings. Version is dynamically extracted from CMakeLists.txt. Enables
  abi3 on Linux/macOS via SWIG 4.2+ and NumPy 2.0+ Limited API support.
  Enables LTO and dead-code stripping for smaller wheel sizes.
- `.github/workflows/build-pip.yml`: CI workflow using cibuildwheel to build
  binary wheels on 5 platform runners. All GitHub Actions are pinned to
  commit SHAs for supply chain security. Publishes wheels to PyPI via OIDC
  trusted publishers on tag push.
- `THIRD_PARTY_NOTICES`: License notices for bundled dependencies (OpenBLAS,
  LLVM OpenMP).
- `tests/test_wheel_smoke.py`: 12-test smoke suite validating import,
  OpenMP, BLAS (FlatL2/FlatIP), index factory (IVF+PQ), HNSW, serialization
  roundtrip, GC safety, contrib imports, and SIMD level detection.

**Modified files:**
- `faiss/CMakeLists.txt`: Added threaded OpenBLAS fallback — prefers
  `libopenblaso` (OpenMP-threaded) or `libopenblasp` (pthreads-threaded)
  over the serial default on RHEL/Fedora-based manylinux containers.
- `python/CMakeLists.txt`: Added `install()` targets for scikit-build-core
  wheel packaging, including all Dynamic Dispatch SWIG variants
  (avx2/avx512/avx512_spr/sve). Added abi3 support: conditionally uses
  `Python::SABIModule`, defines `Py_LIMITED_API`, and sets `.abi3.so` suffix
  when `SKBUILD_SABI_VERSION` is set.
- `.github/workflows/build.yml`: Wired `build-pip.yml` into the root CI
  trigger so pip builds run alongside conda builds.
- `tests/BUCK`: Added `python_pytest` target for the smoke test.

Reviewed By: mnorris11

Differential Revision: D95258115

Author: alibeklfc

.github/workflows/build-pip.yml

@@ -0,0 +1,58 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+name: Build pip wheels
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  build-wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-14, macos-15-intel, windows-2022, 2-core-ubuntu-arm]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@ee02a1537ce3071a004a6b08c41e72f0fdc42d9a  # v3.4.0
+        env:
+          CIBW_BUILD_VERBOSITY: 1
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: wheels-${{ matrix.os }}
+          path: wheelhouse/*.whl
+
+  publish:
+    name: Publish to PyPI
+    needs: [build-wheels]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    environment:
+      name: pypi
+      url: https://pypi.org/p/faiss-cpu
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
+        with:
+          path: dist
+          merge-multiple: true
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc  # release/v1

.github/workflows/build.yml

@@ -15,3 +15,5 @@ jobs:
     secrets:
       ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+  build-pip:
+    uses: ./.github/workflows/build-pip.yml

THIRD_PARTY_NOTICES

@@ -583,14 +583,28 @@ if(MKL_FOUND)
   target_link_libraries(faiss_avx512_spr PRIVATE ${MKL_LIBRARIES})
   target_link_libraries(faiss_sve PRIVATE ${MKL_LIBRARIES})
 else()
-  find_package(BLAS REQUIRED)
+  # Prefer threaded OpenBLAS (OpenMP or pthreads) over the serial default.
+  # On RHEL/Fedora, the default libopenblas.so is the serial variant;
+  # libopenblaso is OpenMP-threaded and libopenblasp is pthreads-threaded.
+  find_library(BLAS_PREFER_THREADED NAMES openblaso openblasp)
+  if(BLAS_PREFER_THREADED)
+    message(STATUS "Using threaded OpenBLAS: ${BLAS_PREFER_THREADED}")
+    set(BLAS_LIBRARIES "${BLAS_PREFER_THREADED}")
+    # OpenBLAS bundles LAPACK routines, so no separate find_package(LAPACK)
+    # is needed. Calling it would trigger FindBLAS internally, which
+    # unconditionally resets BLAS_LIBRARIES to the serial libopenblas,
+    # causing auditwheel to bundle both serial and threaded variants.
+    set(LAPACK_LIBRARIES "${BLAS_PREFER_THREADED}")
+  else()
+    find_package(BLAS REQUIRED)
+    find_package(LAPACK REQUIRED)
+  endif()
   target_link_libraries(faiss PRIVATE ${BLAS_LIBRARIES})
   target_link_libraries(faiss_avx2 PRIVATE ${BLAS_LIBRARIES})
   target_link_libraries(faiss_avx512 PRIVATE ${BLAS_LIBRARIES})
   target_link_libraries(faiss_avx512_spr PRIVATE ${BLAS_LIBRARIES})
   target_link_libraries(faiss_sve PRIVATE ${BLAS_LIBRARIES})
 
-  find_package(LAPACK REQUIRED)
   target_link_libraries(faiss PRIVATE ${LAPACK_LIBRARIES})
   target_link_libraries(faiss_avx2 PRIVATE ${LAPACK_LIBRARIES})
   target_link_libraries(faiss_avx512 PRIVATE ${LAPACK_LIBRARIES})

faiss/CMakeLists.txt

@@ -184,22 +184,23 @@ set_property(TARGET faiss_example_external_module PROPERTY SWIG_COMPILE_OPTIONS
 if(AIX)
   set_target_properties(swigfaiss PROPERTIES SUFFIX .a)
   set_target_properties(faiss_example_external_module PROPERTIES SUFFIX .a)
-elseif(NOT WIN32)
-  # NOTE: Python does not recognize the dylib extension.
-  set_target_properties(swigfaiss PROPERTIES SUFFIX .so)
-  set_target_properties(swigfaiss_avx2 PROPERTIES SUFFIX .so)
-  set_target_properties(swigfaiss_avx512 PROPERTIES SUFFIX .so)
-  set_target_properties(swigfaiss_avx512_spr PROPERTIES SUFFIX .so)
-  set_target_properties(swigfaiss_sve PROPERTIES SUFFIX .so)
-  set_target_properties(faiss_example_external_module PROPERTIES SUFFIX .so)
-else()
+elseif(WIN32)
   # we need bigobj for the swig wrapper
   target_compile_options(swigfaiss PRIVATE /bigobj)
   target_compile_options(swigfaiss_avx2 PRIVATE /bigobj)
   target_compile_options(swigfaiss_avx512 PRIVATE /bigobj)
   target_compile_options(swigfaiss_avx512_spr PRIVATE /bigobj)
   target_compile_options(swigfaiss_sve PRIVATE /bigobj)
   target_compile_options(faiss_example_external_module PRIVATE /bigobj)
+elseif(NOT SKBUILD_SABI_VERSION)
+  # NOTE: Python does not recognize the dylib extension.
+  # When building abi3, the suffix is already set to .abi3.so above.
+  set_target_properties(swigfaiss PROPERTIES SUFFIX .so)
+  set_target_properties(swigfaiss_avx2 PROPERTIES SUFFIX .so)
+  set_target_properties(swigfaiss_avx512 PROPERTIES SUFFIX .so)
+  set_target_properties(swigfaiss_avx512_spr PROPERTIES SUFFIX .so)
+  set_target_properties(swigfaiss_sve PROPERTIES SUFFIX .so)
+  set_target_properties(faiss_example_external_module PROPERTIES SUFFIX .so)
 endif()
 
 if(FAISS_ENABLE_SVS)
@@ -239,43 +240,80 @@ endif()
 
 find_package(OpenMP REQUIRED)
 
+# When scikit-build-core requests abi3 (Stable ABI), it passes
+# SKBUILD_SABI_VERSION. Use Development.SABIModule to link against the
+# stable Python ABI instead of the version-specific one.
+if(SKBUILD_SABI_VERSION)
+  find_package(Python REQUIRED
+    COMPONENTS Development.SABIModule NumPy
+  )
+else()
+  find_package(Python REQUIRED
+    COMPONENTS Development.Module NumPy
+  )
+endif()
+
+# Select the Python module target based on whether abi3 is requested.
+if(SKBUILD_SABI_VERSION)
+  set(FAISS_PYTHON_MODULE_TARGET Python::SABIModule)
+else()
+  set(FAISS_PYTHON_MODULE_TARGET Python::Module)
+endif()
+
+# When building for abi3, define Py_LIMITED_API and set the .abi3.so suffix
+# so the wheel is tagged as abi3-compatible.
+if(SKBUILD_SABI_VERSION)
+  # 0x030A0000 = Python 3.10
+  set(FAISS_PY_LIMITED_API "0x030A0000")
+  foreach(tgt swigfaiss swigfaiss_avx2 swigfaiss_avx512
+      swigfaiss_avx512_spr swigfaiss_sve)
+    if(TARGET ${tgt})
+      target_compile_definitions(${tgt}
+        PRIVATE Py_LIMITED_API=${FAISS_PY_LIMITED_API})
+      if(NOT WIN32)
+        set_target_properties(${tgt} PROPERTIES SUFFIX ".abi3.so")
+      endif()
+    endif()
+  endforeach()
+endif()
+
 target_link_libraries(swigfaiss PRIVATE
   faiss
-  Python::Module
+  ${FAISS_PYTHON_MODULE_TARGET}
   Python::NumPy
   OpenMP::OpenMP_CXX
 )
 
 target_link_libraries(swigfaiss_avx2 PRIVATE
   faiss_avx2
-  Python::Module
+  ${FAISS_PYTHON_MODULE_TARGET}
   Python::NumPy
   OpenMP::OpenMP_CXX
 )
 
 target_link_libraries(swigfaiss_avx512 PRIVATE
   faiss_avx512
-  Python::Module
+  ${FAISS_PYTHON_MODULE_TARGET}
   Python::NumPy
   OpenMP::OpenMP_CXX
 )
 
 target_link_libraries(swigfaiss_avx512_spr PRIVATE
   faiss_avx512_spr
-  Python::Module
+  ${FAISS_PYTHON_MODULE_TARGET}
   Python::NumPy
   OpenMP::OpenMP_CXX
 )
 
 target_link_libraries(swigfaiss_sve PRIVATE
   faiss_sve
-  Python::Module
+  ${FAISS_PYTHON_MODULE_TARGET}
   Python::NumPy
   OpenMP::OpenMP_CXX
 )
 
 target_link_libraries(faiss_example_external_module PRIVATE
-  Python::Module
+  ${FAISS_PYTHON_MODULE_TARGET}
   Python::NumPy
   OpenMP::OpenMP_CXX
   swigfaiss
@@ -291,16 +329,16 @@ target_include_directories(swigfaiss_avx512_spr PRIVATE ${PROJECT_SOURCE_DIR}/..
 target_include_directories(swigfaiss_sve PRIVATE ${PROJECT_SOURCE_DIR}/../..)
 target_include_directories(faiss_example_external_module PRIVATE ${PROJECT_SOURCE_DIR}/../..)
 
-find_package(Python REQUIRED
-  COMPONENTS Development.Module NumPy
-)
-
-add_library(faiss_python_callbacks EXCLUDE_FROM_ALL
+add_library(faiss_python_callbacks STATIC EXCLUDE_FROM_ALL
   python_callbacks.cpp
 )
 set_property(TARGET faiss_python_callbacks
   PROPERTY POSITION_INDEPENDENT_CODE ON
 )
+if(SKBUILD_SABI_VERSION)
+  target_compile_definitions(faiss_python_callbacks
+    PRIVATE Py_LIMITED_API=${FAISS_PY_LIMITED_API})
+endif()
 
 if ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
 target_link_libraries(faiss_python_callbacks PRIVATE faiss)
@@ -328,3 +366,95 @@ configure_file(array_conversions.py array_conversions.py COPYONLY)
 
 # file(GLOB files "${PROJECT_SOURCE_DIR}/../../contrib/*.py")
 file(COPY ${PROJECT_SOURCE_DIR}/../../contrib  DESTINATION .)
+
+# Install targets for scikit-build-core / pip wheel packaging.
+
+# Set RPATH so the SWIG modules find libfaiss in the same directory.
+# auditwheel/delocate follow RPATH to locate libraries for bundling.
+foreach(tgt swigfaiss swigfaiss_avx2 swigfaiss_avx512
+    swigfaiss_avx512_spr swigfaiss_sve)
+  if(TARGET ${tgt})
+    if(APPLE)
+      set_target_properties(${tgt} PROPERTIES INSTALL_RPATH "@loader_path")
+    else()
+      set_target_properties(${tgt} PROPERTIES INSTALL_RPATH "$ORIGIN")
+    endif()
+  endif()
+endforeach()
+
+# Install the SWIG extension modules (.so/.pyd).
+# In DD mode, swigfaiss_avx2/avx512/avx512_spr/sve are also built and
+# loader.py selects the best one at runtime based on CPU capabilities.
+foreach(tgt swigfaiss swigfaiss_avx2 swigfaiss_avx512
+    swigfaiss_avx512_spr swigfaiss_sve)
+  if(TARGET ${tgt})
+    get_target_property(_excluded ${tgt} EXCLUDE_FROM_ALL)
+    if(NOT _excluded)
+      install(TARGETS ${tgt}
+        LIBRARY DESTINATION faiss
+        RUNTIME DESTINATION faiss
+      )
+    endif()
+  endif()
+endforeach()
+
+# Install the core faiss shared libraries into the Python package directory
+# so that auditwheel/delocate can find and bundle them with the SWIG modules.
+# Guard: when Python bindings are built standalone (e.g. conda), the faiss
+# target is an installed import library, not a build target.
+foreach(tgt faiss faiss_avx2 faiss_avx512 faiss_avx512_spr faiss_sve)
+  if(TARGET ${tgt})
+    get_target_property(_imported ${tgt} IMPORTED)
+    get_target_property(_excluded ${tgt} EXCLUDE_FROM_ALL)
+    if(NOT _imported AND NOT _excluded)
+      install(TARGETS ${tgt}
+        LIBRARY DESTINATION faiss
+        RUNTIME DESTINATION faiss
+      )
+    endif()
+  endif()
+endforeach()
+
+# Install SWIG-generated Python wrappers.
+# In DD mode, each variant has its own .py wrapper that loader.py imports.
+foreach(mod swigfaiss swigfaiss_avx2 swigfaiss_avx512
+    swigfaiss_avx512_spr swigfaiss_sve)
+  if(EXISTS "${CMAKE_CURRENT_BINARY_DIR}/${mod}.py" OR TARGET ${mod})
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${mod}.py"
+      DESTINATION faiss
+      OPTIONAL
+    )
+  endif()
+endforeach()
+
+# Install hand-written Python source files
+install(FILES
+  "${CMAKE_CURRENT_SOURCE_DIR}/__init__.py"
+  "${CMAKE_CURRENT_SOURCE_DIR}/loader.py"
+  "${CMAKE_CURRENT_SOURCE_DIR}/class_wrappers.py"
+  "${CMAKE_CURRENT_SOURCE_DIR}/extra_wrappers.py"
+  "${CMAKE_CURRENT_SOURCE_DIR}/gpu_wrappers.py"
+  "${CMAKE_CURRENT_SOURCE_DIR}/array_conversions.py"
+  DESTINATION faiss
+)
+
+# Install type stubs if present
+if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/__init__.pyi")
+  install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/__init__.pyi" DESTINATION faiss)
+endif()
+if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/py.typed")
+  install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/py.typed" DESTINATION faiss)
+endif()
+
+# Install contrib subpackage (excluding Meta-internal *_fb.py files)
+install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../../contrib/"
+  DESTINATION faiss/contrib
+  FILES_MATCHING PATTERN "*.py"
+  PATTERN "*_fb.py" EXCLUDE
+)
+
+# Install contrib/torch subpackage data files
+install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../../contrib/torch/"
+  DESTINATION faiss/contrib/torch
+  FILES_MATCHING PATTERN "*.md"
+)