Add pip install support via scikit-build-core + cibuildwheel (#4862)

alibeklfc · facebook-github-bot · commit 45bb916a09bc · 2026-03-04T18:49:54.000-08:00
Summary:

Add official pip wheel packaging for `faiss-cpu`, enabling
`pip install faiss-cpu` from PyPI. Builds wheels across Linux x86_64/aarch64,
macOS arm64/x86_64, and Windows x86_64 for Python 3.10-3.13.

**New files:**
- `pyproject.toml`: scikit-build-core build backend config with cibuildwheel
  settings. Uses Dynamic Dispatch (`FAISS_OPT_LEVEL=dd`) for runtime SIMD
  selection, OpenBLAS for BLAS, and disables GPU/MKL/SVS.
- `.github/workflows/build-pip.yml`: CI workflow using cibuildwheel to build
  wheels on 5 platform runners. Publishes to PyPI via OIDC trusted publishers
  on tag push.
- `tests/test_wheel_smoke.py`: 11-check smoke test suite validating import,
  OpenMP, BLAS (FlatL2/FlatIP), index factory (IVF+PQ), HNSW, serialization
  roundtrip, GC safety, contrib imports, and SIMD level detection.

**Modified files:**
- `python/CMakeLists.txt`: Added `install()` targets so scikit-build-core can
  package the SWIG extension module, Python source files, and contrib
  subpackage into wheels.
- `.github/workflows/build.yml`: Wired `build-pip.yml` into the root CI
  trigger so pip builds run alongside conda builds.

Differential Revision: D95258115
diff --git a/.github/workflows/build-pip.yml b/.github/workflows/build-pip.yml
@@ -0,0 +1,82 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+name: Build pip wheels
+
+on:
+  workflow_call:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'pyproject.toml'
+      - 'faiss/python/**'
+      - '.github/workflows/build-pip.yml'
+
+jobs:
+  build-wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-14, windows-2022, 2-core-ubuntu-arm]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.22
+        env:
+          CIBW_BUILD_VERBOSITY: 1
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.os }}
+          path: wheelhouse/*.whl
+
+  build-sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - name: Upload sdist
+        uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+
+  publish:
+    name: Publish to PyPI
+    needs: [build-wheels, build-sdist]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    environment:
+      name: pypi
+      url: https://pypi.org/p/faiss-cpu
+    permissions:
+      id-token: write
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -15,3 +15,5 @@ jobs:
     secrets:
       ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+  build-pip:
+    uses: ./.github/workflows/build-pip.yml
diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt
@@ -299,7 +299,7 @@ find_package(Python REQUIRED
   COMPONENTS Development.Module NumPy
 )
 
-add_library(faiss_python_callbacks EXCLUDE_FROM_ALL
+add_library(faiss_python_callbacks STATIC EXCLUDE_FROM_ALL
   python_callbacks.cpp
 )
 set_property(TARGET faiss_python_callbacks
@@ -332,3 +332,66 @@ configure_file(array_conversions.py array_conversions.py COPYONLY)
 
 # file(GLOB files "${PROJECT_SOURCE_DIR}/../../contrib/*.py")
 file(COPY ${PROJECT_SOURCE_DIR}/../../contrib  DESTINATION .)
+
+# =============================================================================
+# Install targets for scikit-build-core / pip wheel packaging.
+# These install the Python package into the wheel via cmake --install.
+# =============================================================================
+
+# Set RPATH so the SWIG module finds libfaiss in the same directory.
+# auditwheel/delocate follow RPATH to locate libraries for bundling.
+if(APPLE)
+  set_target_properties(swigfaiss PROPERTIES INSTALL_RPATH "@loader_path")
+else()
+  set_target_properties(swigfaiss PROPERTIES INSTALL_RPATH "$ORIGIN")
+endif()
+
+# Install the SWIG extension module (.so/.pyd)
+install(TARGETS swigfaiss
+  LIBRARY DESTINATION faiss
+  RUNTIME DESTINATION faiss
+)
+
+# Install the core faiss shared library into the Python package directory
+# so that auditwheel/delocate can find and bundle it with the SWIG module.
+install(TARGETS faiss
+  LIBRARY DESTINATION faiss
+  RUNTIME DESTINATION faiss
+)
+
+# Install SWIG-generated Python wrapper
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/swigfaiss.py"
+  DESTINATION faiss
+)
+
+# Install hand-written Python source files
+install(FILES
+  "${CMAKE_CURRENT_SOURCE_DIR}/__init__.py"
+  "${CMAKE_CURRENT_SOURCE_DIR}/loader.py"
+  "${CMAKE_CURRENT_SOURCE_DIR}/class_wrappers.py"
+  "${CMAKE_CURRENT_SOURCE_DIR}/extra_wrappers.py"
+  "${CMAKE_CURRENT_SOURCE_DIR}/gpu_wrappers.py"
+  "${CMAKE_CURRENT_SOURCE_DIR}/array_conversions.py"
+  DESTINATION faiss
+)
+
+# Install type stubs if present
+if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/__init__.pyi")
+  install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/__init__.pyi" DESTINATION faiss)
+endif()
+if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/py.typed")
+  install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/py.typed" DESTINATION faiss)
+endif()
+
+# Install contrib subpackage (excluding Meta-internal *_fb.py files)
+install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../../contrib/"
+  DESTINATION faiss/contrib
+  FILES_MATCHING PATTERN "*.py"
+  PATTERN "*_fb.py" EXCLUDE
+)
+
+# Install contrib/torch subpackage data files
+install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../../contrib/torch/"
+  DESTINATION faiss/contrib/torch
+  FILES_MATCHING PATTERN "*.md"
+)
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,108 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+[build-system]
+requires = ["scikit-build-core>=0.10", "numpy>=2.0"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "faiss-cpu"
+version = "1.14.0"
+description = "A library for efficient similarity search and clustering of dense vectors."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+  {name = "Meta AI Research"},
+]
+keywords = ["search", "nearest-neighbors", "clustering", "vectors", "similarity"]
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "Operating System :: MacOS :: MacOS X",
+  "Operating System :: Microsoft :: Windows",
+  "Operating System :: POSIX :: Linux",
+  "Programming Language :: C++",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = ["numpy>=1.25", "packaging", "mkl>=2024.2; platform_system=='Windows'"]
+
+[project.urls]
+Homepage = "https://github.com/facebookresearch/faiss"
+Documentation = "https://github.com/facebookresearch/faiss/wiki"
+Repository = "https://github.com/facebookresearch/faiss"
+Issues = "https://github.com/facebookresearch/faiss/issues"
+
+[tool.scikit-build]
+cmake.build-type = "Release"
+
+# Exclude C++ headers, cmake configs, and pkgconfig from the wheel.
+# Only the Python package (faiss/) and bundled shared libs remain.
+wheel.exclude = ["include/**", "share/**", "lib/cmake/**", "lib/pkgconfig/**"]
+
+[tool.scikit-build.cmake.define]
+FAISS_OPT_LEVEL = "dd"
+FAISS_ENABLE_GPU = "OFF"
+FAISS_ENABLE_PYTHON = "ON"
+FAISS_ENABLE_MKL = "OFF"
+FAISS_ENABLE_C_API = "OFF"
+FAISS_ENABLE_EXTRAS = "OFF"
+FAISS_ENABLE_SVS = "OFF"
+BUILD_TESTING = "OFF"
+BUILD_SHARED_LIBS = "ON"
+
+[tool.cibuildwheel]
+build = "cp310-* cp311-* cp312-* cp313-*"
+skip = "*-win32 *-manylinux_i686 *-musllinux_*"
+test-requires = ["numpy>=2.0,<3.0", "pytest"]
+test-command = "python -m pytest {project}/tests/test_wheel_smoke.py -v"
+
+[tool.cibuildwheel.linux]
+# EPEL needed for openblas-devel on aarch64 (not in CentOS 7 base repos)
+before-all = "yum install -y epel-release && yum install -y openblas-devel swig"
+
+[tool.cibuildwheel.macos]
+before-all = "brew install libomp openblas swig"
+
+[tool.cibuildwheel.macos.environment]
+# Help CMake find brew-installed OpenBLAS and libomp on macOS.
+# FindOpenMP cannot auto-detect AppleClang + brew libomp, so we pass
+# the flags and library path explicitly.
+CMAKE_PREFIX_PATH = "$(brew --prefix openblas);$(brew --prefix libomp)"
+CMAKE_ARGS = "-DOpenMP_CXX_FLAGS=-Xpreprocessor;-fopenmp;-I$(brew --prefix libomp)/include -DOpenMP_CXX_LIB_NAMES=omp -DOpenMP_omp_LIBRARY=$(brew --prefix libomp)/lib/libomp.dylib"
+# Brew libomp targets macOS 14.0+; the wheel must declare the same floor
+# so that delocate does not reject the library.
+MACOSX_DEPLOYMENT_TARGET = "14.0"
+
+[tool.cibuildwheel.windows]
+# Install SWIG and MKL (consistent with conda recipe).
+# MKL is installed via pip into the system Python, then copied to a fixed
+# location (C:\mkl) so FindMKL.cmake can discover it regardless of
+# PEP 517 build isolation.
+before-all = "choco install -y swig && pip install mkl==2024.2.2 mkl-devel==2024.2.2 && python -c \"import os,sys,shutil; os.makedirs('C:/mkl/lib',exist_ok=True); os.makedirs('C:/mkl/bin',exist_ok=True); os.makedirs('C:/mkl/include',exist_ok=True); [shutil.copy2(os.path.join(r,f),'C:/mkl/lib/') for r,_,fs in os.walk(sys.prefix) for f in fs if f.endswith('.lib') and 'mkl' in f]; [shutil.copy2(os.path.join(r,f),'C:/mkl/bin/') for r,_,fs in os.walk(sys.prefix) for f in fs if f.endswith('.dll') and 'mkl' in f]; [shutil.copy2(os.path.join(r,f),'C:/mkl/include/') for r,_,fs in os.walk(sys.prefix) for f in fs if f=='mkl.h']; print('MKL lib:',os.listdir('C:/mkl/lib')); print('MKL bin:',os.listdir('C:/mkl/bin'))\""
+# Skip delvewheel repair — faiss.dll is co-located with _swigfaiss.pyd in
+# the faiss/ package dir (installed by python/CMakeLists.txt), so Python's
+# DLL loader finds it automatically.  MKL runtime DLLs come from the mkl
+# pip package declared in [project] dependencies.
+repair-wheel-command = ""
+
+[tool.cibuildwheel.windows.environment]
+# Dynamic Dispatch is not yet supported on Windows (the DD cmake block is
+# gated behind if(NOT WIN32)).  Fall back to generic baseline, matching the
+# conda recipe which also builds "vanilla (no avx)" on Windows.
+# Use MKL for BLAS (consistent with conda).  MKLROOT points at the fixed
+# copy created in before-all.  We intentionally do NOT override LIB or PATH
+# here — $VAR expansion can fail in cibuildwheel on Windows, clobbering
+# MSVC toolchain paths and breaking pip in the test virtualenv.
+# FindMKL.cmake discovers libraries via MKLROOT alone; MKL runtime DLLs
+# are provided by the mkl pip dependency declared in [project].
+MKLROOT = "C:/mkl"
+CMAKE_ARGS = "-DFAISS_OPT_LEVEL=generic -DFAISS_ENABLE_MKL=ON -DBLA_VENDOR=Intel10_64_dyn"
diff --git a/tests/test_wheel_smoke.py b/tests/test_wheel_smoke.py
