Skip to content

Commit f1994e2

Browse files
alibeklfcfacebook-github-bot
alibeklfc
authored and
facebook-github-bot
committed
Add pip install support via scikit-build-core + cibuildwheel (facebookresearch#4862)
Summary: Add official pip wheel packaging for `faiss-cpu`, enabling `pip install faiss-cpu` from PyPI. Builds wheels across Linux x86_64/aarch64, macOS arm64/x86_64, and Windows x86_64 for Python 3.10-3.13. **New files:** - `pyproject.toml`: scikit-build-core build backend config with cibuildwheel settings. Uses Dynamic Dispatch (`FAISS_OPT_LEVEL=dd`) for runtime SIMD selection, OpenBLAS for BLAS, and disables GPU/MKL/SVS. - `.github/workflows/build-pip.yml`: CI workflow using cibuildwheel to build wheels on 5 platform runners. Publishes to PyPI via OIDC trusted publishers on tag push. - `tests/test_wheel_smoke.py`: 11-check smoke test suite validating import, OpenMP, BLAS (FlatL2/FlatIP), index factory (IVF+PQ), HNSW, serialization roundtrip, GC safety, contrib imports, and SIMD level detection. **Modified files:** - `python/CMakeLists.txt`: Added `install()` targets so scikit-build-core can package the SWIG extension module, Python source files, and contrib subpackage into wheels. - `.github/workflows/build.yml`: Wired `build-pip.yml` into the root CI trigger so pip builds run alongside conda builds. Differential Revision: D95258115
1 parent 8431e04 commit f1994e2

11 files changed

Lines changed: 1169 additions & 1 deletion

File tree

‎.github/scripts/repair_win_wheel.py‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
"""Custom Windows wheel repair: bundle MKL runtime DLLs into the faiss package.
7+
8+
Copies MKL (and Intel OpenMP / TBB) DLLs from a staging directory into the
9+
faiss/ package directory inside the wheel so they are co-located with
10+
_swigfaiss.pyd and faiss.dll. On Python 3.8+ Windows, DLL dependencies are
11+
resolved from the directory containing the loading DLL, so co-location is
12+
sufficient — no .pth file or os.add_dll_directory() call is needed.
13+
14+
Usage:
15+
python repair_win_wheel.py <wheel> <dest_dir> [--dll-dir C:/mkl/bin]
16+
"""
17+
18+
import argparse
19+
import base64
20+
import glob
21+
import hashlib
22+
import os
23+
import shutil
24+
import tempfile
25+
import zipfile
26+
27+
28+
def repair(wheel_path, dest_dir, dll_dir):
29+
wheel_name = os.path.basename(wheel_path)
30+
tmpdir = tempfile.mkdtemp()
31+
32+
try:
33+
# Unpack the wheel (it's a zip file).
34+
with zipfile.ZipFile(wheel_path) as zf:
35+
zf.extractall(tmpdir)
36+
37+
# List DLLs/PYDs already in the wheel (diagnostic).
38+
print("Files in wheel:")
39+
for root, _dirs, files in os.walk(tmpdir):
40+
for f in sorted(files):
41+
if f.lower().endswith((".dll", ".pyd")):
42+
rel = os.path.relpath(os.path.join(root, f), tmpdir)
43+
size = os.path.getsize(os.path.join(root, f))
44+
print(f" {rel} ({size:,} bytes)")
45+
46+
# Locate the faiss package directory inside the wheel.
47+
faiss_dir = os.path.join(tmpdir, "faiss")
48+
if not os.path.isdir(faiss_dir):
49+
raise RuntimeError("faiss/ directory not found in wheel")
50+
51+
# Find the RECORD file (in *.dist-info/).
52+
record_path = None
53+
for root, _dirs, files in os.walk(tmpdir):
54+
if root.endswith(".dist-info") and "RECORD" in files:
55+
record_path = os.path.join(root, "RECORD")
56+
break
57+
if not record_path:
58+
raise RuntimeError("RECORD file not found in wheel")
59+
60+
# Copy runtime DLLs into faiss/ and update RECORD.
61+
new_records = []
62+
for dll in sorted(glob.glob(os.path.join(dll_dir, "*.dll"))):
63+
dll_name = os.path.basename(dll)
64+
dst = os.path.join(faiss_dir, dll_name)
65+
if os.path.exists(dst):
66+
print(f" skip {dll_name} (already in wheel)")
67+
continue
68+
shutil.copy2(dll, dst)
69+
70+
with open(dst, "rb") as f:
71+
digest = hashlib.sha256(f.read()).digest()
72+
b64 = base64.urlsafe_b64encode(digest).rstrip(b"=").decode()
73+
size = os.path.getsize(dst)
74+
new_records.append(f"faiss/{dll_name},sha256={b64},{size}")
75+
print(f" bundled {dll_name} ({size:,} bytes)")
76+
77+
with open(record_path, "a") as f:
78+
for rec in new_records:
79+
f.write(rec + "\n")
80+
81+
# Repack the wheel.
82+
out_path = os.path.join(dest_dir, wheel_name)
83+
with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED) as zf:
84+
for root, _dirs, files in os.walk(tmpdir):
85+
for fname in sorted(files):
86+
fullpath = os.path.join(root, fname)
87+
arcname = os.path.relpath(fullpath, tmpdir)
88+
zf.write(fullpath, arcname)
89+
90+
print(f" repaired wheel written to {out_path}")
91+
finally:
92+
shutil.rmtree(tmpdir, ignore_errors=True)
93+
94+
95+
if __name__ == "__main__":
96+
parser = argparse.ArgumentParser()
97+
parser.add_argument("wheel", help="Path to the input wheel")
98+
parser.add_argument("dest_dir", help="Directory to write the repaired wheel")
99+
parser.add_argument(
100+
"--dll-dir",
101+
default="C:/mkl/bin",
102+
help="Directory containing DLLs to bundle",
103+
)
104+
args = parser.parse_args()
105+
repair(args.wheel, args.dest_dir, args.dll_dir)

‎.github/workflows/build-pip-gpu.yml‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
name: Build GPU pip wheels
7+
8+
on:
9+
workflow_call:
10+
workflow_dispatch:
11+
pull_request:
12+
branches:
13+
- main
14+
paths:
15+
- 'pyproject-gpu.toml'
16+
- 'pyproject-gpu-cuvs.toml'
17+
- 'faiss/python/**'
18+
- 'faiss/gpu/**'
19+
- '.github/workflows/build-pip-gpu.yml'
20+
21+
jobs:
22+
build-faiss-gpu:
23+
name: Build faiss-gpu wheels
24+
runs-on: 4-core-ubuntu-gpu-t4
25+
env:
26+
CIBW_BUILD_VERBOSITY: 1
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v4
30+
with:
31+
fetch-depth: 0
32+
fetch-tags: true
33+
34+
- name: Set up Python
35+
uses: actions/setup-python@v5
36+
with:
37+
python-version: '3.12'
38+
39+
- name: Install cibuildwheel
40+
run: pip install cibuildwheel==2.22.0
41+
42+
- name: Activate GPU pyproject.toml
43+
run: cp pyproject-gpu.toml pyproject.toml
44+
45+
- name: Build wheels
46+
run: cibuildwheel --output-dir wheelhouse
47+
48+
- name: Upload wheels
49+
uses: actions/upload-artifact@v4
50+
with:
51+
name: wheels-gpu
52+
path: wheelhouse/*.whl
53+
54+
build-faiss-gpu-cuvs:
55+
name: Build faiss-gpu-cuvs wheels
56+
runs-on: 4-core-ubuntu-gpu-t4
57+
env:
58+
CIBW_BUILD_VERBOSITY: 1
59+
steps:
60+
- name: Checkout
61+
uses: actions/checkout@v4
62+
with:
63+
fetch-depth: 0
64+
fetch-tags: true
65+
66+
- name: Set up Python
67+
uses: actions/setup-python@v5
68+
with:
69+
python-version: '3.12'
70+
71+
- name: Install cibuildwheel
72+
run: pip install cibuildwheel==2.22.0
73+
74+
- name: Activate cuVS pyproject.toml
75+
run: cp pyproject-gpu-cuvs.toml pyproject.toml
76+
77+
- name: Build wheels
78+
run: cibuildwheel --output-dir wheelhouse
79+
80+
- name: Upload wheels
81+
uses: actions/upload-artifact@v4
82+
with:
83+
name: wheels-gpu-cuvs
84+
path: wheelhouse/*.whl
85+
86+
publish-gpu:
87+
name: Publish faiss-gpu to PyPI
88+
needs: [build-faiss-gpu]
89+
runs-on: ubuntu-latest
90+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
91+
environment:
92+
name: pypi
93+
url: https://pypi.org/p/faiss-gpu
94+
permissions:
95+
id-token: write
96+
steps:
97+
- name: Download wheels
98+
uses: actions/download-artifact@v4
99+
with:
100+
name: wheels-gpu
101+
path: dist
102+
103+
- name: Publish to PyPI
104+
uses: pypa/gh-action-pypi-publish@release/v1
105+
106+
publish-gpu-cuvs:
107+
name: Publish faiss-gpu-cuvs to PyPI
108+
needs: [build-faiss-gpu-cuvs]
109+
runs-on: ubuntu-latest
110+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
111+
environment:
112+
name: pypi
113+
url: https://pypi.org/p/faiss-gpu-cuvs
114+
permissions:
115+
id-token: write
116+
steps:
117+
- name: Download wheels
118+
uses: actions/download-artifact@v4
119+
with:
120+
name: wheels-gpu-cuvs
121+
path: dist
122+
123+
- name: Publish to PyPI
124+
uses: pypa/gh-action-pypi-publish@release/v1

‎.github/workflows/build-pip.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
name: Build pip wheels
7+
8+
on:
9+
workflow_call:
10+
workflow_dispatch:
11+
pull_request:
12+
branches:
13+
- main
14+
paths:
15+
- 'pyproject.toml'
16+
- 'faiss/python/**'
17+
- '.github/workflows/build-pip.yml'
18+
19+
jobs:
20+
build-wheels:
21+
name: Build wheels on ${{ matrix.os }}
22+
runs-on: ${{ matrix.os }}
23+
strategy:
24+
fail-fast: false
25+
matrix:
26+
os: [ubuntu-latest, macos-14, windows-2022, 2-core-ubuntu-arm]
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v4
30+
with:
31+
fetch-depth: 0
32+
fetch-tags: true
33+
34+
- name: Build wheels
35+
uses: pypa/cibuildwheel@v2.22
36+
env:
37+
CIBW_BUILD_VERBOSITY: 1
38+
39+
- name: Upload wheels
40+
uses: actions/upload-artifact@v4
41+
with:
42+
name: wheels-${{ matrix.os }}
43+
path: wheelhouse/*.whl
44+
45+
build-sdist:
46+
name: Build source distribution
47+
runs-on: ubuntu-latest
48+
steps:
49+
- name: Checkout
50+
uses: actions/checkout@v4
51+
with:
52+
fetch-depth: 0
53+
fetch-tags: true
54+
55+
- name: Build sdist
56+
run: pipx run build --sdist
57+
58+
- name: Upload sdist
59+
uses: actions/upload-artifact@v4
60+
with:
61+
name: sdist
62+
path: dist/*.tar.gz
63+
64+
publish:
65+
name: Publish to PyPI
66+
needs: [build-wheels, build-sdist]
67+
runs-on: ubuntu-latest
68+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
69+
environment:
70+
name: pypi
71+
url: https://pypi.org/p/faiss-cpu
72+
permissions:
73+
id-token: write
74+
steps:
75+
- name: Download all artifacts
76+
uses: actions/download-artifact@v4
77+
with:
78+
path: dist
79+
merge-multiple: true
80+
81+
- name: Publish to PyPI
82+
uses: pypa/gh-action-pypi-publish@release/v1

‎.github/workflows/build.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,7 @@ jobs:
1515
secrets:
1616
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
1717
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
18+
build-pip:
19+
uses: ./.github/workflows/build-pip.yml
20+
build-pip-gpu:
21+
uses: ./.github/workflows/build-pip-gpu.yml

0 commit comments

Comments
 (0)