Skip to content

Commit 8ac42f7

Browse files
alibeklfcfacebook-github-bot
alibeklfc
authored and
facebook-github-bot
committed
Add pip install support via scikit-build-core + cibuildwheel (#4862)
Summary: Add official pip wheel packaging for `faiss-cpu`, enabling `pip install faiss-cpu` from PyPI. Builds wheels across Linux x86_64/aarch64, macOS arm64/x86_64, and Windows x86_64 for Python 3.10-3.13. **New files:** - `pyproject.toml`: scikit-build-core build backend config with cibuildwheel settings. Uses Dynamic Dispatch (`FAISS_OPT_LEVEL=dd`) for runtime SIMD selection, OpenBLAS for BLAS, and disables GPU/MKL/SVS. - `.github/workflows/build-pip.yml`: CI workflow using cibuildwheel to build wheels on 5 platform runners. Publishes to PyPI via OIDC trusted publishers on tag push. - `tests/test_wheel_smoke.py`: 11-check smoke test suite validating import, OpenMP, BLAS (FlatL2/FlatIP), index factory (IVF+PQ), HNSW, serialization roundtrip, GC safety, contrib imports, and SIMD level detection. **Modified files:** - `python/CMakeLists.txt`: Added `install()` targets so scikit-build-core can package the SWIG extension module, Python source files, and contrib subpackage into wheels. - `.github/workflows/build.yml`: Wired `build-pip.yml` into the root CI trigger so pip builds run alongside conda builds. Differential Revision: D95258115
1 parent d0434be commit 8ac42f7

12 files changed

Lines changed: 1310 additions & 1 deletion

‎.github/scripts/repair_win_wheel.py‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
"""Custom Windows wheel repair: bundle MKL runtime DLLs into the faiss package.
7+
8+
Copies MKL (and Intel OpenMP / TBB) DLLs from a staging directory into the
9+
faiss/ package directory inside the wheel so they are co-located with
10+
_swigfaiss.pyd and faiss.dll. On Python 3.8+ Windows, DLL dependencies are
11+
resolved from the directory containing the loading DLL, so co-location is
12+
sufficient — no .pth file or os.add_dll_directory() call is needed.
13+
14+
Usage:
15+
python repair_win_wheel.py <wheel> <dest_dir> [--dll-dir C:/mkl/bin]
16+
"""
17+
18+
import argparse
19+
import base64
20+
import glob
21+
import hashlib
22+
import os
23+
import shutil
24+
import tempfile
25+
import zipfile
26+
27+
28+
def repair(wheel_path, dest_dir, dll_dir):
29+
wheel_name = os.path.basename(wheel_path)
30+
tmpdir = tempfile.mkdtemp()
31+
32+
try:
33+
with zipfile.ZipFile(wheel_path) as zf:
34+
zf.extractall(tmpdir)
35+
36+
# List DLLs/PYDs already in the wheel (diagnostic).
37+
print("Files in wheel:")
38+
for root, _dirs, files in os.walk(tmpdir):
39+
for f in sorted(files):
40+
if f.lower().endswith((".dll", ".pyd")):
41+
rel = os.path.relpath(os.path.join(root, f), tmpdir)
42+
size = os.path.getsize(os.path.join(root, f))
43+
print(f" {rel} ({size:,} bytes)")
44+
45+
# Locate the faiss package directory inside the wheel.
46+
faiss_dir = os.path.join(tmpdir, "faiss")
47+
if not os.path.isdir(faiss_dir):
48+
raise RuntimeError("faiss/ directory not found in wheel")
49+
50+
# Find the RECORD file (in *.dist-info/).
51+
record_path = None
52+
for root, _dirs, files in os.walk(tmpdir):
53+
if root.endswith(".dist-info") and "RECORD" in files:
54+
record_path = os.path.join(root, "RECORD")
55+
break
56+
if not record_path:
57+
raise RuntimeError("RECORD file not found in wheel")
58+
59+
# Copy runtime DLLs into faiss/ and update RECORD.
60+
new_records = []
61+
for dll in sorted(glob.glob(os.path.join(dll_dir, "*.dll"))):
62+
dll_name = os.path.basename(dll)
63+
dst = os.path.join(faiss_dir, dll_name)
64+
if os.path.exists(dst):
65+
print(f" skip {dll_name} (already in wheel)")
66+
continue
67+
shutil.copy2(dll, dst)
68+
69+
with open(dst, "rb") as f:
70+
digest = hashlib.sha256(f.read()).digest()
71+
b64 = base64.urlsafe_b64encode(digest).rstrip(b"=").decode()
72+
size = os.path.getsize(dst)
73+
new_records.append(f"faiss/{dll_name},sha256={b64},{size}")
74+
print(f" bundled {dll_name} ({size:,} bytes)")
75+
76+
with open(record_path, "a") as f:
77+
for rec in new_records:
78+
f.write(rec + "\n")
79+
80+
# Repack the wheel.
81+
out_path = os.path.join(dest_dir, wheel_name)
82+
with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED) as zf:
83+
for root, _dirs, files in os.walk(tmpdir):
84+
for fname in sorted(files):
85+
fullpath = os.path.join(root, fname)
86+
arcname = os.path.relpath(fullpath, tmpdir)
87+
zf.write(fullpath, arcname)
88+
89+
print(f" repaired wheel written to {out_path}")
90+
finally:
91+
shutil.rmtree(tmpdir, ignore_errors=True)
92+
93+
94+
if __name__ == "__main__":
95+
parser = argparse.ArgumentParser()
96+
parser.add_argument("wheel", help="Path to the input wheel")
97+
parser.add_argument(
98+
"dest_dir",
99+
help="Directory to write the repaired wheel",
100+
)
101+
parser.add_argument(
102+
"--dll-dir",
103+
default="C:/mkl/bin",
104+
help="Directory containing DLLs to bundle",
105+
)
106+
args = parser.parse_args()
107+
repair(args.wheel, args.dest_dir, args.dll_dir)

‎.github/workflows/build-pip-gpu.yml‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
name: Build GPU pip wheels
7+
8+
on:
9+
workflow_call:
10+
workflow_dispatch:
11+
pull_request:
12+
branches:
13+
- main
14+
paths:
15+
- 'pyproject-gpu.toml'
16+
- 'pyproject-gpu-cuvs.toml'
17+
- 'faiss/python/**'
18+
- 'faiss/gpu/**'
19+
- '.github/workflows/build-pip-gpu.yml'
20+
21+
jobs:
22+
build-faiss-gpu:
23+
name: Build faiss-gpu ${{ matrix.python }}
24+
runs-on: 4-core-ubuntu-gpu-t4
25+
strategy:
26+
fail-fast: false
27+
matrix:
28+
python: ["cp310", "cp311", "cp312", "cp313"]
29+
env:
30+
CIBW_BUILD: "${{ matrix.python }}-manylinux_x86_64"
31+
CIBW_BUILD_VERBOSITY: 1
32+
steps:
33+
- name: Checkout
34+
uses: actions/checkout@v4
35+
with:
36+
fetch-depth: 0
37+
fetch-tags: true
38+
39+
- name: Set up Python
40+
uses: actions/setup-python@v5
41+
with:
42+
python-version: '3.12'
43+
44+
- name: Install cibuildwheel
45+
run: pip install cibuildwheel==2.22.0
46+
47+
- name: Activate GPU pyproject.toml
48+
run: cp pyproject-gpu.toml pyproject.toml
49+
50+
- name: Build wheels
51+
run: cibuildwheel --output-dir wheelhouse
52+
53+
- name: Upload wheels
54+
uses: actions/upload-artifact@v4
55+
with:
56+
name: wheels-gpu-${{ matrix.python }}
57+
path: wheelhouse/*.whl
58+
59+
build-faiss-gpu-cuvs:
60+
name: Build faiss-gpu-cuvs ${{ matrix.python }}
61+
runs-on: 4-core-ubuntu-gpu-t4
62+
strategy:
63+
fail-fast: false
64+
matrix:
65+
python: ["cp310", "cp311", "cp312", "cp313"]
66+
env:
67+
CIBW_BUILD: "${{ matrix.python }}-manylinux_x86_64"
68+
CIBW_BUILD_VERBOSITY: 1
69+
steps:
70+
- name: Checkout
71+
uses: actions/checkout@v4
72+
with:
73+
fetch-depth: 0
74+
fetch-tags: true
75+
76+
- name: Set up Python
77+
uses: actions/setup-python@v5
78+
with:
79+
python-version: '3.12'
80+
81+
- name: Install cibuildwheel
82+
run: pip install cibuildwheel==2.22.0
83+
84+
- name: Activate cuVS pyproject.toml
85+
run: cp pyproject-gpu-cuvs.toml pyproject.toml
86+
87+
- name: Build wheels
88+
run: cibuildwheel --output-dir wheelhouse
89+
90+
- name: Upload wheels
91+
uses: actions/upload-artifact@v4
92+
with:
93+
name: wheels-gpu-cuvs-${{ matrix.python }}
94+
path: wheelhouse/*.whl
95+
96+
publish-gpu:
97+
name: Publish faiss-gpu to PyPI
98+
needs: [build-faiss-gpu]
99+
runs-on: ubuntu-latest
100+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
101+
environment:
102+
name: pypi
103+
url: https://pypi.org/p/faiss-gpu
104+
permissions:
105+
id-token: write
106+
steps:
107+
- name: Download wheels
108+
uses: actions/download-artifact@v4
109+
with:
110+
pattern: wheels-gpu-cp*
111+
path: dist
112+
merge-multiple: true
113+
114+
- name: Publish to PyPI
115+
uses: pypa/gh-action-pypi-publish@release/v1
116+
117+
publish-gpu-cuvs:
118+
name: Publish faiss-gpu-cuvs to PyPI
119+
needs: [build-faiss-gpu-cuvs]
120+
runs-on: ubuntu-latest
121+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
122+
environment:
123+
name: pypi
124+
url: https://pypi.org/p/faiss-gpu-cuvs
125+
permissions:
126+
id-token: write
127+
steps:
128+
- name: Download wheels
129+
uses: actions/download-artifact@v4
130+
with:
131+
pattern: wheels-gpu-cuvs-*
132+
path: dist
133+
merge-multiple: true
134+
135+
- name: Publish to PyPI
136+
uses: pypa/gh-action-pypi-publish@release/v1

‎.github/workflows/build-pip.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
name: Build pip wheels
7+
8+
on:
9+
workflow_call:
10+
workflow_dispatch:
11+
pull_request:
12+
branches:
13+
- main
14+
paths:
15+
- 'pyproject.toml'
16+
- 'faiss/python/**'
17+
- '.github/workflows/build-pip.yml'
18+
19+
jobs:
20+
build-wheels:
21+
name: Build wheels on ${{ matrix.os }}
22+
runs-on: ${{ matrix.os }}
23+
strategy:
24+
fail-fast: false
25+
matrix:
26+
os: [ubuntu-latest, macos-14, windows-2022, 2-core-ubuntu-arm]
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v4
30+
with:
31+
fetch-depth: 0
32+
fetch-tags: true
33+
34+
- name: Build wheels
35+
uses: pypa/cibuildwheel@v2.22
36+
env:
37+
CIBW_BUILD_VERBOSITY: 1
38+
39+
- name: Upload wheels
40+
uses: actions/upload-artifact@v4
41+
with:
42+
name: wheels-${{ matrix.os }}
43+
path: wheelhouse/*.whl
44+
45+
build-sdist:
46+
name: Build source distribution
47+
runs-on: ubuntu-latest
48+
steps:
49+
- name: Checkout
50+
uses: actions/checkout@v4
51+
with:
52+
fetch-depth: 0
53+
fetch-tags: true
54+
55+
- name: Build sdist
56+
run: pipx run build --sdist
57+
58+
- name: Upload sdist
59+
uses: actions/upload-artifact@v4
60+
with:
61+
name: sdist
62+
path: dist/*.tar.gz
63+
64+
publish:
65+
name: Publish to PyPI
66+
needs: [build-wheels, build-sdist]
67+
runs-on: ubuntu-latest
68+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
69+
environment:
70+
name: pypi
71+
url: https://pypi.org/p/faiss-cpu
72+
permissions:
73+
id-token: write
74+
steps:
75+
- name: Download all artifacts
76+
uses: actions/download-artifact@v4
77+
with:
78+
path: dist
79+
merge-multiple: true
80+
81+
- name: Publish to PyPI
82+
uses: pypa/gh-action-pypi-publish@release/v1

‎.github/workflows/build.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,7 @@ jobs:
1515
secrets:
1616
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
1717
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
18+
build-pip:
19+
uses: ./.github/workflows/build-pip.yml
20+
build-pip-gpu:
21+
uses: ./.github/workflows/build-pip-gpu.yml

0 commit comments

Comments
 (0)