Skip to content

Commit 874592b

Browse files
alibeklfcfacebook-github-bot
alibeklfc
authored and
facebook-github-bot
committed
Add pip install support via scikit-build-core + cibuildwheel (facebookresearch#4862)
Summary: Add official pip wheel packaging for `faiss-cpu`, enabling `pip install faiss-cpu` from PyPI. Builds wheels across Linux x86_64/aarch64, macOS arm64/x86_64, and Windows x86_64 for Python 3.10-3.13. **New files:** - `pyproject.toml`: scikit-build-core build backend config with cibuildwheel settings. Uses Dynamic Dispatch (`FAISS_OPT_LEVEL=dd`) for runtime SIMD selection, OpenBLAS for BLAS, and disables GPU/MKL/SVS. - `.github/workflows/build-pip.yml`: CI workflow using cibuildwheel to build wheels on 5 platform runners. Publishes to PyPI via OIDC trusted publishers on tag push. - `tests/test_wheel_smoke.py`: 11-check smoke test suite validating import, OpenMP, BLAS (FlatL2/FlatIP), index factory (IVF+PQ), HNSW, serialization roundtrip, GC safety, contrib imports, and SIMD level detection. **Modified files:** - `python/CMakeLists.txt`: Added `install()` targets so scikit-build-core can package the SWIG extension module, Python source files, and contrib subpackage into wheels. - `.github/workflows/build.yml`: Wired `build-pip.yml` into the root CI trigger so pip builds run alongside conda builds. Differential Revision: D95258115
1 parent bb298a2 commit 874592b

12 files changed

Lines changed: 1337 additions & 1 deletion

‎.github/scripts/repair_win_wheel.py‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
"""Custom Windows wheel repair: bundle MKL runtime DLLs into the faiss package.
7+
8+
Copies MKL (and Intel OpenMP / TBB) DLLs from a staging directory into the
9+
faiss/ package directory inside the wheel so they are co-located with
10+
_swigfaiss.pyd and faiss.dll. On Python 3.8+ Windows, DLL dependencies are
11+
resolved from the directory containing the loading DLL, so co-location is
12+
sufficient — no .pth file or os.add_dll_directory() call is needed.
13+
14+
Usage:
15+
python repair_win_wheel.py <wheel> <dest_dir> [--dll-dir C:/mkl/bin]
16+
"""
17+
18+
import argparse
19+
import base64
20+
import glob
21+
import hashlib
22+
import os
23+
import shutil
24+
import tempfile
25+
import zipfile
26+
27+
28+
def repair(wheel_path, dest_dir, dll_dir):
29+
wheel_name = os.path.basename(wheel_path)
30+
tmpdir = tempfile.mkdtemp()
31+
32+
try:
33+
# Unpack the wheel (it's a zip file).
34+
with zipfile.ZipFile(wheel_path) as zf:
35+
zf.extractall(tmpdir)
36+
37+
# List DLLs/PYDs already in the wheel (diagnostic).
38+
print("Files in wheel:")
39+
for root, _dirs, files in os.walk(tmpdir):
40+
for f in sorted(files):
41+
if f.lower().endswith((".dll", ".pyd")):
42+
rel = os.path.relpath(os.path.join(root, f), tmpdir)
43+
size = os.path.getsize(os.path.join(root, f))
44+
print(f" {rel} ({size:,} bytes)")
45+
46+
# Locate the faiss package directory inside the wheel.
47+
faiss_dir = os.path.join(tmpdir, "faiss")
48+
if not os.path.isdir(faiss_dir):
49+
raise RuntimeError("faiss/ directory not found in wheel")
50+
51+
# Find the RECORD file (in *.dist-info/).
52+
record_path = None
53+
for root, _dirs, files in os.walk(tmpdir):
54+
if root.endswith(".dist-info") and "RECORD" in files:
55+
record_path = os.path.join(root, "RECORD")
56+
break
57+
if not record_path:
58+
raise RuntimeError("RECORD file not found in wheel")
59+
60+
# Copy runtime DLLs into faiss/ and update RECORD.
61+
new_records = []
62+
for dll in sorted(glob.glob(os.path.join(dll_dir, "*.dll"))):
63+
dll_name = os.path.basename(dll)
64+
dst = os.path.join(faiss_dir, dll_name)
65+
if os.path.exists(dst):
66+
print(f" skip {dll_name} (already in wheel)")
67+
continue
68+
shutil.copy2(dll, dst)
69+
70+
with open(dst, "rb") as f:
71+
digest = hashlib.sha256(f.read()).digest()
72+
b64 = base64.urlsafe_b64encode(digest).rstrip(b"=").decode()
73+
size = os.path.getsize(dst)
74+
new_records.append(f"faiss/{dll_name},sha256={b64},{size}")
75+
print(f" bundled {dll_name} ({size:,} bytes)")
76+
77+
with open(record_path, "a") as f:
78+
for rec in new_records:
79+
f.write(rec + "\n")
80+
81+
# Repack the wheel.
82+
out_path = os.path.join(dest_dir, wheel_name)
83+
with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED) as zf:
84+
for root, _dirs, files in os.walk(tmpdir):
85+
for fname in sorted(files):
86+
fullpath = os.path.join(root, fname)
87+
arcname = os.path.relpath(fullpath, tmpdir)
88+
zf.write(fullpath, arcname)
89+
90+
print(f" repaired wheel written to {out_path}")
91+
finally:
92+
shutil.rmtree(tmpdir, ignore_errors=True)
93+
94+
95+
if __name__ == "__main__":
96+
parser = argparse.ArgumentParser()
97+
parser.add_argument("wheel", help="Path to the input wheel")
98+
parser.add_argument("dest_dir", help="Directory to write the repaired wheel")
99+
parser.add_argument(
100+
"--dll-dir",
101+
default="C:/mkl/bin",
102+
help="Directory containing DLLs to bundle",
103+
)
104+
args = parser.parse_args()
105+
repair(args.wheel, args.dest_dir, args.dll_dir)

‎.github/workflows/build-pip-gpu.yml‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
name: Build GPU pip wheels
7+
8+
on:
9+
workflow_call:
10+
workflow_dispatch:
11+
pull_request:
12+
branches:
13+
- main
14+
paths:
15+
- 'pyproject-gpu.toml'
16+
- 'pyproject-gpu-cuvs.toml'
17+
- 'faiss/python/**'
18+
- 'faiss/gpu/**'
19+
- '.github/workflows/build-pip-gpu.yml'
20+
21+
jobs:
22+
build-faiss-gpu:
23+
name: Build faiss-gpu ${{ matrix.python }}
24+
runs-on: 4-core-ubuntu-gpu-t4
25+
strategy:
26+
fail-fast: false
27+
matrix:
28+
python: ["cp310", "cp311", "cp312", "cp313"]
29+
env:
30+
CIBW_BUILD: "${{ matrix.python }}-manylinux_x86_64"
31+
CIBW_BUILD_VERBOSITY: 1
32+
steps:
33+
- name: Checkout
34+
uses: actions/checkout@v4
35+
with:
36+
fetch-depth: 0
37+
fetch-tags: true
38+
39+
- name: Set up Python
40+
uses: actions/setup-python@v5
41+
with:
42+
python-version: '3.12'
43+
44+
- name: Install cibuildwheel
45+
run: pip install cibuildwheel==2.22.0
46+
47+
- name: Activate GPU pyproject.toml
48+
run: cp pyproject-gpu.toml pyproject.toml
49+
50+
- name: Build wheels
51+
run: cibuildwheel --output-dir wheelhouse
52+
53+
- name: Upload wheels
54+
uses: actions/upload-artifact@v4
55+
with:
56+
name: wheels-gpu-${{ matrix.python }}
57+
path: wheelhouse/*.whl
58+
59+
build-faiss-gpu-cuvs:
60+
name: Build faiss-gpu-cuvs ${{ matrix.python }}
61+
runs-on: 4-core-ubuntu-gpu-t4
62+
strategy:
63+
fail-fast: false
64+
matrix:
65+
python: ["cp310", "cp311", "cp312", "cp313"]
66+
env:
67+
CIBW_BUILD: "${{ matrix.python }}-manylinux_x86_64"
68+
CIBW_BUILD_VERBOSITY: 1
69+
steps:
70+
- name: Checkout
71+
uses: actions/checkout@v4
72+
with:
73+
fetch-depth: 0
74+
fetch-tags: true
75+
76+
- name: Set up Python
77+
uses: actions/setup-python@v5
78+
with:
79+
python-version: '3.12'
80+
81+
- name: Install cibuildwheel
82+
run: pip install cibuildwheel==2.22.0
83+
84+
- name: Activate cuVS pyproject.toml
85+
run: cp pyproject-gpu-cuvs.toml pyproject.toml
86+
87+
- name: Build wheels
88+
run: cibuildwheel --output-dir wheelhouse
89+
90+
- name: Upload wheels
91+
uses: actions/upload-artifact@v4
92+
with:
93+
name: wheels-gpu-cuvs-${{ matrix.python }}
94+
path: wheelhouse/*.whl
95+
96+
publish-gpu:
97+
name: Publish faiss-gpu to PyPI
98+
needs: [build-faiss-gpu]
99+
runs-on: ubuntu-latest
100+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
101+
environment:
102+
name: pypi
103+
url: https://pypi.org/p/faiss-gpu
104+
permissions:
105+
id-token: write
106+
steps:
107+
- name: Download wheels
108+
uses: actions/download-artifact@v4
109+
with:
110+
pattern: wheels-gpu-cp*
111+
path: dist
112+
merge-multiple: true
113+
114+
- name: Publish to PyPI
115+
uses: pypa/gh-action-pypi-publish@release/v1
116+
117+
publish-gpu-cuvs:
118+
name: Publish faiss-gpu-cuvs to PyPI
119+
needs: [build-faiss-gpu-cuvs]
120+
runs-on: ubuntu-latest
121+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
122+
environment:
123+
name: pypi
124+
url: https://pypi.org/p/faiss-gpu-cuvs
125+
permissions:
126+
id-token: write
127+
steps:
128+
- name: Download wheels
129+
uses: actions/download-artifact@v4
130+
with:
131+
pattern: wheels-gpu-cuvs-*
132+
path: dist
133+
merge-multiple: true
134+
135+
- name: Publish to PyPI
136+
uses: pypa/gh-action-pypi-publish@release/v1

‎.github/workflows/build-pip.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
#
3+
# This source code is licensed under the MIT license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
name: Build pip wheels
7+
8+
on:
9+
workflow_call:
10+
workflow_dispatch:
11+
pull_request:
12+
branches:
13+
- main
14+
paths:
15+
- 'pyproject.toml'
16+
- 'faiss/python/**'
17+
- '.github/workflows/build-pip.yml'
18+
19+
jobs:
20+
build-wheels:
21+
name: Build wheels on ${{ matrix.os }}
22+
runs-on: ${{ matrix.os }}
23+
strategy:
24+
fail-fast: false
25+
matrix:
26+
os: [ubuntu-latest, macos-14, windows-2022, 2-core-ubuntu-arm]
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v4
30+
with:
31+
fetch-depth: 0
32+
fetch-tags: true
33+
34+
- name: Build wheels
35+
uses: pypa/cibuildwheel@v2.22
36+
env:
37+
CIBW_BUILD_VERBOSITY: 1
38+
39+
- name: Upload wheels
40+
uses: actions/upload-artifact@v4
41+
with:
42+
name: wheels-${{ matrix.os }}
43+
path: wheelhouse/*.whl
44+
45+
build-sdist:
46+
name: Build source distribution
47+
runs-on: ubuntu-latest
48+
steps:
49+
- name: Checkout
50+
uses: actions/checkout@v4
51+
with:
52+
fetch-depth: 0
53+
fetch-tags: true
54+
55+
- name: Build sdist
56+
run: pipx run build --sdist
57+
58+
- name: Upload sdist
59+
uses: actions/upload-artifact@v4
60+
with:
61+
name: sdist
62+
path: dist/*.tar.gz
63+
64+
publish:
65+
name: Publish to PyPI
66+
needs: [build-wheels, build-sdist]
67+
runs-on: ubuntu-latest
68+
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
69+
environment:
70+
name: pypi
71+
url: https://pypi.org/p/faiss-cpu
72+
permissions:
73+
id-token: write
74+
steps:
75+
- name: Download all artifacts
76+
uses: actions/download-artifact@v4
77+
with:
78+
path: dist
79+
merge-multiple: true
80+
81+
- name: Publish to PyPI
82+
uses: pypa/gh-action-pypi-publish@release/v1

‎.github/workflows/build.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,7 @@ jobs:
1515
secrets:
1616
ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
1717
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
18+
build-pip:
19+
uses: ./.github/workflows/build-pip.yml
20+
build-pip-gpu:
21+
uses: ./.github/workflows/build-pip-gpu.yml

0 commit comments

Comments
 (0)