Skip to content

Commit 96a4ecf

Browse files
limqiyinglimqiying
authored
Merge branch 'main' into cuvs-filter
2 parents 5a1ac9f + 9d6b2e7 commit 96a4ecf

87 files changed

Lines changed: 2948 additions & 2134 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/build_cmake/action.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ runs:
5252
conda install -y -q "conda<=25.07"
5353
echo "$CONDA/bin" >> $GITHUB_PATH
5454
55-
conda install -y -q python=3.12 cmake=3.30.4 make=4.2 swig=4.0 "numpy>=2.0,<3.0" scipy=1.16 pytest=7.4 gflags=2.2
55+
conda install -y -q python=3.12 cmake=3.30.4 make=4.2 swig=4.0 "numpy>=2.0,<3.0" scipy=1.16 pytest=7.4 gflags=2.2 setuptools
5656
5757
# install base packages for ARM64
5858
if [ "${{ runner.arch }}" = "ARM64" ]; then
@@ -79,7 +79,7 @@ runs:
7979
8080
# install SVS runtime for SVS builds
8181
if [ "${{ inputs.svs }}" = "ON" ]; then
82-
conda install -y -q libsvs-runtime -c https://software.repos.intel.com/python/conda -c conda-forge
82+
conda install -y -q libsvs-runtime -c conda-forge
8383
fi
8484
8585
# install test packages

‎.github/actions/build_conda/action.yml‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,6 @@ runs:
4545
conda list --show-channel-urls
4646
conda install -y -q "conda!=24.11.0,<=25.07"
4747
conda install -y -q "conda-build=25.3.1" "liblief=0.14.1"
48-
# Add Intel channel with lowest priority for libsvs-runtime only
49-
conda config --append channels https://software.repos.intel.com/python/conda
50-
conda config --set channel_priority flexible
5148
- name: Enable anaconda uploads
5249
if: inputs.label != ''
5350
shell: ${{ steps.choose_shell.outputs.shell }}
@@ -72,7 +69,7 @@ runs:
7269
PACKAGE_TYPE: ${{ inputs.label }}
7370
run: |
7471
conda list --show-channel-urls
75-
conda build faiss --user pytorch --label ${{ inputs.label }} -c pytorch
72+
conda build faiss --python 3.12 --user pytorch --label ${{ inputs.label }} -c pytorch
7673
- name: Conda build (GPU)
7774
if: inputs.label == '' && inputs.cuda != '' && inputs.cuvs == ''
7875
shell: ${{ steps.choose_shell.outputs.shell }}
@@ -89,7 +86,7 @@ runs:
8986
PACKAGE_TYPE: ${{ inputs.label }}
9087
run: |
9188
conda list --show-channel-urls
92-
conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
89+
conda build faiss-gpu --python 3.12 --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
9390
--user pytorch --label ${{ inputs.label }} -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia
9491
- name: Conda build (GPU w/ cuVS)
9592
if: inputs.label == '' && inputs.cuda != '' && inputs.cuvs != ''
@@ -107,7 +104,7 @@ runs:
107104
PACKAGE_TYPE: ${{ inputs.label }}
108105
run: |
109106
conda list --show-channel-urls
110-
conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
107+
conda build faiss-gpu-cuvs --python 3.12 --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
111108
--user pytorch --label ${{ inputs.label }} -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia
112109
- name: Check installed packages channel
113110
shell: ${{ steps.choose_shell.outputs.shell }}

‎.github/workflows/index-io-backward-compatibility.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@ jobs:
5050
shell: bash
5151
run: |
5252
eval "$(conda shell.bash hook)"
53-
conda create -n faiss_conda_read -y python=3.11
53+
conda create -n faiss_conda_read -y python=3.12
5454
conda activate faiss_conda_read
55-
conda install -y -c pytorch -c conda-forge faiss-cpu=1.13.2
55+
conda install -y -c pytorch -c conda-forge faiss-cpu=1.14.1
5656
conda list
5757
5858
- name: Run Conda reader (read Faiss index and verify)
@@ -87,9 +87,9 @@ jobs:
8787
shell: bash
8888
run: |
8989
eval "$(conda shell.bash hook)"
90-
conda create -n faiss_conda_write -y python=3.11
90+
conda create -n faiss_conda_write -y python=3.12
9191
conda activate faiss_conda_write
92-
conda install -y -c pytorch -c conda-forge faiss-cpu=1.13.2
92+
conda install -y -c pytorch -c conda-forge faiss-cpu=1.14.1
9393
conda list
9494
9595
- name: Create shared data directory

‎conda/faiss-gpu-cuvs/meta.yaml‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ outputs:
4747
- cmake >=3.30.4
4848
- make =4.2 # [not win]
4949
- _openmp_mutex =4.5=2_kmp_llvm # [x86_64]
50-
- mkl >=2024.2.2,<2025.0a0 # [x86_64]
51-
- mkl-devel >=2024.2.2,<2025.0a0 # [x86_64]
50+
- mkl >=2024.2.2 # [x86_64]
51+
- mkl-devel >=2024.2.2 # [x86_64]
5252
- cuda-toolkit {{ cudatoolkit }}
5353
- cuda-cudart {{ cudart_constraints }}
5454
- cuda-cudart-dev {{ cudart_constraints }}
@@ -58,21 +58,21 @@ outputs:
5858
- cuda-cudart-static_linux-64 {{ cudart_constraints }} # [linux64]
5959
host:
6060
- _openmp_mutex =4.5=2_kmp_llvm # [x86_64]
61-
- mkl >=2024.2.2,<2025.0a0 # [x86_64]
61+
- mkl >=2024.2.2 # [x86_64]
6262
- openblas =0.3.30 # [not x86_64]
6363
- libcuvs =25.10
6464
- cuda-version {{ cuda_constraints }}
65-
- libsvs-runtime =0.1.0 # [x86_64 and linux]
65+
- libsvs-runtime =0.2.0 # [x86_64 and linux]
6666
run:
6767
- _openmp_mutex =4.5=2_kmp_llvm # [x86_64]
68-
- mkl >=2024.2.2,<2025.0a0 # [x86_64]
68+
- mkl >=2024.2.2 # [x86_64]
6969
- openblas =0.3.30 # [not x86_64]
7070
- cuda-cudart {{ cuda_constraints }}
7171
- libcublas {{ libcublas_constraints }}
7272
- libcuvs =25.10
7373
- cuda-version {{ cuda_constraints }}
7474
- libnvjitlink
75-
- libsvs-runtime =0.1.0 # [x86_64 and linux]
75+
- libsvs-runtime =0.2.0 # [x86_64 and linux]
7676
test:
7777
requires:
7878
- conda-build
@@ -97,16 +97,17 @@ outputs:
9797
- cmake >=3.26.4
9898
- make =4.2 # [not win]
9999
- _openmp_mutex =4.5=2_kmp_llvm # [x86_64]
100-
- mkl >=2024.2.2,<2025.0a0 # [x86_64]
100+
- mkl >=2024.2.2 # [x86_64]
101101
- cuda-toolkit {{ cudatoolkit }}
102102
host:
103-
- mkl >=2024.2.2,<2025.0a0 # [x86_64]
103+
- mkl >=2024.2.2 # [x86_64]
104104
- _openmp_mutex =4.5=2_kmp_llvm # [x86_64]
105105
- python {{ python }}
106106
- numpy >=2.0,<2.3
107+
- setuptools
107108
- {{ pin_subpackage('libfaiss', exact=True) }}
108109
run:
109-
- mkl >=2024.2.2,<2025.0a0 # [x86_64]
110+
- mkl >=2024.2.2 # [x86_64]
110111
- _openmp_mutex =4.5=2_kmp_llvm # [x86_64]
111112
- python {{ python }}
112113
- numpy >=2.0,<2.3

‎conda/faiss-gpu/meta.yaml‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -48,18 +48,18 @@ outputs:
4848
- cmake >=3.24.0
4949
- make =4.2 # [not win and not (osx and arm64)]
5050
- make =4.4 # [osx and arm64]
51-
- mkl-devel >=2024.2.2,<2025.0a0 # [x86_64]
51+
- mkl-devel >=2024.2.2 # [x86_64]
5252
- cuda-toolkit {{ cudatoolkit }}
5353
host:
54-
- mkl >=2024.2.2,<2025.0a0 # [x86_64]
54+
- mkl >=2024.2.2 # [x86_64]
5555
- openblas =0.3.30 # [not x86_64]
56-
- libsvs-runtime =0.1.0 # [x86_64 and linux]
56+
- libsvs-runtime =0.2.0 # [x86_64 and linux]
5757
run:
58-
- mkl >=2024.2.2,<2025.0a0 # [x86_64]
58+
- mkl >=2024.2.2 # [x86_64]
5959
- openblas =0.3.30 # [not x86_64]
6060
- cuda-cudart {{ cuda_constraints }}
6161
- libcublas {{ libcublas_constraints }}
62-
- libsvs-runtime =0.1.0 # [x86_64 and linux]
62+
- libsvs-runtime =0.2.0 # [x86_64 and linux]
6363
test:
6464
requires:
6565
- conda-build
@@ -86,15 +86,16 @@ outputs:
8686
- make =4.4 # [osx and arm64]
8787
- _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win]
8888
- cuda-toolkit {{ cudatoolkit }}
89-
- mkl-devel >=2024.2.2,<2025.0a0 # [x86_64]
89+
- mkl-devel >=2024.2.2 # [x86_64]
9090
host:
91-
- mkl >=2024.2.2,<2025.0a0 # [x86_64]
91+
- mkl >=2024.2.2 # [x86_64]
9292
- python {{ python }}
9393
- numpy >=2.0,<3.0
94+
- setuptools
9495
- _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win]
9596
- {{ pin_subpackage('libfaiss', exact=True) }}
9697
run:
97-
- mkl >=2024.2.2,<2025.0a0 # [x86_64]
98+
- mkl >=2024.2.2 # [x86_64]
9899
- python {{ python }}
99100
- numpy >=2.0,<3.0
100101
- packaging

‎conda/faiss/meta.yaml‎

Lines changed: 28 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -45,38 +45,42 @@ outputs:
4545
- make =4.2 # [not win and not (osx and arm64)]
4646
- make =4.4 # [osx and arm64]
4747
{% if PY_VER == '3.10' or PY_VER == '3.11' %}
48-
- mkl-devel =2024.2.2 # [x86_64]
48+
- mkl-devel >=2024.2.2 # [x86_64]
4949
- python_abi <3.12
5050
{% elif PY_VER == '3.12' %}
51-
- mkl-devel =2024.2.2 # [x86_64 and not win]
52-
- mkl-devel =2024.2.2 # [x86_64 and win]
51+
- mkl-devel >=2024.2.2 # [x86_64 and not win]
52+
- mkl-devel >=2024.2.2 # [x86_64 and win]
5353
- python_abi =3.12
54+
{% else %}
55+
- mkl-devel >=2024.2.2 # [x86_64 and not win]
56+
- mkl-devel >=2024.2.2 # [x86_64 and win]
57+
- python_abi
5458
{% endif %}
5559
- libopenblas=0.3.30=openmp_h60d53f8_1 # [osx]
5660
host:
5761
- python {{ python }}
5862
- libcxx =20.1.1 # [osx and arm64]
59-
- libsvs-runtime =0.1.0 # [x86_64 and linux]
63+
- libsvs-runtime =0.2.0 # [x86_64 and linux]
6064
{% if PY_VER == '3.10' or PY_VER == '3.11' %}
61-
- mkl =2024.2.2 # [x86_64]
65+
- mkl >=2024.2.2 # [x86_64]
6266
- python_abi <3.12
6367
{% elif PY_VER == '3.12' %}
64-
- mkl =2024.2.2 # [x86_64 and not win]
65-
- mkl =2024.2.2 # [x86_64 and win]
68+
- mkl >=2024.2.2 # [x86_64 and not win]
69+
- mkl >=2024.2.2 # [x86_64 and win]
6670
- python_abi =3.12
6771
{% endif %}
6872
- openblas =0.3.30 # [not x86_64]
6973
- libopenblas=0.3.30=openmp_h60d53f8_1 # [osx]
7074
run:
7175
- python {{ python }}
7276
- libcxx =20.1.1 # [osx and arm64]
73-
- libsvs-runtime =0.1.0 # [x86_64 and linux]
77+
- libsvs-runtime =0.2.0 # [x86_64 and linux]
7478
{% if PY_VER == '3.10' or PY_VER == '3.11' %}
75-
- mkl =2024.2.2 # [x86_64]
79+
- mkl >=2024.2.2 # [x86_64]
7680
- python_abi <3.12
7781
{% elif PY_VER == '3.12' %}
78-
- mkl =2024.2.2 # [x86_64 and not win]
79-
- mkl =2024.2.2 # [x86_64 and win]
82+
- mkl >=2024.2.2 # [x86_64 and not win]
83+
- mkl >=2024.2.2 # [x86_64 and win]
8084
- python_abi =3.12
8185
{% endif %}
8286
- openblas =0.3.30 # [not x86_64]
@@ -109,26 +113,27 @@ outputs:
109113
- make =4.4 # [osx and arm64]
110114
- _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win]
111115
{% if PY_VER == '3.10' or PY_VER == '3.11' %}
112-
- mkl =2024.2.2 # [x86_64]
116+
- mkl >=2024.2.2 # [x86_64]
113117
- python_abi <3.12
114118
{% elif PY_VER == '3.12' %}
115-
- mkl =2024.2.2 # [x86_64 and not win]
116-
- mkl =2024.2.2 # [x86_64 and win]
119+
- mkl >=2024.2.2 # [x86_64 and not win]
120+
- mkl >=2024.2.2 # [x86_64 and win]
117121
- python_abi =3.12
118122
{% endif %}
119123
- libopenblas=0.3.30=openmp_h60d53f8_1 # [osx]
120124
host:
121125
- python {{ python }}
122126
- numpy >=2.0,<3.0
127+
- setuptools
123128
- {{ pin_subpackage('libfaiss', exact=True) }}
124129
- libcxx =20.1.1 # [osx and arm64]
125130
- _openmp_mutex =4.5=2_kmp_llvm # [x86_64 and not win]
126131
{% if PY_VER == '3.10' or PY_VER == '3.11' %}
127-
- mkl =2024.2.2 # [x86_64]
132+
- mkl >=2024.2.2 # [x86_64]
128133
- python_abi <3.12
129134
{% elif PY_VER == '3.12' %}
130-
- mkl =2024.2.2 # [x86_64 and not win]
131-
- mkl =2024.2.2 # [x86_64 and win]
135+
- mkl >=2024.2.2 # [x86_64 and not win]
136+
- mkl >=2024.2.2 # [x86_64 and win]
132137
- python_abi =3.12
133138
{% endif %}
134139
- libopenblas=0.3.30=openmp_h60d53f8_1 # [osx]
@@ -139,11 +144,11 @@ outputs:
139144
- {{ pin_subpackage('libfaiss', exact=True) }}
140145
- libcxx =20.1.1 # [osx and arm64]
141146
{% if PY_VER == '3.10' or PY_VER == '3.11' %}
142-
- mkl =2024.2.2 # [x86_64]
147+
- mkl >=2024.2.2 # [x86_64]
143148
- python_abi <3.12
144149
{% elif PY_VER == '3.12' %}
145-
- mkl =2024.2.2 # [x86_64 and not win]
146-
- mkl =2024.2.2 # [x86_64 and win]
150+
- mkl >=2024.2.2 # [x86_64 and not win]
151+
- mkl >=2024.2.2 # [x86_64 and win]
147152
- python_abi =3.12
148153
{% endif %}
149154
- libopenblas=0.3.30=openmp_h60d53f8_1 # [osx]
@@ -153,11 +158,11 @@ outputs:
153158
- scipy
154159
- pytorch-cpu >=2.7
155160
{% if PY_VER == '3.10' or PY_VER == '3.11' %}
156-
- mkl =2024.2.2 # [x86_64]
161+
- mkl >=2024.2.2 # [x86_64]
157162
- python_abi <3.12
158163
{% elif PY_VER == '3.12' %}
159-
- mkl =2024.2.2 # [x86_64 and not win]
160-
- mkl =2024.2.2 # [x86_64 and win]
164+
- mkl >=2024.2.2 # [x86_64 and not win]
165+
- mkl >=2024.2.2 # [x86_64 and win]
161166
- python_abi =3.12
162167
{% endif %}
163168
commands:

‎faiss/CMakeLists.txt‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -117,9 +117,9 @@ set(FAISS_SRC
117117
impl/kmeans1d.cpp
118118
impl/lattice_Zn.cpp
119119
impl/mapped_io.cpp
120-
impl/pq4_fast_scan.cpp
121-
impl/pq4_fast_scan_search_1.cpp
122-
impl/pq4_fast_scan_search_qbs.cpp
120+
impl/fast_scan/pq4_fast_scan.cpp
121+
impl/fast_scan/pq4_fast_scan_search_1.cpp
122+
impl/fast_scan/pq4_fast_scan_search_qbs.cpp
123123
impl/residual_quantizer_encode_steps.cpp
124124
impl/zerocopy_io.cpp
125125
impl/NNDescent.cpp
@@ -227,8 +227,8 @@ set(FAISS_HEADERS
227227
impl/HNSW.h
228228
impl/LocalSearchQuantizer.h
229229
impl/ProductAdditiveQuantizer.h
230-
impl/LookupTableScaler.h
231-
impl/FastScanDistancePostProcessing.h
230+
impl/fast_scan/LookupTableScaler.h
231+
impl/fast_scan/FastScanDistancePostProcessing.h
232232
impl/maybe_owned_vector.h
233233
impl/NNDescent.h
234234
impl/NSG.h
@@ -262,10 +262,13 @@ set(FAISS_HEADERS
262262
impl/kmeans1d.h
263263
impl/lattice_Zn.h
264264
impl/platform_macros.h
265-
impl/pq4_fast_scan.h
265+
impl/fast_scan/pq4_fast_scan.h
266+
impl/fast_scan/decompose_qbs.h
267+
impl/fast_scan/kernels_simd256.h
268+
impl/fast_scan/kernels_simd512.h
266269
impl/residual_quantizer_encode_steps.h
267270
impl/simd_dispatch.h
268-
impl/simd_result_handlers.h
271+
impl/fast_scan/simd_result_handlers.h
269272
impl/zerocopy_io.h
270273
utils/pq_code_distance.h
271274
impl/pq_code_distance/pq_code_distance-inl.h
@@ -296,12 +299,13 @@ set(FAISS_HEADERS
296299
utils/rabitq_simd.h
297300
utils/random.h
298301
utils/sorting.h
299-
utils/simdlib.h
300-
utils/simdlib_avx2.h
301-
utils/simdlib_avx512.h
302-
utils/simdlib_emulated.h
303-
utils/simdlib_neon.h
304-
utils/simdlib_ppc64.h
302+
impl/simdlib/simdlib.h
303+
impl/simdlib/simdlib_dispatch.h
304+
impl/simdlib/simdlib_avx2.h
305+
impl/simdlib/simdlib_avx512.h
306+
impl/simdlib/simdlib_emulated.h
307+
impl/simdlib/simdlib_neon.h
308+
impl/simdlib/simdlib_ppc64.h
305309
utils/utils.h
306310
utils/simd_levels.h
307311
utils/distances_fused/avx512.h
@@ -321,6 +325,7 @@ set(FAISS_HEADERS
321325
utils/hamming_distance/avx2-inl.h
322326
utils/hamming_distance/avx512-inl.h
323327
utils/simd_impl/distances_autovec-inl.h
328+
utils/simd_impl/distances_simdlib256.h
324329
utils/simd_impl/distances_sse-inl.h
325330
)
326331

@@ -459,6 +464,14 @@ if(FAISS_OPT_LEVEL STREQUAL "dd")
459464
endif()
460465
endif()
461466

467+
# NEON is mandatory on ARM64 — ensure COMPILE_SIMD_ARM_NEON is always defined
468+
# and NEON source files are always compiled into the main faiss target.
469+
# (On x86, AVX2/AVX512 are optional and only compiled per opt_level above.)
470+
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64|arm64|ARM64)")
471+
target_compile_definitions(faiss PRIVATE COMPILE_SIMD_ARM_NEON)
472+
target_sources(faiss PRIVATE ${FAISS_SIMD_NEON_SRC})
473+
endif()
474+
462475
if(FAISS_ENABLE_SVS)
463476
find_package(svs_runtime 0.2.0 REQUIRED)
464477

0 commit comments

Comments
 (0)