Let the binary Hamming paths reach the VPOPCNT kernels

benchs/bench_rabitq.py

@@ -23,8 +23,8 @@
 #   d=512  -> size=64  -> 512-bit zmm, 1 iteration per bit-plane
 #   d=768  -> size=96  -> 512-bit zmm + 256-bit tail
 #   d=1024 -> size=128 -> 512-bit zmm only, 2 iterations per bit-plane
-# Sweeping these is useful for verifying the AVX512_SPR (vpopcntdq)
-# specialization in faiss/utils/simd_impl/rabitq_avx512_spr.cpp and for
+# Sweeping these is useful for verifying the AVX512_VPOPCNT specialization in
+# faiss/utils/simd_impl/rabitq_avx512_vpopcnt.cpp and for
 # profiling perf-record annotations across SIMD-width tiers.
 DIMENSIONS = [256, 512, 768, 1024]
 nlist: int = 1000

faiss/CMakeLists.txt

@@ -39,15 +39,16 @@ set(FAISS_SIMD_AVX512_SRC
   utils/simd_impl/rabitq_avx512.cpp
   utils/simd_impl/super_kmeans_kernels_avx512.cpp
 )
-# AVX-512 sources that additionally require AVX512_VPOPCNTDQ
-# (Sapphire Rapids and later). Compiled into faiss_avx512_spr only,
-# and into the DD faiss target with an extra per-file -mavx512vpopcntdq
-# flag. Not compiled into faiss_avx512 — that target stops at the
-# baseline AVX-512F/CD/VL/DQ/BW feature set.
+# AVX-512 sources that only require AVX512_VPOPCNTDQ on top of the baseline
+# AVX-512F/CD/VL/DQ/BW feature set. In DD builds these are available on CPUs
+# such as Ice Lake and Zen 4 without requiring the full SPR feature set.
+set(FAISS_SIMD_AVX512_VPOPCNT_SRC
+  utils/simd_impl/rabitq_avx512_vpopcnt.cpp
+  utils/hamming_distance/hamming_avx512_vpopcnt.cpp
+)
+# AVX-512 sources that require the full Sapphire Rapids feature set.
 set(FAISS_SIMD_AVX512_SPR_SRC
   impl/scalar_quantizer/sq-avx512-spr.cpp
-  utils/simd_impl/rabitq_avx512_spr.cpp
-  utils/hamming_distance/hamming_avx512_spr.cpp
 )
 set(FAISS_SIMD_NEON_SRC
   impl/fast_scan/impl-neon.cpp
@@ -80,7 +81,8 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64|amd64|AMD64)")
   set(FAISS_SIMD_SRC
     ${FAISS_SIMD_AVX2_SRC}
     ${FAISS_SIMD_AVX512_SRC}
-    ${FAISS_SIMD_AVX512_SPR_SRC} ${FAISS_SIMD_AVX512_SPR_SRC})
+    ${FAISS_SIMD_AVX512_VPOPCNT_SRC}
+    ${FAISS_SIMD_AVX512_SPR_SRC})
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64|arm64|ARM64)")
   set(FAISS_SIMD_SRC ${FAISS_SIMD_NEON_SRC} ${FAISS_SIMD_SVE_SRC})
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(riscv64|riscv)")
@@ -404,7 +406,7 @@ set(FAISS_HEADERS
   utils/hamming_distance/hamming_computer.h
   utils/hamming_distance/hamming_computer-avx2.h
   utils/hamming_distance/hamming_computer-avx512.h
-  utils/hamming_distance/hamming_computer-avx512_spr.h
+  utils/hamming_distance/hamming_computer-avx512_vpopcnt.h
   utils/hamming_distance/hamming_computer-generic.h
   utils/hamming_distance/hamming_computer-neon.h
   utils/hamming_distance/hamming_computer-rvv.h
@@ -496,8 +498,16 @@ else()
   # we need bigobj for the swig wrapper
   add_compile_options(/bigobj)
 endif()
-target_sources(faiss_avx512_spr PRIVATE ${FAISS_SIMD_AVX2_SRC} ${FAISS_SIMD_AVX512_SRC} ${FAISS_SIMD_AVX512_SPR_SRC})
-target_compile_definitions(faiss_avx512_spr PRIVATE COMPILE_SIMD_AVX2 COMPILE_SIMD_AVX512 COMPILE_SIMD_AVX512_SPR)
+target_sources(faiss_avx512_spr PRIVATE
+  ${FAISS_SIMD_AVX2_SRC}
+  ${FAISS_SIMD_AVX512_SRC}
+  ${FAISS_SIMD_AVX512_VPOPCNT_SRC}
+  ${FAISS_SIMD_AVX512_SPR_SRC})
+target_compile_definitions(faiss_avx512_spr PRIVATE
+  COMPILE_SIMD_AVX2
+  COMPILE_SIMD_AVX512
+  COMPILE_SIMD_AVX512_VPOPCNT
+  COMPILE_SIMD_AVX512_SPR)
 
 add_library(faiss_sve ${FAISS_SRC})
 if(NOT FAISS_OPT_LEVEL STREQUAL "sve")
@@ -539,7 +549,10 @@ if(FAISS_OPT_LEVEL STREQUAL "dd")
   # Architecture-specific SIMD definitions for Dynamic Dispatch
   if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64|amd64|AMD64)")
     target_compile_definitions(faiss PRIVATE
-      COMPILE_SIMD_AVX2 COMPILE_SIMD_AVX512 COMPILE_SIMD_AVX512_SPR)
+      COMPILE_SIMD_AVX2
+      COMPILE_SIMD_AVX512
+      COMPILE_SIMD_AVX512_VPOPCNT
+      COMPILE_SIMD_AVX512_SPR)
     if(NOT WIN32)
       # Baseline flags for common files (prevents auto-vectorization)
       target_compile_options(faiss PRIVATE
@@ -554,9 +567,13 @@ if(FAISS_OPT_LEVEL STREQUAL "dd")
         PROPERTIES COMPILE_OPTIONS
           "-mavx512f;-mavx512cd;-mavx512vl;-mavx512dq;-mavx512bw;-mfma;-mf16c;-mpopcnt"
       )
-      # SPR-only sources additionally require AVX512_VPOPCNTDQ.
-      # vpopcntq is the whole point of these files, so the extra flag
-      # is mandatory; without it the intrinsics fail to compile.
+      # VPOPCNT sources only require VPOPCNTDQ in addition to baseline AVX-512.
+      set_source_files_properties(${FAISS_SIMD_AVX512_VPOPCNT_SRC}
+        TARGET_DIRECTORY faiss
+        PROPERTIES COMPILE_OPTIONS
+          "-mavx2;-mfma;-mf16c;-mavx512f;-mavx512cd;-mavx512vl;-mavx512dq;-mavx512bw;-mavx512vpopcntdq;-mpopcnt"
+      )
+      # The remaining SPR sources require the full SPR feature set.
       set_source_files_properties(${FAISS_SIMD_AVX512_SPR_SRC}
         TARGET_DIRECTORY faiss
         PROPERTIES COMPILE_OPTIONS
@@ -573,11 +590,11 @@ if(FAISS_OPT_LEVEL STREQUAL "dd")
         TARGET_DIRECTORY faiss
         PROPERTIES COMPILE_OPTIONS "/arch:AVX512"
       )
-      # MSVC has no per-feature flag for VPOPCNTDQ; /arch:AVX512
-      # enables it as part of the AVX-512 feature set used by recent
-      # toolchains. (Newer MSVC supports __isa_available checks at
-      # runtime; the SPR specialization is gated by COMPILE_SIMD_AVX512_SPR
-      # at the source level, so this is safe.)
+      # MSVC has no per-feature flag for VPOPCNTDQ.
+      set_source_files_properties(${FAISS_SIMD_AVX512_VPOPCNT_SRC}
+        TARGET_DIRECTORY faiss
+        PROPERTIES COMPILE_OPTIONS "/arch:AVX512"
+      )
       set_source_files_properties(${FAISS_SIMD_AVX512_SPR_SRC}
         TARGET_DIRECTORY faiss
         PROPERTIES COMPILE_OPTIONS "/arch:AVX512"

faiss/IndexBinaryIVF.cpp

@@ -348,10 +348,6 @@ void IndexBinaryIVF::replace_invlists(InvertedLists* il, bool own) {
     own_invlists = own;
 }
 
-// IVFBinaryScannerL2, search_knn_hamming_count, BlockSearch,
-// BlockSearchVariableK, search_knn_hamming_per_invlist are now in
-// impl/binary_hamming/IndexBinaryIVF_impl.h (compiled per-ISA)
-
 namespace {
 
 void search_knn_hamming_heap(
@@ -461,13 +457,10 @@ void search_knn_hamming_heap(
 
 } // anonymous namespace
 
-// The remaining template code (search_knn_hamming_count,
-// search_knn_hamming_per_invlist, etc.) has been moved to
-// impl/binary_hamming/IndexBinaryIVF_impl.h
-
 BinaryInvertedListScanner* IndexBinaryIVF::get_InvertedListScanner(
         bool store_pairs) const {
-    return with_simd_level([&]<SIMDLevel SL>() {
+    // VPOPCNT: HammingComputer32/64/Default have VPOPCNTDQ kernels only there.
+    return with_simd_level_with_vpopcnt([&]<SIMDLevel SL>() {
         return make_binary_ivf_scanner_fixSL<SL>(code_size, store_pairs);
     });
 }
@@ -483,7 +476,7 @@ void IndexBinaryIVF::search_preassigned(
         bool store_pairs,
         const IVFSearchParameters* params) const {
     if (per_invlist_search) {
-        with_simd_level([&]<SIMDLevel SL>() {
+        with_simd_level_with_vpopcnt([&]<SIMDLevel SL>() {
             search_knn_hamming_per_invlist_fixSL<SL>(
                     code_size,
                     this,
@@ -501,7 +494,7 @@ void IndexBinaryIVF::search_preassigned(
         search_knn_hamming_heap(
                 this, n, x, k, cidx, cdis, dis, idx, store_pairs, params);
     } else {
-        with_simd_level([&]<SIMDLevel SL>() {
+        with_simd_level_with_vpopcnt([&]<SIMDLevel SL>() {
             search_knn_hamming_count_fixSL<SL>(
                     code_size,
                     store_pairs,

faiss/IndexFlat.cpp

@@ -269,10 +269,15 @@ struct FlatIPDis : FlatCodesDistanceComputer {
 
 FlatCodesDistanceComputer* IndexFlat::get_FlatCodesDistanceComputer() const {
     FlatCodesDistanceComputer* dc = nullptr;
+    // BASE_WITH_SVE rather than plain with_simd_level: the fvec_* kernels
+    // these computers call have dedicated ARM_SVE specializations, which the
+    // BASE mask would skip in favour of ARM_NEON.
     if (metric_type == METRIC_L2) {
-        with_simd_level([&]<SIMDLevel SL>() { dc = new FlatL2Dis<SL>(*this); });
+        with_simd_level_with_sve(
+                [&]<SIMDLevel SL>() { dc = new FlatL2Dis<SL>(*this); });
     } else if (metric_type == METRIC_INNER_PRODUCT) {
-        with_simd_level([&]<SIMDLevel SL>() { dc = new FlatIPDis<SL>(*this); });
+        with_simd_level_with_sve(
+                [&]<SIMDLevel SL>() { dc = new FlatIPDis<SL>(*this); });
     } else {
         dc = get_extra_distance_computer(d, metric_type, metric_arg, get_xb());
     }
@@ -404,7 +409,7 @@ FlatCodesDistanceComputer* IndexFlatL2::get_FlatCodesDistanceComputer() const {
     if (metric_type == METRIC_L2) {
         if (!cached_l2norms.empty()) {
             FlatCodesDistanceComputer* dc = nullptr;
-            with_simd_level([&]<SIMDLevel SL>() {
+            with_simd_level_with_sve([&]<SIMDLevel SL>() {
                 dc = new FlatL2WithNormsDis<SL>(*this);
             });
             return dc;
@@ -784,7 +789,9 @@ void IndexFlatPanorama::search_subset(
         idx_t k,
         float* distances,
         idx_t* labels) const {
-    with_simd_level([&]<SIMDLevel SL>() {
+    // BASE_WITH_SVE: the only SIMD-templated call in the loop body is
+    // fvec_inner_product, which has an ARM_SVE specialization.
+    with_simd_level_with_sve([&]<SIMDLevel SL>() {
         with_metric_type(metric_type, [&]<MetricType M>() {
             constexpr bool is_sim = is_similarity_metric(M);
             using C = std::conditional_t<

faiss/SuperKMeans.cpp

@@ -620,8 +620,11 @@ void super_kmeans_assign_iteration(
                        &ldc);
             }
 
-            // One SIMD dispatch per (xi, yj) tile.
-            with_simd_level([&]<SIMDLevel SL>() {
+            // One SIMD dispatch per (xi, yj) tile. BASE_WITH_SVE rather than
+            // plain with_simd_level: block_l2 has a dedicated ARM_SVE kernel,
+            // which the BASE mask would skip in favour of ARM_NEON (no
+            // specialization, so the scalar primary template).
+            with_simd_level_with_sve([&]<SIMDLevel SL>() {
                 [[maybe_unused]] const int omp_chunk_local = cp.omp_chunk;
                 int64_t tile_total = 0;
                 int64_t tile_pruned = 0;

faiss/docs/simd_dynamic_dispatch_migration.md

@@ -215,8 +215,8 @@ ARM_NEON + RISCV_RVV implementations exist. If your function has another subset
 of available implementations, it can be passed with
 `with_selected_simd_levels<mask>` with a bitmask of available levels. Missing
 levels in the mask cause the dispatch to **fall through** to the next lower
-level in the same architecture family (x86: AVX512_SPR → AVX512 → AVX2 →
-NONE; ARM: ARM_SVE → ARM_NEON → NONE; RISC-V: RISCV_RVV → NONE —
+level in the same architecture family (x86: AVX512_SPR → AVX512_VPOPCNT →
+AVX512 → AVX2 → NONE; ARM: ARM_SVE → ARM_NEON → NONE; RISC-V: RISCV_RVV → NONE —
 architecture chains are independent):
 
 ```cpp
@@ -238,8 +238,11 @@ your own with `(1 << int(SIMDLevel::X)) | ...`):
 |------|--------|---------|
 | `AVAILABLE_SIMD_LEVELS_NONE` | NONE only | Scalar-only functions |
 | `AVAILABLE_SIMD_LEVELS_AVX2_NEON` | NONE, AVX2, ARM_NEON | 256-bit `simdlib` ops (`with_simd_level_256bit`) |
-| `AVAILABLE_SIMD_LEVELS_A0` | NONE, AVX2, AVX512, ARM_NEON, RISCV_RVV | Default (`with_simd_level`) |
-| `AVAILABLE_SIMD_LEVELS_A1` | A0 + ARM_SVE | Functions with dedicated SVE implementations |
+| `AVAILABLE_SIMD_LEVELS_BASE` | NONE, AVX2, AVX512, ARM_NEON, RISCV_RVV | Default (`with_simd_level`). ARM_NEON is part of BASE: NEON is mandatory on aarch64, SVE is optional |
+| `AVAILABLE_SIMD_LEVELS_BASE_WITH_VPOPCNT` | BASE + AVX512_VPOPCNT | Kernels needing only VPOPCNTDQ (`with_simd_level_with_vpopcnt`) |
+| `AVAILABLE_SIMD_LEVELS_BASE_WITH_SPR` | BASE + AVX512_SPR | Kernels needing the whole SPR feature set (`with_simd_level_with_spr`) |
+| `AVAILABLE_SIMD_LEVELS_BASE_WITH_SVE` | BASE + ARM_SVE | Functions with dedicated SVE implementations (`with_simd_level_with_sve`) |
+| `AVAILABLE_SIMD_LEVELS_BASE_WITH_SPR_AND_SVE` | BASE + AVX512_SPR + ARM_SVE | Functions with both (`with_simd_level_with_spr_and_sve`) |
 | `AVAILABLE_SIMD_LEVELS_ALL` | All levels | Identity / diagnostic functions |
 
 ### Step 4: Register in build system

faiss/impl/AdditiveQuantizer.cpp

@@ -376,7 +376,8 @@ void AdditiveQuantizer::compute_centroid_norms(float* norms) const {
     size_t ntotal = (size_t)1 << tot_bits;
     int64_t ntotal_signed = ntotal;
     // TODO: make tree of partial sums
-    with_simd_level([&]<SIMDLevel SL>() {
+    // BASE_WITH_SVE: fvec_norm_L2sqr has an ARM_SVE specialization.
+    with_simd_level_with_sve([&]<SIMDLevel SL>() {
 #pragma omp parallel
         {
             std::vector<float> tmp(d);

faiss/impl/ProductQuantizer.cpp

@@ -280,8 +280,9 @@ void compute_1_code(const ProductQuantizer& pq, const float* x, uint8_t* code) {
 } // namespace
 
 void ProductQuantizer::compute_code(const float* x, uint8_t* code) const {
-    // a1: fvec_L2sqr_ny_nearest / _y_transposed have ARM_SVE specializations
-    with_simd_level_a1([&]<SIMDLevel SL>() {
+    // with_sve: fvec_L2sqr_ny_nearest / _y_transposed have ARM_SVE
+    // specializations
+    with_simd_level_with_sve([&]<SIMDLevel SL>() {
         switch (nbits) {
             case 8:
                 compute_1_code<PQEncoder8, SL>(*this, x, code);
@@ -295,7 +296,7 @@ void ProductQuantizer::compute_code(const float* x, uint8_t* code) const {
                 compute_1_code<PQEncoderGeneric, SL>(*this, x, code);
                 break;
         }
-    }); // with_simd_level_a1
+    }); // with_simd_level_with_sve
 }
 
 template <class PQDecoder>
@@ -443,8 +444,8 @@ void ProductQuantizer::compute_codes(const float* x, uint8_t* codes, size_t n)
 
 void ProductQuantizer::compute_distance_table(const float* x, float* dis_table)
         const {
-    // a1: fvec_L2sqr_ny / _transposed have ARM_SVE specializations
-    with_simd_level_a1([&]<SIMDLevel SL>() {
+    // with_sve: fvec_L2sqr_ny / _transposed have ARM_SVE specializations
+    with_simd_level_with_sve([&]<SIMDLevel SL>() {
         if (transposed_centroids.empty()) {
             // use regular version
             for (size_t m = 0; m < M; m++) {
@@ -826,8 +827,8 @@ void ProductQuantizer::compute_sdc_table() {
     sdc_table.resize(M * ksub * ksub);
 
     if (dsub < 4) {
-        // a1: fvec_L2sqr_ny has an ARM_SVE specialization
-        with_simd_level_a1([&]<SIMDLevel SL>() {
+        // with_sve: fvec_L2sqr_ny has an ARM_SVE specialization
+        with_simd_level_with_sve([&]<SIMDLevel SL>() {
 #pragma omp parallel for
             for (int64_t mk = 0; mk < static_cast<int64_t>(M * ksub); mk++) {
                 // allow omp to schedule in a more fine-grained way

faiss/impl/RaBitQUtils.cpp

@@ -329,7 +329,7 @@ float compute_full_multibit_distance(
         size_t d,
         size_t ex_bits,
         MetricType metric_type) {
-    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_A0_SPR>(
+    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE_WITH_VPOPCNT>(
             [&]<SIMDLevel SL>() {
                 return compute_full_multibit_distance<SL>(
                         sign_bits,

faiss/impl/RaBitQuantizer.cpp

@@ -665,12 +665,10 @@ FlatCodesDistanceComputer* RaBitQuantizer::get_distance_computer(
     // call the SIMD-specialized rabitq functions directly (no per-call
     // with_simd_level overhead).
     //
-    // Use A0_SPR (which includes AVX512_SPR) so that on Sapphire Rapids
-    // and later x86 microarchitectures the VPOPCNTDQ-based RaBitQ
-    // specialization in rabitq_avx512_spr.cpp is selected. On AVX-512
-    // CPUs without VPOPCNTDQ, dispatch falls through to the AVX512
-    // specialization in rabitq_avx512.cpp.
-    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_A0_SPR>(
+    // VPOPCNT rather than SPR: Ice Lake and Zen 4 have VPOPCNTDQ without the
+    // rest of the SPR feature set. Below it, dispatch falls through to
+    // rabitq_avx512.cpp.
+    return with_selected_simd_levels<AVAILABLE_SIMD_LEVELS_BASE_WITH_VPOPCNT>(
             [&]<SIMDLevel SL>() -> FlatCodesDistanceComputer* {
                 if (qb == 0) {
                     auto dc =

faiss/impl/ScalarQuantizer.cpp

@@ -665,7 +665,7 @@ ScalarQuantizer::SQuantizer* ScalarQuantizer::select_quantizer() const {
     // A SIMD level's factory returns nullptr when the dimension is
     // incompatible (e.g. AVX-512 needs d % 16 == 0); the dispatcher then falls
     // back to the next-lower level (AVX-512 -> AVX2 -> scalar).
-    return with_simd_level_fallback<AVAILABLE_SIMD_LEVELS_A0_SPR>(
+    return with_simd_level_fallback<AVAILABLE_SIMD_LEVELS_BASE_WITH_SPR>(
             [&]<SIMDLevel SL>() -> SQuantizer* {
                 return scalar_quantizer::sq_select_quantizer<SL>(
                         qtype, d, trained);
@@ -702,7 +702,7 @@ void ScalarQuantizer::decode(const uint8_t* codes, float* x, size_t n) const {
 ScalarQuantizer::SQDistanceComputer* ScalarQuantizer::get_distance_computer(
         MetricType metric) const {
     FAISS_THROW_IF_NOT(metric == METRIC_L2 || metric == METRIC_INNER_PRODUCT);
-    return with_simd_level_fallback<AVAILABLE_SIMD_LEVELS_A0_SPR>(
+    return with_simd_level_fallback<AVAILABLE_SIMD_LEVELS_BASE_WITH_SPR>(
             [&]<SIMDLevel SL>() -> SQDistanceComputer* {
                 return scalar_quantizer::sq_select_distance_computer<SL>(
                         metric, qtype, d, trained);
@@ -715,7 +715,7 @@ InvertedListScanner* ScalarQuantizer::select_InvertedListScanner(
         bool store_pairs,
         const IDSelector* sel,
         bool by_residual) const {
-    return with_simd_level_fallback<AVAILABLE_SIMD_LEVELS_A0_SPR>(
+    return with_simd_level_fallback<AVAILABLE_SIMD_LEVELS_BASE_WITH_SPR>(
             [&]<SIMDLevel SL>() -> InvertedListScanner* {
                 return scalar_quantizer::sq_select_InvertedListScanner<SL>(
                         qtype,

faiss/impl/binary_hamming/IndexBinaryIVF_impl.h

@@ -70,12 +70,22 @@ struct IVFBinaryScannerL2 : BinaryInvertedListScanner {
             size_t k) const override {
         using C = CMax<int32_t, idx_t>;
 
+        // An empty heap holds no top to read and can accept nothing.
+        if (k == 0) {
+            return 0;
+        }
+
+        // Only an accepted code can lower the heap top, so the bound is
+        // reread in the branch rather than on every iteration.
+        uint32_t bound = static_cast<uint32_t>(simi[0]);
+
         size_t nup = 0;
         for (size_t j = 0; j < n; j++) {
             uint32_t dis = hc.hamming(codes);
-            if (dis < static_cast<uint32_t>(simi[0])) {
+            if (dis < bound) {
                 idx_t id = store_pairs ? lo_build(list_no, j) : ids[j];
                 heap_replace_top<C>(k, simi, idxi, dis, id);
+                bound = static_cast<uint32_t>(simi[0]);
                 nup++;
             }
             codes += code_size;