Delete old pq4_accumulate_loop files (replaced by dispatching.h) (facebookresearch#4905)

Summary:
Pull Request resolved: facebookresearch#4905

Now that all search paths use FastScanCodeScanner (via dispatching.h),
the old pq4_accumulate_loop and pq4_accumulate_loop_qbs free functions
have no callers. Delete pq4_fast_scan_search_1.cpp and
pq4_fast_scan_search_qbs.cpp entirely. Relocate the remaining utility
functions (pq4_qbs_to_nq, pq4_preferred_qbs, accumulate_to_mem) to
pq4_fast_scan.cpp. Remove dead declarations from pq4_fast_scan.h.

This completes the code deduplication: dispatching.h uses shared helpers
from accumulate_loops.h and decompose_qbs.h, and no duplicate
implementations remain.

Reviewed By: mdouze

Differential Revision: D96131176

Author: algoriddle

faiss/CMakeLists.txt

@@ -121,8 +121,6 @@ set(FAISS_SRC
   impl/lattice_Zn.cpp
   impl/mapped_io.cpp
   impl/fast_scan/fast_scan.cpp
-  impl/fast_scan/pq4_fast_scan_search_1.cpp
-  impl/fast_scan/pq4_fast_scan_search_qbs.cpp
   impl/residual_quantizer_encode_steps.cpp
   impl/zerocopy_io.cpp
   impl/NNDescent.cpp

faiss/impl/fast_scan/fast_scan.cpp

@@ -6,6 +6,8 @@
  */
 
 #include <faiss/impl/FaissAssert.h>
+#include <faiss/impl/fast_scan/LookupTableScaler.h>
+#include <faiss/impl/fast_scan/decompose_qbs.h>
 #include <faiss/impl/fast_scan/fast_scan.h>
 #include <faiss/impl/fast_scan/simd_result_handlers.h>
 #include <faiss/impl/simd_dispatch.h>
@@ -351,6 +353,52 @@ int pq4_pack_LUT_qbs_q_map(
     return i0;
 }
 
+// declared in simd_result_handlers.h
+bool simd_result_handlers_accept_virtual = true;
+
+int pq4_qbs_to_nq(int qbs) {
+    int i0 = 0;
+    int qi = qbs;
+    while (qi) {
+        int nq = qi & 15;
+        qi >>= 4;
+        i0 += nq;
+    }
+    return i0;
+}
+
+int pq4_preferred_qbs(int n) {
+    // from timings in P141901742, P141902828
+    static int map[12] = {
+            0, 1, 2, 3, 0x13, 0x23, 0x33, 0x223, 0x233, 0x333, 0x2233, 0x2333};
+    if (n <= 11) {
+        return map[n];
+    } else if (n <= 24) {
+        // override qbs: all first stages with 3 steps
+        // then 1 stage with the rest
+        int nbit = 4 * (n / 3); // nbits with only 3s
+        int qbs = 0x33333333 & ((1 << nbit) - 1);
+        qbs |= (n % 3) << nbit;
+        return qbs;
+    } else {
+        FAISS_THROW_FMT("number of queries %d too large", n);
+    }
+}
+
+void accumulate_to_mem(
+        int nq,
+        size_t ntotal2,
+        int nsq,
+        const uint8_t* codes,
+        const uint8_t* LUT,
+        uint16_t* accu) {
+    using namespace simd_result_handlers;
+    FAISS_THROW_IF_NOT(ntotal2 % 32 == 0);
+    StoreResultHandler<> handler(accu, ntotal2);
+    DummyScaler<> scaler;
+    accumulate(nq, ntotal2, nsq, codes, LUT, handler, scaler, 32 * nsq / 2);
+}
+
 } // namespace faiss
 
 /***************************************************************

faiss/impl/fast_scan/fast_scan.h

@@ -29,7 +29,6 @@ namespace faiss {
 struct IDSelector;
 struct RangeSearchResult;
 struct RangeSearchPartialResult;
-struct SIMDResultHandler;
 struct SIMDResultHandlerToFloat;
 
 /** Pack codes for consumption by the SIMD kernels.
@@ -125,42 +124,6 @@ struct CodePackerPQ4 : CodePacker {
  */
 void pq4_pack_LUT(int nq, int nsq, const uint8_t* src, uint8_t* dest);
 
-/** Loop over database elements and accumulate results into result handler
- *
- * @param nq      number of queries
- * @param nb      number of database elements
- * @param bbs     size of database blocks (multiple of 32)
- * @param nsq     number of sub-quantizers (multiple of 2)
- * @param codes   packed codes array
- * @param LUT     packed look-up table
- * @param scaler  scaler to scale the encoded norm
- * @param block_stride  stride in bytes between consecutive blocks.
- */
-void pq4_accumulate_loop(
-        int nq,
-        size_t nb,
-        int bbs,
-        int nsq,
-        const uint8_t* codes,
-        const uint8_t* LUT,
-        SIMDResultHandler& res,
-        int pq2x4_scale,
-        size_t block_stride);
-
-/* qbs versions, supported only for bbs=32.
- *
- * The kernel function runs the kernel for *several* query blocks
- * and bbs database vectors. The sizes of the blocks are encoded in qbs as
- * base-16 digits.
- *
- * For example, qbs = 0x1223 means that the kernel will be run 4 times, the
- * first time with 3 query vectors, second time with 2 query vectors, then 2
- * vectors again and finally with 1 query vector. The output block will thus be
- * nq = 3 + 2 + 2 + 1 = 6 queries. For a given total block size, the optimal
- * decomposition into sub-blocks (measured empirically) is given by
- * preferred_qbs().
- */
-
 /* compute the number of queries from a base-16 decomposition */
 int pq4_qbs_to_nq(int qbs);
 
@@ -187,28 +150,7 @@ int pq4_pack_LUT_qbs_q_map(
         const int* q_map,
         uint8_t* dest);
 
-/** Run accumulation loop.
- *
- * @param qbs     4-bit encoded number of queries
- * @param nb      number of database codes (multiple of bbs)
- * @param nsq     number of sub-quantizers
- * @param codes   encoded database vectors (packed)
- * @param LUT     look-up table (packed)
- * @param res     call-back for the results
- * @param pq2x4_scale  scaler to scale the encoded norm
- * @param block_stride  stride in bytes between consecutive blocks.
- */
-void pq4_accumulate_loop_qbs(
-        int qbs,
-        size_t nb,
-        int nsq,
-        const uint8_t* codes,
-        const uint8_t* LUT,
-        SIMDResultHandler& res,
-        int pq2x4_scale,
-        size_t block_stride);
-
-/** Wrapper of pq4_accumulate_loop_qbs using simple StoreResultHandler
+/** Wrapper using simple StoreResultHandler
  *  and DummyScaler
  *
  * @param nq      number of queries