Enable dynamic dispatch build on Windows

Summary:
Attempt to enable FAISS dynamic dispatch (DD) mode on Windows/MSVC.

Changes:
- CMakeLists.txt: Remove if(NOT WIN32) guard from DD section, add MSVC per-file
  SIMD flags (/arch:AVX2, /arch:AVX512, /bigobj) alongside existing GCC/Clang flags
- build-pull-request.yml: Add windows-x86_64-DD-cmake job that runs immediately
  (no dependency on linux-x86_64-cmake), builds with MSVC and FAISS_OPT_LEVEL=dd,
  runs C++ tests and Python tests

This diff is expected to fail on Windows due to MSVC requiring explicit template
specialization declarations (C++ §17.8.3) which GCC/Clang don't enforce. The CI
failure will surface the exact errors to guide the fix.

Differential Revision: D101649751

Author: mdouze

.github/workflows/build-pull-request.yml

@@ -170,6 +170,47 @@ jobs:
           fetch-tags: true
       - name: Build and Package (conda)
         uses: ./.github/actions/build_conda
+  windows-x86_64-DD-cmake:
+    name: Windows x86_64 Dynamic Dispatch (cmake)
+    runs-on: windows-2022
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: '3.12'
+          miniforge-version: latest
+          channels: conda-forge
+      - name: Install dependencies
+        shell: pwsh
+        run: |
+          conda install -y -q cmake swig "numpy>=2.0,<3.0" scipy pytest gflags setuptools
+          conda install -y -q mkl=2024.2.2 mkl-devel=2024.2.2
+      - name: Build
+        shell: pwsh
+        run: |
+          cmake -B build -T v143 -A x64 -G "Visual Studio 17 2022" `
+            -DBUILD_TESTING=ON `
+            -DBUILD_SHARED_LIBS=ON `
+            -DFAISS_ENABLE_GPU=OFF `
+            -DFAISS_OPT_LEVEL=dd `
+            -DFAISS_ENABLE_PYTHON=OFF `
+            -DFAISS_ENABLE_C_API=ON `
+            -DBLA_VENDOR=Intel10_64_dyn `
+            -DCMAKE_BUILD_TYPE=Release `
+            .
+          # Add faiss.dll and dependency DLLs to PATH for GoogleTest discovery
+          $env:PATH = "$PWD\build\faiss\Release;$env:CONDA_PREFIX\Library\bin;$env:PATH"
+          cmake --build build --config Release -j $env:NUMBER_OF_PROCESSORS
+      - name: C++ tests
+        shell: pwsh
+        run: |
+          # Copy faiss.dll next to the test executable so it can be found
+          Copy-Item build\faiss\Release\faiss.dll build\tests\Release\
+          $env:PATH = "$env:CONDA_PREFIX\Library\bin;$env:PATH"
+          cd build
+          ctest --output-on-failure -C Release
   windows-x86_64-conda:
     name: Windows x86_64 (conda)
     needs: linux-x86_64-cmake

CMakeLists.txt

@@ -115,7 +115,9 @@ endif()
 include(CTest)
 if(BUILD_TESTING)
   add_subdirectory(tests)
-  add_subdirectory(perf_tests)
+  if(NOT WIN32)
+    add_subdirectory(perf_tests)
+  endif()
   if(FAISS_ENABLE_GPU)
     if(FAISS_ENABLE_ROCM)
       add_subdirectory(faiss/gpu/test)

faiss/CMakeLists.txt

@@ -453,16 +453,16 @@ target_compile_definitions(faiss_sve PRIVATE COMPILE_SIMD_ARM_NEON COMPILE_SIMD_
 if(FAISS_OPT_LEVEL STREQUAL "dd")
   # Add SIMD source files to main faiss target for DD builds
   target_sources(faiss PRIVATE ${FAISS_SIMD_SRC})
-  if(NOT WIN32)
-    target_compile_definitions(faiss PRIVATE FAISS_ENABLE_DD)
-    # Architecture-specific SIMD definitions for Dynamic Dispatch
-    if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64|amd64|AMD64)")
-      target_compile_definitions(faiss PRIVATE
-        COMPILE_SIMD_AVX2 COMPILE_SIMD_AVX512 COMPILE_SIMD_AVX512_SPR)
+  target_compile_definitions(faiss PRIVATE FAISS_ENABLE_DD)
+  # Architecture-specific SIMD definitions for Dynamic Dispatch
+  if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64|amd64|AMD64)")
+    target_compile_definitions(faiss PRIVATE
+      COMPILE_SIMD_AVX2 COMPILE_SIMD_AVX512 COMPILE_SIMD_AVX512_SPR)
+    if(NOT WIN32)
       # Baseline flags for common files (prevents auto-vectorization)
       target_compile_options(faiss PRIVATE
         $<$<COMPILE_LANGUAGE:CXX>:-mpopcnt -msse4 -mno-avx -mno-avx2>)
-      # Per-file SIMD flags
+      # Per-file SIMD flags (GCC/Clang)
       set_source_files_properties(${FAISS_SIMD_AVX2_SRC}
         TARGET_DIRECTORY faiss
         PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c;-mpopcnt"
@@ -472,15 +472,26 @@ if(FAISS_OPT_LEVEL STREQUAL "dd")
         PROPERTIES COMPILE_OPTIONS
           "-mavx512f;-mavx512cd;-mavx512vl;-mavx512dq;-mavx512bw;-mfma;-mf16c;-mpopcnt"
       )
-    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64|arm64|ARM64)")
-      # ARM NEON is always available on aarch64, no special compiler flags needed
-      target_compile_definitions(faiss PRIVATE COMPILE_SIMD_ARM_NEON COMPILE_SIMD_ARM_SVE)
-      # Per-file SVE flags (NEON needs no flags on aarch64)
-      set_source_files_properties(${FAISS_SIMD_SVE_SRC}
+    else()
+      # Per-file SIMD flags (MSVC)
+      add_compile_options(/bigobj)
+      set_source_files_properties(${FAISS_SIMD_AVX2_SRC}
+        TARGET_DIRECTORY faiss
+        PROPERTIES COMPILE_OPTIONS "/arch:AVX2"
+      )
+      set_source_files_properties(${FAISS_SIMD_AVX512_SRC}
         TARGET_DIRECTORY faiss
-        PROPERTIES COMPILE_OPTIONS "-march=armv8.2-a+sve"
+        PROPERTIES COMPILE_OPTIONS "/arch:AVX512"
       )
     endif()
+  elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64|arm64|ARM64)")
+    # ARM NEON is always available on aarch64, no special compiler flags needed
+    target_compile_definitions(faiss PRIVATE COMPILE_SIMD_ARM_NEON COMPILE_SIMD_ARM_SVE)
+    # Per-file SVE flags (NEON needs no flags on aarch64)
+    set_source_files_properties(${FAISS_SIMD_SVE_SRC}
+      TARGET_DIRECTORY faiss
+      PROPERTIES COMPILE_OPTIONS "-march=armv8.2-a+sve"
+    )
   endif()
 endif()
 

faiss/utils/distances_fused/avx512.cpp

@@ -78,7 +78,7 @@ void kernel(
     const float* const __restrict xd_0 = x + i * DIM;
 
     // prefetch the next point
-    _mm_prefetch(xd_0 + DIM * sizeof(float), _MM_HINT_NTA);
+    _mm_prefetch((char*)(xd_0 + DIM * sizeof(float)), _MM_HINT_NTA);
 
     // load a single point from x
     // load -2 * value
@@ -262,10 +262,10 @@ void exhaustive_L2sqr_fused_cmax(
         }
     }
 
-    const size_t nx_p = (nx / NX_POINTS_PER_LOOP) * NX_POINTS_PER_LOOP;
+    const idx_t nx_p = (nx / NX_POINTS_PER_LOOP) * NX_POINTS_PER_LOOP;
     // the main loop.
 #pragma omp parallel for schedule(dynamic)
-    for (size_t i = 0; i < nx_p; i += NX_POINTS_PER_LOOP) {
+    for (idx_t i = 0; i < nx_p; i += NX_POINTS_PER_LOOP) {
         kernel<DIM, NX_POINTS_PER_LOOP, NY_POINTS_PER_LOOP>(
                 x, y, y_transposed.data(), ny, res, y_norms, i);
     }

tests/CMakeLists.txt

@@ -10,9 +10,7 @@ set(FAISS_TEST_SRC
   test_ivfpq_indexing.cpp
   test_lowlevel_ivf.cpp
   test_ivf_index.cpp
-  test_merge.cpp
   test_omp_threads.cpp
-  test_ondisk_ivf.cpp
   test_pairs_decoding.cpp
   test_params_override.cpp
   test_pq_encoding.cpp
@@ -31,7 +29,6 @@ set(FAISS_TEST_SRC
   test_partitioning.cpp
   test_fastscan_perf.cpp
   test_pqfs_unaligned.cpp
-  test_disable_pq_sdc_tables.cpp
   test_common_ivf_empty_index.cpp
   test_callback.cpp
   test_utils.cpp
@@ -49,6 +46,15 @@ set(FAISS_TEST_SRC
   test_fast_scan_distance_to_code.cpp
 )
 
+# Tests that use POSIX APIs (pthread, mkstemp, unistd.h) not available on Windows
+if(NOT WIN32)
+  list(APPEND FAISS_TEST_SRC
+    test_merge.cpp
+    test_ondisk_ivf.cpp
+    test_disable_pq_sdc_tables.cpp
+  )
+endif()
+
 if(FAISS_ENABLE_SVS)
   list(APPEND FAISS_TEST_SRC test_svs.cpp)
 endif()
@@ -138,4 +144,10 @@ target_link_libraries(faiss_test PRIVATE
 
 # Defines `gtest_discover_tests()`.
 include(GoogleTest)
-gtest_discover_tests(faiss_test)
+if(WIN32)
+  # On Windows, defer test discovery to ctest time (PRE_TEST) instead of
+  # build time (POST_BUILD) because faiss.dll is not in PATH during build.
+  gtest_discover_tests(faiss_test DISCOVERY_MODE PRE_TEST)
+else()
+  gtest_discover_tests(faiss_test)
+endif()

tests/test_fastscan_perf.cpp

@@ -7,6 +7,7 @@
 
 #include <gtest/gtest.h>
 
+#include <chrono>
 #include <cstddef>
 #include <cstdint>
 #include <memory>

tests/test_mmap.cpp

@@ -46,7 +46,7 @@ std::vector<uint8_t> make_binary_data(
         size_t seed) {
     std::vector<uint8_t> database(n * d);
     std::mt19937 rng(seed);
-    std::uniform_int_distribution<uint8_t> distrib(0, 255);
+    std::uniform_int_distribution<int> distrib(0, 255);
 
     for (size_t i = 0; i < n * d; i++) {
         database[i] = distrib(rng);

tests/test_pq_code_distance.cpp

@@ -51,7 +51,7 @@ void test(
 
     // rng
     std::minstd_rand rng(123);
-    std::uniform_int_distribution<uint8_t> u(0, 255);
+    std::uniform_int_distribution<int> u(0, 255);
     std::uniform_real_distribution<float> uf(0, 1);
 
     // initialize lookup
@@ -65,10 +65,10 @@ void test(
 #pragma omp parallel
     {
         std::minstd_rand rng0(123);
-        std::uniform_int_distribution<uint8_t> u1(0, 255);
+        std::uniform_int_distribution<int> u1(0, 255);
 
 #pragma omp for schedule(guided)
-        for (size_t i = 0; i < codes.size(); i++) {
+        for (int64_t i = 0; i < (int64_t)codes.size(); i++) {
             codes[i] = u1(rng0);
         }
     }
@@ -77,7 +77,7 @@ void test(
     std::vector<float> resultsRef(n, 0);
     for (size_t k = 0; k < 10; k++) {
 #pragma omp parallel for schedule(guided)
-        for (size_t i = 0; i < n; i++) {
+        for (int64_t i = 0; i < (int64_t)n; i++) {
             resultsRef[i] = faiss::PQCodeDistance<
                     faiss::PQDecoder8,
                     faiss::SIMDLevel::NONE>::
@@ -93,7 +93,7 @@ void test(
         const auto startingTimepoint = std::chrono::steady_clock::now();
         for (size_t k = 0; k < 1000; k++) {
 #pragma omp parallel for schedule(guided)
-            for (size_t i = 0; i < n; i++) {
+            for (int64_t i = 0; i < (int64_t)n; i++) {
                 resultsScalar1x[i] = faiss::PQCodeDistance<
                         faiss::PQDecoder8,
                         faiss::SIMDLevel::NONE>::
@@ -118,7 +118,7 @@ void test(
         const auto startingTimepoint = std::chrono::steady_clock::now();
         for (size_t k = 0; k < 1000; k++) {
 #pragma omp parallel for schedule(guided)
-            for (size_t i = 0; i < n; i += 4) {
+            for (int64_t i = 0; i < (int64_t)n; i += 4) {
                 faiss::PQCodeDistance<
                         faiss::PQDecoder8,
                         faiss::SIMDLevel::NONE>::
@@ -151,7 +151,7 @@ void test(
         const auto startingTimepoint = std::chrono::steady_clock::now();
         for (size_t k = 0; k < 1000; k++) {
 #pragma omp parallel for schedule(guided)
-            for (size_t i = 0; i < n; i++) {
+            for (int64_t i = 0; i < (int64_t)n; i++) {
                 resultsDispatched1x[i] = faiss::pq_code_distance_single(
                         subq, 8, lookup.data(), codes.data() + subq * i);
             }
@@ -170,7 +170,7 @@ void test(
         const auto startingTimepoint = std::chrono::steady_clock::now();
         for (size_t k = 0; k < 1000; k++) {
 #pragma omp parallel for schedule(guided)
-            for (size_t i = 0; i < n; i += 4) {
+            for (int64_t i = 0; i < (int64_t)n; i += 4) {
                 faiss::pq_code_distance_four(
                         subq,
                         8,

tests/test_pqfs_unaligned.cpp

@@ -59,9 +59,7 @@ TEST(FastScanUnaligned, TestUnalignedCodesAccess) {
 #endif
     std::random_device rd;
     std::mt19937 gen(rd());
-    std::uniform_int_distribution<uint8_t> dist(
-            std::numeric_limits<uint8_t>::min(),
-            std::numeric_limits<uint8_t>::max());
+    std::uniform_int_distribution<int> dist(0, 255);
 
     std::vector<uint8_t> aligned_buffer =
             create_aligned_buffer(code_size, alignment);

tests/test_scalar_quantizer.cpp

@@ -9,6 +9,7 @@
 
 #include <cmath>
 #include <memory>
+#include <random>
 #include <vector>
 
 #include <faiss/IndexFlat.h>
@@ -110,12 +111,14 @@ TEST(TestSQ0bit, CoarseOnlySearch) {
     int nq = 10;
     int k = 5;
 
+    std::mt19937 rng(42);
+    std::uniform_real_distribution<float> distrib(0.0f, 1.0f);
     std::vector<float> xb(nb * d), xq(nq * d);
     for (int i = 0; i < nb * d; i++) {
-        xb[i] = drand48();
+        xb[i] = distrib(rng);
     }
     for (int i = 0; i < nq * d; i++) {
-        xq[i] = drand48();
+        xq[i] = distrib(rng);
     }
 
     faiss::IndexFlatL2 quantizer(d);
@@ -180,12 +183,14 @@ TEST(TestSQ0bit, InnerProduct) {
     int nq = 5;
     int k = 3;
 
+    std::mt19937 rng2(43);
+    std::uniform_real_distribution<float> distrib2(0.0f, 1.0f);
     std::vector<float> xb(nb * d), xq(nq * d);
     for (int i = 0; i < nb * d; i++) {
-        xb[i] = drand48();
+        xb[i] = distrib2(rng2);
     }
     for (int i = 0; i < nq * d; i++) {
-        xq[i] = drand48();
+        xq[i] = distrib2(rng2);
     }
 
     faiss::IndexFlatIP quantizer(d);