Merge branch 'main' into sq-avx512-spr-opt

Author: mdouze

.github/actions/build_cmake/action.yml

@@ -40,7 +40,7 @@ runs:
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.12'
-        miniforge-version: latest # ensures conda-forge channel is used.
+        miniforge-version: latest
         channels: conda-forge
         conda-remove-defaults: 'true'
         # Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64.
@@ -50,18 +50,15 @@ runs:
       if: inputs.metal != 'ON'
       shell: bash
       run: |
-        # initialize Conda
-        conda config --set solver libmamba
         # Ensure starting packages are from conda-forge.
         conda list --show-channel-urls
-        conda install -y -q "conda<=25.07"
         echo "$CONDA/bin" >> $GITHUB_PATH
 
         conda install -y -q python=3.12 cmake=3.30.4 make=4.2 swig=4.0 "numpy>=2.0,<3.0" scipy=1.16 pytest=7.4 gflags=2.2 setuptools
 
         # install base packages for ARM64
         if [ "${{ runner.arch }}" = "ARM64" ]; then
-          conda install -y -q -c conda-forge openblas=0.3.29 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
+          conda install -y -q -c conda-forge openblas=0.3.33 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
         fi
 
         # install base packages for X86_64
@@ -79,7 +76,11 @@ runs:
           conda install -y -q cuda-libraries-dev=12.6 cuda-nvcc=12.6 cuda-nvtx=12.6 cuda-cupti=12.6 cuda-cudart-dev=12.6 gxx_linux-64=12.4 -c "nvidia/label/cuda-12.6"
         # and CUDA from cuVS channel for cuVS builds
         elif [ "${{ inputs.cuvs }}" = "ON" ]; then
-          conda install -y -q libcuvs=26.02 'cuda-version=12.9' cuda-toolkit=12.9 sysroot_linux-64=2.34 -c rapidsai -c rapidsai-nightly -c conda-forge
+          conda install -y -q libcuvs=26.02 'cuda-version=12.9' sysroot_linux-64=2.34 -c rapidsai -c rapidsai-nightly -c conda-forge
+          # Clean index cache to prevent sqlite3 "database is locked" errors
+          # from conda-libmamba-solver's shards cache between consecutive installs.
+          # See: https://github.com/conda/conda-libmamba-solver/issues/667
+          conda clean --index-cache 2>/dev/null || true
         fi
 
         # install SVS runtime for SVS builds
@@ -241,6 +242,7 @@ runs:
       shell: bash
       run: |
         conda list --show-channel-urls
+        conda install -y -q "pip<26"
         pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.1
     - name: Python tests (CPU only)
       if: inputs.gpu == 'OFF' && inputs.metal != 'ON'

.github/actions/build_conda/action.yml

@@ -30,9 +30,10 @@ runs:
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.12'
-        miniforge-version: latest # ensures conda-forge channel is used.
+        miniforge-version: latest
         channels: conda-forge
         conda-remove-defaults: 'true'
+        activate-environment: 'base'
         # Set to runner.arch=aarch64 if we're on arm64 because
         # there's no miniforge ARM64 package, just aarch64.
         # They are the same thing, just named differently.
@@ -43,7 +44,7 @@ runs:
       run: |
         # Ensure starting packages are from conda-forge.
         conda list --show-channel-urls
-        conda install -y -q "conda!=24.11.0,<=25.07"
+        conda install -y -q "conda!=24.11.0,<25.7"
         conda install -y -q "conda-build=25.3.1" "liblief=0.14.1"
     - name: Enable anaconda uploads
       if: inputs.label != ''
@@ -80,7 +81,7 @@ runs:
       working-directory: conda
       run: |
         conda list --show-channel-urls
-        conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
+        conda build faiss-gpu --python 3.12 --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
             -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia
     - name: Conda build (GPU) w/ anaconda upload
       if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs == ''
@@ -98,7 +99,7 @@ runs:
       working-directory: conda
       run: |
         conda list --show-channel-urls
-        conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
+        conda build faiss-gpu-cuvs --python 3.12 --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
             -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia
     - name: Conda build (GPU w/ cuVS) w/ anaconda upload
       if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs != ''

.github/workflows/build-pip.yml

@@ -0,0 +1,85 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+name: Build pip wheels
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  build-wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-14, macos-15-intel, windows-2022, 2-core-ubuntu-arm]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@ee02a1537ce3071a004a6b08c41e72f0fdc42d9a  # v3.4.0
+        env:
+          CIBW_BUILD_VERBOSITY: 1
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: wheels-${{ matrix.os }}
+          path: wheelhouse/*.whl
+
+  publish:
+    name: Publish to PyPI
+    needs: [build-wheels]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    environment:
+      name: pypi
+      url: https://pypi.org/p/faiss-cpu
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Download wheel artifacts
+        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
+        with:
+          path: dist
+          pattern: wheels-*
+          merge-multiple: true
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc  # release/v1
+
+  publish-testpypi:
+    name: Publish to TestPyPI
+    needs: [build-wheels]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch'
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/faiss-cpu
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Download wheel artifacts
+        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
+        with:
+          path: dist
+          pattern: wheels-*
+          merge-multiple: true
+
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc  # release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
+          verbose: true

.github/workflows/build-pull-request.yml

@@ -276,6 +276,21 @@ jobs:
           fetch-tags: true
       - name: Build and Package (conda)
         uses: ./.github/actions/build_conda
+  linux-x86_64-GPU-conda:
+    name: Linux x86_64 GPU (conda, CUDA 12.6)
+    runs-on: 4-core-ubuntu-gpu-t4
+    env:
+      CUDA_ARCHS: "70-real;72-real;75-real;80;86-real"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - name: Build and Package (conda)
+        uses: ./.github/actions/build_conda
+        with:
+          cuda: "12.6"
   linux-x86_64-svs:
     name: Linux x86_64 w/ SVS (cmake)
     needs: linux-x86_64-cmake
@@ -287,6 +302,109 @@ jobs:
         uses: ./.github/actions/build_cmake
         with:
           svs: ON
+  linux-riscv64-DD-cmake:
+    name: Linux riscv64 Dynamic Dispatch cross-compile (cmake)
+    needs: linux-x86_64-cmake
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up riscv64 cross-compilation environment
+        run: |
+          sudo dpkg --add-architecture riscv64
+
+          # Restrict existing apt sources to amd64/i386 so that apt does not
+          # try (and fail) to fetch riscv64 packages from the main Ubuntu
+          # mirror, which does not carry riscv64.
+          if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then
+            # Ubuntu 24.04+ deb822 format: insert Architectures field.
+            sudo sed -i '/^Types: deb$/a Architectures: amd64 i386' \
+              /etc/apt/sources.list.d/ubuntu.sources
+          fi
+          if grep -q '^deb http' /etc/apt/sources.list 2>/dev/null; then
+            # Ubuntu 22.04 traditional .list format.
+            sudo sed -i 's|^deb http|deb [arch=amd64,i386] http|g' \
+              /etc/apt/sources.list
+          fi
+
+          # Add Ubuntu Ports as the riscv64 package source.
+          CODENAME=$(. /etc/os-release && echo "$VERSION_CODENAME")
+          echo "deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports ${CODENAME} main restricted universe" \
+            | sudo tee    /etc/apt/sources.list.d/riscv64-ports.list
+          echo "deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports ${CODENAME}-updates main restricted universe" \
+            | sudo tee -a /etc/apt/sources.list.d/riscv64-ports.list
+
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq \
+            cmake \
+            gcc-riscv64-linux-gnu \
+            g++-riscv64-linux-gnu \
+            libopenblas-dev:riscv64 \
+            libgomp1:riscv64 \
+            qemu-user-static
+
+          # Bridge Ubuntu's multiarch library directory into the sysroot so
+          # that CMake's find_library() (ONLY mode, root=/usr/riscv64-linux-gnu)
+          # can reach packages installed as :riscv64.
+          # CMake searches ${root}/usr/lib/<multiarch-tuple>/ thanks to the
+          # compiler's -print-multiarch output (riscv64-linux-gnu).
+          sudo mkdir -p /usr/riscv64-linux-gnu/usr/lib
+          sudo ln -sfn /usr/lib/riscv64-linux-gnu \
+            /usr/riscv64-linux-gnu/usr/lib/riscv64-linux-gnu
+
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@v1
+        with:
+          key: linux-riscv64-dd
+          max-size: 2G
+          update-package-index: true
+
+      - name: Configure (cmake)
+        run: |
+          cmake -B build \
+            -DCMAKE_TOOLCHAIN_FILE=cmake/toolchains/riscv64-linux-gnu.cmake \
+            -DBUILD_TESTING=ON \
+            -DBUILD_SHARED_LIBS=ON \
+            -DFAISS_ENABLE_GPU=OFF \
+            -DFAISS_OPT_LEVEL=dd \
+            -DFAISS_ENABLE_PYTHON=OFF \
+            -DFAISS_ENABLE_C_API=OFF \
+            -DBLA_VENDOR=OpenBLAS \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_BUILD_RPATH=/usr/lib/riscv64-linux-gnu \
+            .
+
+      - name: Build faiss_test (includes test_factory_tools)
+        run: cmake --build build --target faiss_test -j$(nproc)
+
+      - name: Verify binary with ldd (via qemu)
+        run: |
+          file build/tests/faiss_test
+          file build/tests/faiss_test | grep -q "RISC-V" || \
+            { echo "ERROR: not a RISC-V binary"; exit 1; }
+
+          # Expose riscv64 libs inside the qemu sysroot so the riscv64 ld.so
+          # launched by qemu can resolve NEEDED entries.
+          sudo mkdir -p /usr/riscv64-linux-gnu/usr/lib
+          sudo ln -sfn /usr/lib/riscv64-linux-gnu \
+            /usr/riscv64-linux-gnu/usr/lib/riscv64-linux-gnu
+
+          # Invoke the riscv64 dynamic linker under qemu with --list, which
+          # is exactly what ldd does on native hardware.
+          RISCV64_LD=$(find /usr/riscv64-linux-gnu/lib \
+            -name "ld-linux-riscv64-*.so.1" | head -1)
+          qemu-riscv64-static -L /usr/riscv64-linux-gnu \
+            "$RISCV64_LD" --list build/tests/faiss_test
+
+      - name: Run test_simd_levels (via QEMU)
+        run: |
+          qemu-riscv64-static -L /usr/riscv64-linux-gnu \
+            build/tests/faiss_test \
+            --gtest_filter="SIMDConfig.*:SIMDLevel.*:CompileOptions.*"
+
   index-io-backward-compatibility:
     needs: linux-x86_64-cmake
     name: Index serialization backward compatibility

.github/workflows/build.yml

@@ -15,3 +15,5 @@ jobs:
     secrets:
       ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+  build-pip:
+    uses: ./.github/workflows/build-pip.yml

.github/workflows/index-io-backward-compatibility.yml

@@ -52,7 +52,7 @@ jobs:
           eval "$(conda shell.bash hook)"
           conda create -n faiss_conda_read -y python=3.12
           conda activate faiss_conda_read
-          conda install -y -c pytorch -c conda-forge faiss-cpu=1.14.1
+          conda install -y -c pytorch -c conda-forge faiss-cpu=1.14.1 "mkl>=2024.2.2,<2026"
           conda list
 
       - name: Run Conda reader (read Faiss index and verify)
@@ -89,7 +89,7 @@ jobs:
           eval "$(conda shell.bash hook)"
           conda create -n faiss_conda_write -y python=3.12
           conda activate faiss_conda_write
-          conda install -y -c pytorch -c conda-forge faiss-cpu=1.14.1
+          conda install -y -c pytorch -c conda-forge faiss-cpu=1.14.1 "mkl>=2024.2.2,<2026"
           conda list
 
       - name: Create shared data directory

CMakeLists.txt

@@ -127,7 +127,7 @@ endif()
 include(CTest)
 if(BUILD_TESTING)
   add_subdirectory(tests)
-  if(NOT WIN32)
+  if(NOT WIN32 AND NOT CMAKE_CROSSCOMPILING)
     add_subdirectory(perf_tests)
   endif()
   if(FAISS_ENABLE_GPU)