Merge branch 'facebookresearch:main' into polysemous-avx512

Author: mulugetam

.github/actions/build_cmake/action.yml

@@ -16,10 +16,23 @@ inputs:
     description: 'Enable ROCm support.'
     required: false
     default: OFF
+  svs:
+    description: 'Enable SVS support.'
+    required: false
+    default: OFF
+  setup_conda:
+    description: 'Setup miniconda environment.'
+    required: false
+    default: 'true'
+  upload_artifacts:
+    description: 'Upload test artifacts. Prevents collisions when multiple jobs need to run build_cmake.'
+    required: false
+    default: 'true'
 runs:
   using: composite
   steps:
     - name: Setup miniconda
+      if: inputs.setup_conda == 'true'
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.11'
@@ -36,10 +49,10 @@ runs:
         conda config --set solver libmamba
         # Ensure starting packages are from conda-forge.
         conda list --show-channel-urls
-        conda update -y -q conda
+        conda install -y -q "conda<=25.07"
         echo "$CONDA/bin" >> $GITHUB_PATH
 
-        conda install -y -q python=3.11 cmake=3.30.4 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2
+        conda install -y -q python=3.11 cmake=3.30.4 make=4.2 swig=4.0 "numpy>=2.0,<3.0" scipy=1.16 pytest=7.4 gflags=2.2
 
         # install base packages for ARM64
         if [ "${{ runner.arch }}" = "ARM64" ]; then
@@ -50,27 +63,33 @@ runs:
         if [ "${{ runner.arch }}" = "X64" ]; then
           # TODO: merge this with ARM64
           conda install -y -q -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17
-          conda install -y -q mkl=2022.2.1 mkl-devel=2022.2.1
+          conda install -y -q mkl=2024.2.2 mkl-devel=2024.2.2
         fi
 
         # no CUDA needed for ROCm so skip this
         if [ "${{ inputs.rocm }}" = "ON" ]; then
           :
         # regular CUDA for GPU builds
         elif [ "${{ inputs.gpu }}" = "ON" ] && [ "${{ inputs.cuvs }}" = "OFF" ]; then
-          conda install -y -q cuda-toolkit=12.4 -c "nvidia/label/cuda-12.4.0"
+          conda install -y -q cuda-toolkit=12.6 gxx_linux-64=12.4 -c "nvidia/label/cuda-12.6"
         # and CUDA from cuVS channel for cuVS builds
         elif [ "${{ inputs.cuvs }}" = "ON" ]; then
-          conda install -y -q libcuvs=25.08 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c rapidsai-nightly -c conda-forge
+          conda install -y -q libcuvs=25.10 'cuda-version=12.6' cuda-toolkit=12.6 gxx_linux-64=13.4.0 -c rapidsai -c rapidsai-nightly -c conda-forge
+        fi
+
+        # install SVS runtime for SVS builds
+        if [ "${{ inputs.svs }}" = "ON" ]; then
+          conda install -y -q libsvs-runtime -c https://software.repos.intel.com/python/conda -c conda-forge
         fi
 
         # install test packages
         if [ "${{ inputs.rocm }}" = "ON" ]; then
           : # skip torch install via conda, we need to install via pip to get
             #  ROCm-enabled version until it's supported in conda by PyTorch
         elif [ "${{ inputs.gpu }}" = "ON" ]; then
-          conda install -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c "nvidia/label/cuda-12.4.0"
+          conda install -y -q "pytorch>=2.7" "pytorch-gpu>=2.7" -c pytorch -c "nvidia/label/12.6"
         else
+          # TestLowLevelIVF.IVFRQ hangs on pytorch>=2.7, so left it as <2.5 for now.
           conda install -y -q "pytorch<2.5" -c pytorch
         fi
     - name: ROCm - Install dependencies
@@ -121,13 +140,15 @@ runs:
       run: |
         eval "$(conda shell.bash hook)"
         conda activate
+        conda list --show-channel-urls
         cmake -B build \
               -DBUILD_TESTING=ON \
               -DBUILD_SHARED_LIBS=ON \
               -DFAISS_ENABLE_GPU=${{ inputs.gpu }} \
               -DFAISS_ENABLE_CUVS=${{ inputs.cuvs }} \
               -DFAISS_ENABLE_ROCM=${{ inputs.rocm }} \
               -DFAISS_OPT_LEVEL=${{ inputs.opt_level }} \
+              -DFAISS_ENABLE_SVS=${{ inputs.svs }} \
               -DFAISS_ENABLE_C_API=ON \
               -DPYTHON_EXECUTABLE=$CONDA/bin/python \
               -DCMAKE_BUILD_TYPE=Release \
@@ -138,33 +159,39 @@ runs:
     - name: C++ tests
       shell: bash
       run: |
+        conda list --show-channel-urls
         export GTEST_OUTPUT="xml:$(realpath .)/test-results/googletest/"
         make -C build test
     - name: C++ perf benchmarks
       shell: bash
       if: inputs.rocm == 'OFF'
       run: |
+        conda list --show-channel-urls
         find ./build/perf_tests/ -executable -type f -name "bench*" -exec '{}' -v \;
     - name: Install Python extension
       shell: bash
       working-directory: build/faiss/python
       run: |
+        conda list --show-channel-urls
         $CONDA/bin/python setup.py install
     - name: ROCm - install ROCm-enabled torch via pip
       if: inputs.rocm == 'ON'
       shell: bash
       run: |
+        conda list --show-channel-urls
         pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.1
     - name: Python tests (CPU only)
       if: inputs.gpu == 'OFF'
       shell: bash
       run: |
+        conda list --show-channel-urls
         pytest --junitxml=test-results/pytest/results.xml tests/test_*.py
         pytest --junitxml=test-results/pytest/results-torch.xml tests/torch_*.py
     - name: Python tests (CPU + GPU)
       if: inputs.gpu == 'ON'
       shell: bash
       run: |
+        conda list --show-channel-urls
         pytest --junitxml=test-results/pytest/results.xml tests/test_*.py
         pytest --junitxml=test-results/pytest/results-torch.xml tests/torch_*.py
         cp tests/common_faiss_tests.py faiss/gpu/test
@@ -174,13 +201,14 @@ runs:
       if: inputs.opt_level == 'avx2'
       shell: bash
       run: |
+        conda list --show-channel-urls
         FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs $CONDA/bin/python -c "import faiss" 2>&1 | grep faiss.so
         LD_DEBUG=libs $CONDA/bin/python -c "import faiss" 2>&1 | grep faiss_avx2.so
     - name: Upload test results
-      if: always()
+      if: inputs.upload_artifacts == 'true'
       uses: actions/upload-artifact@v4
       with:
-        name: test-results-arch=${{ runner.arch }}-opt=${{ inputs.opt_level }}-gpu=${{ inputs.gpu }}-cuvs=${{ inputs.cuvs }}-rocm=${{ inputs.rocm }}
+        name: test-results-arch=${{ runner.arch }}-opt=${{ inputs.opt_level }}-gpu=${{ inputs.gpu }}-cuvs=${{ inputs.cuvs }}-rocm=${{ inputs.rocm }}-svs=${{ inputs.svs }}
         path: test-results
     - name: Check installed packages channel
       shell: bash

.github/actions/build_conda/action.yml

@@ -43,22 +43,26 @@ runs:
       run: |
         # Ensure starting packages are from conda-forge.
         conda list --show-channel-urls
-        conda install -y -q "conda!=24.11.0"
+        conda install -y -q "conda!=24.11.0,<=25.07"
         conda install -y -q "conda-build=25.3.1" "liblief=0.14.1"
-        conda list --show-channel-urls
+        # Add Intel channel with lowest priority for libsvs-runtime only
+        conda config --append channels https://software.repos.intel.com/python/conda
+        conda config --set channel_priority flexible
     - name: Enable anaconda uploads
       if: inputs.label != ''
       shell: ${{ steps.choose_shell.outputs.shell }}
       env:
         PACKAGE_TYPE: ${{ inputs.label }}
       run: |
+        conda list --show-channel-urls
         conda install -y -q anaconda-client
         conda config --set anaconda_upload yes
     - name: Conda build (CPU)
       if: inputs.label == '' && inputs.cuda == ''
       shell: ${{ steps.choose_shell.outputs.shell }}
       working-directory: conda
       run: |
+        conda list --show-channel-urls
         conda build faiss --python 3.11 -c pytorch
     - name: Conda build (CPU) w/ anaconda upload
       if: inputs.label != '' && inputs.cuda == ''
@@ -67,12 +71,14 @@ runs:
       env:
         PACKAGE_TYPE: ${{ inputs.label }}
       run: |
+        conda list --show-channel-urls
         conda build faiss --user pytorch --label ${{ inputs.label }} -c pytorch
     - name: Conda build (GPU)
       if: inputs.label == '' && inputs.cuda != '' && inputs.cuvs == ''
       shell: ${{ steps.choose_shell.outputs.shell }}
       working-directory: conda
       run: |
+        conda list --show-channel-urls
         conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
             -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia
     - name: Conda build (GPU) w/ anaconda upload
@@ -82,13 +88,15 @@ runs:
       env:
         PACKAGE_TYPE: ${{ inputs.label }}
       run: |
+        conda list --show-channel-urls
         conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
             --user pytorch --label ${{ inputs.label }} -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia
     - name: Conda build (GPU w/ cuVS)
       if: inputs.label == '' && inputs.cuda != '' && inputs.cuvs != ''
       shell: ${{ steps.choose_shell.outputs.shell }}
       working-directory: conda
       run: |
+        conda list --show-channel-urls
         conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
             -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia
     - name: Conda build (GPU w/ cuVS) w/ anaconda upload
@@ -98,6 +106,7 @@ runs:
       env:
         PACKAGE_TYPE: ${{ inputs.label }}
       run: |
+        conda list --show-channel-urls
         conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
             --user pytorch --label ${{ inputs.label }} -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia
     - name: Check installed packages channel

.github/workflows/autoclose.yml

@@ -2,9 +2,24 @@ name: Close Inactive Issues
 on:
   schedule:
     - cron: "30 1 * * *"
+  issues:
+    types: [labeled]
 
 jobs:
+  notify-on-label:
+    if: github.event_name == 'issues' && github.event.label.name == 'autoclose'
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+      - name: Comment on autoclose label
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+        run: gh issue comment ${{ github.event.issue.number }} --body "This issue has been marked for autoclose and will be closed after 14 days of inactivity."
+
   close-issues:
+    if: github.event_name == 'schedule'
     runs-on: ubuntu-latest
     permissions:
       issues: write

.github/workflows/build-pull-request.yml

@@ -17,11 +17,11 @@ jobs:
             sudo apt install -y lsb-release wget software-properties-common gnupg
             wget https://apt.llvm.org/llvm.sh
             chmod u+x llvm.sh
-            sudo ./llvm.sh 18
-            sudo apt-get install -y git-core clang-format-18
+            sudo ./llvm.sh 21
+            sudo apt-get install -y git-core clang-format-21
       - name: Verify clang-format
         run: |
-            git ls-files | grep -E  '\.(cpp|h|cu|cuh)$' | xargs clang-format-18 -i
+            git ls-files | grep -E  '\.(cpp|h|cu|cuh)$' | xargs clang-format-21 -i
             if git diff --quiet; then
               echo "Formatting OK!"
             else
@@ -38,6 +38,9 @@ jobs:
         uses: actions/checkout@v4
       - name: Build and Test (cmake)
         uses: ./.github/actions/build_cmake
+      - name: Build C++ demos
+        run: |
+          make -C build demo_diversity_result_handler
   linux-x86_64-AVX2-cmake:
     name: Linux x86_64 AVX2 (cmake)
     needs: linux-x86_64-cmake
@@ -71,6 +74,17 @@ jobs:
         uses: ./.github/actions/build_cmake
         with:
           opt_level: avx512_spr
+  linux-x86_64-DD-cmake:
+    name: Linux x86_64 Dynamic Dispatch (cmake)
+    needs: linux-x86_64-cmake
+    runs-on: faiss-aws-m7i.large
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Build and Test (cmake)
+        uses: ./.github/actions/build_cmake
+        with:
+          opt_level: dd
   linux-x86_64-GPU-cmake:
     name: Linux x86_64 GPU (cmake)
     needs: linux-x86_64-cmake
@@ -94,31 +108,22 @@ jobs:
         with:
           gpu: ON
           cuvs: ON
-  linux-x86_64-GPU-w-ROCm-cmake:
-    name: Linux x86_64 GPU w/ ROCm (cmake)
+  linux-arm64-SVE-cmake:
+    name: Linux arm64 SVE (cmake)
     needs: linux-x86_64-cmake
-    runs-on: faiss-amd-MI200
-    container:
-      image: ubuntu:22.04
-      options: --device=/dev/kfd --device=/dev/dri --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN
+    runs-on: faiss-aws-r8g.large
     steps:
-      - name: Container setup
-        run: |
-            if [ -f /.dockerenv ]; then
-              apt-get update && apt-get install -y sudo && apt-get install -y git
-              git config --global --add safe.directory '*'
-            else
-              echo 'Skipping. Current job is not running inside a container.'
-            fi
       - name: Checkout
         uses: actions/checkout@v4
       - name: Build and Test (cmake)
         uses: ./.github/actions/build_cmake
         with:
-          gpu: ON
-          rocm: ON
-  linux-arm64-SVE-cmake:
-    name: Linux arm64 SVE (cmake)
+          opt_level: sve
+        env:
+          # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction
+          OPENBLAS_NUM_THREADS: '1'
+  linux-arm64-DD-cmake:
+    name: Linux arm64 Dynamic Dispatch (cmake)
     needs: linux-x86_64-cmake
     runs-on: faiss-aws-r8g.large
     steps:
@@ -127,9 +132,8 @@ jobs:
       - name: Build and Test (cmake)
         uses: ./.github/actions/build_cmake
         with:
-          opt_level: sve
+          opt_level: dd
         env:
-          # Context: https://github.com/facebookresearch/faiss/wiki/Troubleshooting#surprising-faiss-openmp-and-openblas-interaction
           OPENBLAS_NUM_THREADS: '1'
   linux-x86_64-conda:
     name: Linux x86_64 (conda)
@@ -167,3 +171,18 @@ jobs:
           fetch-tags: true
       - name: Build and Package (conda)
         uses: ./.github/actions/build_conda
+  linux-x86_64-svs:
+    name: Linux x86_64 w/ SVS (cmake)
+    needs: linux-x86_64-cmake
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Build and Test (cmake)
+        uses: ./.github/actions/build_cmake
+        with:
+          svs: ON
+  index-io-backward-compatibility:
+    needs: linux-x86_64-cmake
+    name: Index serialization backward compatibility
+    uses: ./.github/workflows/index-io-backward-compatibility.yml

.github/workflows/build-release.yml

@@ -23,7 +23,7 @@ jobs:
         with:
           label: main
   linux-x86_64-GPU-packages-CUDA-12-4-0:
-    name: Linux x86_64 GPU packages (CUDA 12.4.0)
+    name: Linux x86_64 GPU packages (CUDA 12.6)
     runs-on: 4-core-ubuntu-gpu-t4
     env:
       CUDA_ARCHS: "70-real;72-real;75-real;80;86-real"
@@ -39,9 +39,9 @@ jobs:
           ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
         with:
           label: main
-          cuda: "12.4.0"
+          cuda: "12.6"
   linux-x86_64-GPU-CUVS-packages-CUDA12-4-0:
-    name: Linux x86_64 GPU w/ cuVS packages (CUDA 12.4.0)
+    name: Linux x86_64 GPU w/ cuVS packages (CUDA 12.6)
     runs-on: 4-core-ubuntu-gpu-t4
     env:
       CUDA_ARCHS: "70-real;72-real;75-real;80;86-real"
@@ -58,7 +58,7 @@ jobs:
         with:
           label: main
           cuvs: "ON"
-          cuda: "12.4.0"
+          cuda: "12.6"
   windows-x86_64-packages:
     name: Windows x86_64 packages
     runs-on: windows-2022