Merge branch 'main' into avx512_vnni_opt

Author: mnorris11

.github/actions/build_cmake/action.yml

@@ -20,6 +20,10 @@ inputs:
     description: 'Enable SVS support.'
     required: false
     default: OFF
+  metal:
+    description: 'Enable Metal GPU backend (macOS Apple Silicon).'
+    required: false
+    default: OFF
   setup_conda:
     description: 'Setup miniconda environment.'
     required: false
@@ -32,31 +36,29 @@ runs:
   using: composite
   steps:
     - name: Setup miniconda
-      if: inputs.setup_conda == 'true'
+      if: inputs.setup_conda == 'true' && inputs.metal != 'ON'
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.12'
-        miniforge-version: latest # ensures conda-forge channel is used.
+        miniforge-version: latest
         channels: conda-forge
         conda-remove-defaults: 'true'
         # Set to aarch64 if we're on arm64 because there's no miniforge ARM64 package, just aarch64.
         # They are the same thing, just named differently.
         architecture: ${{ runner.arch  == 'ARM64' && 'aarch64' || runner.arch }}
     - name: Configure build environment
+      if: inputs.metal != 'ON'
       shell: bash
       run: |
-        # initialize Conda
-        conda config --set solver libmamba
         # Ensure starting packages are from conda-forge.
         conda list --show-channel-urls
-        conda install -y -q "conda<=25.07"
         echo "$CONDA/bin" >> $GITHUB_PATH
 
         conda install -y -q python=3.12 cmake=3.30.4 make=4.2 swig=4.0 "numpy>=2.0,<3.0" scipy=1.16 pytest=7.4 gflags=2.2 setuptools
 
         # install base packages for ARM64
         if [ "${{ runner.arch }}" = "ARM64" ]; then
-          conda install -y -q -c conda-forge openblas=0.3.29 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
+          conda install -y -q -c conda-forge openblas=0.3.33 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
         fi
 
         # install base packages for X86_64
@@ -71,10 +73,14 @@ runs:
           :
         # regular CUDA for GPU builds
         elif [ "${{ inputs.gpu }}" = "ON" ] && [ "${{ inputs.cuvs }}" = "OFF" ]; then
-          conda install -y -q cuda-libraries-dev=12.6 cuda-nvcc=12.6 cuda-nvtx=12.6 cuda-cupti=12.6 cuda-cudart-dev=12.6 gxx_linux-64=12.4 -c "nvidia/label/cuda-12.6"
+          conda install -y -q cuda-libraries-dev=13.2 cuda-nvcc=13.2 cuda-nvtx=13.2 cuda-cupti=13.2 cuda-cudart-dev=13.2 gxx_linux-64=14.2 -c "nvidia/label/cuda-13.2"
         # and CUDA from cuVS channel for cuVS builds
         elif [ "${{ inputs.cuvs }}" = "ON" ]; then
-          conda install -y -q libcuvs=26.02 'cuda-version=12.9' cuda-toolkit=12.9 sysroot_linux-64=2.34 -c rapidsai -c rapidsai-nightly -c conda-forge
+          conda install -y -q libcuvs=26.02 'cuda-version=13.2' sysroot_linux-64=2.34 -c rapidsai -c rapidsai-nightly -c conda-forge
+          # Clean index cache to prevent sqlite3 "database is locked" errors
+          # from conda-libmamba-solver's shards cache between consecutive installs.
+          # See: https://github.com/conda/conda-libmamba-solver/issues/667
+          conda clean --index-cache 2>/dev/null || true
         fi
 
         # install SVS runtime for SVS builds
@@ -87,7 +93,10 @@ runs:
           : # skip torch install via conda, we need to install via pip to get
             #  ROCm-enabled version until it's supported in conda by PyTorch
         elif [ "${{ inputs.gpu }}" = "ON" ]; then
-          conda install -y -q "pytorch>=2.7" "pytorch-gpu>=2.7" -c pytorch -c "nvidia/label/12.9"
+          # PyTorch deprecated its official Anaconda channel (pytorch/pytorch#138506),
+          # so we install via pip for CUDA builds. --break-system-packages is needed
+          # because the CI runner's Python is PEP 668 externally-managed.
+          pip install "torch>=2.7" --break-system-packages --index-url https://download.pytorch.org/whl/cu132
         else
           # TestLowLevelIVF.IVFRQ hangs on pytorch>=2.7, so left it as <2.5 for now.
           conda install -y -q "pytorch<2.5" -c pytorch
@@ -159,7 +168,18 @@ runs:
       if: inputs.gpu == 'ON' && inputs.rocm == 'ON'
       shell: bash
       run: rocm-smi
+    - name: Setup ccache
+      uses: hendrikmuhs/ccache-action@v1
+      with:
+        key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.opt_level }}-gpu${{ inputs.gpu }}-cuvs${{ inputs.cuvs }}-rocm${{ inputs.rocm }}-svs${{ inputs.svs }}
+        max-size: 2G
+        update-package-index: true
+    - name: Setup macOS Metal environment
+      if: inputs.metal == 'ON'
+      shell: bash
+      run: brew install cmake libomp gflags
     - name: Build all targets
+      if: inputs.metal != 'ON'
       shell: bash
       run: |
         eval "$(conda shell.bash hook)"
@@ -176,23 +196,45 @@ runs:
               -DFAISS_ENABLE_C_API=ON \
               -DPYTHON_EXECUTABLE=$CONDA/bin/python \
               -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+              -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+              -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
               -DBLA_VENDOR=${{ runner.arch == 'X64' && 'Intel10_64_dyn' || '' }} \
               -DCMAKE_CUDA_FLAGS=${{ runner.arch == 'X64' && '"-gencode arch=compute_75,code=sm_75"' || '' }} \
               .
         make -k -C build -j$(nproc)
+    - name: Build Metal targets
+      if: inputs.metal == 'ON'
+      shell: bash
+      run: |
+        cmake -B build \
+              -DFAISS_ENABLE_METAL=ON \
+              -DFAISS_ENABLE_GPU=OFF \
+              -DFAISS_ENABLE_PYTHON=OFF \
+              -DBUILD_TESTING=ON \
+              -DCMAKE_BUILD_TYPE=Release \
+              -DCMAKE_PREFIX_PATH="$(brew --prefix libomp)" \
+              .
+        cmake --build build --target faiss_metal TestMetalIndexFlat -j$(sysctl -n hw.logicalcpu)
     - name: C++ tests
+      if: inputs.metal != 'ON'
       shell: bash
       run: |
         conda list --show-channel-urls
         export GTEST_OUTPUT="xml:$(realpath .)/test-results/googletest/"
         make -C build test
+    - name: C++ tests (Metal)
+      if: inputs.metal == 'ON'
+      shell: bash
+      run: cd build && ctest -R TestMetalIndexFlat --output-on-failure
     - name: C++ perf benchmarks
+      if: inputs.rocm == 'OFF' && inputs.metal != 'ON'
       shell: bash
-      if: inputs.rocm == 'OFF'
       run: |
         conda list --show-channel-urls
         find ./build/perf_tests/ -executable -type f -name "bench*" -exec '{}' -v \;
     - name: Install Python extension
+      if: inputs.metal != 'ON'
       shell: bash
       working-directory: build/faiss/python
       run: |
@@ -203,9 +245,10 @@ runs:
       shell: bash
       run: |
         conda list --show-channel-urls
+        conda install -y -q "pip<26"
         pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.1
     - name: Python tests (CPU only)
-      if: inputs.gpu == 'OFF'
+      if: inputs.gpu == 'OFF' && inputs.metal != 'ON'
       shell: bash
       run: |
         conda list --show-channel-urls
@@ -235,6 +278,7 @@ runs:
         name: test-results-arch=${{ runner.arch }}-opt=${{ inputs.opt_level }}-gpu=${{ inputs.gpu }}-cuvs=${{ inputs.cuvs }}-rocm=${{ inputs.rocm }}-svs=${{ inputs.svs }}
         path: test-results
     - name: Check installed packages channel
+      if: inputs.metal != 'ON'
       shell: bash
       run: |
         # Shows that all installed packages are from conda-forge.

.github/actions/build_conda/action.yml

@@ -30,9 +30,10 @@ runs:
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.12'
-        miniforge-version: latest # ensures conda-forge channel is used.
+        miniforge-version: latest
         channels: conda-forge
         conda-remove-defaults: 'true'
+        activate-environment: 'base'
         # Set to runner.arch=aarch64 if we're on arm64 because
         # there's no miniforge ARM64 package, just aarch64.
         # They are the same thing, just named differently.
@@ -43,7 +44,7 @@ runs:
       run: |
         # Ensure starting packages are from conda-forge.
         conda list --show-channel-urls
-        conda install -y -q "conda!=24.11.0,<=25.07"
+        conda install -y -q "conda!=24.11.0,<25.7"
         conda install -y -q "conda-build=25.3.1" "liblief=0.14.1"
     - name: Enable anaconda uploads
       if: inputs.label != ''
@@ -80,7 +81,7 @@ runs:
       working-directory: conda
       run: |
         conda list --show-channel-urls
-        conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
+        conda build faiss-gpu --python 3.12 --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
             -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia
     - name: Conda build (GPU) w/ anaconda upload
       if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs == ''
@@ -98,7 +99,7 @@ runs:
       working-directory: conda
       run: |
         conda list --show-channel-urls
-        conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
+        conda build faiss-gpu-cuvs --python 3.12 --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
             -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia
     - name: Conda build (GPU w/ cuVS) w/ anaconda upload
       if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs != ''

.github/workflows/build-pip.yml

@@ -0,0 +1,88 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+name: Build pip wheels
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  build-wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-14, macos-15-intel, windows-2022, 2-core-ubuntu-arm]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@ee02a1537ce3071a004a6b08c41e72f0fdc42d9a  # v3.4.0
+        env:
+          CIBW_BUILD_VERBOSITY: 1
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: wheels-${{ matrix.os }}
+          path: wheelhouse/*.whl
+
+  publish:
+    name: Publish to PyPI
+    needs: [build-wheels]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    environment:
+      name: pypi
+      url: https://pypi.org/p/faiss-cpu
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Download wheel artifacts
+        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
+        with:
+          path: dist
+          pattern: wheels-*
+          merge-multiple: true
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc  # release/v1
+        with:
+          verbose: true
+          attestations: false
+
+  publish-testpypi:
+    name: Publish to TestPyPI
+    needs: [build-wheels]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch'
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/faiss-cpu
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Download wheel artifacts
+        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e  # v4.1.7
+        with:
+          path: dist
+          pattern: wheels-*
+          merge-multiple: true
+
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc  # release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
+          verbose: true