Add Faiss library as an optional dependency (facebookincubator#13575)

Summary:
Pull Request resolved: facebookincubator#13575

This PR introduces FAISS as an optional dependency, making the first step to support Faiss vector search in velox.

Reviewed By: xiaoxmeng

Differential Revision: D75707431

fbshipit-source-id: fc153cfc0acf3a7c12219b27a13c307257c1de5c

Author: zhichenxu-meta

.github/workflows/linux-build-base.yml

@@ -39,6 +39,7 @@ jobs:
       GTest_SOURCE: BUNDLED
       cudf_SOURCE: BUNDLED
       CUDA_VERSION: '12.8'
+      faiss_SOURCE: SYSTEM
       USE_CLANG: "${{ inputs.use-clang && 'true' || 'false' }}"
     steps:
       - uses: actions/checkout@v4
@@ -93,6 +94,7 @@ jobs:
             "-DVELOX_ENABLE_EXAMPLES=ON"
             "-DVELOX_ENABLE_ARROW=ON"
             "-DVELOX_ENABLE_GEO=ON"
+            "-DVELOX_ENABLE_FAISS=ON"
             "-DVELOX_ENABLE_PARQUET=ON"
             "-DVELOX_ENABLE_HDFS=ON"
             "-DVELOX_ENABLE_S3=ON"

.github/workflows/macos.yml

@@ -70,6 +70,8 @@ jobs:
           source scripts/setup-macos.sh
           install_build_prerequisites
           install_velox_deps_from_brew
+          # Needed for faiss to find BLAS
+          install_faiss_deps
           install_double_conversion
 
           echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV
@@ -86,6 +88,7 @@ jobs:
       - name: Configure Build
         env:
           fmt_SOURCE: BUNDLED #brew fmt11 is not supported
+          faiss_SOURCE: BUNDLED #brew faiss is not supported
           CMAKE_POLICY_VERSION_MINIMUM: '3.5'
         run: |
           ccache -sz -M 5Gi
@@ -97,7 +100,8 @@ jobs:
               -DVELOX_ENABLE_PARQUET=ON \
               -DVELOX_MONO_LIBRARY=ON \
               -DVELOX_BUILD_SHARED=ON \
-              -DCMAKE_BUILD_TYPE=$BUILD_TYPE
+              -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
+              -DVELOX_ENABLE_FAISS=ON
 
       - name: Build
         run: |

CMake/resolve_dependency_modules/README.md

@@ -31,6 +31,7 @@ by Velox. See details on bundling below.
 | re2               | 2021-04-01      | Yes      |
 | fmt               | 10.1.1          | Yes      |
 | simdjson          | 3.9.3           | Yes      |
+| faiss             | 1.11.0          | Yes      |
 | folly             | v2025.04.28.00  | Yes      |
 | fizz              | v2025.04.28.00  | No       |
 | wangle            | v2025.04.28.00  | No       |

CMake/resolve_dependency_modules/faiss.cmake

@@ -0,0 +1,65 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+include_guard(GLOBAL)
+
+set(VELOX_FAISS_BUILD_VERSION 1.11.0)
+set(VELOX_FAISS_BUILD_SHA256_CHECKSUM
+    c5d517da6deb6a6d74290d7145331fc7474426025e2d826fa4a6d40670f4493c)
+set(VELOX_FAISS_SOURCE_URL
+    "https://github.com/facebookresearch/faiss/archive/refs/tags/v${VELOX_FAISS_BUILD_VERSION}.tar.gz"
+)
+
+velox_resolve_dependency_url(FAISS)
+
+# We need these hints for macos to build.
+if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
+  message(STATUS "Detected Apple platform")
+  execute_process(
+    COMMAND brew --prefix libomp
+    RESULT_VARIABLE BREW_LIBOMP_RESULT
+    OUTPUT_VARIABLE BREW_LIBOMP_PREFIX
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+  if(BREW_LIBOMP_RESULT EQUAL 0 AND EXISTS "${BREW_LIBOMP_PREFIX}")
+    list(APPEND CMAKE_PREFIX_PATH "${BREW_LIBOMP_PREFIX}")
+  endif()
+
+  execute_process(
+    COMMAND brew --prefix openblas
+    RESULT_VARIABLE BREW_OPENBLAS_RESULT
+    OUTPUT_VARIABLE BREW_OPENBLAS_PREFIX
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+  if(BREW_OPENBLAS_RESULT EQUAL 0 AND EXISTS "${BREW_OPENBLAS_PREFIX}")
+    list(APPEND CMAKE_PREFIX_PATH "${BREW_OPENBLAS_PREFIX}")
+  endif()
+endif()
+
+FetchContent_Declare(
+  faiss
+  URL ${VELOX_FAISS_SOURCE_URL}
+  URL_HASH ${VELOX_FAISS_BUILD_SHA256_CHECKSUM}
+  SYSTEM # Once there are targets depending on FAISS we can add
+         # `EXCLUDE_FROM_ALL` back in
+)
+
+# Set build options
+block()
+set(BUILD_SHARED_LIBS OFF)
+set(CMAKE_BUILD_TYPE Release)
+set(FAISS_ENABLE_GPU OFF)
+set(FAISS_ENABLE_PYTHON OFF)
+set(FAISS_ENABLE_GPU_TESTS OFF)
+# Make FAISS available
+FetchContent_MakeAvailable(faiss)
+add_library(FAISS::faiss ALIAS faiss)
+endblock()

CMakeLists.txt

@@ -160,6 +160,7 @@ option(
   OFF)
 option(VELOX_SIMDJSON_SKIPUTF8VALIDATION
        "Skip simdjson utf8 validation in JSON parsing" OFF)
+option(VELOX_ENABLE_FAISS "Build faiss vector search support" OFF)
 
 # Explicitly force compilers to generate colored output. Compilers usually do
 # this by default if they detect the output is a terminal, but this assumption
@@ -229,6 +230,11 @@ if(${VELOX_ENABLE_DUCKDB})
   velox_resolve_dependency(DuckDB)
 endif()
 
+if(VELOX_ENABLE_FAISS)
+  velox_set_source(faiss)
+  velox_resolve_dependency(faiss)
+endif()
+
 if(DEFINED ENV{INSTALL_PREFIX})
   # Allow installed package headers to be picked up before brew/system package
   # headers. We set this after DuckDB bundling since DuckDB uses its own

scripts/setup-centos9.sh

@@ -142,6 +142,11 @@ function install_adapters {
   run_and_time install_hdfs
 }
 
+function install_faiss_deps {
+  dnf install -y openblas-devel
+  dnf install -y libomp
+}
+
 function install_velox_deps {
   run_and_time install_velox_deps_from_dnf
   run_and_time install_conda
@@ -164,6 +169,7 @@ function install_velox_deps {
   run_and_time install_xsimd
   run_and_time install_simdjson
   run_and_time install_geos
+  run_and_time install_faiss
 }
 
 (return 2> /dev/null) && return # If script was sourced, don't run commands.

scripts/setup-common.sh

@@ -22,6 +22,7 @@ VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared f
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
 BUILD_GEOS="${BUILD_GEOS:-true}"
+BUILD_FAISS="${BUILD_FAISS:-true}"
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
 EXTRA_ARROW_OPTIONS=${EXTRA_ARROW_OPTIONS:-""}
 
@@ -224,6 +225,26 @@ function install_geos {
   fi
 }
 
+function install_faiss_deps {
+  echo "Unsupported platform for faiss"
+}
+
+function install_faiss {
+  if [[ "$BUILD_FAISS" == "true" ]]; then
+    # Install OpenBLAS and libomp if not already installed
+    install_faiss_deps
+
+    wget_and_untar "https://github.com/facebookresearch/faiss/archive/refs/tags/v${FAISS_VERSION}.tar.gz" faiss
+    cmake_install_dir faiss \
+      -DFAISS_ENABLE_GPU=OFF \
+      -DFAISS_ENABLE_PYTHON=OFF \
+      -DFAISS_ENABLE_REMOTE=OFF \
+      -DFAISS_ENABLE_GPU_TESTS=OFF \
+      -DFAISS_ENABLE_BENCHMARKS=OFF
+  fi
+}
+
+
 # Adapters that can be installed.
 
 function install_aws_deps {

scripts/setup-helper-functions.sh

@@ -196,11 +196,12 @@ function wget_and_untar {
 }
 
 function cmake_install_dir {
-  pushd "${DEPENDENCY_DIR}/$1"
+  pushd "${DEPENDENCY_DIR}/$1" || return 1
   # remove the directory argument
   shift
+  # shellcheck disable=SC2068
   cmake_install $@
-  popd
+  popd || return 1
 }
 
 function cmake_install {

scripts/setup-macos.sh

@@ -40,6 +40,7 @@ export CMAKE_POLICY_VERSION_MINIMUM="3.5"
 
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
 MACOS_VELOX_DEPS="bison flex gflags glog googletest icu4c libevent libsodium lz4 openssl protobuf@21 simdjson snappy xz xxhash zstd"
+
 MACOS_BUILD_DEPS="ninja cmake"
 
 SUDO="${SUDO:-""}"
@@ -133,6 +134,38 @@ function install_duckdb_clang {
   fi
 }
 
+function install_faiss_deps {
+  brew install openblas
+  brew install libomp
+}
+
+function install_faiss {
+  if [[ "$BUILD_FAISS" == "true" ]]; then
+    # Install OpenBLAS and libomp if not already installed
+    install_faiss_deps
+
+    wget_and_untar "https://github.com/facebookresearch/faiss/archive/refs/tags/v${FAISS_VERSION}.tar.gz" faiss
+
+    local cmake_args
+    cmake_args=(
+      -DFAISS_ENABLE_GPU=OFF
+      -DFAISS_ENABLE_PYTHON=OFF
+      -DFAISS_ENABLE_REMOTE=OFF
+      -DFAISS_ENABLE_GPU_TESTS=OFF
+      -DFAISS_ENABLE_BENCHMARKS=OFF
+      -DFAISS_ENABLE_GPU=OFF
+      -FAISS_ENABLE_MKL=OFF
+    )
+
+    local libomp_prefix
+    libomp_prefix=$(brew --prefix libomp)
+    cmake_args+=(
+      "-DCMAKE_PREFIX_PATH=${libomp_prefix}"
+    )
+    cmake_install_dir faiss "${cmake_args[@]}"
+  fi
+}
+
 function install_velox_deps {
   run_and_time install_velox_deps_from_brew
   run_and_time install_ranges_v3
@@ -152,8 +185,9 @@ function install_velox_deps {
 # See https://github.com/facebook/fbthrift/pull/317 for an explanation.
 # run_and_time install_thrift
   run_and_time install_arrow
-  run_and_time install_duckdb_clang 
+  run_and_time install_duckdb_clang
   run_and_time install_geos
+  run_and_time install_faiss
 }
 
 (return 2> /dev/null) && return # If script was sourced, don't run commands.

scripts/setup-ubuntu.sh

@@ -227,6 +227,10 @@ function install_adapters {
   run_and_time install_hdfs
 }
 
+function install_faiss_deps {
+  sudo apt-get install -y libopenblas-dev libomp-dev
+}
+
 function install_velox_deps {
   run_and_time install_velox_deps_from_apt
   run_and_time install_fmt
@@ -246,6 +250,7 @@ function install_velox_deps {
   run_and_time install_xsimd
   run_and_time install_simdjson
   run_and_time install_geos
+  run_and_time install_faiss
 }
 
 function install_apt_deps {