Merge pull request #1014 from marek-hradil/allow-dynamic-backends

Allow dynamic backends

Author: MarcusDunn

llama-cpp-2/Cargo.toml

@@ -41,6 +41,7 @@ mtmd = ["llama-cpp-sys-2/mtmd"]
 system-ggml = ["llama-cpp-sys-2/system-ggml"]
 system-ggml-static = ["llama-cpp-sys-2/system-ggml-static"]
 llguidance = ["dep:llguidance", "dep:toktrie"]
+dynamic-backends = ["llama-cpp-sys-2/dynamic-backends"]
 
 
 [target.'cfg(all(target_os = "macos", any(target_arch = "aarch64", target_arch = "arm64")))'.dependencies]

llama-cpp-2/build.rs

@@ -0,0 +1,5 @@
+fn main() {
+    if let Ok(dir) = std::env::var("DEP_LLAMA_BACKENDS_DIR") {
+        println!("cargo:rustc-env=GGML_BACKENDS_DIR={}", dir);
+    }
+}

llama-cpp-2/src/llama_backend.rs

@@ -180,6 +180,34 @@ impl Drop for LlamaBackend {
     }
 }
 
+/// Compile-time path to the built GGML backend modules directory.
+/// Populated by build.rs from `DEP_LLAMA_BACKENDS_DIR` (emitted by llama-cpp-sys-2).
+/// None on static builds or when the feature is disabled.
+#[cfg(feature = "dynamic-backends")]
+pub const BACKENDS_DIR: Option<&str> = option_env!("GGML_BACKENDS_DIR");
+
+/// Load GGML backend modules from the given directory.
+///
+/// Call this before [`LlamaBackend::init`] to enable runtime hardware selection
+/// (Vulkan, CPU-AVX512, CPU-AVX2, etc.) when built with the `dynamic-backends` feature.
+#[cfg(feature = "dynamic-backends")]
+pub fn load_backends_from_path(path: &std::path::Path) {
+    let s = std::ffi::CString::new(path.to_str().expect("path must be valid UTF-8"))
+        .expect("path must not contain null bytes");
+    unsafe { llama_cpp_sys_2::ggml_backend_load_all_from_path(s.as_ptr()) }
+}
+
+/// Load GGML backend modules from the compile-time default directory ([`BACKENDS_DIR`]).
+///
+/// This is a no-op when `BACKENDS_DIR` is `None` (static builds or development builds
+/// that have not set `GGML_BACKENDS_DIR`).
+#[cfg(feature = "dynamic-backends")]
+pub fn load_backends() {
+    if let Some(dir) = BACKENDS_DIR {
+        load_backends_from_path(std::path::Path::new(dir));
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

llama-cpp-sys-2/Cargo.toml

@@ -96,3 +96,4 @@ static-stdcxx = []
 system-ggml = []
 system-ggml-static = ["system-ggml"]
 mtmd = []
+dynamic-backends = ["dynamic-link"]

llama-cpp-sys-2/build.rs

@@ -820,6 +820,18 @@ fn main() {
         config.define("LLAMA_USE_SYSTEM_GGML", "ON");
     }
 
+    if cfg!(feature = "dynamic-backends") {
+        // Pre-create the backends directory so CMake can install MODULE libs there.
+        // GGML_BACKEND_DIR causes backends to install to this known path instead of
+        // CMAKE_INSTALL_BINDIR, making them easy to locate in downstream build scripts.
+        let backends_dir = out_dir.join("backends");
+        std::fs::create_dir_all(&backends_dir).unwrap();
+        config.define("GGML_BACKEND_DL", "ON");
+        config.define("GGML_CPU_ALL_VARIANTS", "ON");
+        config.define("GGML_BACKEND_DIR", backends_dir.to_str().unwrap());
+        // BUILD_SHARED_LIBS=ON is already set above via the dynamic-link feature.
+    }
+
     // General
     config
         .profile(&profile)
@@ -828,6 +840,10 @@ fn main() {
 
     let build_dir = config.build();
 
+    if cfg!(feature = "dynamic-backends") {
+        println!("cargo:backends_dir={}", out_dir.join("backends").display());
+    }
+
     // Build mtmd directly with cc::Build, bypassing the cmake tools build.
     // Using LLAMA_BUILD_TOOLS=ON would pull in all tools (batched-bench, quantize, etc.)
     // and their CMakeLists.txt files, which are not included in the crate package.