feat: add llama.cpp OpenVINO backend for Linux

src/cpp/resources/backend_versions.json

@@ -6,7 +6,11 @@
     "rocm-nightly": "b1274",
     "cuda": "b9436",
     "metal": "b9253",
-    "cpu": "b9253"
+    "cpu": "b9253",
+    "openvino": "b9488"
+  },
+  "openvino": {
+    "runtime_version": "2026.0"
   },
   "whispercpp": {
     "cpu": "v1.8.4",

src/cpp/resources/defaults.json

@@ -21,11 +21,13 @@
     "rocm_args": "",
     "vulkan_args": "",
     "cpu_args": "",
+    "openvino_args": "",
     "prefer_system": true,
     "rocm_bin": "builtin",
     "vulkan_bin": "builtin",
     "cuda_bin": "builtin",
-    "cpu_bin": "builtin"
+    "cpu_bin": "builtin",
+    "openvino_bin": "builtin"
   },
   "whispercpp": {
     "backend": "auto",

src/cpp/server/backends/llamacpp_server.cpp

@@ -117,6 +117,10 @@ static bool is_llamacpp_cuda_backend(const std::string& backend) {
     return backend == "cuda";
 }
 
+static bool is_llamacpp_openvino_backend(const std::string& backend) {
+    return backend == "openvino";
+}
+
 static std::string trim_version_prefix(const std::string& version) {
     if (!version.empty() && version[0] == 'v') {
         return version.substr(1);
@@ -143,6 +147,15 @@ static std::string get_therock_version() {
     return trim_to_major_minor(config["therock"]["version"].get<std::string>());
 }
 
+static std::string get_openvino_runtime_version() {
+    auto config = JsonUtils::load_from_file(utils::get_resource_path("resources/backend_versions.json"));
+    if (!config.contains("openvino") || !config["openvino"].is_object() ||
+        !config["openvino"].contains("runtime_version") || !config["openvino"]["runtime_version"].is_string()) {
+        throw std::runtime_error("backend_versions.json is missing 'openvino.runtime_version'");
+    }
+    return config["openvino"]["runtime_version"].get<std::string>();
+}
+
 InstallParams LlamaCppServer::get_install_params(const std::string& backend, const std::string& version) {
     InstallParams params;
 
@@ -211,6 +224,14 @@ InstallParams LlamaCppServer::get_install_params(const std::string& backend, con
         params.filename = "llama-" + version + "-bin-macos-arm64.tar.gz";
 #else
         throw std::runtime_error("Metal llamacpp only supported on macOS");
+#endif
+    } else if (resolved_backend == "openvino") {
+        params.repo = "lemonade-sdk/llama.cpp";
+#ifdef __linux__
+        std::string openvino_ver = get_openvino_runtime_version();
+        params.filename = "llama-" + version + "-bin-ubuntu-openvino-" + openvino_ver + "-x64.tar.gz";
+#else
+        throw std::runtime_error("OpenVINO llamacpp is currently supported on Linux only");
 #endif
     } else if (resolved_backend == "cpu") {
         params.repo = "ggml-org/llama.cpp";
@@ -342,9 +363,9 @@ void LlamaCppServer::load(const std::string& model_name,
     }
     push_reserved(reserved_flags, "--mmproj", std::vector<std::string>{"-mm", "-mmu", "--mmproj-url", "--no-mmproj", "--mmproj-auto", "--no-mmproj-auto", "--mmproj-offload", "--no-mmproj-offload"});
 
-    // Enable context shift for vulkan/rocm/cuda (not supported on Metal)
+    // Enable context shift for vulkan/rocm/cuda/openvino (not supported on Metal)
     if (llamacpp_backend == "vulkan" || is_llamacpp_rocm_backend(llamacpp_backend) ||
-        is_llamacpp_cuda_backend(llamacpp_backend)) {
+        is_llamacpp_cuda_backend(llamacpp_backend) || is_llamacpp_openvino_backend(llamacpp_backend)) {
         push_overridable_arg(args, llamacpp_args, "--context-shift");
         push_overridable_arg(args, llamacpp_args, "--keep", "16");
     } else {
@@ -431,10 +452,9 @@ void LlamaCppServer::load(const std::string& model_name,
 
         env_vars.push_back({"LD_LIBRARY_PATH", lib_path});
         LOG(DEBUG, "LlamaCpp") << "Setting LD_LIBRARY_PATH=" << lib_path << std::endl;
-    } else if (is_llamacpp_cuda_backend(llamacpp_backend)) {
-        // The llama.cpp-builds Linux tarballs ship the bundled CUDA runtime
-        // (libcudart.so, libcublas.so, etc.) alongside llama-server, so add the
-        // executable's directory to LD_LIBRARY_PATH like we do for ROCm.
+    } else if (is_llamacpp_cuda_backend(llamacpp_backend) || is_llamacpp_openvino_backend(llamacpp_backend)) {
+        // CUDA and OpenVINO tarballs both bundle their runtime libraries (.so files)
+        // alongside llama-server, so add the executable's directory to LD_LIBRARY_PATH.
         fs::path exe_dir = fs::path(executable).parent_path();
         std::string lib_path = exe_dir.string();
 

src/cpp/server/config_file.cpp

@@ -200,12 +200,14 @@ static const EnvMapping env_mappings[] = {
     {"LEMONADE_LLAMACPP_ROCM_ARGS",      "llamacpp",  "rocm_args"},
     {"LEMONADE_LLAMACPP_VULKAN_ARGS",    "llamacpp",  "vulkan_args"},
     {"LEMONADE_LLAMACPP_CPU_ARGS",       "llamacpp",  "cpu_args"},
+    {"LEMONADE_LLAMACPP_OPENVINO_ARGS",  "llamacpp",  "openvino_args"},
     {"LEMONADE_LLAMACPP_DEVICE",         "llamacpp",  "device"},
     {"LEMONADE_LLAMACPP_PREFER_SYSTEM",  "llamacpp",  "prefer_system"},
     {"LEMONADE_LLAMACPP_ROCM_BIN",       "llamacpp",  "rocm_bin"},
     {"LEMONADE_LLAMACPP_VULKAN_BIN",     "llamacpp",  "vulkan_bin"},
     {"LEMONADE_LLAMACPP_CUDA_BIN",       "llamacpp",  "cuda_bin"},
     {"LEMONADE_LLAMACPP_CPU_BIN",        "llamacpp",  "cpu_bin"},
+    {"LEMONADE_LLAMACPP_OPENVINO_BIN",   "llamacpp",  "openvino_bin"},
     // whispercpp
     {"LEMONADE_WHISPERCPP",              "whispercpp", "backend"},
     {"LEMONADE_WHISPERCPP_ARGS",         "whispercpp", "args"},

src/cpp/server/system_info.cpp

@@ -469,6 +469,9 @@ static const std::vector<RecipeBackendDef> RECIPE_DEFS = {
         {"cpu", {"x86_64"}},
         {"amd_gpu", {}},      // all AMD GPU families
     }},
+    {"llamacpp", "openvino", {"linux"}, {
+        {"cpu", {"x86_64"}},
+    }},
     {"llamacpp", "rocm", {"windows", "linux"}, {
         {"amd_gpu", {"gfx1150", "gfx1151", "gfx103X", "gfx110X", "gfx120X"}},  // STX iGPUs + RDNA2/3/4 dGPUs
     }},