chore: ⬆️ Update ggml-org/llama.cpp to f280b26983ad0fdb705a0d9ebf0503e76f2899b0 (mudler#11646)

* ⬆️ Update ggml-org/llama.cpp

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* fix(llama-cpp): adapt to the common JSON API

The llama.cpp bump replaces its nlohmann JSON alias with common_json. Update the gRPC adapter for the new exception, iterator, conversion, and container APIs.

Assisted-by: Codex:gpt-5.6 [systematic-debugging]

* fix(turboquant): adapt the JSON exception type

The shared gRPC source now follows the upstream common_json API. The
TurboQuant fork still exposes nlohmann JSON and cannot compile the new
exception type.

Translate that exception in the fork-specific source patch so both
llama.cpp variants compile from the shared adapter.

Assisted-by: Codex:gpt-5.6 [systematic-debugging]

* fix(bonsai): adapt the JSON exception type

The shared gRPC source uses upstream's common_json wrapper. The Bonsai fork still exposes nlohmann JSON and cannot compile that exception type.\n\nTranslate the exception in the fork-specific preparation step and verify that repeated preparation stays idempotent.\n\nAssisted-by: Codex:gpt-5.6 [systematic-debugging]

* fix(llama-cpp): let prepare register gRPC

The score patch duplicated the gRPC CMake registration that prepare.sh already owns. Its stale context rejects the current upstream tools file on Darwin before compilation starts.

Assisted-by: Codex:gpt-5

---------

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>
Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

Author: 3 people

backend/cpp/bonsai/Makefile

@@ -41,6 +41,7 @@ define bonsai-build
 	# and are applied by apply-patches.sh below.
 	rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/patches
 	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build purge
+	bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp
 	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp
 	bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp
 	$(info $(GREEN)I bonsai build info:$(1)$(RESET))
@@ -79,6 +80,7 @@ bonsai-cpu-all:
 	# and are applied by apply-patches.sh below.
 	rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/patches
 	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build purge
+	bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp
 	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp
 	bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp
 	$(info $(GREEN)I bonsai build info:cpu-all-variants$(RESET))

backend/cpp/bonsai/patch-grpc-server.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Adapt the shared llama.cpp gRPC source to the older JSON API in Bonsai.
+
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+    echo "usage: $0 <grpc-server.cpp>" >&2
+    exit 2
+fi
+
+SRC=$1
+if [[ ! -f "$SRC" ]]; then
+    echo "grpc-server.cpp not found at $SRC" >&2
+    exit 2
+fi
+
+if grep -q 'common_json_error' "$SRC"; then
+    echo "==> patching $SRC to use the Bonsai JSON exception type"
+    awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp"
+    mv "$SRC.tmp" "$SRC"
+    echo "==> Bonsai JSON exception patch OK"
+else
+    echo "==> $SRC already uses a Bonsai-compatible JSON exception type, skipping"
+fi

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd
+LLAMA_VERSION?=f280b26983ad0fdb705a0d9ebf0503e76f2899b0
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/grpc-server.cpp

@@ -294,7 +294,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
             } else {
                 SRV_WRN("[TOOLS DEBUG] parse_options: Parsed tools JSON is not an array: %s\n", tools_json.dump().c_str());
             }
-        } catch (const json::parse_error& e) {
+        } catch (const common_json_error& e) {
             SRV_WRN("Failed to parse tools JSON from proto: %s\n", e.what());
             SRV_WRN("[TOOLS DEBUG] parse_options: Tools string that failed to parse: %s\n", predict->tools().c_str());
         }
@@ -324,7 +324,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
                 SRV_DBG("[TOOLS DEBUG] Received tool_choice object from Go layer: %s\n", tool_choice_json.dump().c_str());
             }
             SRV_INF("Extracted tool_choice from proto: %s\n", predict->toolchoice().c_str());
-        } catch (const json::parse_error& e) {
+        } catch (const common_json_error& e) {
             // If parsing fails, treat as string
             data["tool_choice"] = predict->toolchoice();
             SRV_INF("Extracted tool_choice as string: %s\n", predict->toolchoice().c_str());
@@ -353,7 +353,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
             // Add to data - llama.cpp server expects it as an object (map)
             data["logit_bias"] = logit_bias_json;
             SRV_INF("Using logit_bias: %s\n", predict->logitbias().c_str());
-        } catch (const json::parse_error& e) {
+        } catch (const common_json_error& e) {
             SRV_ERR("Failed to parse logit_bias JSON from proto: %s\n", e.what());
         }
     }
@@ -398,7 +398,10 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
             });
     }
 
-    data["stop"] = predict->stopprompts();
+    data["stop"] = json::array();
+    for (const auto & stop : predict->stopprompts()) {
+        data["stop"].push_back(stop);
+    }
     // data["n_probs"] = predict->nprobs();
     //TODO: images,
 
@@ -1795,7 +1798,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
                         for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j));
                         for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j));
                     }
-                    messages_json.push_back(llama_grpc::build_reconstructed_message(rin));
+                    messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump()));
                 }
 
                 // Final safety check: Ensure no message has null content (Jinja templates require strings)
@@ -1988,7 +1991,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
                             if (!body_json.contains("chat_template_kwargs")) {
                                 body_json["chat_template_kwargs"] = json::object();
                             }
-                            for (auto& el : ctk.items()) {
+                            for (auto el : ctk.items()) {
                                 body_json["chat_template_kwargs"][el.key()] = el.value();
                             }
                         }
@@ -2074,30 +2077,27 @@ class BackendServiceImpl final : public backend::Backend::Service {
             // If not using chat templates, extract files from image_data/audio_data fields
             // (If using chat templates, files were already extracted by oaicompat_chat_params_parse)
             if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) {
-                const auto &images_data = data.find("image_data");
-                if (images_data != data.end() && images_data->is_array())
+                if (data.contains("image_data") && data.at("image_data").is_array())
                 {
-                    for (const auto &img : *images_data)
+                    for (const auto &img : data.at("image_data"))
                     {
                         auto decoded_data = base64_decode(img["data"].get<std::string>());
                         files.push_back(decoded_data);
                     }
                 }
 
-                const auto &audio_data = data.find("audio_data");
-                if (audio_data != data.end() && audio_data->is_array())
+                if (data.contains("audio_data") && data.at("audio_data").is_array())
                 {
-                    for (const auto &audio : *audio_data)
+                    for (const auto &audio : data.at("audio_data"))
                     {
                         auto decoded_data = base64_decode(audio["data"].get<std::string>());
                         files.push_back(decoded_data);
                     }
                 }
 
-                const auto &video_data = data.find("video_data");
-                if (video_data != data.end() && video_data->is_array())
+                if (data.contains("video_data") && data.at("video_data").is_array())
                 {
-                    for (const auto &video : *video_data)
+                    for (const auto &video : data.at("video_data"))
                     {
                         auto decoded_data = base64_decode(video["data"].get<std::string>());
                         files.push_back(decoded_data);
@@ -2370,7 +2370,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
                         for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j));
                         for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j));
                     }
-                    messages_json.push_back(llama_grpc::build_reconstructed_message(rin));
+                    messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump()));
                 }
 
                 // Final safety check: Ensure no message has null content (Jinja templates require strings)
@@ -2563,7 +2563,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
                             if (!body_json.contains("chat_template_kwargs")) {
                                 body_json["chat_template_kwargs"] = json::object();
                             }
-                            for (auto& el : ctk.items()) {
+                            for (auto el : ctk.items()) {
                                 body_json["chat_template_kwargs"][el.key()] = el.value();
                             }
                         }
@@ -2649,32 +2649,29 @@ class BackendServiceImpl final : public backend::Backend::Service {
             // If not using chat templates, extract files from image_data/audio_data fields
             // (If using chat templates, files were already extracted by oaicompat_chat_params_parse)
             if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) {
-                const auto &images_data = data.find("image_data");
-                if (images_data != data.end() && images_data->is_array())
+                if (data.contains("image_data") && data.at("image_data").is_array())
                 {
-                    std::cout << "[PREDICT] Processing " << images_data->size() << " images" << std::endl;
-                    for (const auto &img : *images_data)
+                    std::cout << "[PREDICT] Processing " << data.at("image_data").size() << " images" << std::endl;
+                    for (const auto &img : data.at("image_data"))
                     {
                         std::cout << "[PREDICT] Processing image" << std::endl;
                         auto decoded_data = base64_decode(img["data"].get<std::string>());
                         files.push_back(decoded_data);
                     }
                 }
 
-                const auto &audio_data = data.find("audio_data");
-                if (audio_data != data.end() && audio_data->is_array())
+                if (data.contains("audio_data") && data.at("audio_data").is_array())
                 {
-                    for (const auto &audio : *audio_data)
+                    for (const auto &audio : data.at("audio_data"))
                     {
                         auto decoded_data = base64_decode(audio["data"].get<std::string>());
                         files.push_back(decoded_data);
                     }
                 }
 
-                const auto &video_data = data.find("video_data");
-                if (video_data != data.end() && video_data->is_array())
+                if (data.contains("video_data") && data.at("video_data").is_array())
                 {
-                    for (const auto &video : *video_data)
+                    for (const auto &video : data.at("video_data"))
                     {
                         auto decoded_data = base64_decode(video["data"].get<std::string>());
                         files.push_back(decoded_data);
@@ -3005,7 +3002,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
         }
 
         // Collect responses
-        json responses = json::array();
+        std::vector<json> responses;
         for (auto & res : all_results.results) {
             GGML_ASSERT(dynamic_cast<server_task_result_rerank*>(res.get()) != nullptr);
             responses.push_back(res->to_json());
@@ -3018,7 +3015,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
         // Crop results by request.top_n if specified
         int top_n = request->top_n();
         if (top_n > 0 && top_n < static_cast<int>(responses.size())) {
-            responses = json(responses.begin(), responses.begin() + top_n);
+            responses.resize(top_n);
         }
         // Set usage information
         backend::Usage* usage = rerankResult->mutable_usage();

backend/cpp/llama-cpp/message_content.h

@@ -52,14 +52,15 @@ inline nlohmann::ordered_json normalize_message_content(const std::string& role,
 // (#7528). A multimodal user message legitimately carries a typed-part array
 // ({type:text}, {type:image_url}, ...), which must be left intact. Shared by the
 // streaming and non-streaming paths so this invariant cannot drift between them.
-inline void normalize_template_message(nlohmann::ordered_json& msg) {
+template <typename Json>
+inline void normalize_template_message(Json& msg) {
     if (!msg.contains("content")) {
         msg["content"] = ""; // templates expect the field to exist
         return;
     }
-    nlohmann::ordered_json& content = msg["content"];
+    auto& content = msg["content"];
     const std::string role = (msg.contains("role") && msg["role"].is_string())
-                                 ? msg["role"].get<std::string>()
+                                 ? msg["role"].template get<std::string>()
                                  : std::string();
     if (content.is_null()) {
         content = ""; // #7324: null would crash content[:N] slicing

backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch

@@ -6,10 +6,9 @@ Subject: [PATCH 1/2] score-patch
 ---
  common/common.cpp               |   6 +-
  common/common.h                 |   3 +
- tools/CMakeLists.txt            |   1 +
  tools/server/server-context.cpp | 358 +++++++++++++++++++++++++++++++-
  tools/server/server-task.h      |  47 +++++
- 5 files changed, 406 insertions(+), 9 deletions(-)
+ 4 files changed, 405 insertions(+), 9 deletions(-)
 
 diff --git a/common/common.cpp b/common/common.cpp
 index 2e3f14c..0cec0dc 100644
@@ -42,15 +41,6 @@ index 878534d..4001df2 100644
      int32_t n_sequences           =     1; // number of sequences to decode
      int32_t n_outputs_max         =     0; // max outputs in a batch (0 = n_batch)
      int32_t n_outputs_max_per_seq =     1; // max outputs per sequence
-diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
-index 780df32..1d2fe8f 100644
---- a/tools/CMakeLists.txt
-+++ b/tools/CMakeLists.txt
-@@ -41,3 +41,4 @@ else()
-     add_subdirectory(fit-params)
-     add_subdirectory(results)
- endif()
-+add_subdirectory(grpc-server)
 diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
 index 3b5f6a1..d0e18e6 100644
 --- a/tools/server/server-context.cpp

backend/cpp/turboquant/patch-grpc-server.sh

@@ -8,6 +8,8 @@
 #      so the grpc-server option parser skips the two references to
 #      common_params::checkpoint_min_step (the default and the option handler).
 #      That field does not exist in the fork yet; drop this once it does.
+#   3. Use nlohmann's parse_error type in JSON catch clauses because the fork
+#      predates upstream's common_json_error wrapper.
 #
 # The fork used to lag upstream on the whole common_params_speculative refactor
 # (ggml-org/llama.cpp#22397/#22838/#22964), the model_tgt rename (#22838) and
@@ -100,4 +102,16 @@ else
     echo "==> LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP define OK"
 fi
 
+# 3. The shared source follows current upstream and catches common_json_error.
+#    TurboQuant still exposes nlohmann::json directly, so its equivalent parse
+#    failures use json::parse_error instead.
+if grep -q 'common_json_error' "$SRC"; then
+    echo "==> patching $SRC to use the TurboQuant JSON exception type"
+    awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp"
+    mv "$SRC.tmp" "$SRC"
+    echo "==> TurboQuant JSON exception patch OK"
+else
+    echo "==> $SRC already uses a TurboQuant-compatible JSON exception type, skipping"
+fi
+
 echo "==> all patches applied"

scripts/build/bonsai-json-compat_test.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -euo pipefail
+
+ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+PATCHER="$ROOT/backend/cpp/bonsai/patch-grpc-server.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+cat > "$WORK/grpc-server.cpp" <<'EOF'
+try {
+    json::parse("{");
+} catch (const common_json_error& e) {
+}
+EOF
+
+bash "$PATCHER" "$WORK/grpc-server.cpp"
+grep -q 'catch (const json::parse_error& e)' "$WORK/grpc-server.cpp"
+! grep -q 'common_json_error' "$WORK/grpc-server.cpp"
+
+# A repeated preparation pass must not change the generated source.
+cp "$WORK/grpc-server.cpp" "$WORK/once.cpp"
+bash "$PATCHER" "$WORK/grpc-server.cpp"
+cmp "$WORK/once.cpp" "$WORK/grpc-server.cpp"
+
+echo "PASS: Bonsai uses its fork-compatible JSON exception"