save

Author: dkalinowski

src/llm/apis/openai_completions.cpp

@@ -17,6 +17,7 @@
 #include "openai_completions.hpp"
 
 #include <cmath>
+#include <limits>
 #include <memory>
 #include "src/port/rapidjson_stringbuffer.hpp"
 #include "src/port/rapidjson_writer.hpp"
@@ -44,6 +45,51 @@ namespace ovms {
 
 constexpr size_t DEFAULT_MAX_STOP_WORDS = 16;  // same as deep-seek
 
+namespace {
+
+ov::genai::JsonContainer rapidJsonValueToJsonContainer(const rapidjson::Value& value) {
+    if (value.IsNull()) {
+        return ov::genai::JsonContainer(nullptr);
+    }
+    if (value.IsBool()) {
+        return ov::genai::JsonContainer(value.GetBool());
+    }
+    if (value.IsInt64()) {
+        return ov::genai::JsonContainer(value.GetInt64());
+    }
+    if (value.IsUint64()) {
+        auto uintValue = value.GetUint64();
+        if (uintValue <= static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) {
+            return ov::genai::JsonContainer(static_cast<int64_t>(uintValue));
+        }
+        return ov::genai::JsonContainer(static_cast<double>(uintValue));
+    }
+    if (value.IsDouble()) {
+        return ov::genai::JsonContainer(value.GetDouble());
+    }
+    if (value.IsString()) {
+        return ov::genai::JsonContainer(std::string(value.GetString(), value.GetStringLength()));
+    }
+    if (value.IsArray()) {
+        ov::genai::JsonContainer arrayContainer = ov::genai::JsonContainer::array();
+        for (const auto& item : value.GetArray()) {
+            arrayContainer.push_back(rapidJsonValueToJsonContainer(item));
+        }
+        return arrayContainer;
+    }
+    if (value.IsObject()) {
+        ov::genai::JsonContainer objectContainer = ov::genai::JsonContainer::object();
+        for (auto member = value.MemberBegin(); member != value.MemberEnd(); ++member) {
+            const std::string key(member->name.GetString(), member->name.GetStringLength());
+            objectContainer[key] = rapidJsonValueToJsonContainer(member->value);
+        }
+        return objectContainer;
+    }
+    throw std::invalid_argument("Unsupported JSON value type");
+}
+
+}  // namespace
+
 absl::Status OpenAIChatCompletionsHandler::parseCompletionsPart() {
     // prompt: string
     auto it = doc.FindMember("prompt");
@@ -430,6 +476,23 @@ absl::Status OpenAIChatCompletionsHandler::parseTools() {
     }
 
     request.toolChoice = tool_choice;
+    request.tools = std::nullopt;
+    if (it != doc.MemberEnd() && !it->value.IsNull()) {
+        try {
+            request.tools = rapidJsonValueToJsonContainer(it->value);
+        } catch (const std::exception& e) {
+            SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Direct tools conversion to JsonContainer failed: {}. Falling back to JSON string conversion.", e.what());
+            try {
+                rapidjson::StringBuffer toolsBuffer;
+                rapidjson::Writer<rapidjson::StringBuffer> toolsWriter(toolsBuffer);
+                it->value.Accept(toolsWriter);
+                request.tools = ov::genai::JsonContainer::from_json_string(toolsBuffer.GetString());
+            } catch (const std::exception& fallbackEx) {
+                SPDLOG_LOGGER_DEBUG(llm_calculator_logger, "Fallback tools conversion failed: {}", fallbackEx.what());
+                return absl::InvalidArgumentError(absl::StrCat("Invalid tools payload: ", fallbackEx.what()));
+            }
+        }
+    }
     if (jsonChanged) {
         StringBuffer buffer;
         Writer<StringBuffer> writer(buffer);
@@ -466,6 +529,10 @@ std::optional<std::string> OpenAIChatCompletionsHandler::getResponseFormat() con
     return request.responseFormat;
 }
 
+const std::optional<ov::genai::JsonContainer>& OpenAIChatCompletionsHandler::getTools() const {
+    return request.tools;
+}
+
 std::string convertOpenAIResponseFormatToStructuralTagStringFormat(const rapidjson::Value& openAIFormat) {
     // Build the new object: {"type": "structural_tag", "format": <openAIFormat>}
     // If response_format has {"json_schema": {"schema": {...}}}, flatten it to {"json_schema": {...}}

src/llm/apis/openai_completions.hpp

@@ -102,6 +102,7 @@ class OpenAIChatCompletionsHandler {
     ov::genai::ChatHistory& getChatHistory();
     std::optional<int> getMaxTokens() const;
     std::optional<std::string> getResponseFormat() const;
+    const std::optional<ov::genai::JsonContainer>& getTools() const;
 
     bool isStream() const;
     std::string getModel() const;

src/llm/apis/openai_request.hpp

@@ -25,6 +25,7 @@
 #include <vector>
 
 #include <openvino/runtime/tensor.hpp>
+#include <openvino/genai/json_container.hpp>
 #include <openvino/genai/tokenizer.hpp>
 
 #include "src/port/rapidjson_document.hpp"
@@ -78,6 +79,8 @@ struct OpenAIChatCompletionsRequest {
     std::optional<std::string> responseFormat{std::nullopt};
     // Map that holds tool names and schemas for their arguments
     ToolsSchemas_t toolNameSchemaMap;
+    // Full tools payload in JSON form for passing directly to tokenizer chat template.
+    std::optional<ov::genai::JsonContainer> tools{std::nullopt};
     // Holds value for tool_choice field as described in https://platform.openai.com/docs/api-reference/chat/create#chat_create-tool_choice
     std::string toolChoice;
 

src/llm/servable.cpp

@@ -156,6 +156,7 @@ absl::Status GenAiServable::parseRequest(std::shared_ptr<GenAiServableExecutionC
     return absl::OkStatus();
 }
 
+// Continuous batching LLM
 absl::Status GenAiServable::prepareInputs(std::shared_ptr<GenAiServableExecutionContext>& executionContext) {
     if (executionContext->apiHandler == nullptr) {
         return absl::Status(absl::StatusCode::kInvalidArgument, "API handler is not initialized");

src/llm/visual_language_model/continuous_batching/servable.cpp

@@ -62,6 +62,7 @@ std::shared_ptr<GenAiServableProperties> VisualLanguageModelServable::getPropert
     return properties;
 }
 
+// Continuous Batching VLM
 absl::Status VisualLanguageModelServable::prepareInputs(std::shared_ptr<GenAiServableExecutionContext>& executionContext) {
     auto vlmExecutionContext = std::static_pointer_cast<VisualLanguageModelServableExecutionContext>(executionContext);
     if (vlmExecutionContext->apiHandler == nullptr) {
@@ -93,24 +94,12 @@ absl::Status VisualLanguageModelServable::prepareInputs(std::shared_ptr<GenAiSer
         }
 
         constexpr bool add_generation_prompt = true;  // confirm it should be hardcoded
-        ov::genai::JsonContainer tools = ov::genai::JsonContainer::from_json_string(R"([
-  {
-    "type": "function",
-    "function": {
-      "name": "get_weather",
-      "description": "Get current weather by city",
-      "parameters": {
-        "type": "object",
-        "properties": {
-          "city": {"type": "string"}
-        },
-        "required": ["city"]
-      }
-    }
-  }
-])");
-        vlmExecutionContext->inputText = properties->tokenizer.apply_chat_template(chatHistory, add_generation_prompt, {}, tools);
-        //vlmExecutionContext->inputText = properties->tokenizer.apply_chat_template(chatHistory, add_generation_prompt, {});
+        const auto& tools = vlmExecutionContext->apiHandler->getTools();
+        if (tools.has_value()) {
+            vlmExecutionContext->inputText = properties->tokenizer.apply_chat_template(chatHistory, add_generation_prompt, {}, tools);
+        } else {
+            vlmExecutionContext->inputText = properties->tokenizer.apply_chat_template(chatHistory, add_generation_prompt, {});
+        }
     } else {
         return absl::InvalidArgumentError("Unsupported endpoint");
     }

src/llm/visual_language_model/legacy/servable.cpp

@@ -222,6 +222,7 @@ absl::Status VisualLanguageModelLegacyServable::preparePartialResponse(std::shar
     return absl::OkStatus();
 }
 
+// Legacy VLM
 absl::Status VisualLanguageModelLegacyServable::prepareInputs(std::shared_ptr<GenAiServableExecutionContext>& executionContext) {
     auto vlmExecutionContext = std::static_pointer_cast<VisualLanguageModelLegacyServableExecutionContext>(executionContext);
     if (vlmExecutionContext->apiHandler == nullptr) {