feat(cpp): VLM image support in C++ SDK (#858)

Closes #785

## Changes

- Adds `gaia::Image` with `fromBytes` / `fromFile` factories, RFC 4648
base64 encoding, magic-byte MIME detection (PNG/JPEG/GIF/WebP/BMP), 20
MiB size cap, `O_NOFOLLOW` + post-open `fstat` TOCTOU guard on POSIX,
and a whitelist enforcing only the five supported MIME types
- Adds `gaia::ContentPart` (text / image_url parts with `toJson()`
producing the OpenAI vision wire format)
- Extends `gaia::Message` with an additive
`std::optional<std::vector<ContentPart>> parts` field; `toJson()`
dispatches to array or string accordingly — fully backward-compatible
with existing aggregate-init sites
- Adds two new `processQuery` overloads (`string + vector<Image>`
ergonomic; `vector<Message>` caller-composed) unified through a private
`processQueryInternal` that is the sole writer of `conversationHistory_`
- Image parts are stripped from history at end-of-turn (base64 never
retained across calls)
- RAII `InFlightGuard` via `std::atomic<bool> inFlight_` and
`compare_exchange_strong` — concurrent `processQuery` calls on the same
Agent throw `std::runtime_error`
- Empty-input validation fires **before** `ensureModelLoaded` so no
`/load` fires on invalid input
- Lifts `cpp/benchmarks/mock_llm_server.h` to
`cpp/tests/support/mock_llm_server.h` and extends it with
`receivedBodies()`, `loadRequestCount()`, `holdNextResponse()`, and a
`reportModelLoaded` constructor flag; benchmark header is now a thin
shim
- Adds `cpp/examples/vlm_agent.cpp` end-to-end demo
- Adds `cpp/tests/integration/test_integration_vlm.cpp` (3 tests: live
Lemonade VLM smoke, messages-list overload, ctx-overflow error surface)
with Lemonade version-pin probe via `GAIA_PINNED_LEMONADE_VERSION`
- Updates `docs/cpp/api-reference.mdx`, `docs/cpp/overview.mdx`,
`docs/cpp/quickstart.mdx` with VLM section, new overloads, thread-safety
update, and example invocation

## Test coverage

| Layer | Tests | Status |
|-------|-------|--------|
| Unit — MIME / base64 / Image | 20 tests in `test_image.cpp` | ✅
331/331 pass |
| Unit — ContentPart / Message | 15 new tests in `test_types.cpp` | ✅ |
| Agent-level (mock HTTP) | 13 tests in `test_agent_vlm.cpp` | ✅ |
| Integration (live Lemonade) | 3 tests in `test_integration_vlm.cpp` |
gated, opt-in |

## Reviewer notes

- The `Message::parts` field is **additive** — all existing code
compiles unchanged; consumers linked against a prebuilt `gaia_core` must
rebuild (noted in docs)
- `detectImageMimeType` returns `""` (empty string) for ≥ 12-byte
buffers with unrecognized magic; returns `"image/png"` only for
null/short buffers (AC-15e safe-fallback contract)
- Integration tests require `-DGAIA_BUILD_INTEGRATION_TESTS=ON` and a
live Lemonade server with `Qwen3-VL-4B-Instruct-GGUF`; they are
non-blocking in CI
- The pre-existing `errorCount` unused-variable warning in
`agent.cpp:692` is not introduced by this PR

Author: itomek

.gitattributes

@@ -0,0 +1,3 @@
+# Binary test fixtures — prevent LF/CRLF mangling
+cpp/tests/fixtures/*.png binary
+cpp/tests/fixtures/*.jpg binary

cpp/CMakeLists.txt

@@ -93,6 +93,7 @@ endif()
 # ---------------------------------------------------------------------------
 add_library(gaia_core
     src/types.cpp
+    src/image.cpp
     src/tool_registry.cpp
     src/console.cpp
     src/clean_console.cpp
@@ -191,6 +192,9 @@ endif()
 if(GAIA_BUILD_EXAMPLES)
     add_executable(security_demo examples/security_demo.cpp)
     target_link_libraries(security_demo PRIVATE gaia::gaia_core)
+
+    add_executable(vlm_agent examples/vlm_agent.cpp)
+    target_link_libraries(vlm_agent PRIVATE gaia::gaia_core)
 endif()
 
 # ---------------------------------------------------------------------------
@@ -201,9 +205,11 @@ if(GAIA_BUILD_TESTS)
 
     add_executable(tests_mock
         tests/test_types.cpp
+        tests/test_image.cpp
         tests/test_tool_registry.cpp
         tests/test_json_utils.cpp
         tests/test_agent.cpp
+        tests/test_agent_vlm.cpp
         tests/test_mcp_client.cpp
         tests/test_console.cpp
         tests/test_lemonade_client.cpp
@@ -219,6 +225,23 @@ if(GAIA_BUILD_TESTS)
         GTest::gtest_main
     )
 
+    # VLM tests need httplib (mock LLM server) and the fixtures directory.
+    if(httplib_FOUND)
+        target_link_libraries(tests_mock PRIVATE httplib::httplib)
+    else()
+        target_include_directories(tests_mock SYSTEM PRIVATE
+            $<TARGET_PROPERTY:httplib::httplib,INTERFACE_INCLUDE_DIRECTORIES>)
+    endif()
+    if(OpenSSL_FOUND)
+        target_compile_definitions(tests_mock PRIVATE CPPHTTPLIB_OPENSSL_SUPPORT)
+        target_link_libraries(tests_mock PRIVATE OpenSSL::SSL OpenSSL::Crypto)
+    endif()
+
+    target_compile_definitions(tests_mock PRIVATE
+        GAIA_TEST_FIXTURES_DIR="${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures"
+    )
+    target_include_directories(tests_mock PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests)
+
     include(GoogleTest)
     gtest_discover_tests(tests_mock)
 endif()
@@ -236,13 +259,18 @@ if(GAIA_BUILD_INTEGRATION_TESTS)
         tests/integration/test_integration_mcp.cpp
         tests/integration/test_integration_wifi.cpp
         tests/integration/test_integration_health.cpp
+        tests/integration/test_integration_vlm.cpp
     )
 
     target_link_libraries(tests_integration PRIVATE
         gaia::gaia_core
         GTest::gtest
     )
 
+    target_compile_definitions(tests_integration PRIVATE
+        GAIA_TEST_FIXTURES_DIR="${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures"
+    )
+
     include(GoogleTest)
     gtest_discover_tests(tests_integration
         PROPERTIES TIMEOUT 300

cpp/benchmarks/mock_llm_server.h

@@ -1,154 +1,10 @@
 // Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
 // SPDX-License-Identifier: MIT
 //
-// In-process mock HTTP server mimicking the Lemonade Server API.
-// Used by benchmarks to avoid requiring a real LLM backend.
+// The canonical mock server now lives under cpp/tests/support/. This header
+// remains as a thin re-include so existing benchmark sources compile
+// unchanged. Do not add new contents here.
 
 #pragma once
 
-#include <atomic>
-#include <chrono>
-#include <deque>
-#include <mutex>
-#include <stdexcept>
-#include <string>
-#include <thread>
-
-#include <httplib.h>
-
-namespace bench {
-
-// Default chat completion response — agent returns a final answer immediately.
-static const std::string kDefaultAnswer = R"({"choices":[{"message":{"content":"{\"thought\":\"done\",\"goal\":\"complete\",\"answer\":\"benchmark result\"}"}}]})";
-
-// Tool-call response — agent calls the echo tool first.
-static const std::string kToolCall = R"({"choices":[{"message":{"content":"{\"thought\":\"calling tool\",\"goal\":\"test\",\"tool\":\"echo\",\"tool_args\":{\"message\":\"bench\"}}"}}]})";
-
-// Health response — reports mock-model as already loaded so ensureModelLoaded() skips /load.
-static const std::string kHealthOk = R"({"status":"ok","all_models_loaded":[{"model_name":"mock-model","recipe_options":{"ctx_size":16384}}]})";
-
-// Models list response
-static const std::string kModelsList = R"({"data":[{"id":"mock-model"}]})";
-
-// Load response
-static const std::string kLoadOk = R"({"status":"ok"})";
-
-class MockLlmServer {
-public:
-    /// Start server on an OS-assigned port.
-    /// Constructor blocks until the server is accepting connections.
-    MockLlmServer() : server_(std::make_unique<httplib::Server>()) {
-        registerHandlers();
-
-        // bind_to_any_port returns the OS-assigned port (avoids CI port conflicts)
-        port_ = server_->bind_to_any_port("127.0.0.1");
-        if (port_ <= 0) {
-            throw std::runtime_error("MockLlmServer: failed to bind to any port");
-        }
-
-        thread_ = std::thread([this]() { server_->listen_after_bind(); });
-
-        waitUntilReady();
-    }
-
-    ~MockLlmServer() {
-        server_->stop();
-        if (thread_.joinable()) {
-            thread_.join();
-        }
-    }
-
-    // Non-copyable, non-movable
-    MockLlmServer(const MockLlmServer&) = delete;
-    MockLlmServer& operator=(const MockLlmServer&) = delete;
-
-    /// The port the server is listening on.
-    int port() const { return port_; }
-
-    /// Base URL suitable for AgentConfig::baseUrl (without /api/v1 — LemonadeClient adds it).
-    std::string baseUrl() const { return "http://127.0.0.1:" + std::to_string(port_); }
-
-    /// Push a response to return for the next POST /chat/completions call.
-    /// When the queue is empty the default answer response is returned.
-    void pushResponse(const std::string& body) {
-        std::lock_guard<std::mutex> lk(mu_);
-        responseQueue_.push_back(body);
-    }
-
-    /// Push N copies of a response.
-    void pushResponses(const std::string& body, int n) {
-        std::lock_guard<std::mutex> lk(mu_);
-        for (int i = 0; i < n; ++i) {
-            responseQueue_.push_back(body);
-        }
-    }
-
-    /// Clear pending queued responses.
-    void clearQueue() {
-        std::lock_guard<std::mutex> lk(mu_);
-        responseQueue_.clear();
-    }
-
-    /// Number of chat completion requests handled so far.
-    int requestCount() const { return requestCount_.load(); }
-
-private:
-    void registerHandlers() {
-        // Health check — always reports mock-model loaded
-        server_->Get("/api/v1/health", [](const httplib::Request&, httplib::Response& res) {
-            res.set_content(kHealthOk, "application/json");
-        });
-
-        // Load model — no-op safety fallback
-        server_->Post("/api/v1/load", [](const httplib::Request&, httplib::Response& res) {
-            res.set_content(kLoadOk, "application/json");
-        });
-
-        // Models list
-        server_->Get("/api/v1/models", [](const httplib::Request&, httplib::Response& res) {
-            res.set_content(kModelsList, "application/json");
-        });
-
-        // Chat completions — dequeue a pre-loaded response or return default answer
-        server_->Post("/api/v1/chat/completions",
-                      [this](const httplib::Request&, httplib::Response& res) {
-                          ++requestCount_;
-                          std::string body;
-                          {
-                              std::lock_guard<std::mutex> lk(mu_);
-                              if (!responseQueue_.empty()) {
-                                  body = responseQueue_.front();
-                                  responseQueue_.pop_front();
-                              } else {
-                                  body = kDefaultAnswer;
-                              }
-                          }
-                          res.set_content(body, "application/json");
-                      });
-    }
-
-    void waitUntilReady() {
-        // Poll health endpoint until the server responds
-        httplib::Client cli("127.0.0.1", port_);
-        cli.set_connection_timeout(1);
-        cli.set_read_timeout(1);
-
-        for (int attempt = 0; attempt < 50; ++attempt) {
-            auto res = cli.Get("/api/v1/health");
-            if (res && res->status == 200) {
-                return;
-            }
-            std::this_thread::sleep_for(std::chrono::milliseconds(20));
-        }
-        throw std::runtime_error("MockLlmServer: server did not become ready");
-    }
-
-    std::unique_ptr<httplib::Server> server_;
-    std::thread thread_;
-    int port_ = 0;
-    std::mutex mu_;
-    std::deque<std::string> responseQueue_;
-    std::atomic<int> requestCount_{0};
-};
-
-} // namespace bench
+#include "../tests/support/mock_llm_server.h"

cpp/examples/vlm_agent.cpp

@@ -0,0 +1,52 @@
+// Copyright(C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+// Minimal VLM example: loads an image from disk and asks a vision model
+// about it via the OpenAI-compatible /chat/completions endpoint.
+//
+// Usage:  vlm_agent <image_path> [prompt]
+//
+// Requires a Lemonade server running with a VLM model loaded.
+//   Environment:
+//     LEMONADE_BASE_URL   (default: http://localhost:8000/api/v1)
+//     GAIA_MODEL_ID       (default: Qwen3-VL-4B-Instruct-GGUF)
+
+#include <cstdlib>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include <gaia/agent.h>
+#include <gaia/types.h>
+
+int main(int argc, char** argv) {
+    if (argc < 2) {
+        std::cerr << "Usage: " << (argc > 0 ? argv[0] : "vlm_agent")
+                  << " <image_path> [prompt]\n";
+        return 2;
+    }
+    std::string imagePath = argv[1];
+    std::string prompt = (argc >= 3) ? argv[2] : "Describe this image.";
+
+    try {
+        gaia::Image img = gaia::Image::fromFile(imagePath);
+        std::cout << "Loaded " << img.size() << " bytes, MIME: "
+                  << img.mimeType() << "\n";
+
+        gaia::AgentConfig cfg;
+        cfg.modelId = gaia::getEnvVar("GAIA_MODEL_ID", "Qwen3-VL-4B-Instruct-GGUF");
+        cfg.contextSize = 32768;   // VLM-recommended
+        cfg.maxSteps = 3;
+        cfg.silentMode = false;
+
+        gaia::Agent agent(cfg);
+        gaia::json result = agent.processQuery(prompt, {img});
+
+        std::cout << "\n== Answer ==\n"
+                  << result.value("result", "<no result>") << "\n";
+        return 0;
+    } catch (const std::exception& e) {
+        std::cerr << "Error: " << e.what() << "\n";
+        return 1;
+    }
+}

cpp/include/gaia/agent.h

@@ -57,6 +57,20 @@ class GAIA_API Agent {
     /// @return JSON result with "result" key containing the final answer
     json processQuery(const std::string& userInput, int maxSteps = 0);
 
+    /// VLM convenience overload: text + images in a single user turn.
+    /// Images are sent as base64 data-URIs inside an OpenAI-compatible
+    /// image_url content part. Stateful and symmetric with the string
+    /// overload: history is appended with text-only stripped messages.
+    json processQuery(const std::string& userInput,
+                      const std::vector<Image>& images,
+                      int maxSteps = 0);
+
+    /// Low-level overload: caller composes the turn as a vector of
+    /// Messages (which may include pre-set `parts` for mixed content).
+    /// The messages are appended to conversationHistory_ (stripped of
+    /// image parts on store). Throws std::invalid_argument on empty input.
+    json processQuery(const std::vector<Message>& messages, int maxSteps = 0);
+
     /// Connect to an MCP server and register its tools.
     /// Mirrors Python MCPClientMixin.connect_mcp_server().
     ///
@@ -140,6 +154,12 @@ class GAIA_API Agent {
     virtual std::string getSystemPrompt() const { return ""; }
 
 private:
+    /// Unified entry point for all processQuery overloads. Owns the full
+    /// conversation turn: concurrency guard, empty-input validation,
+    /// ensureModelLoaded, history prepend, LLM loop, and end-of-turn
+    /// history write (text-only; image parts stripped).
+    json processQueryInternal(const std::vector<Message>& userMessages, int maxSteps);
+
     // ---- LLM Communication ----
 
     /// Send messages to the LLM and get a response.
@@ -172,6 +192,10 @@ class GAIA_API Agent {
     LemonadeClient lemonade_;
     std::atomic<bool> modelEnsured_{false};
 
+    // Concurrency guard — Agent is NOT re-entrant. A second processQuery
+    // call on the same Agent (from any thread) throws std::runtime_error.
+    std::atomic<bool> inFlight_{false};
+
     AgentState executionState_ = AgentState::PLANNING;
     json currentPlan_;
     int currentStep_ = 0;