refactor(local-inference): rewrite vision shim against mtmd ABI

Upstream llama.cpp removed `examples/llava/` and consolidated multimodal
under `tools/mtmd/`. Rewrite the desktop dylib build, shim wrappers, and
TS adapter bindings against the mtmd surface.

- Build script: drop `LLAMA_BUILD_EXAMPLES=ON` for vision; enable
  `LLAMA_BUILD_MTMD=ON` instead. Replace the cmake target probe loop
  (`llava_static`/`llava`/`mtmd`) with a single `--target mtmd` build.
  Drop static-lib find for `libllava*.a`/`libclip*.a`; stage shared
  `libmtmd.<ext>` next to libllama and link the shim with `-lmtmd`.
  Drop `examples/llava` include path; add `tools/mtmd`.

- Shim .h/.c: replace the llava/clip wrappers (eliza_clip_load /
  _free, eliza_llava_image_embed_load / _free / _eval) with the mtmd
  pointer-style surface (eliza_mtmd_init / _free, _bitmap_init_rgb /
  _bitmap_free, _input_chunks_init / _free / _size / _get,
  _input_chunk_type / _n_tokens, _tokenize, _encode_chunk,
  _output_embd). All gated behind #ifdef ELIZA_ENABLE_VISION.

- Desktop adapter: replace VisionShimSymbols + bindVision with the
  mtmd bindings. Rename clip ctx state to mtmd ctx. Stub
  describeImage with an actionable error citing the two missing
  pieces (JS-side RGB decode via `sharp` — already in app-core but
  not this plugin — and an embeddings-batch wrapper for llama_batch).
  Add u64 to the FFITypeEnum for the size_t mtmd args.

- FFI_BACKEND_WIREUP_PLAN.md: document mtmd ABI status, remaining
  gaps, and runtime smoke-test gate before defaulting on.

Non-vision builds are unchanged: ELIZA_ENABLE_VISION defaults off, no
mtmd target is built, bindVision returns null, and describeImage
throws the existing "vision build flag not set" error.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: Shaw

packages/app-core/scripts/build-llama-cpp-desktop-dylib.mjs

@@ -198,21 +198,26 @@ function buildTarget(targetKey) {
   fs.mkdirSync(buildDir, { recursive: true });
 
   // ── Step 1: build libllama as a shared library ────────────────────────────
-  // Vision (mmproj/llava) is opt-in: setting `ELIZA_ENABLE_VISION=1` in the
-  // build env flips `LLAMA_BUILD_EXAMPLES=ON` so the llava + clip libraries
-  // get built alongside libllama. The shim then compiles with
-  // `-DELIZA_ENABLE_VISION=1` and links against them. Default builds skip
-  // vision entirely (no examples target, no shim vision wrappers).
+  // Vision (mmproj/mtmd) is opt-in: setting `ELIZA_ENABLE_VISION=1` in the
+  // build env flips `LLAMA_BUILD_MTMD=ON` so the mtmd shared library
+  // gets built alongside libllama. The shim then compiles with
+  // `-DELIZA_ENABLE_VISION=1` and links against `libmtmd`. Default builds
+  // skip vision entirely (no mtmd target, no shim vision wrappers).
+  //
+  // Upstream renamed the multimodal surface from `examples/llava/` to
+  // `tools/mtmd/` and consolidated llava + clip into a single mtmd target
+  // built as a shared library when `BUILD_SHARED_LIBS=ON`.
   const ENABLE_VISION = process.env.ELIZA_ENABLE_VISION === "1";
   const cmakeArgs = [
     srcDir,
     "-DCMAKE_BUILD_TYPE=Release",
     "-DBUILD_SHARED_LIBS=ON",
     "-DGGML_NATIVE=OFF",
     "-DLLAMA_BUILD_TESTS=OFF",
-    `-DLLAMA_BUILD_EXAMPLES=${ENABLE_VISION ? "ON" : "OFF"}`,
+    "-DLLAMA_BUILD_EXAMPLES=OFF",
     "-DLLAMA_BUILD_SERVER=OFF",
     "-DLLAMA_CURL=OFF",
+    ...(ENABLE_VISION ? ["-DLLAMA_BUILD_MTMD=ON"] : []),
     ...t.cmakeFlags,
   ];
   log(`cmake configure ${targetKey} (shared libllama)`);
@@ -233,46 +238,27 @@ function buildTarget(targetKey) {
     buildDir,
   );
 
-  // ── Step 1b: build llava + clip when vision is enabled ───────────────────
-  // llama.cpp's llava lives under `examples/llava/` (older checkouts) or
-  // `tools/mtmd/` (recent). The `llava` target builds both the llava and
-  // clip object/static libraries when LLAMA_BUILD_EXAMPLES=ON. We build
-  // both static libs here and link them into the shim below.
+  // ── Step 1b: build mtmd when vision is enabled ───────────────────────────
+  // llama.cpp HEAD exposes multimodal under `tools/mtmd/`. The `mtmd`
+  // cmake target builds `libmtmd.<ext>` as a shared library (because
+  // BUILD_SHARED_LIBS=ON). LLAMA_BUILD_MTMD=ON enables the target. The
+  // shim links against this shared lib via `-lmtmd`.
   if (ENABLE_VISION) {
-    log(`cmake build ${targetKey} (llava + clip for vision)`);
-    // The target name has historically been `llava` (older) or
-    // `llava_static`/`mtmd` (newer). Try the common ones; let cmake
-    // surface a clear error if neither exists.
-    const visionTargets = ["llava_static", "llava", "mtmd"];
-    let visionTargetBuilt = false;
-    for (const vt of visionTargets) {
-      const probe = spawnSync(
-        "cmake",
-        [
-          "--build",
-          ".",
-          "--config",
-          "Release",
-          "--target",
-          vt,
-          "--parallel",
-          String(os.cpus().length),
-        ],
-        { cwd: buildDir, stdio: "inherit" },
-      );
-      if (probe.status === 0) {
-        visionTargetBuilt = true;
-        log(`built vision target ${vt}`);
-        break;
-      }
-    }
-    if (!visionTargetBuilt) {
-      die(
-        `ELIZA_ENABLE_VISION=1 but could not build any of {${visionTargets.join(", ")}}; ` +
-          `the llama.cpp checkout may not expose llava/mtmd. Check ` +
-          `${path.join(srcDir, "examples", "llava")} or ${path.join(srcDir, "tools", "mtmd")}.`,
-      );
-    }
+    log(`cmake build ${targetKey} (mtmd for vision)`);
+    run(
+      "cmake",
+      [
+        "--build",
+        ".",
+        "--config",
+        "Release",
+        "--target",
+        "mtmd",
+        "--parallel",
+        String(os.cpus().length),
+      ],
+      buildDir,
+    );
   }
 
   // ── Step 2: locate the built libllama.<ext> and stage it ─────────────────
@@ -304,6 +290,34 @@ function buildTarget(targetKey) {
   log(`staging ${libllamaSrcPath} → ${outDir}`);
   fs.copyFileSync(libllamaSrcPath, path.join(outDir, libllamaName));
 
+  // ── Step 2b: stage libmtmd.<ext> when vision is enabled ──────────────────
+  // mtmd is built as a shared lib (BUILD_SHARED_LIBS=ON propagates to all
+  // targets). The shim's `-lmtmd` link will need it resolvable next to
+  // libllama at load time via the same rpath.
+  if (ENABLE_VISION) {
+    const libmtmdName = `libmtmd.${t.libExt}`;
+    const mtmdCandidates = [
+      path.join(buildDir, libmtmdName),
+      path.join(buildDir, "bin", libmtmdName),
+      path.join(buildDir, "tools", "mtmd", libmtmdName),
+    ];
+    let libmtmdSrcPath = mtmdCandidates.find((p) => fs.existsSync(p));
+    if (!libmtmdSrcPath) {
+      const found = spawnSync("find", [buildDir, "-name", libmtmdName, "-print"], {
+        encoding: "utf8",
+      });
+      libmtmdSrcPath = found.stdout.split("\n").find((s) => s.trim());
+    }
+    if (!libmtmdSrcPath) {
+      die(
+        `ELIZA_ENABLE_VISION=1 but ${libmtmdName} not found in ${buildDir}; ` +
+          `check that -DLLAMA_BUILD_MTMD=ON + -DBUILD_SHARED_LIBS=ON took effect.`,
+      );
+    }
+    log(`staging ${libmtmdSrcPath} → ${outDir}`);
+    fs.copyFileSync(libmtmdSrcPath, path.join(outDir, libmtmdName));
+  }
+
   // ── Step 3: stage headers ────────────────────────────────────────────────
   const incDir = path.join(outDir, "include");
   fs.mkdirSync(incDir, { recursive: true });
@@ -320,6 +334,15 @@ function buildTarget(targetKey) {
     path.join(SHIM_DIR, "eliza_llama_shim.h"),
     path.join(incDir, "eliza_llama_shim.h"),
   );
+  // mtmd.h is staged into include/ for debug/reference; the shim compile
+  // also has `-I${srcDir}/tools/mtmd` so this copy is optional but matches
+  // how llama.h is staged.
+  if (ENABLE_VISION) {
+    const mtmdH = path.join(srcDir, "tools", "mtmd", "mtmd.h");
+    if (fs.existsSync(mtmdH)) {
+      fs.copyFileSync(mtmdH, path.join(incDir, "mtmd.h"));
+    }
+  }
 
   // ── Step 4: compile the shim and NEEDED-link libllama ────────────────────
   const shimOut = path.join(outDir, `libeliza-llama-shim.${t.libExt}`);
@@ -340,52 +363,16 @@ function buildTarget(targetKey) {
     "-lllama",
   ];
 
-  // Vision opt-in: link against the static llava + clip libraries built
-  // in Step 1b. We add their include dirs + static-lib paths to the
-  // compile invocation, and define ELIZA_ENABLE_VISION so the shim's
-  // `#ifdef` block compiles.
+  // Vision opt-in: link against the shared `libmtmd.<ext>` built in
+  // Step 1b. We add the mtmd include dir to the compile invocation and
+  // define ELIZA_ENABLE_VISION so the shim's `#ifdef` block compiles.
   if (ENABLE_VISION) {
     compilerArgs.push("-DELIZA_ENABLE_VISION=1");
-    // Header search paths for llava.h / clip.h. Older llama.cpp keeps them
-    // under examples/llava/; newer puts them under tools/mtmd/. Add both —
-    // missing dirs are harmless.
-    compilerArgs.push(
-      `-I${path.join(srcDir, "examples", "llava")}`,
-      `-I${path.join(srcDir, "tools", "mtmd")}`,
-    );
-    // Find the built static libs and link them. They land somewhere
-    // under buildDir depending on cmake generator; do a shallow scan
-    // for `libllava*.a` and `libclip*.a` plus `libmtmd*.a`.
-    const visionLibCandidates = spawnSync(
-      "find",
-      [
-        buildDir,
-        "-name",
-        "libllava*.a",
-        "-o",
-        "-name",
-        "libclip*.a",
-        "-o",
-        "-name",
-        "libmtmd*.a",
-      ],
-      { encoding: "utf8" },
-    );
-    const visionLibs = visionLibCandidates.stdout
-      .split("\n")
-      .filter((s) => s.trim());
-    if (visionLibs.length === 0) {
-      die(
-        `ELIZA_ENABLE_VISION=1 but no llava/clip/mtmd static lib found in ${buildDir} ` +
-          `after the vision target build step. Check Step 1b's cmake output.`,
-      );
-    }
-    for (const lib of visionLibs) {
-      log(`linking vision lib ${lib}`);
-      compilerArgs.push(lib);
-    }
-    // Linker also needs C++ stdlib for llava (it's C++ under the hood).
-    compilerArgs.push(platform === "darwin" ? "-lc++" : "-lstdc++");
+    // Header search path for mtmd.h.
+    compilerArgs.push(`-I${path.join(srcDir, "tools", "mtmd")}`);
+    // NEEDED-link against the staged libmtmd next to libllama. The shared
+    // lib pulls in its own C++ transitive deps, so no -lc++/-lstdc++ here.
+    compilerArgs.push("-lmtmd");
   }
 
   // Set rpath so libeliza-llama-shim resolves libllama from its own dir at

packages/app-core/scripts/desktop-llama-shim/eliza_llama_shim.c

@@ -607,67 +607,64 @@ void* eliza_llama_sampler_init_prefill_plan(const uint8_t* plan_bytes, size_t pl
     return llama_sampler_init(&prefill_plan_sampler_i, s);
 }
 
-// ── vision: mmproj-driven image describe (OPT-IN) ────────────────────────────
+// ── vision: mmproj-driven image describe (OPT-IN, mtmd ABI) ──────────────────
 //
 // Gated by `-DELIZA_ENABLE_VISION=1` at shim compile time. The build script
 // flips this when `ELIZA_ENABLE_VISION=1` is set in the build env, AND
-// supplements the compile invocation with the llava + clip object files
-// (or static lib) so the symbols exist to link.
+// adds `-I${srcDir}/tools/mtmd` + `-lmtmd` so the symbols below resolve.
 //
-// llama.cpp's vision API has shifted between releases: older checkouts
-// expose llava via `examples/llava/{llava.h, clip.h}`; recent ones moved
-// to `tools/mtmd/mtmd.h` with a slightly different surface. The wrappers
-// below target the older but more stable API. The build script's vision
-// step is responsible for putting those headers on the include path.
+// llama.cpp HEAD consolidated the historical `examples/llava/{llava.h,clip.h}`
+// path into a single `tools/mtmd/mtmd.h` surface. The wrappers below target
+// that ABI and assume `libmtmd.<ext>` is staged next to libllama so the
+// runtime loader can resolve it via the shim's rpath.
 
 #ifdef ELIZA_ENABLE_VISION
+#include "mtmd.h"
 
-#include "clip.h"
-#include "llava.h"
-
-void* eliza_clip_load(const char* path) {
-    if (!path) return NULL;
-    // verbosity=0 silences clip_model_load's stderr chatter (it prints
-    // per-layer load progress otherwise, which spams the host).
-    return (void*)clip_model_load(path, /*verbosity=*/0);
+void* eliza_mtmd_init(const char* mmproj_path, void* text_model, bool use_gpu, int n_threads) {
+    if (!mmproj_path || !text_model) return NULL;
+    struct mtmd_context_params p = mtmd_context_params_default();
+    p.use_gpu = use_gpu;
+    p.n_threads = n_threads;
+    p.print_timings = false;
+    return (void*)mtmd_init_from_file(mmproj_path, (const struct llama_model*)text_model, p);
 }
+void eliza_mtmd_free(void* ctx) { if (ctx) mtmd_free((mtmd_context*)ctx); }
 
-void eliza_clip_free(void* ctx_clip) {
-    if (ctx_clip) clip_free((struct clip_ctx*)ctx_clip);
+void* eliza_mtmd_bitmap_init_rgb(uint32_t nx, uint32_t ny, const uint8_t* rgb) {
+    return (!rgb) ? NULL : (void*)mtmd_bitmap_init(nx, ny, rgb);
 }
+void eliza_mtmd_bitmap_free(void* bm) { if (bm) mtmd_bitmap_free((mtmd_bitmap*)bm); }
 
-void* eliza_llava_image_embed_load(
-    void* ctx_clip,
-    int32_t n_threads,
-    const uint8_t* image_bytes,
-    int32_t image_bytes_length)
-{
-    if (!ctx_clip || !image_bytes || image_bytes_length <= 0) return NULL;
-    return (void*)llava_image_embed_make_with_bytes(
-        (struct clip_ctx*)ctx_clip,
-        n_threads,
-        image_bytes,
-        image_bytes_length);
+void* eliza_mtmd_input_chunks_init(void) { return (void*)mtmd_input_chunks_init(); }
+void  eliza_mtmd_input_chunks_free(void* c) { if (c) mtmd_input_chunks_free((mtmd_input_chunks*)c); }
+
+int32_t eliza_mtmd_tokenize(void* ctx, void* out_chunks,
+                            const char* text, bool add_special, bool parse_special,
+                            void* const* bitmaps, size_t n_bitmaps) {
+    if (!ctx || !out_chunks || !text) return -1;
+    struct mtmd_input_text t = { text, add_special, parse_special };
+    return mtmd_tokenize((mtmd_context*)ctx, (mtmd_input_chunks*)out_chunks,
+                         &t, (const mtmd_bitmap**)bitmaps, n_bitmaps);
 }
 
-void eliza_llava_image_embed_free(void* embed) {
-    if (embed) llava_image_embed_free((struct llava_image_embed*)embed);
+size_t eliza_mtmd_input_chunks_size(void* c) { return mtmd_input_chunks_size((const mtmd_input_chunks*)c); }
+void*  eliza_mtmd_input_chunks_get(void* c, size_t i) {
+    return (void*)mtmd_input_chunks_get((const mtmd_input_chunks*)c, i);
+}
+int32_t eliza_mtmd_input_chunk_type(void* ch) {
+    return (int32_t)mtmd_input_chunk_get_type((const mtmd_input_chunk*)ch);
+}
+size_t eliza_mtmd_input_chunk_n_tokens(void* ch) {
+    return mtmd_input_chunk_get_n_tokens((const mtmd_input_chunk*)ch);
 }
 
-int32_t eliza_llava_image_embed_eval(
-    void* ctx_llama,
-    void* embed,
-    int32_t n_batch,
-    int32_t* n_past)
-{
-    if (!ctx_llama || !embed || !n_past) return -1;
-    // llava_eval_image_embed returns bool; map to int32 for the ABI.
-    bool ok = llava_eval_image_embed(
-        (struct llama_context*)ctx_llama,
-        (const struct llava_image_embed*)embed,
-        n_batch,
-        n_past);
-    return ok ? 0 : -1;
+int32_t eliza_mtmd_encode_chunk(void* ctx, void* chunk) {
+    if (!ctx || !chunk) return -1;
+    return mtmd_encode_chunk((mtmd_context*)ctx, (const mtmd_input_chunk*)chunk);
 }
 
+const float* eliza_mtmd_output_embd(void* ctx) {
+    return ctx ? mtmd_get_output_embd((mtmd_context*)ctx) : NULL;
+}
 #endif  // ELIZA_ENABLE_VISION