Skip to content

Commit 3791bdf

Browse files
committed
feat: add standalone qwen3vl vision-encoder benchmark tool
tools/mtmd/qwen3vl-encoder: a small executable that runs only the Qwen3-VL vision encoder (ViT + patch-merge projection) through ggml, with no libllama dependency (it compiles the clip sources directly). It loads an mmproj gguf, preprocesses an image, and times clip_image_batch_encode across CPU / Vulkan / OpenCL, with a cosine-similarity check (--dump / --ref) to keep changes lossless. This gives a fast edit -> build -> run loop for optimizing the shared Vulkan/OpenCL/CPU kernels of the vision encoder without the heavy llama build. See tools/mtmd/qwen3vl-encoder/standalone_loop.md for the local + Android + Firebase workflow.
1 parent dceb04c commit 3791bdf

3 files changed

Lines changed: 791 additions & 0 deletions

File tree

tools/mtmd/CMakeLists.txt

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,3 +127,35 @@ set_target_properties(llama-mtmd-debug PROPERTIES OUTPUT_NAME llama-mtmd-debug)
127127
target_link_libraries(llama-mtmd-debug PRIVATE llama-common mtmd Threads::Threads)
128128
target_compile_features(llama-mtmd-debug PRIVATE cxx_std_17)
129129
endif()
130+
131+
# qvac: standalone Qwen3-VL vision encoder benchmark.
132+
# Compiles the clip encoder sources DIRECTLY (bypassing the mtmd lib, which
133+
# links llama) so the resulting binary depends only on ggml + backends. This
134+
# is the fast desktop/on-device iteration loop for optimizing the shared
135+
# Vulkan/OpenCL kernels. Enabled by default; toggle with LLAMA_QWEN3VL_ENCODER.
136+
option(LLAMA_QWEN3VL_ENCODER "build the standalone qwen3vl vision encoder benchmark" ON)
137+
if (LLAMA_QWEN3VL_ENCODER)
138+
file(GLOB MTMD_MODEL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/models/*.cpp)
139+
add_executable(qwen3vl-encoder
140+
qwen3vl-encoder/main.cpp
141+
clip.cpp
142+
mtmd-image.cpp
143+
${MTMD_MODEL_SOURCES}
144+
)
145+
target_include_directories(qwen3vl-encoder PRIVATE
146+
${CMAKE_CURRENT_SOURCE_DIR}
147+
${CMAKE_CURRENT_SOURCE_DIR}/../.. # repo root (ggml headers, etc.)
148+
${CMAKE_CURRENT_SOURCE_DIR}/../../include # llama.h (header-only, pulled via mtmd.h)
149+
${CMAKE_CURRENT_SOURCE_DIR}/../../vendor # vendored third-party
150+
${CMAKE_CURRENT_SOURCE_DIR}/../../vendor/stb # stb_image.h
151+
)
152+
target_link_libraries(qwen3vl-encoder PRIVATE ggml Threads::Threads)
153+
target_compile_features(qwen3vl-encoder PRIVATE cxx_std_17)
154+
if (NOT MSVC)
155+
# stb_image.h + clip.cpp cast-qual warnings, same as the mtmd lib
156+
target_compile_options(qwen3vl-encoder PRIVATE -Wno-cast-qual)
157+
endif()
158+
if (ANDROID)
159+
target_compile_options(qwen3vl-encoder PRIVATE -Wno-missing-prototypes)
160+
endif()
161+
endif()

0 commit comments

Comments
 (0)