Merge pull request #5 from nyo16/feat/qwen35-support

  Add Qwen 3.5 support with reasoning_content parsing (v0.6.0)

Author: nyo16

.gitignore

@@ -26,3 +26,4 @@ llama_cpp_ex-*.tar
 /priv/*.so
 /priv/*.dylib
 /priv/*.dll
+models/

CHANGELOG.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## v0.6.0
+
+### Added
+
+- **Qwen 3.5 support** — llama.cpp updated to c5a778891 (35 commits since v0.5.0).
+- **`reasoning_content` in ChatCompletion** — `chat_completion/3` now splits `<think>...</think>` blocks from the response when `enable_thinking: true`. The choice message includes `reasoning_content` (the thinking text) and `content` (the final answer). Returns `nil` when thinking is not enabled or no thinking block is present.
+- **`reasoning_content` in ChatCompletionChunk** — `stream_chat_completion/3` emits chunks with `reasoning_content` in the delta while the model is thinking, then switches to `content` after `</think>`.
+- **`LlamaCppEx.Thinking`** — New module with `parse/1` for one-shot parsing and `stream_parser/1` + `feed/2` for streaming token-boundary-safe parsing of think blocks. Handles the real-world Qwen3/3.5 template behavior where `<think>` is opened by the template itself.
+
+### Changed
+
+- **llama.cpp submodule** — Updated from 7f5ee54 to c5a778891.
+  - ggml: add GATED_DELTA_NET op for Qwen 3.5 hybrid architecture
+  - model: update Qwen 3.5 model type detection
+  - convert: register Qwen 3.5 ForCausalLM for text only
+  - CUDA: use shared mem for ssm_conv, improve performance via fewer synchronizations
+  - Hexagon: add f32 ssm_conv, fp16 binary ops, Flash Attention optimizations
+  - OpenCL: add l2_norm, neg, exp, diag ops
+  - CPU: skip redundant ROPE cache updates, fix data race for debug asserts
+  - quants: add memsets and other fixes for IQ quants
+  - kv-cache: fix M-RoPE checkpoints, checkpoint every n tokens
+  - server: preserve Anthropic thinking blocks in conversion
+
+### Unchanged
+
+- `chat/3` and `stream_chat/3` continue returning raw text (no breaking change).
+
 ## v0.5.0
 
 ### Added

README.md

@@ -28,7 +28,7 @@ Add `llama_cpp_ex` to your list of dependencies in `mix.exs`:
 ```elixir
 def deps do
   [
-    {:llama_cpp_ex, "~> 0.5.0"}
+    {:llama_cpp_ex, "~> 0.6.0"}
   ]
 end
 ```
@@ -220,17 +220,28 @@ Multiple callers are batched into a single forward pass per tick, improving thro
 
 ## Benchmarks
 
-Measured on Apple M4 Max (64 GB) with Qwen3-4B Q4_K_M, Metal backend (`n_gpu_layers: -1`).
+Measured on Apple M4 Max (64 GB), Metal backend (`n_gpu_layers: -1`).
 
-### Single-sequence generation
+### Single-model generation speed
+
+| Model | Quantization | Tokens/sec |
+|-------|-------------|------------|
+| Llama 3.2 3B Instruct | Q4_K_XL | 125.6 |
+| Ministral 3 3B Reasoning | Q4_K_XL | 113.0 |
+| Ministral 3 3B Instruct | Q4_K_XL | 104.3 |
+| GPT-OSS 20B | Q4_K_XL | 79.4 |
+| Qwen3.5-35B-A3B | Q6_K | 56.0 |
+| Qwen3.5-27B | Q4_K_XL | 17.5 |
+
+### Single-sequence generation (Qwen3-4B Q4_K_M)
 
 | Prompt | 32 tokens | 128 tokens |
 |--------|-----------|------------|
 | short (6 tok) | 0.31s (3.19 ips) | 1.01s (0.98 ips) |
 | medium (100 tok) | 0.36s (2.79 ips) | 1.06s (0.94 ips) |
 | long (500 tok) | 0.65s (1.53 ips) | 1.29s (0.77 ips) |
 
-### Continuous batching throughput
+### Continuous batching throughput (Qwen3-4B Q4_K_M)
 
 ```
 max_tokens: 32, prompt: "short"

checksum.exs

@@ -1,4 +1 @@
-%{
-  "llama_cpp_ex-nif-2.17-aarch64-apple-darwin-0.5.0.tar.gz" => "sha256:c38dac814cb760820d45101a4910ce859105267ba09a1f3e85ff7ea5c045b7bc",
-  "llama_cpp_ex-nif-2.17-x86_64-linux-gnu-0.5.0.tar.gz" => "sha256:98f22f5d2e1de1307eb27df611b7ae5d7c436c44359cad816ad2ee36300a516c",
-}
+%{}

lib/llama_cpp_ex.ex

@@ -36,7 +36,8 @@ defmodule LlamaCppEx do
     Embedding,
     Grammar,
     ChatCompletion,
-    ChatCompletionChunk
+    ChatCompletionChunk,
+    Thinking
   }
 
   @doc """
@@ -327,6 +328,17 @@ defmodule LlamaCppEx do
 
       {texts, finish_reason, completion_tokens} = collect_completion_tokens(ref, timeout)
 
+      raw_text = Enum.join(texts)
+      enable_thinking = Keyword.get(chat_opts, :enable_thinking, false)
+
+      {reasoning_content, content} =
+        if enable_thinking do
+          {rc, c} = Thinking.parse(raw_text)
+          {if(rc == "", do: nil, else: rc), c}
+        else
+          {nil, raw_text}
+        end
+
       completion = %ChatCompletion{
         id: "chatcmpl-" <> random_hex(12),
         object: "chat.completion",
@@ -335,7 +347,11 @@ defmodule LlamaCppEx do
         choices: [
           %{
             index: 0,
-            message: %{role: "assistant", content: Enum.join(texts)},
+            message: %{
+              role: "assistant",
+              content: content,
+              reasoning_content: reasoning_content
+            },
             finish_reason: finish_reason
           }
         ],
@@ -427,14 +443,19 @@ defmodule LlamaCppEx do
             )
           end)
 
+        enable_thinking = Keyword.get(chat_opts, :enable_thinking, false)
+
         %{
           ref: ref,
           gen_pid: gen_pid,
           timeout: timeout,
           id: id,
           created: created,
           model: model_name,
-          phase: :first
+          phase: :first,
+          enable_thinking: enable_thinking,
+          thinking_parser:
+            if(enable_thinking, do: Thinking.stream_parser(thinking: true), else: nil)
         }
       end,
       fn
@@ -452,15 +473,45 @@ defmodule LlamaCppEx do
         %{phase: :streaming, ref: ref, timeout: timeout} = state ->
           receive do
             {^ref, {:token, _id, text}} ->
-              chunk = %ChatCompletionChunk{
-                id: state.id,
-                object: "chat.completion.chunk",
-                created: state.created,
-                model: state.model,
-                choices: [%{index: 0, delta: %{content: text}, finish_reason: nil}]
-              }
-
-              {[chunk], state}
+              if state.enable_thinking do
+                {events, new_parser} = Thinking.feed(state.thinking_parser, text)
+                state = %{state | thinking_parser: new_parser}
+
+                chunks =
+                  Enum.map(events, fn
+                    {:thinking, t} ->
+                      %ChatCompletionChunk{
+                        id: state.id,
+                        object: "chat.completion.chunk",
+                        created: state.created,
+                        model: state.model,
+                        choices: [
+                          %{index: 0, delta: %{reasoning_content: t}, finish_reason: nil}
+                        ]
+                      }
+
+                    {:content, t} ->
+                      %ChatCompletionChunk{
+                        id: state.id,
+                        object: "chat.completion.chunk",
+                        created: state.created,
+                        model: state.model,
+                        choices: [%{index: 0, delta: %{content: t}, finish_reason: nil}]
+                      }
+                  end)
+
+                {chunks, state}
+              else
+                chunk = %ChatCompletionChunk{
+                  id: state.id,
+                  object: "chat.completion.chunk",
+                  created: state.created,
+                  model: state.model,
+                  choices: [%{index: 0, delta: %{content: text}, finish_reason: nil}]
+                }
+
+                {[chunk], state}
+              end
 
             {^ref, :eog} ->
               final_chunk = %ChatCompletionChunk{

lib/llama_cpp_ex/chat_completion.ex

@@ -16,7 +16,11 @@ defmodule LlamaCppEx.ChatCompletion do
 
   @type choice :: %{
           index: integer(),
-          message: %{role: String.t(), content: String.t()},
+          message: %{
+            role: String.t(),
+            content: String.t(),
+            reasoning_content: String.t() | nil
+          },
           finish_reason: String.t()
         }
 

lib/llama_cpp_ex/chat_completion_chunk.ex

@@ -15,7 +15,11 @@ defmodule LlamaCppEx.ChatCompletionChunk do
 
   @type chunk_choice :: %{
           index: integer(),
-          delta: %{optional(:role) => String.t(), optional(:content) => String.t()},
+          delta: %{
+            optional(:role) => String.t(),
+            optional(:content) => String.t(),
+            optional(:reasoning_content) => String.t()
+          },
           finish_reason: String.t() | nil
         }