Skip to content
This repository was archived by the owner on Nov 24, 2025. It is now read-only.

Commit 615fe87

Browse files
thxCodethxCode
committed
fix: tool call parsing in qwen3 coder
Signed-off-by: thxCode <thxcode0824@gmail.com>
1 parent 2f74ef5 commit 615fe87

4 files changed

Lines changed: 135 additions & 51 deletions

File tree

‎llama-box/httpserver.hpp‎

Lines changed: 61 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2919,12 +2919,15 @@ struct httpserver {
29192919
{
29202920
chat_templates = common_chat_templates_init(llm_model, params.llm_params.chat_template);
29212921

2922+
const auto * source = common_chat_templates_source(chat_templates.get());
29222923
// NB(thxCode): llama_chat_template_alias is a patch.
2923-
std::string alias = llama_chat_template_alias(common_chat_templates_source(chat_templates.get()));
2924+
std::string alias = llama_chat_template_alias(source);
29242925

29252926
if (params.llm_params.use_jinja) {
29262927
// NB(thxCode): common_chat_templates_supports_tool_calls is a patch.
2927-
support_tool_calls = common_chat_templates_supports_tool_calls(chat_templates.get());
2928+
support_tool_calls = common_chat_templates_supports_tool_calls(chat_templates.get());
2929+
// NB(thxCode): common_chat_templates_supports_parallel_tool_calls is a patch.
2930+
support_parallel_tool_calls = common_chat_templates_supports_parallel_tool_calls(chat_templates.get());
29282931
} else {
29292932
bool get_token = false;
29302933
// chatml / chatglm4
@@ -3009,7 +3012,7 @@ struct httpserver {
30093012
}
30103013

30113014
{
3012-
if (alias == "deepseek3" || string_starts_with(llm_model_arch_name, "qwen3")) {
3015+
if (alias == "deepseek3" || alias == "granite" || string_starts_with(llm_model_arch_name, "qwen3")) {
30133016
llama_tokens ids = common_tokenize(llm_vocab, "<think>", false, true);
30143017
if (ids.size() == 1) {
30153018
reasoning_start_token = ids[0];
@@ -3018,6 +3021,12 @@ struct httpserver {
30183021
if (ids.size() == 1) {
30193022
reasoning_end_token = ids[0];
30203023
}
3024+
// If the template does not contain the end token, we disable reasoning.
3025+
auto source_str = std::string(source);
3026+
if (source_str.find("</think>") == std::string::npos) {
3027+
reasoning_start_token = LLAMA_TOKEN_NULL;
3028+
reasoning_end_token = LLAMA_TOKEN_NULL;
3029+
}
30213030
} else if (alias == "command-r") {
30223031
llama_tokens ids = common_tokenize(llm_vocab, "<|START_THINKING|>", false, true);
30233032
if (ids.size() == 1) {
@@ -3027,6 +3036,12 @@ struct httpserver {
30273036
if (ids.size() == 1) {
30283037
reasoning_end_token = ids[0];
30293038
}
3039+
// If the template does not contain the end token, we disable reasoning.
3040+
auto source_str = std::string(source);
3041+
if (source_str.find("<|END_THINKING|>") == std::string::npos) {
3042+
reasoning_start_token = LLAMA_TOKEN_NULL;
3043+
reasoning_end_token = LLAMA_TOKEN_NULL;
3044+
}
30303045
}
30313046
support_reasoning = params.llm_params.reasoning_budget != 0 &&
30323047
reasoning_start_token != LLAMA_TOKEN_NULL &&
@@ -3371,6 +3386,7 @@ struct httpserver {
33713386

33723387
// tool calls
33733388
bool support_tool_calls = false;
3389+
bool support_parallel_tool_calls = true;
33743390
// non-jinja tool calls
33753391
llama_tokens tool_call_start_tokens = {};
33763392
std::vector<std::string> tool_call_start_words = {};
@@ -5344,45 +5360,46 @@ struct httpserver {
53445360
/* LLAMA */
53455361
else {
53465362
metadata_json = {
5347-
{ "vocab_type", llama_vocab_type(llm_vocab) },
5348-
{ "n_vocab", llama_vocab_n_tokens(llm_vocab) },
5349-
{ "n_ctx_train", llama_model_n_ctx_train(llm_model) },
5350-
{ "n_embd", llama_model_n_embd(llm_model) },
5351-
{ "n_params", llama_model_n_params(llm_model) },
5352-
{ "size", llama_model_size(llm_model) },
5353-
{ "n_ctx", llm_ctx_size },
5354-
{ "n_slot", 1 },
5355-
{ "n_slot_ctx", llm_ctx_size },
5356-
{ "ctx_shift", shift_context },
5357-
{ "prompt_cache", cache_prompt },
5358-
{ "seed", int32_t(params.llm_params.sampling.seed) },
5359-
{ "temperature", params.llm_params.sampling.temp },
5360-
{ "dynatemp_range", params.llm_params.sampling.dynatemp_range },
5361-
{ "dynatemp_exponent", params.llm_params.sampling.dynatemp_exponent },
5362-
{ "top_k", params.llm_params.sampling.top_k },
5363-
{ "top_p", params.llm_params.sampling.top_p },
5364-
{ "min_p", params.llm_params.sampling.min_p },
5365-
{ "top_n_sigma", params.llm_params.sampling.top_n_sigma },
5366-
{ "xtc_probability", params.llm_params.sampling.xtc_probability },
5367-
{ "xtc_threshold", params.llm_params.sampling.xtc_threshold },
5368-
{ "typical_p", params.llm_params.sampling.typ_p },
5369-
{ "repeat_last_n", params.llm_params.sampling.penalty_last_n },
5370-
{ "repeat_penalty", params.llm_params.sampling.penalty_repeat },
5371-
{ "presence_penalty", params.llm_params.sampling.penalty_present },
5372-
{ "frequency_penalty", params.llm_params.sampling.penalty_freq },
5373-
{ "dry_multiplier", params.llm_params.sampling.dry_multiplier },
5374-
{ "dry_base", params.llm_params.sampling.dry_base },
5375-
{ "dry_allowed_length", params.llm_params.sampling.dry_allowed_length },
5376-
{ "dry_penalty_last_n", params.llm_params.sampling.dry_penalty_last_n },
5377-
{ "dry_sequence_breakers", params.llm_params.sampling.dry_sequence_breakers },
5378-
{ "mirostat", params.llm_params.sampling.mirostat },
5379-
{ "mirostat_tau", params.llm_params.sampling.mirostat_tau },
5380-
{ "mirostat_eta", params.llm_params.sampling.mirostat_eta },
5381-
{ "support_vision", llm_ctx_clip_v != nullptr },
5382-
{ "support_audio", llm_ctx_clip_a != nullptr },
5383-
{ "support_speculative", llm_ctx_draft != nullptr },
5384-
{ "support_tool_calls", support_tool_calls },
5385-
{ "support_reasoning", support_reasoning },
5363+
{ "vocab_type", llama_vocab_type(llm_vocab) },
5364+
{ "n_vocab", llama_vocab_n_tokens(llm_vocab) },
5365+
{ "n_ctx_train", llama_model_n_ctx_train(llm_model) },
5366+
{ "n_embd", llama_model_n_embd(llm_model) },
5367+
{ "n_params", llama_model_n_params(llm_model) },
5368+
{ "size", llama_model_size(llm_model) },
5369+
{ "n_ctx", llm_ctx_size },
5370+
{ "n_slot", 1 },
5371+
{ "n_slot_ctx", llm_ctx_size },
5372+
{ "ctx_shift", shift_context },
5373+
{ "prompt_cache", cache_prompt },
5374+
{ "seed", int32_t(params.llm_params.sampling.seed) },
5375+
{ "temperature", params.llm_params.sampling.temp },
5376+
{ "dynatemp_range", params.llm_params.sampling.dynatemp_range },
5377+
{ "dynatemp_exponent", params.llm_params.sampling.dynatemp_exponent },
5378+
{ "top_k", params.llm_params.sampling.top_k },
5379+
{ "top_p", params.llm_params.sampling.top_p },
5380+
{ "min_p", params.llm_params.sampling.min_p },
5381+
{ "top_n_sigma", params.llm_params.sampling.top_n_sigma },
5382+
{ "xtc_probability", params.llm_params.sampling.xtc_probability },
5383+
{ "xtc_threshold", params.llm_params.sampling.xtc_threshold },
5384+
{ "typical_p", params.llm_params.sampling.typ_p },
5385+
{ "repeat_last_n", params.llm_params.sampling.penalty_last_n },
5386+
{ "repeat_penalty", params.llm_params.sampling.penalty_repeat },
5387+
{ "presence_penalty", params.llm_params.sampling.penalty_present },
5388+
{ "frequency_penalty", params.llm_params.sampling.penalty_freq },
5389+
{ "dry_multiplier", params.llm_params.sampling.dry_multiplier },
5390+
{ "dry_base", params.llm_params.sampling.dry_base },
5391+
{ "dry_allowed_length", params.llm_params.sampling.dry_allowed_length },
5392+
{ "dry_penalty_last_n", params.llm_params.sampling.dry_penalty_last_n },
5393+
{ "dry_sequence_breakers", params.llm_params.sampling.dry_sequence_breakers },
5394+
{ "mirostat", params.llm_params.sampling.mirostat },
5395+
{ "mirostat_tau", params.llm_params.sampling.mirostat_tau },
5396+
{ "mirostat_eta", params.llm_params.sampling.mirostat_eta },
5397+
{ "support_vision", llm_ctx_clip_v != nullptr },
5398+
{ "support_audio", llm_ctx_clip_a != nullptr },
5399+
{ "support_speculative", llm_ctx_draft != nullptr },
5400+
{ "support_tool_calls", support_tool_calls },
5401+
{ "support_parallel_tool_calls", support_parallel_tool_calls },
5402+
{ "support_reasoning", support_reasoning },
53865403
};
53875404
}
53885405

@@ -5952,7 +5969,8 @@ struct httpserver {
59525969
}
59535970
}
59545971

5955-
bool tool_call_stop_fast = !req->parallel_tool_calls || req->tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;
5972+
bool tool_call_stop_fast = !(support_parallel_tool_calls && req->parallel_tool_calls) ||
5973+
req->tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED || req->tools.size() <= 1;
59565974

59575975
// NB(thxCode): disable reasoning process if we need to generate tool calls in jinja.
59585976
bool reasoning_finished = !support_reasoning || (params.llm_params.use_jinja &&

‎llama-box/patches/llama.cpp/tool_calling.patch‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
diff --git a/common/chat.cpp b/common/chat.cpp
2-
index b1a1218ca..dea9e8727 100644
2+
index b1a1218ca..b0f097bd9 100644
33
--- a/common/chat.cpp
44
+++ b/common/chat.cpp
5-
@@ -502,6 +502,13 @@ void common_chat_templates_free(struct common_chat_templates * tmpls) {
5+
@@ -502,6 +502,20 @@ void common_chat_templates_free(struct common_chat_templates * tmpls) {
66
delete tmpls;
77
}
88

@@ -12,11 +12,18 @@ index b1a1218ca..dea9e8727 100644
1212
+ : *tmpls->template_default;
1313
+ return tmpl.original_caps().supports_tool_calls;
1414
+}
15+
+
16+
+bool common_chat_templates_supports_parallel_tool_calls(const struct common_chat_templates * tmpls) {
17+
+ const auto & tmpl = tmpls->template_tool_use
18+
+ ? *tmpls->template_tool_use
19+
+ : *tmpls->template_default;
20+
+ return tmpl.original_caps().supports_parallel_tool_calls;
21+
+}
1522
+
1623
bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls) {
1724
return tmpls->has_explicit_template;
1825
}
19-
@@ -2006,6 +2013,83 @@ static common_chat_params common_chat_templates_apply_jinja(
26+
@@ -2006,6 +2020,83 @@ static common_chat_params common_chat_templates_apply_jinja(
2027
return common_chat_params_init_generic(tmpl, params);
2128
}
2229

@@ -100,7 +107,7 @@ index b1a1218ca..dea9e8727 100644
100107
// Legacy template route (adhoc C++ implementation of known templates), forward to llama_chat_apply_template.
101108
static common_chat_params common_chat_templates_apply_legacy(
102109
const struct common_chat_templates * tmpls,
103-
@@ -2065,6 +2149,17 @@ static common_chat_params common_chat_templates_apply_legacy(
110+
@@ -2065,6 +2156,17 @@ static common_chat_params common_chat_templates_apply_legacy(
104111
return params;
105112
}
106113

@@ -118,7 +125,7 @@ index b1a1218ca..dea9e8727 100644
118125
common_chat_params common_chat_templates_apply(
119126
const struct common_chat_templates * tmpls,
120127
const struct common_chat_templates_inputs & inputs)
121-
@@ -2133,7 +2228,7 @@ common_chat_msg common_chat_parse(const std::string & input, bool is_partial, co
128+
@@ -2133,7 +2235,7 @@ common_chat_msg common_chat_parse(const std::string & input, bool is_partial, co
122129
try {
123130
common_chat_parse(builder);
124131
} catch (const common_chat_msg_partial_exception & ex) {
@@ -127,7 +134,7 @@ index b1a1218ca..dea9e8727 100644
127134
if (!is_partial) {
128135
builder.clear_tools();
129136
builder.move_to(0);
130-
@@ -2141,8 +2236,8 @@ common_chat_msg common_chat_parse(const std::string & input, bool is_partial, co
137+
@@ -2141,8 +2243,8 @@ common_chat_msg common_chat_parse(const std::string & input, bool is_partial, co
131138
}
132139
}
133140
auto msg = builder.result();
@@ -140,14 +147,15 @@ index b1a1218ca..dea9e8727 100644
140147
return msg;
141148
}
142149
diff --git a/common/chat.h b/common/chat.h
143-
index c4d6b2e85..a95fa3176 100644
150+
index c4d6b2e85..eddd3dbf2 100644
144151
--- a/common/chat.h
145152
+++ b/common/chat.h
146-
@@ -168,10 +168,15 @@ common_chat_templates_ptr common_chat_templates_init(
153+
@@ -168,10 +168,16 @@ common_chat_templates_ptr common_chat_templates_init(
147154
const std::string & bos_token_override = "",
148155
const std::string & eos_token_override = "");
149156

150157
+bool common_chat_templates_supports_tool_calls(const struct common_chat_templates * tmpls);
158+
+bool common_chat_templates_supports_parallel_tool_calls(const struct common_chat_templates * tmpls);
151159
bool common_chat_templates_was_explicit(const struct common_chat_templates * tmpls);
152160
const char * common_chat_templates_source(const struct common_chat_templates * tmpls, const char * variant = nullptr);
153161

‎llama-box/tools/chat_tool_square_of_number.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
3+
#
4+
# MIT license
5+
# Copyright (c) 2025 llama-box authors
6+
# SPDX-License-Identifier: MIT
7+
#
8+
9+
function square_of_number() {
10+
ARGS="${1}"
11+
ID="${2}"
12+
13+
INPUT_NUM=$(echo "${ARGS}" | jq -cr '.input_num')
14+
RESPONSE="$(echo "${INPUT_NUM}^2" | bc)"
15+
if [[ -z "${RESPONSE}" ]]; then
16+
MESSAGE="{\"role\":\"tool\",\"content\":\"{\\\"error\\\":\\\"Failed to calculate.\\\"}\",\"tool_call_id\":\"${ID}\"}"
17+
else
18+
MESSAGE="{\"role\":\"tool\",\"content\":\"{\\\"result\\\":\\\"${RESPONSE}\\\"}\",\"tool_call_id\":\"${ID}\"}"
19+
fi
20+
21+
echo "${MESSAGE}"
22+
}
23+
24+
function register_square_of_number() {
25+
TOOLNAMES+=("square_of_number")
26+
TOOLS+=("{\"type\":\"function\",\"function\":{\"name\":\"square_of_number\",\"description\":\"Output the square of the number.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"input_num\":{\"type\":\"number\",\"description\":\"input_num is a number that will be squared.\"}},\"required\":[\"input_num\"]}}}")
27+
}
28+
29+
register_square_of_number

‎llama-box/tools/chat_tool_square_root_of_number.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
3+
#
4+
# MIT license
5+
# Copyright (c) 2025 llama-box authors
6+
# SPDX-License-Identifier: MIT
7+
#
8+
9+
function square_root_of_number() {
10+
ARGS="${1}"
11+
ID="${2}"
12+
13+
INPUT_NUM=$(echo "${ARGS}" | jq -cr '.input_num')
14+
RESPONSE="$(echo "scale=4; sqrt(${INPUT_NUM})" | bc)"
15+
if [[ -z "${RESPONSE}" ]]; then
16+
MESSAGE="{\"role\":\"tool\",\"content\":\"{\\\"error\\\":\\\"Failed to calculate.\\\"}\",\"tool_call_id\":\"${ID}\"}"
17+
else
18+
MESSAGE="{\"role\":\"tool\",\"content\":\"{\\\"result\\\":\\\"${RESPONSE}\\\"}\",\"tool_call_id\":\"${ID}\"}"
19+
fi
20+
21+
echo "${MESSAGE}"
22+
}
23+
24+
function register_square_root_of_number() {
25+
TOOLNAMES+=("square_root_of_number")
26+
TOOLS+=("{\"type\":\"function\",\"function\":{\"name\":\"square_root_of_number\",\"description\":\"Output the square root of the number.\",\"parameters\":{\"type\":\"object\",\"properties\":{\"input_num\":{\"type\":\"number\",\"description\":\"input_num is the radicand.\"}},\"required\":[\"input_num\"]}}}")
27+
}
28+
29+
register_square_root_of_number

0 commit comments

Comments
 (0)